calog/tests/testExit.c

// testExit.c -- calogAbortAll: one native stops every script in the runtime.
//
// This is the contract API.md states for the runner's calogExit: it DOES NOT RETURN. A native that
// hands back calogAbortAll's value must unwind the CALLING script out of its chunk right at the call
// site, on every engine -- so each script below calls stopAll() and then spins in an infinite loop.
// The loop is the evidence: if the abort unwound the script it is never entered and the context
// retires within milliseconds, while a native that merely returned would pin that context forever.
// Native calls cannot serve as evidence here, because the latch refuses them either way.
//
// The same run also proves the three properties the abort must have: an aborted script is NOT
// reported to the error handler (it did not fail, and its engine must print nothing either), the
// runtime stays latched afterwards, and every later native call is refused instead of served.

#define _POSIX_C_SOURCE 200809L

#include "calog.h"

#include <stdatomic.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdint.h>
#include <string.h>
#include <time.h>

#define CHECK(cond, msg) checkImpl((cond), (msg), __LINE__)

// s7 interns a small, bounded set of "permanent" strings it never reclaims (an s7 trait, not a
// leak), so suppress exactly that site to keep the leak check meaningful. LSan calls this weak hook.
const char *__lsan_default_suppressions(void);
const char *__lsan_default_suppressions(void) {
    return "leak:make_permanent_string\n";
}

// A retiring context is joined by its own thread returning, so the wait is short in the passing
// case; the budget only has to cover a loaded machine (4000 * 0.5 ms = 2 s).
#define PUMP_INTERVAL_NS 500000
#define PUMP_LIMIT       4000

// One engine's script: reach mark(), abort, then loop forever if the abort let the script continue.
typedef struct EngineCaseT {
    const CalogEngineT *engine;
    const char         *name;
    const char         *source;
} EngineCaseT;

// One engine's "hand it something broken" case. `reports` is whether that engine can still produce a
// diagnostic once the runtime is latched aborting. It can when its PARSER rejects the source before
// any instruction runs; it cannot when the engine only discovers the problem by running the script,
// because the interpreter hook unwinds it first. That is a property of the engine's architecture,
// not a policy choice -- see the table in main.
typedef struct BrokenCaseT {
    const CalogEngineT *engine;
    const char         *name;
    const char         *source;
    bool                reports;
} BrokenCaseT;

static _Atomic int32_t markCount   = 0;
static _Atomic int32_t errorCount  = 0;
static int32_t         testsRun    = 0;
static int32_t         testsFailed = 0;

static void    checkEngine(bool condition, const char *engineName, const char *what, int32_t line);
static void    checkImpl(bool condition, const char *message, int32_t line);
static int32_t nativeMark(CalogValueT *args, int32_t argCount, CalogValueT *result, void *userData);
static int32_t nativeStopAll(CalogValueT *args, int32_t argCount, CalogValueT *result, void *userData);
static void    onError(uint64_t contextId, const char *message, void *userData);
static void    runBrokenCase(const BrokenCaseT *item);
static void    runCase(const EngineCaseT *item);
static void    testLatchIsPerRuntime(void);


static void checkEngine(bool condition, const char *engineName, const char *what, int32_t line) {
    char message[160];

    snprintf(message, sizeof(message), "%s: %s", engineName, what);
    checkImpl(condition, message, line);
}


static void checkImpl(bool condition, const char *message, int32_t line) {
    testsRun++;
    if (!condition) {
        testsFailed++;
        printf("FAIL testExit.c:%d  %s\n", line, message);
    }
}


// Host native: records that the script got this far. Registered like the runner's calogPrint, so the
// script blocks on the host thread for it -- the ordinary path a script reaches C through.
static int32_t nativeMark(CalogValueT *args, int32_t argCount, CalogValueT *result, void *userData) {
    (void)args;
    (void)argCount;
    (void)userData;
    atomic_fetch_add(&markCount, 1);
    calogValueNil(result);
    return calogOkE;
}


// Inline native: what the runner's calogExit does, minus recording an exit code. Inline is what
// makes the abort reach THIS script -- an error raised on the host thread could not unwind it.
//
// calogCurrentRetire is the test's probe, not part of what calogExit does. Retirement is serviced
// only AFTER the current eval returns, so "this context's thread exited" is precisely "the eval
// returned" -- which is the property under test, and the one thing a script cannot report itself
// once every native call is refused. A script that kept running would sit in its loop forever and
// its thread would never exit.
static int32_t nativeStopAll(CalogValueT *args, int32_t argCount, CalogValueT *result, void *userData) {
    (void)args;
    (void)argCount;
    (void)userData;
    calogValueNil(result);
    calogCurrentRetire();
    return calogAbortAll(calogCurrent(), result);
}


static void onError(uint64_t contextId, const char *message, void *userData) {
    (void)contextId;
    (void)userData;
    atomic_fetch_add(&errorCount, 1);
    printf("       (error handler saw: %s)\n", message != NULL ? message : "(null)");
}


// The other half of the contract: an abort must swallow only the errors it CAUSED. Here the runtime
// is latched aborting BEFORE a script with a syntax error is handed to the engine -- the exact
// window a real run hits when one script calls calogExit while another is still being loaded. The
// script did not fail because of the abort; it was broken to begin with, and its author needs to be
// told so. Checked on all ten engines: the four whose eval API compiles and runs in a single call
// (QuickJS, Tcl, mruby, s7) could not tell the two apart and silently ate the diagnostic, while the
// engines that split load from run were already reporting it. See BrokenCaseT for the one engine
// that cannot report here at all, and why.
static void runBrokenCase(const BrokenCaseT *item) {
    CalogT         *calog;
    CalogContextT  *ctx;
    CalogValueT     result;
    struct timespec tick = { 0, PUMP_INTERVAL_NS };
    int32_t         index;

    atomic_store(&errorCount, 0);

    calog = calogCreate();
    if (calog == NULL) {
        checkEngine(false, item->name, "runtime create failed", __LINE__);
        return;
    }
    calogSetErrorHandler(calog, onError, NULL);

    ctx = calogContextOpen(calog, item->engine);
    if (ctx == NULL) {
        checkEngine(false, item->name, "context open failed", __LINE__);
        calogDestroy(calog);
        return;
    }
    // Latch first: everything after this runs in the teardown window where the diagnostic used to
    // disappear. Nothing has run in this context, so no abort has been raised into its VM.
    calogValueNil(&result);
    calogAbortAll(calog, &result);
    calogValueFree(&result);

    calogContextEval(ctx, item->source);
    for (index = 0; index < PUMP_LIMIT; index++) {
        calogPump(calog);
        if (atomic_load(&errorCount) > 0) {
            break;
        }
        nanosleep(&tick, NULL);
    }
    calogPump(calog);

    if (item->reports) {
        checkEngine(atomic_load(&errorCount) > 0, item->name, "a script that is broken still reports, mid-abort", __LINE__);
    } else {
        checkEngine(atomic_load(&errorCount) == 0, item->name, "engine finds syntax errors only by running, so the abort wins", __LINE__);
    }

    calogContextClose(ctx);
    calogDestroy(calog);
}


// Run one engine's script on its own runtime (the latch is one-way, so each engine needs a fresh
// one), then check what the abort did.
static void runCase(const EngineCaseT *item) {
    CalogT         *calog;
    CalogContextT  *ctx;
    CalogValueT     result;
    struct timespec tick = { 0, PUMP_INTERVAL_NS };
    int32_t         index;
    bool            finished;

    atomic_store(&markCount, 0);
    atomic_store(&errorCount, 0);

    calog = calogCreate();
    if (calog == NULL) {
        checkEngine(false, item->name, "runtime create failed", __LINE__);
        return;
    }
    calogSetErrorHandler(calog, onError, NULL);
    calogRegister(calog, "mark", nativeMark, NULL);
    calogRegisterInline(calog, "stopAll", nativeStopAll, NULL);

    ctx = calogContextOpen(calog, item->engine);
    if (ctx == NULL) {
        checkEngine(false, item->name, "context open failed", __LINE__);
        calogDestroy(calog);
        return;
    }
    calogContextEval(ctx, item->source);
    finished = false;
    for (index = 0; index < PUMP_LIMIT; index++) {
        calogPump(calog);
        if (calogContextFinished(ctx)) {
            finished = true;
            break;
        }
        nanosleep(&tick, NULL);
    }
    calogPump(calog);

    checkEngine(atomic_load(&markCount) == 1, item->name, "the script ran up to the abort", __LINE__);
    checkEngine(finished, item->name, "the abort unwound the script -- the loop after it never ran", __LINE__);
    checkEngine(atomic_load(&errorCount) == 0, item->name, "an aborted script is not reported as an error", __LINE__);
    checkEngine(calogAborting(calog), item->name, "the runtime stays latched aborting", __LINE__);

    calogValueNil(&result);
    checkEngine(calogCall(calog, "mark", NULL, 0, &result) == calogErrAbortE, item->name, "a later native call is refused", __LINE__);
    calogValueFree(&result);
    checkEngine(atomic_load(&markCount) == 1, item->name, "the refused call never reached the native", __LINE__);

    // A context that did NOT retire is still spinning in that infinite loop, and closing it would
    // join a thread that never returns -- hanging the test instead of reporting the failure. Leak it
    // (the check above has already failed) and move on to the next engine.
    if (finished) {
        calogContextClose(ctx);
        calogDestroy(calog);
    }
}


// The latch belongs to ONE runtime: aborting a runtime must not stop scripts in a second one that
// happens to share the process. This also covers calogAbortAll called from the host thread, which
// has no script of its own to unwind.
static void testLatchIsPerRuntime(void) {
    CalogT     *first;
    CalogT     *second;
    CalogValueT result;

    first  = calogCreate();
    second = calogCreate();
    if (first == NULL || second == NULL) {
        CHECK(false, "per-runtime latch: runtime create failed");
        calogDestroy(first);
        calogDestroy(second);
        return;
    }
    calogRegister(first, "mark", nativeMark, NULL);
    calogRegister(second, "mark", nativeMark, NULL);
    atomic_store(&markCount, 0);

    calogValueNil(&result);
    CHECK(calogAbortAll(first, &result) == calogErrAbortE, "calogAbortAll reports calogErrAbortE");
    calogValueFree(&result);
    CHECK(calogAborting(first), "the aborted runtime is latched");
    CHECK(!calogAborting(second), "a second runtime is untouched");

    calogValueNil(&result);
    CHECK(calogCall(first, "mark", NULL, 0, &result) == calogErrAbortE, "the aborted runtime refuses a native");
    calogValueFree(&result);
    calogValueNil(&result);
    CHECK(calogCall(second, "mark", NULL, 0, &result) == calogOkE, "the second runtime still serves natives");
    calogValueFree(&result);
    CHECK(atomic_load(&markCount) == 1, "exactly one of the two calls ran the native");

    calogDestroy(first);
    calogDestroy(second);
}


int main(void) {
    // Each script: reach the host, abort, then a loop that must never be entered. The loop bodies
    // differ only in each language's syntax for "forever".
    static const EngineCaseT cases[] = {
        { &calogLuaEngine,      "lua",      "mark()\nstopAll()\nwhile true do end" },
        { &calogJsEngine,       "js",       "mark(); stopAll(); while (true) {}" },
        { &calogSquirrelEngine, "squirrel", "mark(); stopAll(); while(true){}" },
        { &calogMyBasicEngine,  "my-basic", "mark()\nstopAll()\nWHILE 1\nWEND" },
        { &calogBerryEngine,    "berry",    "mark()\nstopAll()\nwhile true\nend" },
        { &calogS7Engine,       "s7",       "(begin (mark) (stopAll) (do () (#f)))" },   // s7 reads ONE top-level form
        { &calogWrenEngine,     "wren",     "Calog.call(\"mark\", [])\nCalog.call(\"stopAll\", [])\nwhile (true) {}" },
        { &calogMrubyEngine,    "mruby",    "mark()\nstopAll()\nwhile true do end" },
        { &calogTclEngine,      "tcl",      "mark\nstopAll\nwhile {1} {}" },
        { &calogJanetEngine,    "janet",    "(mark) (stopAll) (var i 0) (while true (set i (+ i 1)))" }
    };
    // The same ten engines, each handed something it can only reject.
    //
    // Nine of them find the problem with their PARSER, before a single instruction runs, so the
    // diagnostic survives the abort. my-basic is the exception, and not by choice: mb_load_string
    // accepts almost anything and the error only appears once the statement RUNS -- by which point
    // the step hook has already unwound the script, exactly as it must for Ctrl-C to work on a
    // runaway loop. The two cannot both hold on an engine with no separate parse step, so the
    // asymmetry is asserted here rather than papered over.
    static const BrokenCaseT broken[] = {
        { &calogLuaEngine,      "lua",      "if if if",          true  },
        { &calogJsEngine,       "js",       "function ( { )",    true  },
        { &calogSquirrelEngine, "squirrel", "function ( { )",    true  },
        { &calogMyBasicEngine,  "my-basic", "IF IF IF",          false },
        { &calogBerryEngine,    "berry",    "def def def",       true  },
        { &calogS7Engine,       "s7",       "(((",               true  },
        { &calogWrenEngine,     "wren",     "class class class", true  },
        { &calogMrubyEngine,    "mruby",    "def def def",       true  },
        { &calogTclEngine,      "tcl",      "set x [",           true  },
        { &calogJanetEngine,    "janet",    "(((",               true  }
    };
    size_t index;

    for (index = 0; index < sizeof(cases) / sizeof(cases[0]); index++) {
        runCase(&cases[index]);
    }
    for (index = 0; index < sizeof(broken) / sizeof(broken[0]); index++) {
        runBrokenCase(&broken[index]);
    }
    testLatchIsPerRuntime();

    printf("testExit: %d checks, %d failed\n", testsRun, testsFailed);
    return testsFailed == 0 ? 0 : 1;
}