From fe0fe73884b646223949d3bc3fcc5ca5752b51fc Mon Sep 17 00:00:00 2001 From: Scott Duensing Date: Fri, 24 Jul 2026 23:03:13 -0500 Subject: [PATCH] Starting to dogfood this. Many issues fixed. --- API.md | 9 +- Makefile | 98 +++++++- PORTING.md | 106 ++++++++ README.md | 25 +- design.md | 428 +++++++++++++++++++++++++++++++++ examples/scripts/README.md | 9 +- libs/calogExport.c | 14 ++ libs/calogPubsub.c | 13 + src/berry/berryAdapter.c | 23 +- src/broker.c | 7 + src/calog.h | 25 +- src/calogInternal.h | 31 +++ src/calogMain.c | 151 +++++++++--- src/context.c | 301 +++++++++++++++++++++-- src/janet/janetAdapter.c | 44 +++- src/js/jsAdapter.c | 9 +- src/lua/luaAdapter.c | 6 + src/mruby/mrubyAdapter.c | 8 + src/mybasic/mybasicAdapter.c | 5 + src/s7/s7Adapter.c | 5 + src/squirrel/squirrelAdapter.c | 6 + src/tcl/tclAdapter.c | 5 + src/value.c | 68 +++++- src/wren/wrenAdapter.c | 8 + tests/testBroker.c | 4 +- tests/testExit.c | 238 ++++++++++++++++++ tests/testHooks.c | 29 ++- tests/testTask.c | 50 ++++ tests/testTeardown.c | 347 ++++++++++++++++++++++++++ tools/crossBuild.sh | 167 ++++++++++++- tools/crossDeps.sh | 69 +++++- tools/crossMacFull.sh | 4 +- tools/crossWinFull.sh | 4 +- vendor/ourbasic/CHANGELOG | 18 ++ vendor/ourbasic/ourBasic.c | 43 +++- 35 files changed, 2280 insertions(+), 97 deletions(-) create mode 100644 tests/testExit.c create mode 100644 tests/testTeardown.c diff --git a/API.md b/API.md index 14733ea8..f77c779e 100644 --- a/API.md +++ b/API.md @@ -14,6 +14,12 @@ etc. -- see the README and `src/calog.h`.) - **Output & exit** (provided by the `calog` runner): `calogPrint(...)` writes to stdout; `calogExit([code])` tears everything down and exits. calog is event-driven, so a script must call `calogExit` (or be interrupted) to end -- a finished top level does not exit. + `calogExit` **does not return**: it stops the calling script right there (the statement after + it never runs), and every other live script stops at its next native call. So the first + `calogExit` decides the process's status -- a `calogExit(1)` on a failed check cannot be + followed by more work, nor overwritten by a later `calogExit(0)`. + `calogEnd([code])` ends **only the calling script**, the same way: it does not return, its context + is reaped, and the other scripts carry on. When the last one ends, `calog` exits. - **Values.** Arguments and results marshal through one canonical type: `nil`, `bool`, `int`, `real`, `string`, `list`, and `map` (keyed record). Strings are **binary-safe** (may contain embedded NULs) everywhere the underlying library allows it. @@ -107,7 +113,8 @@ egress (JavaScript, Berry, s7, mruby); Janet loses `symbol`/`keyword` subtype (i | Function | Description | |---|---| | `calogPrint(...values: any)` | Write each argument to stdout, space-separated, with a trailing newline. | -| `calogExit([code: int])` | Tear down the runtime and exit the process with `code` (default `0`). Does not return. | +| `calogEnd([code: int])` | End **this script only**, leaving every other script running. **Does not return** -- it unwinds the calling script at the call site. Its context is then reaped, and once every launched script has ended, `calog` exits. A `code` given here names the process's status like `calogExit`'s does (first request wins); omitting it claims nothing. | +| `calogExit([code: int])` | Tear down the runtime and exit the process with `code` (default `0`). **Does not return** -- it unwinds the calling script at the call site, so nothing after it runs, and stops every other live script at its next native call. The first caller's `code` is the one reported; a script that catches the unwind still cannot call another native. | ## archive diff --git a/Makefile b/Makefile index f6101ee6..2ca5d2e1 100644 --- a/Makefile +++ b/Makefile @@ -340,7 +340,7 @@ $(PCRE2LIB): BINS = bin/testBroker bin/testLua bin/testMyBasic bin/testPolyglot bin/testActor bin/testHooks \ bin/testEngineLua bin/testEngineMyBasic bin/testSquirrel bin/testEngineSquirrel bin/testJs bin/testEngineJs \ - bin/testEngineBerry bin/testEngineS7 bin/testEngineWren bin/testEngineMruby bin/testEngineTcl bin/testEngineJanet bin/testLoad bin/testDb bin/testNet bin/testTask bin/testExport bin/testJson bin/testXml bin/testTime bin/testFs bin/testCrypto bin/testKv bin/testTimer bin/testPubsub bin/testHttp bin/testHttps bin/testArchive bin/testUtil bin/testSandbox bin/testTrace bin/testHttpdLua bin/embed bin/calog + bin/testEngineBerry bin/testEngineS7 bin/testEngineWren bin/testEngineMruby bin/testEngineTcl bin/testEngineJanet bin/testLoad bin/testDb bin/testNet bin/testTask bin/testExport bin/testJson bin/testXml bin/testTime bin/testFs bin/testCrypto bin/testKv bin/testTimer bin/testPubsub bin/testHttp bin/testHttps bin/testArchive bin/testUtil bin/testSandbox bin/testExit bin/testTeardown bin/testTrace bin/testHttpdLua bin/embed bin/calog all: $(BINS) @@ -352,7 +352,7 @@ $(STRICTOBJ): obj/%.o: %.c | obj $(CC) $(COREFLAGS) $(INC) -c -o $@ $< # strict C, threaded -THREADOBJ = obj/context.o obj/mybasicEngine.o obj/testActor.o obj/testEngineLua.o obj/testEngineSquirrel.o obj/testEngineJs.o obj/testEngineMyBasic.o obj/testEngineBerry.o obj/testEngineS7.o obj/testEngineWren.o obj/testEngineMruby.o obj/testEngineTcl.o obj/testEngineJanet.o obj/calogHandle.o obj/testDb.o obj/testNet.o obj/testTask.o obj/calogExport.o obj/testExport.o obj/calogJson.o obj/testJson.o obj/calogFs.o obj/testFs.o obj/calogTime.o obj/testTime.o obj/calogKv.o obj/testKv.o obj/testCrypto.o obj/calogTimer.o obj/testTimer.o obj/calogPubsub.o obj/testPubsub.o obj/testHttp.o obj/testSsh.o obj/calogCsv.o obj/calogProc.o obj/testSandbox.o obj/testTrace.o obj/testHttpdLua.o obj/testHooks.o +THREADOBJ = obj/context.o obj/mybasicEngine.o obj/testActor.o obj/testEngineLua.o obj/testEngineSquirrel.o obj/testEngineJs.o obj/testEngineMyBasic.o obj/testEngineBerry.o obj/testEngineS7.o obj/testEngineWren.o obj/testEngineMruby.o obj/testEngineTcl.o obj/testEngineJanet.o obj/calogHandle.o obj/testDb.o obj/testNet.o obj/testTask.o obj/calogExport.o obj/testExport.o obj/calogJson.o obj/testJson.o obj/calogFs.o obj/testFs.o obj/calogTime.o obj/testTime.o obj/calogKv.o obj/testKv.o obj/testCrypto.o obj/calogTimer.o obj/testTimer.o obj/calogPubsub.o obj/testPubsub.o obj/testHttp.o obj/testSsh.o obj/calogCsv.o obj/calogProc.o obj/testSandbox.o obj/testExit.o obj/testTeardown.o obj/testTrace.o obj/testHttpdLua.o obj/testHooks.o $(THREADOBJ): obj/%.o: %.c | obj $(CC) $(COREFLAGS) $(INC) -pthread -c -o $@ $< @@ -741,6 +741,15 @@ bin/testUtil: obj/testUtil.o obj/calogCsv.o obj/calogProc.o obj/calogRegex.o lib bin/testSandbox: obj/testSandbox.o lib/libcalog.a lib/liblua.a lib/libquickjs.a lib/libsquirrel.a lib/libmybasic.a lib/libberry.a lib/libs7.a lib/libwren.a lib/libjanet.a $(MRUBYLIB) $(TCLLIB) | bin $(CC) $(LDFLAGS) -pthread -o $@ $^ $(LUALIBS) $(CXXLIB) -lm $(MRUBYLIBS) $(TCLLIBS) +# Destroy-hook phases: a registry holding script functions must drain while contexts are alive. +# JavaScript on purpose -- QuickJS aborts at JS_FreeRuntime if anything of its own is still live. +bin/testTeardown: obj/testTeardown.o obj/calogPubsub.o obj/calogExport.o obj/calogTimer.o lib/libcalog.a lib/liblua.a lib/libquickjs.a | bin + $(CC) $(LDFLAGS) -pthread -o $@ $^ $(LUALIBS) -lm + +# calogAbortAll on every engine (what the runner's calogExit rides on), so it links all ten. +bin/testExit: obj/testExit.o lib/libcalog.a lib/liblua.a lib/libquickjs.a lib/libsquirrel.a lib/libmybasic.a lib/libberry.a lib/libs7.a lib/libwren.a lib/libjanet.a $(MRUBYLIB) $(TCLLIB) | bin + $(CC) $(LDFLAGS) -pthread -o $@ $^ $(LUALIBS) $(CXXLIB) -lm $(MRUBYLIBS) $(TCLLIBS) + bin/testTrace: obj/testTrace.o obj/calogExport.o lib/libcalog.a lib/liblua.a lib/libquickjs.a | bin $(CC) $(LDFLAGS) -pthread -o $@ $^ $(LUALIBS) @@ -765,10 +774,10 @@ ssh-test: bin/testSsh obj bin lib: mkdir -p $@ -test: all +test: all cross-lint ./bin/testBroker && ./bin/testLua && ./bin/testMyBasic && ./bin/testPolyglot && \ ./bin/testActor && ./bin/testHooks && ./bin/testEngineLua && ./bin/testEngineMyBasic && ./bin/testSquirrel && ./bin/testEngineSquirrel && \ - ./bin/testJs && ./bin/testEngineJs && ./bin/testEngineBerry && ./bin/testEngineS7 && ./bin/testEngineWren && ./bin/testEngineMruby && ./bin/testEngineTcl && ./bin/testEngineJanet && ./bin/testLoad && ./bin/testDb && ./bin/testNet && ./bin/testTask && ./bin/testExport && ./bin/testJson && ./bin/testXml && ./bin/testTime && ./bin/testFs && ./bin/testCrypto && ./bin/testKv && ./bin/testTimer && ./bin/testPubsub && ./bin/testHttp && ./bin/testHttps && ./bin/testArchive && ./bin/testUtil && ./bin/testSandbox && ./bin/testTrace && ./bin/testHttpdLua + ./bin/testJs && ./bin/testEngineJs && ./bin/testEngineBerry && ./bin/testEngineS7 && ./bin/testEngineWren && ./bin/testEngineMruby && ./bin/testEngineTcl && ./bin/testEngineJanet && ./bin/testLoad && ./bin/testDb && ./bin/testNet && ./bin/testTask && ./bin/testExport && ./bin/testJson && ./bin/testXml && ./bin/testTime && ./bin/testFs && ./bin/testCrypto && ./bin/testKv && ./bin/testTimer && ./bin/testPubsub && ./bin/testHttp && ./bin/testHttps && ./bin/testArchive && ./bin/testUtil && ./bin/testSandbox && ./bin/testExit && ./bin/testTeardown && ./bin/testTrace && ./bin/testHttpdLua # ThreadSanitizer build of the actor core and the Lua engine path (cannot combine # with ASan). Recompiled from source under TSan; the vendored Lua objects are @@ -928,10 +937,91 @@ fuzz: bin/fuzzJson bin/fuzzCsv bin/fuzzXml fuzz-smoke: fuzz @for f in Json Csv Xml; do echo "== fuzz-smoke $$f ($(FUZZSECS)s) =="; ./bin/fuzz$$f -max_total_time=$(FUZZSECS) -print_final_stats=1 || exit 1; done +# ---- cross-compilation: Windows PE, macOS Mach-O (x86_64 + arm64), musl-static ---------------- +# NOT part of `make test`: these take minutes and need a zig toolchain, which a contributor building +# calog natively does not have to install -- `make`, `make test`, the tsan targets and `make fuzz` +# stay zig-free. The ONE exception is cross-lint below: zig-free, sub-second, and wired into `make +# test`, because the drift it catches is exactly how the cross builds went stale unnoticed for weeks +# (a source file or an ABI-visible define that never reached the cross scripts). What it cannot catch +# -- a target-specific runtime bug, a stale pinned dep -- needs a real cross build: `make cross`. +# +# Nothing here writes into obj/ bin/ lib/; all cross output lands under build/cross/ (gitignored), so +# a cross build never disturbs the sanitized native build in the same tree. +# +# zig lives outside the repo, resolved the way tools/cross*.sh do. ZIGBIN is recursive (`=`), so a +# plain `make` or `make test` never even runs the lookup. +ZIGDEFAULT = /home/scott/zig/current/zig +ZIG ?= $(firstword $(CALOG_ZIG) $(ZIGDEFAULT)) +ZIGBIN = $(shell command -v $(ZIG) 2>/dev/null || { [ -x $(ZIG) ] && echo $(ZIG); }) +CROSSENV = ZIG="$(ZIGBIN)" CALOG_ZIG="$(ZIGBIN)" + +# Compile macros that are ABI-visible (object layout / integer width) rather than cosmetic: an +# adapter and its VM MUST agree on every one. MRB_USE_DEBUG_HOOK is the one that silently broke the +# Windows and macOS CLIs for weeks -- mrb_state grew a member the cross libmruby.a did not have. +CROSSFULL = tools/crossWinFull.sh tools/crossMacFull.sh +CROSSDEFS = _SQ64 SQUSEDOUBLE MB_DOUBLE_FLOAT MRB_USE_DEBUG_HOOK + +# Zig-free, hermetic, sub-second. Every calog source the native build compiles must also be compiled +# by both full-CLI cross scripts, and every ABI define + engine selector must reach them. Text-level +# on purpose: it needs no toolchain, so `make test` can run it. +cross-lint: + @bad=0; \ + for f in libs/*.c src/*.c src/*/*.c; do \ + n=`basename $$f .c`; \ + for s in $(CROSSFULL); do \ + grep -q "$$n" $$s || { echo "cross-lint: $$f is compiled natively but not by $$s" >&2; bad=1; }; \ + done; \ + done; \ + for d in $(CROSSDEFS) `grep -oE 'CALOG_WITH_[A-Z0-9_]+' Makefile | sort -u`; do \ + for s in Makefile $(CROSSFULL); do \ + grep -q -- "$$d" $$s || { echo "cross-lint: -D$$d is missing from $$s" >&2; bad=1; }; \ + done; \ + done; \ + if [ $$bad -eq 0 ]; then echo "== cross-lint ok: cross scripts cover every calog source, ABI define and engine =="; \ + else echo "== cross-lint FAILED: tools/cross*Full.sh drifted from the native build; fix them or run 'make cross' ==" >&2; exit 1; fi + +# One place that decides whether a cross build can run. A hard error, never a silent skip: each +# target below was asked for explicitly, and a skip that reports success is how a broken cross build +# hides. (`make test` never reaches here -- it needs no zig.) +zig-check: + @test -n "$(ZIGBIN)" || { \ + echo "error: zig not found (looked for '$(ZIG)')." >&2; \ + echo " The cross-* targets need a zig toolchain: https://ziglang.org/download/" >&2; \ + echo " Point make at a PERMANENT install: make ZIG=/path/to/zig $(MAKECMDGOALS)" >&2; \ + echo " The native build, 'make test', tsan and fuzz do NOT need zig." >&2; \ + exit 1; } + @echo "== zig: $(ZIGBIN) (`$(ZIGBIN) version`) ==" + +# The representative matrix: all ten engines for musl (built AND RUN here -- the only run-verified +# cross target), Windows PE and macOS Mach-O (both arches), plus calogNet and a real TLS run. +cross-smoke: zig-check + $(CROSSENV) bash ./tools/crossBuild.sh + +# The full CLI per target. These consume the pinned heavy deps under build/cross//, which are +# gitignored -- build them first with the cross-deps-* targets if a fresh checkout lacks them. +cross-win: zig-check + $(CROSSENV) bash ./tools/crossWinFull.sh + +cross-mac: zig-check + $(CROSSENV) bash ./tools/crossMacFull.sh + +cross-deps-win: zig-check + $(CROSSENV) bash ./tools/crossDeps.sh win +cross-deps-mac: zig-check + $(CROSSENV) bash ./tools/crossDeps.sh mac-x64 + $(CROSSENV) bash ./tools/crossDeps.sh mac-arm64 +# musl is a partial target: the deps the cross ENGINE tests need (see tools/crossDeps.sh usage). +cross-deps-musl: zig-check + $(CROSSENV) bash ./tools/crossDeps.sh musl codecs openssl tcl mruby + +# Everything that can be verified here, in the order that fails fastest. +cross: cross-lint cross-smoke cross-win cross-mac + clean: rm -rf obj bin lib -include $(wildcard obj/*.d) -include $(wildcard obj/rel/*.d) +.PHONY: cross cross-lint cross-smoke cross-win cross-mac cross-deps-win cross-deps-mac cross-deps-musl zig-check .PHONY: all test tsan tsansq tsanjs tsanmb tsanberry tsans7 tsanwren tsanmruby tsantcl tsanjanet tsanlibs tsanhttpd release fuzz fuzz-smoke clean diff --git a/PORTING.md b/PORTING.md index 0ae49997..23408095 100644 --- a/PORTING.md +++ b/PORTING.md @@ -32,6 +32,22 @@ A socket `fd` is unsigned on Windows, so the code compares against `CALOG_INVALI `fd < 0`). The abstraction is behavior-identical on POSIX (it maps to the same names), so the Linux and macOS builds are unchanged by it. +### What the core assumes beyond C11 + +Sockets are the only surface that needed abstracting, but the core is not pure C11 either -- it +leans on three POSIX facilities that a new target must supply, none of them behind a wrapper: + +| Facility | Used for | On Windows | +|---|---|---| +| `pthread_*` (threads, mutexes, condvars) | one thread per context, the message queues, the context registry | vendored **winpthreads** | +| `clock_gettime(CLOCK_MONOTONIC)` | `calogMonotonicMillis`: timer deadlines and the sandbox wall-clock budget | mingw-w64 | +| `nanosleep` | the host pump's park between drains, and the teardown's wait for an in-flight context close | mingw-w64 | + +All three already resolve on the mingw-w64 / zig toolchain: `src/calogMain.c` calls `nanosleep` +unconditionally from its pump loop and is part of the Windows CLI that cross-built successfully, so +the teardown's use of it adds no new requirement. A target lacking any of them needs a shim in +`calogPlatform.h` rather than a change to the core. + ## Build matrix | Target | Command | Runtime dependencies | DNS | @@ -186,6 +202,96 @@ Everything lands under `build/cross/` (gitignored); the vendored sources and the ## Status +### Cross coverage as of 2026-07-24 (zig 0.16.0) + +`tools/crossBuild.sh` now reports **39 ok, 0 failed**, and covers **all ten engines on all four +targets** rather than four engines on some: + +| Target | What runs | +|---|---| +| musl static | all ten engine suites **RUN** (fully static, on this host), plus `testNet` and `testHttps` | +| Windows x64 PE | all ten engine suites + `testNet` build and link as PE32+ | +| macOS x86_64 + arm64 | all ten engine suites + `testNet` build and link as Mach-O | + +Two of those are new kinds of evidence, not just more of the same: + +- **TLS is exercised at runtime**, not merely linked. `testHttps` cross-builds fully static for musl + and runs here (6/6): an in-process RSA key + self-signed cert, a loopback TLS server, a rejected + untrusted cert, and the pinned-trust path -- all through the cross-built OpenSSL. It runs under a + strict runner (`mrunStrict`) because the ordinary one counts a nonzero exit as a pass, which would + have reported a broken handshake as success. +- **`tools/crossDeps.sh musl` is a full target**: every dep except `winpthreads` (Windows-only) and + `libarchive` (musl goes through `tools/crossArchive.sh`, which handles its iconv gotcha). Tcl and + mruby for musl are what let the last two engines into the matrix. Tcl needed two musl-specific + turns: configure picks the epoll notifier from the *host's* headers and its `tclEpollNotfy.c` wants + ``, which musl does not ship (so the select notifier is selected instead), and with no + system zlib it must be pointed at the vendored one. + +And the drift that started all of this is now caught mechanically: **`make cross-lint`** is a +zig-free, sub-second parity check wired into `make test`. It fails if a calog source the native build +compiles never reached both full-CLI cross scripts, or if an ABI-visible define (`MRB_USE_DEBUG_HOOK` +and friends) is missing from one of them -- exactly the class of drift that broke the Windows CLI for +weeks. `make cross` runs the real thing; `make cross-smoke`, `cross-win`, `cross-mac` and +`cross-deps-*` are the pieces. None of them are in `make test`, which stays hermetic and zig-free. + +### Re-verified 2026-07-24 (zig 0.16.0) + +The cross-builds were re-run after the teardown/`calogEnd` work. **The core changes port cleanly -- +`src/context.c`, `src/value.c` and `src/broker.c` compile, link and run on every target** -- but the +run exposed three problems that all predate that work. All three are now fixed: + +| Target | Result | +|---|---| +| musl static (built + run) | Lua, JavaScript, Squirrel, my-basic, **and `testNet` (10/10)** all pass | +| Windows x64 PE | `testEngineLua.exe` and `testNet.exe` **build**; the **full `calog.exe` builds** | +| macOS x86_64 + arm64 | `testEngineLua` and `testNet` **build** (both arches) | + +`tools/crossBuild.sh` reports **11 ok, 0 failed**. + +- **`testNet` on every target -- was failing, now fixed.** `libs/calogNet.c` has included + `` since the tcp transport gained TLS, and `tools/crossBuild.sh` passed no OpenSSL + flags: its `testNet` case predated that include. The test itself uses no TLS -- only the library it + links does -- so the fix is to link the per-target OpenSSL that `tools/crossDeps.sh` already + produces for win/mac, plus a new **`musl` target** in that script (OpenSSL only -- the full musl + CLI is `make static` on Alpine, not a cross build). `crossBuild.sh` now links it, and when a + target's OpenSSL has not been built it SKIPS with the exact command to produce it rather than + reporting a failure it cannot fix. The musl `testNet` is fully static and **runs 10/10 on the + build host**, matching the native run. +- **`testEngineMyBasic` on musl -- was 13 of 20 failing, now fixed (20/20).** The fork decided "this + symbol is a number" from `strtoll`/`strtod` alone reaching the string terminator. That is not + portable: **musl's `strtoll` advances `endptr` past leading whitespace and a sign even when no + conversion happens**, while glibc leaves it at the start. So on musl `+` and `-` (the operators) + and `\n` (the statement separator) each classified as the integer 0 -- every expression containing + an operator failed to parse, and every numeric assignment failed to run, both reported as + "Operator expected". Reproduced with a pure my-basic program containing no calog code at all + (`x = 1` -> `MB_FUNC_ERR` on musl, `MB_FUNC_OK` on glibc). Fixed in `vendor/ourbasic/ourBasic.c` + by rejecting a consumed span that is nothing but whitespace and sign; see that fork's CHANGELOG. + Worth remembering when porting: **a failed `strtol` does not leave `endptr` where you assume.** +- **The full CLI on Windows and macOS -- was broken, now fixed.** `tools/crossWinFull.sh` stopped at + `mrubyAdapter.c: no member named 'code_fetch_hook' in 'struct mrb_state'`. `src/mruby/build_config.rb` + defines `MRB_USE_DEBUG_HOOK` (the per-instruction hook the sandbox wall-clock budget needs) and the + Makefile passes the same macro to the adapter compile, but the cross build_config that + `tools/crossDeps.sh` generated defined only `MRB_INT64`, and neither `cross*Full.sh` passed the + macro -- so the cross `libmruby.a` had a different `mrb_state` layout than the adapter expected. + It had been broken since the sandbox-parity work landed (the previous `calog.exe` dated from just + before it), and failing loudly at compile time was the good outcome: the alternative was an ABI + mismatch. The macro is now in the generated cross config **and** in both `cross*Full.sh` adapter + compiles, and mruby was rebuilt for all three targets. **Rebuilt and re-verified:** + + | Artifact | Result | + |---|---| + | `build/cross/win/calog.exe` | PE32+ x86-64, imports only Windows system DLLs | + | `build/cross/mac-x64/calog` | Mach-O 64-bit x86_64, PIE | + | `build/cross/mac-arm64/calog` | Mach-O 64-bit arm64, PIE | + + Each links all ten engines and every library, and each carries this session's runtime changes + (`calogEnd` and the abort message are in the binaries). Still build-verified only -- running them + needs a Windows/wine or Mac host. + +The bullets below record the state at the time each port was done; where they disagree with the +table above, the table is current. + + - **Platform abstraction** (`calogPlatform.h`) and the net/ssh/http conversion: **done**, verified behavior-identical on Linux (`make test` 30/30, socket + HTTP loopback smoke, ASan- and TSan-clean). - **Linux release** target: **done and verified** -- `ldd bin/calog` shows only `libc`/`libm` + loader. diff --git a/README.md b/README.md index cf479fcf..3713036d 100644 --- a/README.md +++ b/README.md @@ -149,8 +149,11 @@ bin/calog config # no extension -> search config.lua / .js / bin/calog producer.js consumer.lua # several files share one runtime (kv, pubsub, exports) ``` -Scripts print with `calogPrint(...)` and end by calling `calogExit([code])` (calog is -event-driven, so a script asks to exit; `Ctrl-C` also works). [`API.md`](API.md) documents +Scripts print with `calogPrint(...)` and stop by calling `calogExit([code])`, which tears +everything down, or `calogEnd([code])`, which ends only the calling script and lets the others +run on (calog is event-driven, so a script asks to stop; `Ctrl-C` also works). Neither +returns -- the statement after it never runs, so a build script's `calogExit(1)` is final -- +and once every script has ended, `calog` exits on its own. [`API.md`](API.md) documents every native a script can call; [`examples/scripts/`](examples/scripts/) has runnable examples across every engine and library, plus polyglot and multi-file demos. @@ -317,6 +320,10 @@ calogValueFree(&arg); calogFnRelease(savedCb); ``` +A reference you hold stays yours to release even after the script that made it is gone: as a +context stops it releases the engine handle behind every callable it owns, so `calogFnInvoke` +then reports `calogErrDeadE` instead of reaching into a torn-down interpreter. + It works the other way too: wrap one of your natives as a function value with `calogFnFromNative` and return it from a native, and the script gets a callable it can invoke (which routes back to your host thread). Every engine supports this; a script invokes @@ -406,6 +413,11 @@ void calogContextClose(CalogContextT *); uint64_t calogContextId(const CalogContextT *); uint64_t calogCurrentId(void); // 0 on the host thread CalogT *calogCurrent(void); + +// stopping scripts (what bin/calog's calogExit and calogEnd are built on) +int32_t calogAbortAll(CalogT *, CalogValueT *result); // return this from a native: every script stops +int32_t calogAbortCurrent(CalogValueT *result); // ...or only the calling one, which then retires +bool calogAborting(CalogT *); // true once the caller has been stopped ``` Statuses are `CalogStatusE` (`calogOkE`, `calogErrArgE`, `calogErrOomE`, ...); `calogOkE` @@ -426,6 +438,15 @@ make tsanmb # ThreadSanitizer: MY-BASIC The ThreadSanitizer targets run under `setarch -R` (ASLR off) so TSan's shadow allocator is happy on all kernels. +```sh +make cross # cross-build everything: musl (RUN), Windows PE, macOS Mach-O -- needs zig +make cross-smoke # just the matrix: all ten engines per target, plus net + a real TLS run on musl +make cross-lint # zig-free parity check; `make test` runs this one +``` + +Only `cross-lint` is part of `make test` -- the rest need a [zig](https://ziglang.org/download/) +toolchain, which building calog natively does not. See [`PORTING.md`](PORTING.md). + ## Project layout ``` diff --git a/design.md b/design.md index 0ea48ba9..d4a354fb 100644 --- a/design.md +++ b/design.md @@ -1384,3 +1384,431 @@ that bypassed the per-block charge). Two limitations are accepted and documented Janet's `JANET_OUT_OF_MEMORY` is `exit(1)`). This is the allocation-granular bound already noted, the same class as my-basic's statement-granular cap; the host-OOM crash is inherent to embedding these VMs and predates the cap. + +--- + +## 25. `calogExit` that does not return -- the runtime abort latch + +`API.md` had always said of the runner's `calogExit([code])`: *"Does not return."* It did. The native +set two atomics (`gExitCode`, `gShutdown`) and handed control straight back to the script, which ran +on to the end of its chunk while the host pump loop was still one 0.5 ms tick away from noticing. Two +consequences, both silent: + +- **Work after the exit still happened.** `if (failed) { calogExit(1) } ... deploy()` deployed. +- **A later exit overwrote an earlier one.** The last `calogExit` to run named the exit code, so a + script ending in `calogExit(0)` reported success no matter which failure had asked for `1` first. + +That is a footgun for exactly the use calog is good at -- a build/CI script in whichever language +suits the job -- so the fix restores the documented semantics rather than documenting the behavior. + +### The mechanism already existed + +Every adapter turns a native that returns non-zero into an engine-level raise (`lua_error`, +`JS_Throw`, `sq_throwerror`, `be_raise`, `s7_error`, `wrenAbortFiber`, `mrb_exc_raise`, `TCL_ERROR`, +`janet_panic`, `MB_FUNC_ERR`). That is how the sandbox's wall-clock hook already stops a runaway +script (sec 24). So a native CAN unwind its caller: it only has to return an error and be sure +nothing reports it as a failure. `calogAbortAll` is that, made explicit: + +```c +int32_t calogAbortAll(CalogT *calog, CalogValueT *result); // latch the runtime, return calogErrAbortE +bool calogAborting(CalogT *calog); // true once the caller has been stopped +``` + +`nativeCalogExit` records the code and returns `calogAbortAll(...)`. The engine raises it, the script +unwinds out of its chunk at the call site, and the statement after `calogExit` never runs. The native +must be registered with `calogRegisterInline` for this to work at all -- an abort raised on the host +thread could not unwind a script on another thread. + +### One latch, runtime-wide + +The obvious design -- abort only the calling context -- needs cross-context propagation, because the +error status is flattened into a string every time it crosses an engine boundary (script A calls +script B's export, B calls `calogExit`; A only ever sees B's message). A single `_Atomic bool` on +`CalogT`, checked at the two dispatch choke points every script call already funnels through, is both +simpler and stronger: + +| choke point | what it refuses once latched | +|---|---| +| `calogCall` (broker.c) | every native, from every context and every engine -- including `calogExit` itself | +| `calogFnInvoke` (value.c) | every script function value: a queued timer or pubsub callback never starts a script body | + +So the abort needs no propagation: whichever context is next to touch C is stopped there, and a +script that *catches* the unwind (`pcall`, `try`, `catch`) cannot call a native or invoke a callable +afterwards -- it can only spin until the host joins it. The latch is per-runtime, so independent +`CalogT`s in one process are unaffected, and it is one-way: a latched runtime is a tearing-down +runtime. + +### An aborted script did not fail + +The unwind reaches each adapter's run function as an ordinary engine error, which would print a +diagnostic and report a script error -- two spurious lines for a perfectly normal `calogExit(3)`. So +every adapter's RUN-failure branch (never its compile branch: a syntax error cannot be an abort) does +its usual cleanup and returns `calogOkE` when `calogAborting(context->broker)`. Returning ok is what +keeps `contextDispatchEval` from posting an error, so the error handler never sees it and the runner +never marks that context failed. + +**Janet is the one engine that needed more.** Alone among the ten it reports the failure itself, from +inside `janet_dobytes` (error line plus stack trace, through `janet_eprintf`), before the adapter can +decide anything. `janet_eprintf` honors the `err` dynamic binding, so the adapter binds a capture +buffer for the duration of the run and relays it to stderr verbatim only for a real failure. The +binding has to go in the TOP dyn table -- `janet_dobytes` prints after `janet_continue` has returned, +when no fiber is current -- and Janet does not mark that table, so the buffer and the `:err` keyword +are `janet_gcroot`ed by hand for the run. A script's own `(eprint ...)` is untouched: inside a fiber +the lookup finds the context env's bindings, not the top table. + +### Exit-code precedence in the runner + +`gExitRequested` latches the first request of any kind -- a `calogExit`, a signal, or a script +erroring out -- and everything after it is ignored. First-writer-wins is the only rule under which a +failure cannot be masked: a later `calogExit(0)`, from the same script or a sibling, no longer clears +an error that already happened, and an error thrown up during teardown no longer overwrites the code +a script deliberately asked for. A second `calogExit` never even reaches the native (the abort latch +refuses it), and a failed launch reports `1` without ending the run for the scripts that did launch. + +"First" means first *observed*, and a script error is observed when it reaches the host thread. So in +a multi-script run, a sibling erroring at the same moment one script calls `calogExit(0)` is a race +-- as everything else between concurrently running scripts is. Within one script there is no race at +all, which is the case that matters: its own `calogExit(1)` unwinds it, so no later line of that +script, `calogExit(0)` included, can ever run. + +### Teardown order stays the host's + +`calogAbortAll` deliberately does not close the calling context. Teardown order is the host's -- +`calogDestroy` releases the cross-context reference holders while every context is still alive, and +only then joins them (that is why `calogTimer` registers its shutdown `calogDestroyBeforeContextsE`). +A context that retired itself from inside the abort would destroy its own interpreter first, +stranding the callables those registries still hold: harmless for a VM that frees everything on +close, fatal for QuickJS, which asserts it owns no live objects at `JS_FreeRuntime`. Stopping the +scripts is the latch's job and needs no context to close. + +### Accepted limitations + +- **A signal does not abort scripts.** `SIGINT`/`SIGTERM` still only request shutdown: a signal + handler must not run the actor layer, and an interrupt is not a script asking to stop. A script + finishes its current chunk while the host tears down, exactly as before. +- **A script that deliberately catches the unwind is not preempted mid-computation** -- the same + cooperative-model limit as the sandbox (sec 24), and the same answer: it is stopped at its next + native call, and pure spinning is ended by the host's teardown. +- **Four engines cannot tell a compile failure from a run failure**, so a chunk that begins + compiling *after* the latch has its syntax error swallowed along with the abort: QuickJS + (`JS_Eval` compiles and runs in one call), Tcl (`Tcl_EvalEx`), mruby (`mrb_load_string` reports + both through `mrb->exc`), and s7 (its catch wrapper flags read and run errors alike). The other + six report the two separately and only quiet the run failure. The window is a teardown-only one -- + an eval already queued when the abort landed -- so the cost is a lost diagnostic for a script + that was never going to run anyway. + +### Test + +`tests/testExit.c` runs, on all ten engines, `mark(); stopAll(); `. The loop is the +evidence: native calls cannot prove the point (the latch refuses them whether or not the script +unwound), but a script that kept running would pin its context thread, so the check is that the +context's thread has exited within the pump budget -- the test's own native adds a +`calogCurrentRetire`, serviced only once an eval returns, to make "the eval returned" observable. +The same run asserts the abort reached no error +handler, that the runtime stays latched, and that a later `calogCall` is refused with +`calogErrAbortE` without reaching the native. A final case proves the latch is per-runtime. + +--- + +## 26. Reclaiming a context's callables -- and `calogEnd` + +A `CalogFnT` is a handle to a function living inside one VM. Section 25 made `calogExit` unwind +scripts; this section fixes the class of bug that made the *next* question -- "how does one script +end itself?" -- unsafe to answer. + +### The bug: a handle outliving its VM + +`threadMain` destroys a context's interpreter as that context's thread exits. Anything still holding +a `CalogFnT` owned by that context is then holding a handle into a VM that no longer exists, and the +engine-side release never runs: `calogFnFinalize` sees the owner is gone and frees only the shell. +Most VMs hide it -- they free everything on close -- but QuickJS asserts it owns no live objects and +aborts the process: + +``` +calog: vendor/quickjs/quickjs.c:2682: JS_FreeRuntime: Assertion `list_empty(&rt->gc_obj_list)' failed. +``` + +Three separate holders reproduce it, all deterministically: + +| holder | reproducer (JavaScript, because QuickJS is the VM that checks) | +|---|---| +| a library registry | `psSubscribe('t', function(){});` then exit, error, or `calogEnd` | +| another engine | `calogCall('keep', function(){})` into a Lua script that stores it, then die | +| an in-flight invoke | a delivery holding the last reference across the owner's teardown | + +The first instinct -- move the pubsub and export destroy hooks from `calogDestroyAfterContextsE` to +`calogDestroyBeforeContextsE`, as `calogTimer` already does -- fixes only the whole-runtime teardown +path, and only for the last runtime in the process. It does nothing for a context that dies while the +runtime lives on, which is the common case: a script that errors after subscribing, `taskExit`, and +now `calogEnd`. It also cannot reach a value another VM is holding, where no registry is involved. + +### The fix: the owner reclaims, on its own thread + +The rule that actually holds is **a context releases every engine handle it owns before its +interpreter goes away**, and only the owner's own thread can do that. So each context now lists the +callables it creates (`calogContextTrackFn` / `calogContextUntrackFn`, guarded by that context's +queue mutex, locked in the runtime's established `ctxMutex` -> `queueMutex` order), and `threadMain` +sweeps the list after the context hooks and before `interpDead` / `destroyInterpreter`: + +```c +contextReclaimCallables(context); // per callable: mark dead, run the engine release, clear the hook +``` + +`calogFnReclaim` marks the callable dead and runs the engine release while the VM is still alive, +then clears the hook -- so every later holder, whatever thread it is on and whenever it gets there, +finalizes an empty shell. An invoke of a reclaimed callable fails with `calogErrDeadE`, the same +answer the actor layer already gave for a context that is gone (which is what the timer library +cancels a timer on, and what `calogFnMarkDead` -- until now dead code with no callers -- was for). + +The sweep takes the whole list under the lock before touching anything, and retains each entry across +the pass: an engine release can cascade (a VM finalizer dropping another of this context's callables) +straight back into `calogContextUntrackFn`, and must not find a list being walked or free an entry +the sweep has not reached. + +This makes the destroy-phase question moot -- no phase changed -- because by the time any registry +releases its reference, the handle it names is already gone. Two smaller repairs came with it: +`psSubscribe` and `calogExport` now refuse to add to a registry that has already been drained (the +guard `timerSchedule` already had: `gInitMutex` held across the whole insert, in that lock order, so +it cannot straddle a shutdown), and `calogActorShutdown` destroys `ctxMutex` after the host-queue +drain rather than before, since a callable finalized by that drain resolves its owner under it. + +### Closing the registry before walking it + +The same audit found a second teardown defect, unrelated to callables. `calogActorShutdown` walked +`calog->ctxSlots` and `calog->ctxCount` **without holding `ctxMutex`** -- and that walk is exactly +when scripts are still running and still free to call `taskSpawn` / `taskLoad`, each of which opens a +context and can `realloc` the slot array out from under the walk. A context registered after the walk +had passed its slot was then freed by the third loop with its thread still running, and that thread +outlived the slot array it was registered in. + +`calogContextOpen` made it worse from the other side: it filled the slot *before* `pthread_create` +and published `started` *after*, so even a locked read could catch a context whose thread was already +running but whose `started` flag said otherwise -- skipped by both the shutdown-request and the join +loops. + +Both halves close with one idea -- **latch the registry, then walk it**: + +- `calogActorShutdown` sets `tearingDown` under `ctxMutex` as its first act and reads `ctxCount` in + the same critical section. From that moment `calogContextOpen` refuses (a `taskSpawn` racing the + teardown simply fails), so the slots it is about to walk are the complete and final set and the + array can no longer move. +- Every slot read in the walks takes `ctxMutex` (`contextAtIndex`), because a script thread can + still be unlinking a context of its own. +- `pthread_create` and `started = true` moved inside the registration critical section, so a locked + read sees either an empty slot or a slot whose thread exists. Nothing waits on the new thread while + the lock is held, so starting it there cannot deadlock -- the new thread merely waits out the + handful of instructions to the unlock if its own first act needs the registry. + +`tests/testHooks.c` pins it deterministically: a per-context shutdown hook runs on the context's own +thread *while* `calogDestroy` is walking the registry, which is precisely the window, and opening a +context from inside that hook must be refused. `tests/testTask.c` adds the stress companion, a spawn +loop racing `calogDestroy` on its own runtime. + +### One closer per context + +The latch fixes the registry, not the contexts in it. `taskClose` is an inline native, so a script +thread can call `calogContextClose` on a task at the same moment the teardown is stopping that very +context -- and `pthread_join` from two threads is undefined, quite apart from one of them freeing the +context the other is still inside. A registry lock cannot express that: the join must not be held +under a lock the joined thread may itself need. + +What it needs is a claim. Each context carries a `closing` flag guarded by `ctxMutex`, and +`calogContextClose` takes it or returns: + +```c +if (broker->tearingDown || context->closing) { unlock; return; } /* someone else owns this one */ +context->closing = true; +``` + +`tearingDown` is the teardown's claim on every context at once, so once the latch is up no new close +can start. The only case left is a close that was *already* running when the latch went up, and +`calogActorShutdown` waits for exactly those -- polling the slots it is about to walk until none is +flagged `closing`, since that closer unlinks the slot when it finishes. The latch guarantees the set +only shrinks, so the wait terminates; in practice it never runs a single iteration. + +A script that calls `taskClose` during teardown simply gets a no-op: the context it named is stopped +and freed moments later by the teardown that already owns it. + +`tests/testTask.c` races a loop of `taskSpawn` + `taskClose` against `calogDestroy` on its own +runtime -- clean under ASan across repeated runs, and under ThreadSanitizer with the Lua and +JavaScript engines linked. + +### `calogEnd([code])` -- ending one script + +With reclamation in place a script can safely end itself, which is what the runner's new `calogEnd` +does. It is `calogAbortCurrent`: the same unwind as `calogAbortAll`, scoped to one context. The +calling script stops at the call, that context alone is latched (so a caught unwind cannot call +another native), and the context retires -- its thread ends, and the runner's pump loop closes it, +drops the live count, and exits once the last launched script is gone. Other scripts are untouched. + +No engine adapter changed for any of this. The ten of them ask one question -- `calogAborting` +(sec 25) -- and that question was widened rather than duplicated: it now answers *has the caller been +stopped*, by the runtime latch or by its own context. A second query would have meant a second edit +to ten files and two ways for them to disagree. + +An optional `code` names the process's status on the same first-writer-wins rule as `calogExit`; +omitting it claims nothing. The three ways a script can stop now read as one set: + +| | ends | process status | +|---|---|---| +| `calogEnd([code])` | this script | `code` if given, else unclaimed | +| `error(...)` | this script | 1 (the run did not fully succeed) | +| `calogExit([code])` | everything | `code` (default 0) | + +`taskExit()` is deliberately left alone: it stays deferred, as documented, and remains the +task-scoped way for a spawned task to retire itself. + +### Test + +`tests/testTeardown.c` uses JavaScript throughout, because on any other engine a stranded handle is +invisible -- there, surviving the teardown IS the assertion. It covers a subscriber, an export and a +timer callback left registered at `calogDestroy`; each of those whose script instead errors out +first; a script that ends itself with the `calogEnd` primitive; a closure handed to a Lua script +whose owner then dies (invoking it afterwards must fail cleanly, not reach into a destroyed VM); and +the drained-registry guards. + +--- + +## 27. What the cross-builds caught -- three portability regressions + +Sections 25 and 26 changed the actor core, so the Windows/macOS/musl cross-builds were re-run +afterwards. The core changes ported cleanly. Everything else the run turned up predated them, and had +been sitting there unnoticed for one structural reason worth stating plainly: **the cross-builds are +not part of `make test`.** Nothing else in the project can drift silently for weeks; these can, and +did. The habit that catches it is to run `tools/crossBuild.sh` after touching the core or a vendored +dependency -- it is a couple of minutes, and it is what surfaced all three of these. + +### An ABI-shaped define that only half the build knew about + +`tools/crossWinFull.sh` stopped at `mrubyAdapter.c: no member named 'code_fetch_hook' in +'struct mrb_state'`. `MRB_USE_DEBUG_HOOK` enables the per-instruction hook the sandbox wall-clock +budget needs (sec 24) -- and it **changes the layout of `mrb_state`**. `src/mruby/build_config.rb` +sets it for the library and the Makefile passes it to the adapter compile, so the two agree natively. +The cross config generated by `tools/crossDeps.sh` set only `MRB_INT64`, and neither `cross*Full.sh` +passed the macro, so the cross `libmruby.a` and the adapter disagreed about the struct. + +It had been broken since sec 24 landed. Failing loudly at compile time was the good outcome: had the +member merely moved rather than vanished, this would have been a silent ABI mismatch in a shipped +binary. The macro now lives in the generated cross config **and** in both full-CLI adapter compiles. +The general rule it earns: *a define that changes a vendored library's layout has to be set every +place that compiles against that library, and the cross path is a place.* + +### A libc that returns `endptr` somewhere else + +`testEngineMyBasic` failed 13 of 20 checks on musl while passing under native gcc, native clang +`-O2`, and zig targeting glibc -- so the variable was musl, not the compiler. my-basic classified a +symbol as a number by asking whether `strtoll`/`strtod` had reached the string terminator: + +``` + strtoll("\n") strtoll("+") + glibc: consumed=0, endptr at start consumed=0, endptr at start + musl: consumed=1, endptr at NUL consumed=1, endptr at NUL +``` + +**musl advances `endptr` past leading whitespace and a sign even when no conversion happens.** So on +musl `+`, `-` (the operators) and `\n` (the statement separator) each classified as the integer 0: +every expression containing an operator failed to parse, and every numeric assignment failed to run, +both surfacing as "Operator expected". It reproduces in a pure my-basic program with no calog code at +all (`x = 1` -> `MB_FUNC_ERR` on musl, `MB_FUNC_OK` on glibc), which is how it was pinned down. The +fork now rejects a consumed span that is nothing but whitespace and sign; `vendor/ourbasic/CHANGELOG` +has the detail. The lesson generalizes past my-basic: *a failed `strtol` does not leave `endptr` +where you assume -- test what was consumed, not just where it stopped.* + +### A dependency that grew under a script that had stopped looking + +`testNet` failed to build on all four targets: `libs/calogNet.c` has included `` since +the tcp transport gained TLS, and `tools/crossBuild.sh` passed no OpenSSL flags. The test itself uses +no TLS -- only the library it links does -- so nothing was wrong with the coverage, just with the +link line. It now links the per-target OpenSSL that `tools/crossDeps.sh` already produced for +Windows and macOS, and that script gained a **musl** target for OpenSSL alone (the full musl CLI is +`make static` on Alpine, where the toolchain is already musl -- not a cross build). When a target's +OpenSSL has not been built, the case SKIPS with the command that produces it, and the summary counts +skips separately: a skip must never be able to read as a pass. + +### Where that leaves the ports + +| | Before | After | +|---|---|---| +| `tools/crossBuild.sh` | 7 ok, 4 failed | **11 ok, 0 failed** | +| musl `testNet` | did not build | **runs 10/10**, fully static, matching the native run | +| musl `testEngineMyBasic` | 13 of 20 failing | **20/20** | +| Full CLI: Windows PE, macOS x86_64 + arm64 | Windows did not build | **all three build**, carrying sec 25/26 | + +The standing gap is unchanged and deliberate: Windows and macOS are **build**-verified only. Running +those binaries needs a Windows/wine or Mac host, so "it links" is the strongest claim the evidence +supports, and PORTING.md says exactly that rather than rounding it up. + +--- + +## 28. Three latent edges, closed + +None of these ever fired in a test. They were found by reading the code around sections 26 and 27 -- +two in the actor core, one in the my-basic fork -- and each is the kind that stays quiet until the +day it does not. + +### The release that could run on the wrong thread + +`actorReleaseCallable` marshals a callable's finalize to its owner's thread, because the adapter's +release is an interpreter op (`luaL_unref`, `JS_FreeValue`, ...) and only the owner may touch that +interpreter. Its fallback did not hold that line: + +```c +if (contextPostRelease(runtime, owner, callable) != calogOkE) { + calogFnFinalize(callable); /* on THIS thread -- not the owner's */ +} +``` + +`contextPostRelease` fails for two different reasons, and they are not equally harmless. If the owner +is **gone**, `calogFnFinalize` takes its "owner gone" branch and only frees memory -- fine. But it +also fails when the `calloc` for the message fails while the owner is **alive**, and then +`calogContextRegistered` returns true and the engine op runs from the wrong thread. That is memory +corruption inside a running VM, reached under memory pressure -- the moment a program is least able to +cope with it. + +`calogFnFinalizeForeign` is the fallback now: untrack, free the adapter's block, free the shell, and +deliberately **leak the handle inside the interpreter**. A leak on an OOM path beats corrupting a VM +that is still running. The "owner gone" case is unchanged -- it already did exactly this. + +One subtlety, caught in review before this shipped: it frees `userData` on **exactly** the condition +`calogFnFinalize` does -- only when a release hook exists. The hook is what makes `userData` calog's +to free (an adapter's single heap block, per the `CalogReleaseFnT` contract). A host-owned callable +from `calogFnFromNative` has no hook and a `userData` the **embedder** owns; freeing that would +corrupt their heap, which is worse than the corruption this function exists to prevent. And the leak +is worth stating honestly: the handle is pinned until that interpreter is destroyed, which for a +long-lived context is not "a moment". + +### The window where a queue accepted work nobody would serve + +`serveLoop` returns when `messageDequeue` finds the queue empty and `shuttingDown` set. `interpDead` +-- which is what stops `registryResolveLocked` handing the context out -- is set later, after the +per-context hooks and the callable reclaim. Between those two points the context still resolved, so a +message enqueued there was **accepted and never served**: a fire-and-forget eval silently dropped, and +a blocking call left its sender waiting on a reply that could not come (the reply-box path waits on a +condvar with no timeout, so `calogDestroy`'s join on that sender never returns). + +Moving `interpDead` earlier would have been the wrong fix -- the reclaim depends on its ordering. The +fix is to make *closing the queue* atomic with the decision to stop serving, which is a different +fact from *the interpreter is gone*: + +- `queueClosed` on the context, guarded by that context's `queueMutex`; +- `messageDequeue` sets it in the **same critical section** where it decides to return NULL; +- `enqueueRaw` refuses while holding that mutex, and `contextEnqueue` reports `calogErrDeadE`. + +Every caller already handled that status, so nothing downstream changed. `tests/testHooks.c` pins the +window deterministically: a per-context shutdown hook runs inside it (after `serveLoop`, before +`interpDead`), and queueing work from there must be refused. That check fails against the pre-fix code +and passes after -- the test has teeth, which for a window this narrow is the only evidence worth +having. + +### An index that could go negative + +`_get_priority_index` (my-basic) linearly searches a table of operator function pointers and returns +`-1` when it finds none. `_get_priority` then does `_PRECEDE_TABLE[idx1][idx2]`, and the assert +guarding it checks only the **upper** bound. calog defines no `NDEBUG`, so today the `mb_assert` +fires; an embedder who builds with `NDEBUG` gets a silent out-of-bounds read instead. + +The table already has a marker for "these two cannot operate together" -- a space -- and the +evaluator turns it into a clean `SE_RN_FAILED_TO_OPERATE` script error. So a negative index now +returns `' '`: an unknown operator becomes a reported script error rather than a read off the front of +a static array. (It has never fired -- an instrumented build confirmed the lookup always resolves -- +which is exactly why it was worth closing while it was still theoretical.) diff --git a/examples/scripts/README.md b/examples/scripts/README.md index c53059d2..bee0d62d 100644 --- a/examples/scripts/README.md +++ b/examples/scripts/README.md @@ -21,7 +21,11 @@ Conventions every example follows: engine's own `print`/keyword; on Wren it is `Calog.call("calogPrint", [...])`). - **`calogExit([code])`** ends the run. calog is event-driven -- a script's top level finishing does not exit the process (it may still have timers/subscriptions live), so a - script asks to exit explicitly. `Ctrl-C` also tears things down cleanly. + script asks to exit explicitly. `Ctrl-C` also tears things down cleanly. `calogExit` does + not return: the statement after it never runs, and the first code asked for is the one the + process reports. +- **`calogEnd([code])`** ends just the calling script, leaving the others running; it does not + return either, and its context is reaped. Once every script has ended, `calog` exits on its own. - Extensions map to engines: `.lua .js .nut .bas .be .scm .wren`. ## `languages/` -- one guided tour per engine @@ -79,7 +83,8 @@ Conventions every example follows: ## `multifile/` -- multiple script files in one run Files listed on the `calog` command line run **concurrently in one process** and share the -same runtime (kv store, pubsub bus, exports). One `calogExit()` from any file ends the run. +same runtime (kv store, pubsub bus, exports). One `calogExit()` from any file ends the run -- +it unwinds its own script immediately and stops the others at their next native call. ```sh bin/calog examples/scripts/multifile/producer.js examples/scripts/multifile/consumer.lua diff --git a/libs/calogExport.c b/libs/calogExport.c index 18248417..bc5acb27 100644 --- a/libs/calogExport.c +++ b/libs/calogExport.c @@ -168,6 +168,16 @@ static int32_t exportPublish(CalogValueT *args, int32_t argCount, CalogValueT *r } name = args[0].as.s.bytes; nameLen = args[0].as.s.length; + // Hold gInitMutex across the whole publish so it can never straddle a calogExportShutdown + // (calogRegistryRelease holds the same lock over exportFreeAll): either the map is live and this + // entry will be freed with it, or the map is already gone and this fails outright. Adding to a + // drained map would strand the entry -- nothing frees it a second time. Lock order is gInitMutex + // then gMapMutex, matching shutdown; the reverse would deadlock. + pthread_mutex_lock(&gInitMutex); + if (gRefCount <= 0) { + pthread_mutex_unlock(&gInitMutex); + return calogFail(result, calogErrDeadE, "calogExport: the export registry has been shut down"); + } pthread_mutex_lock(&gMapMutex); index = exportFindLocked(name, nameLen); if (index >= 0) { @@ -176,6 +186,7 @@ static int32_t exportPublish(CalogValueT *args, int32_t argCount, CalogValueT *r calogFnRetain(args[1].as.fn); gEntries[index].fn = args[1].as.fn; pthread_mutex_unlock(&gMapMutex); + pthread_mutex_unlock(&gInitMutex); return calogOkE; } buffer = gEntries; @@ -183,6 +194,7 @@ static int32_t exportPublish(CalogValueT *args, int32_t argCount, CalogValueT *r status = calogGrow(&buffer, &capacity, (int64_t)gCount + 1, sizeof(ExportEntryT)); if (status != calogOkE) { pthread_mutex_unlock(&gMapMutex); + pthread_mutex_unlock(&gInitMutex); return calogFail(result, status, "calogExport: out of memory"); } gEntries = (ExportEntryT *)buffer; @@ -190,6 +202,7 @@ static int32_t exportPublish(CalogValueT *args, int32_t argCount, CalogValueT *r nameCopy = (char *)malloc((size_t)nameLen + 1); if (nameCopy == NULL) { pthread_mutex_unlock(&gMapMutex); + pthread_mutex_unlock(&gInitMutex); return calogFail(result, calogErrOomE, "calogExport: out of memory"); } if (nameLen > 0) { @@ -202,6 +215,7 @@ static int32_t exportPublish(CalogValueT *args, int32_t argCount, CalogValueT *r gEntries[gCount].fn = args[1].as.fn; gCount++; pthread_mutex_unlock(&gMapMutex); + pthread_mutex_unlock(&gInitMutex); return calogOkE; } diff --git a/libs/calogPubsub.c b/libs/calogPubsub.c index 65375ea5..e674aea7 100644 --- a/libs/calogPubsub.c +++ b/libs/calogPubsub.c @@ -172,9 +172,20 @@ static int32_t pubsubSubscribe(CalogValueT *args, int32_t argCount, CalogValueT if (argCount != 2 || args[0].type != calogStringE || args[1].type != calogFnE) { return calogFail(result, calogErrArgE, "psSubscribe expects (topic, function)"); } + // Hold gInitMutex across the whole subscribe so it can never straddle a calogPubsubShutdown + // (calogRegistryRelease holds the same lock over pubsubFreeAll): either the registry is live and + // this entry will be freed with it, or the registry is already gone and this fails outright. + // Adding to a drained registry would strand the entry -- nothing frees it a second time. Lock + // order is gInitMutex then gListMutex, matching shutdown; the reverse would deadlock. + pthread_mutex_lock(&gInitMutex); + if (gRefCount <= 0) { + pthread_mutex_unlock(&gInitMutex); + return calogFail(result, calogErrDeadE, "psSubscribe: pubsub has been shut down"); + } topicLen = args[0].as.s.length; topicCopy = (char *)malloc((size_t)topicLen + 1); if (topicCopy == NULL) { + pthread_mutex_unlock(&gInitMutex); return calogFail(result, calogErrOomE, "psSubscribe: out of memory"); } memcpy(topicCopy, args[0].as.s.bytes, (size_t)topicLen); @@ -185,6 +196,7 @@ static int32_t pubsubSubscribe(CalogValueT *args, int32_t argCount, CalogValueT status = calogGrow(&buffer, &capacity, (int64_t)gCount + 1, sizeof(SubEntryT)); if (status != calogOkE) { pthread_mutex_unlock(&gListMutex); + pthread_mutex_unlock(&gInitMutex); free(topicCopy); return calogFail(result, status, "psSubscribe: out of memory"); } @@ -199,6 +211,7 @@ static int32_t pubsubSubscribe(CalogValueT *args, int32_t argCount, CalogValueT gEntries[gCount].callback = args[1].as.fn; gCount++; pthread_mutex_unlock(&gListMutex); + pthread_mutex_unlock(&gInitMutex); calogValueInt(result, id); return calogOkE; } diff --git a/src/berry/berryAdapter.c b/src/berry/berryAdapter.c index 1b0124c0..bc775cd8 100644 --- a/src/berry/berryAdapter.c +++ b/src/berry/berryAdapter.c @@ -493,18 +493,27 @@ static int32_t berryFromValue(CalogBerryT *context, const CalogValueT *value, in int32_t calogBerryRun(CalogBerryT *context, const char *source) { - bvm *vm; - int base; - int code; + bvm *vm; + bool compiled; + int base; + int code; - vm = context->vm; - base = be_top(vm); - code = be_loadstring(vm, source); - if (code == BE_OK) { + vm = context->vm; + base = be_top(vm); + code = be_loadstring(vm, source); + compiled = (code == BE_OK); + if (compiled) { code = be_pcall(vm, 0); } if (code != BE_OK) { const char *message; + // calogExit (calogAbortAll) unwound this script deliberately: not a failure, and the + // runtime is already tearing down, so report nothing. Only a script that actually RAN can + // have been aborted, so a syntax error still gets its diagnostic even mid-teardown. + if (compiled && calogAborting(context->broker)) { + be_pop(vm, be_top(vm) - base); + return calogOkE; + } message = be_tostring(vm, -1); fprintf(stderr, "berry error: %s\n", message != NULL ? message : "(unknown)"); be_pop(vm, be_top(vm) - base); diff --git a/src/broker.c b/src/broker.c index 8281936e..edba7594 100644 --- a/src/broker.c +++ b/src/broker.c @@ -35,6 +35,13 @@ int32_t calogCall(CalogT *broker, const char *name, CalogValueT *args, int32_t a CalogEntryT *entry; calogValueNil(result); + // The caller has been stopped -- the runtime torn down (calogAbortAll) or this one script ended + // (calogAbortCurrent). Refuse every native here, at the one dispatch point every engine goes + // through, so a script that caught the unwinding error is stopped again at its very next call + // instead of running on past it. + if (calogAborting(broker)) { + return calogFail(result, calogErrAbortE, CALOG_ABORT_MESSAGE); + } entry = calogLookup(broker, name); if (entry == NULL) { return calogFail(result, calogErrNotFoundE, "no such function"); diff --git a/src/calog.h b/src/calog.h index 1a57b7b6..e2366433 100644 --- a/src/calog.h +++ b/src/calog.h @@ -44,7 +44,10 @@ typedef enum CalogStatusE { calogErrArgE = 5, calogErrRangeE = 6, calogErrUnsupportedE = 7, - calogErrDeadE = 8 + calogErrDeadE = 8, + // The runtime has been latched aborting (calogAbortAll): the call was refused because no script + // may run any more. Every engine raises it as an error, which unwinds the script out of its chunk. + calogErrAbortE = 9 } CalogStatusE; typedef enum CalogTypeE { @@ -258,6 +261,26 @@ CalogT *calogCurrent(void); // runtime of the calling context (NULL i void calogCurrentRetire(void); // ask the calling context to retire itself (deferred; see calogTask) bool calogCurrentShuttingDown(void); // true once the calling context has been asked to close +// ---- stopping every script (the runner's calogExit) ---- +// Latch this runtime ABORTING: no context may run another line of script. Call it from inside a +// native and RETURN ITS VALUE -- the calling engine raises that error, so the script unwinds right at +// the call instead of running on, which is what lets a native "not return". Every later native call +// from any context of this runtime fails the same way, so a script that catches the unwind cannot +// keep working, and an aborted script is NOT reported to the error handler (it did not fail). No +// context is closed here -- teardown stays the host's, in its own order. One-way and idempotent: +// a latched runtime never runs script code again, so latch it only when tearing the runtime down. The +// host decides what happens next -- bin/calog exits the process with the code the script asked for. +int32_t calogAbortAll(CalogT *calog, CalogValueT *result); +// Stop only the CALLING script, leaving the runtime and every other script running (the runner's +// calogEnd). Used the same way -- call it from a native and return its value, and the script unwinds +// at the call site. The context also retires itself, so its thread ends and the host can reap it; +// like an aborted script it is not reported as a failure. Fails if there is no calling script. +int32_t calogAbortCurrent(CalogValueT *result); +// True when the caller must stop running script code -- this runtime was latched by calogAbortAll, +// or the calling context ended itself. Engine adapters use it to tell a script that was stopped +// (report nothing) from one that genuinely failed. +bool calogAborting(CalogT *calog); + extern const CalogEngineT calogLuaEngine; extern const CalogEngineT calogJsEngine; extern const CalogEngineT calogSquirrelEngine; diff --git a/src/calogInternal.h b/src/calogInternal.h index 5aeee73c..27852d64 100644 --- a/src/calogInternal.h +++ b/src/calogInternal.h @@ -9,6 +9,12 @@ #include "calog.h" #include +#include + +// What an aborted script sees: the error every engine raises to unwind a script out of its chunk once +// the runtime is latched aborting (calogAbortAll), and the failure every later native call returns. +// One source of truth for both, so the two never drift. +#define CALOG_ABORT_MESSAGE "calog: the runtime is shutting down" #ifdef _WIN32 // memmem is a GNU/BSD extension that mingw-w64's libc does not provide, so calog's own scanners @@ -110,6 +116,10 @@ struct CalogT { int64_t *ctxFree; // recycled slot indices int64_t ctxFreeCount; int64_t ctxFreeCap; + // Set by calogActorShutdown (under ctxMutex) before it walks the registry: calogContextOpen + // refuses from then on, so a script cannot register a context the teardown would miss -- and + // ctxSlots/ctxCount stop moving, since only an open ever grows them. + bool tearingDown; // actor layer (context.c): installed by calogActorInit, all NULL on a bare broker CalogContextT *hostContext; // id 0, no thread; driven by calogPump pthread_t hostThread; // thread that ran calogActorInit (this runtime's host) @@ -118,6 +128,9 @@ struct CalogT { CalogReleaseHookT releaseHook; CalogErrorFnT errorHandler; void *errorUserData; + // calogAbortAll latch: every context stops running script code (each native call is refused, so + // the engine unwinds it). Read on every dispatch from every thread, hence atomic. One-way. + _Atomic bool aborting; // engines available to calogContextLoad (calogRegisterEngine), search-priority order const CalogEngineT **engines; int64_t engineCount; @@ -144,6 +157,16 @@ void calogForEach(CalogT *calog, void (*visit)(const CalogEntryT *entry, // ---- callable lifecycle (driven by the engine adapters) ---- int32_t calogFnCreate(CalogFnT **out, CalogT *runtime, CalogNativeFnT fn, void *userData, CalogReleaseFnT release, uint64_t ownerCtxId); void calogFnFinalize(CalogFnT *fn); +// As calogFnFinalize, but skips the engine release: for a caller that is NOT on the owner's thread +// and could not marshal the release there. Frees the adapter's block and the shell, leaving the +// handle inside the interpreter to die with it -- a leak, deliberately, instead of an interpreter op +// on the wrong thread. +void calogFnFinalizeForeign(CalogFnT *fn); +// The owning context is about to destroy its interpreter: run the engine's release NOW, while that +// interpreter is still alive, and neutralize the callable so a later finalize -- from a library +// registry, from another engine still holding this value, from an invoke in flight -- only frees the +// shell. Runs on the owner's own thread (contextReclaimCallables). See design.md sec 26. +void calogFnReclaim(CalogFnT *fn); CalogNativeFnT calogFnNative(const CalogFnT *fn); void calogFnMarkDead(CalogFnT *fn); uint64_t calogFnOwner(const CalogFnT *fn); @@ -157,6 +180,14 @@ CalogT *calogContextBroker(const CalogContextT *context); void *calogContextInterp(CalogContextT *context); bool calogContextRegistered(CalogT *runtime, uint64_t ctxId); +// Every callable a context creates is listed against that context, so the context can reclaim the +// engine handles it still owns before its interpreter goes away (calogFnReclaim). Both are no-ops +// for a host-owned callable (CALOG_HOST_ID), which has no interpreter to outlive, and for an owner +// that can no longer be resolved. Track failing (OOM) fails the create: an untracked callable would +// be exactly the leak this exists to prevent. +int32_t calogContextTrackFn(CalogT *runtime, uint64_t ownerCtxId, CalogFnT *fn); +void calogContextUntrackFn(CalogT *runtime, uint64_t ownerCtxId, CalogFnT *fn); + // ---- per-context resource limits (sandboxing) ---- // The mutable state a limited context enforces on its OWN thread. memUsed is charged by the engine's // allocator (Lua/QuickJS natively; the rest via a per-VM or process-global counting allocator plus a diff --git a/src/calogMain.c b/src/calogMain.c index 237ba441..ab3a6c5c 100644 --- a/src/calogMain.c +++ b/src/calogMain.c @@ -14,6 +14,16 @@ // once no launched contexts remain live, calog exits on its own. Absent any of these, calog // runs until killed. // +// The exit code is the FIRST one anything asks for -- a script's calogExit, a signal, or a script +// erroring out -- because that is the only rule under which a failure cannot be masked by a later +// calogExit(0), whether it comes from the same script or a sibling. +// +// calogExit does not return. It records the code, then calogAbortAll latches the runtime: the +// engine raises the error it hands back, which unwinds the calling script out of its chunk right +// at the call, and every other context is stopped at its next native call. So the statement after +// calogExit never runs -- a build script's calogExit(1) cannot be followed by more work, nor +// overwritten by a later calogExit(0). +// // Teardown order matters (see the library headers): the cross-context reference holders // (export/pubsub/timer/kv) are released while contexts are still alive, THEN calogDestroy joins // every context thread, THEN the libraries whose registries outlive the runtime (db/net/ssh/ @@ -72,19 +82,25 @@ static BOOL WINAPI consoleHandler(DWORD ctrlType); #endif static const CalogEngineT *engineForExtension(const char *ext); static const char *extensionOf(const char *arg); +static int32_t nativeCalogEnd(CalogValueT *args, int32_t argCount, CalogValueT *result, void *userData); static int32_t nativeCalogExit(CalogValueT *args, int32_t argCount, CalogValueT *result, void *userData); static int32_t nativeCalogPrint(CalogValueT *args, int32_t argCount, CalogValueT *result, void *userData); +static bool readExitCode(CalogValueT *args, int32_t argCount, CalogValueT *result, int32_t *outCode, bool *outGiven, const char *who); +static void noteExitCode(int32_t code); static void onError(uint64_t contextId, const char *message, void *userData); static void onSignal(int sig); static void printUsage(FILE *stream, const char *program); static void printValue(const CalogValueT *value); static char *readFile(const char *path); +static void requestShutdown(int32_t code); static bool resolveArg(const char *arg, const CalogEngineT **outEngine, char **outSource); // Requested by a script (calogExit), a signal, or the last live context erroring out; the host -// pump loop watches gShutdown and exits with gExitCode. -static _Atomic bool gShutdown = false; -static _Atomic int32_t gExitCode = 0; +// pump loop watches gShutdown and exits with gExitCode. gExitRequested latches the first request of +// any kind (see noteExitCode) so nothing that follows can rewrite the code it named. +static _Atomic bool gShutdown = false; +static _Atomic bool gExitRequested = false; +static _Atomic int32_t gExitCode = 0; // One entry per script context the runner launched. onError flags `failed` for a context whose // script errors; the pump loop then closes that context and drops the live count. When the live @@ -195,22 +211,47 @@ static const char *extensionOf(const char *arg) { } -// calogExit([code]) -- a script asks the runner to tear everything down and exit. Inline: it -// only sets two atomics, which is thread-safe, so it needs no host hop. -static int32_t nativeCalogExit(CalogValueT *args, int32_t argCount, CalogValueT *result, void *userData) { +// calogEnd([code]) -- a script says IT is done, without ending the others. Inline for the same +// reason calogExit is: only a native running on the script's own thread can unwind that script. +// Does not return. Its context retires, so the runner reaps it; when the last launched script has +// ended, the runner exits. A code given here names the process's status the same way calogExit's +// does (first request wins); omitting it claims nothing, leaving the status to whatever else happens. +static int32_t nativeCalogEnd(CalogValueT *args, int32_t argCount, CalogValueT *result, void *userData) { + int32_t code; + bool given; + (void)userData; calogValueNil(result); - if (argCount >= 1) { - if (args[0].type == calogIntE) { - atomic_store(&gExitCode, (int32_t)args[0].as.i); - } else if (args[0].type == calogRealE) { - atomic_store(&gExitCode, (int32_t)args[0].as.r); - } else { - return calogFail(result, calogErrArgE, "calogExit expects an optional integer exit code"); - } + if (!readExitCode(args, argCount, result, &code, &given, "calogEnd")) { + return calogErrArgE; } - atomic_store(&gShutdown, true); - return calogOkE; + if (given) { + noteExitCode(code); + } + return calogAbortCurrent(result); +} + + +// calogExit([code]) -- a script asks the runner to tear everything down and exit. Registered inline +// so it runs on the CALLING script's thread: that is what lets calogAbortAll unwind this very script +// (an error raised on the host thread could not), and everything it touches is thread-safe anyway. +// Does not return -- the engine raises the error calogAbortAll hands back, aborting the script here. +static int32_t nativeCalogExit(CalogValueT *args, int32_t argCount, CalogValueT *result, void *userData) { + CalogT *calog; + int32_t code; + bool given; + + (void)userData; + calogValueNil(result); + if (!readExitCode(args, argCount, result, &code, &given, "calogExit")) { + return calogErrArgE; + } + requestShutdown(code); + calog = calogCurrent(); + if (calog == NULL) { + return calogOkE; // no calling context to abort (never from a script); the pump loop still ends + } + return calogAbortAll(calog, result); } @@ -235,6 +276,17 @@ static int32_t nativeCalogPrint(CalogValueT *args, int32_t argCount, CalogValueT } +// Record what the process will report. Whoever asks FIRST names it -- a script's calogExit, a signal, +// or a script erroring out -- and everything after is ignored. That is the only rule under which a +// failure cannot be masked: neither by a sibling's later calogExit(0), nor by a context erroring out +// while the runtime tears down after a deliberate exit. +static void noteExitCode(int32_t code) { + if (!atomic_exchange(&gExitRequested, true)) { + atomic_store(&gExitCode, code); + } +} + + static void onError(uint64_t contextId, const char *message, void *userData) { int32_t index; @@ -251,10 +303,11 @@ static void onError(uint64_t contextId, const char *message, void *userData) { // Signal handler for SIGINT/SIGTERM: request the same orderly shutdown as calogExit. Only -// async-signal-safe atomic stores happen here. +// async-signal-safe atomic operations happen here. It does NOT abort the running scripts the way +// calogExit does -- an interrupt is not a script asking to stop, and a signal handler must not run +// the actor layer -- so a script gets to finish its current chunk while the host tears down. static void onSignal(int sig) { - atomic_store(&gExitCode, (int32_t)(SIGNAL_EXIT_BASE + sig)); - atomic_store(&gShutdown, true); + requestShutdown((int32_t)(SIGNAL_EXIT_BASE + sig)); } @@ -264,8 +317,7 @@ static void onSignal(int sig) { // (reporting a SIGTERM-style exit code) and return TRUE to mark the event handled. static BOOL WINAPI consoleHandler(DWORD ctrlType) { (void)ctrlType; - atomic_store(&gExitCode, (int32_t)(SIGNAL_EXIT_BASE + SIGTERM)); - atomic_store(&gShutdown, true); + requestShutdown((int32_t)(SIGNAL_EXIT_BASE + SIGTERM)); return TRUE; } #endif @@ -365,6 +417,38 @@ static char *readFile(const char *path) { } +// Read the optional exit code shared by calogEnd and calogExit: *outCode is the code (0 when the +// script named none) and *outGiven says whether it named one at all. Returns false having written +// the diagnostic into result, so the caller returns calogErrArgE. +static bool readExitCode(CalogValueT *args, int32_t argCount, CalogValueT *result, int32_t *outCode, bool *outGiven, const char *who) { + char message[96]; + + *outCode = 0; + *outGiven = false; + if (argCount < 1) { + return true; + } + if (args[0].type == calogIntE) { + *outCode = (int32_t)args[0].as.i; + } else if (args[0].type == calogRealE) { + *outCode = (int32_t)args[0].as.r; + } else { + snprintf(message, sizeof(message), "%s expects an optional integer exit code", who); + calogFail(result, calogErrArgE, message); + return false; + } + *outGiven = true; + return true; +} + + +// Ask the pump loop to stop, reporting code if nothing has named one yet (noteExitCode). +static void requestShutdown(int32_t code) { + noteExitCode(code); + atomic_store(&gShutdown, true); +} + + // Resolve one command-line argument to an engine + its script source (a fresh buffer the caller // frees). A recognized extension binds the exact file to that engine; otherwise the argument is // treated as a base name and searched as . across the engines in priority order. On @@ -488,6 +572,7 @@ int main(int argc, char **argv) { calogSetErrorHandler(calog, onError, NULL); if (calogRegister(calog, "calogPrint", nativeCalogPrint, NULL) != calogOkE || + calogRegisterInline(calog, "calogEnd", nativeCalogEnd, NULL) != calogOkE || calogRegisterInline(calog, "calogExit", nativeCalogExit, NULL) != calogOkE) { fprintf(stderr, "calog: failed to register the runner natives\n"); status = 1; @@ -530,11 +615,11 @@ int main(int argc, char **argv) { context = calogContextOpen(calog, engines[index]); if (context == NULL) { fprintf(stderr, "calog: %s: failed to open a context\n", argv[index + 1]); - atomic_store(&gExitCode, 1); // failed launch: never report success on exit + noteExitCode(1); // failed launch: never report success on exit } else if (calogContextEval(context, sources[index]) != calogOkE) { fprintf(stderr, "calog: %s: failed to start script\n", argv[index + 1]); calogContextClose(context); - atomic_store(&gExitCode, 1); // failed launch: never report success on exit + noteExitCode(1); // failed launch: never report success on exit } else { gLaunched[gLaunchedCount].id = calogContextId(context); gLaunched[gLaunchedCount].context = context; @@ -554,19 +639,27 @@ int main(int argc, char **argv) { } // Service script->native calls until a script calls calogExit, we are signalled, or every - // launched context has errored out. A script error retires (closes) its own context; when - // the last live context is gone, the runner exits. + // launched context is gone -- each having either ended itself with calogEnd or errored out. + // Both are reaped here; when the last live context is gone, the runner exits. while (!atomic_load(&gShutdown)) { calogPump(calog); calogTaskReap(); // join + free any task that retired itself via taskExit() for (index = 0; index < gLaunchedCount; index++) { - if (gLaunched[index].context != NULL && atomic_load(&gLaunched[index].failed)) { - int32_t expected; + if (gLaunched[index].context == NULL) { + continue; + } + if (atomic_load(&gLaunched[index].failed)) { + calogContextClose(gLaunched[index].context); + gLaunched[index].context = NULL; + liveCount--; + noteExitCode(1); // a script that errored out: the run did not fully succeed + } else if (calogContextFinished(gLaunched[index].context)) { + // calogEnd: the script said it was done, so its thread has exited. Close joins it + // and frees the context -- the reaping the script asked for. Not a failure, so the + // exit code is left to whatever else names one. calogContextClose(gLaunched[index].context); gLaunched[index].context = NULL; liveCount--; - expected = 0; - atomic_compare_exchange_strong(&gExitCode, &expected, 1); } } if (liveCount == 0) { diff --git a/src/context.c b/src/context.c index 4875762b..2528eaa0 100644 --- a/src/context.c +++ b/src/context.c @@ -106,6 +106,15 @@ struct CalogContextT { char **allow; // sorted allowed-native names, or NULL = every native permitted int32_t allowCount; bool limited; // any limit active (adapter installs an allocator/hook; allow checked) + _Atomic bool aborting; // calogAbortCurrent (calogEnd): this script stops, the runtime lives on + bool closing; // guarded by broker->ctxMutex: a close already owns this context + bool queueClosed; // guarded by queueMutex: serveLoop has stopped serving, refuse new messages + // Every callable this context created, so it can reclaim their engine handles before its + // interpreter is destroyed (contextReclaimCallables). Guarded by queueMutex -- a finalize can + // reach it from a foreign thread on the best-effort path in actorReleaseCallable. + CalogFnT **ownedFns; + int64_t ownedCount; + int64_t ownedCap; }; // currentContext is the ONLY process-global actor state -- and it is thread-local: it @@ -140,12 +149,14 @@ static void contextDispatchRelease(MessageT *message); static void contextDrainQueue(CalogContextT *context); static void contextFreeAllow(CalogContextT *context); static int32_t contextEnqueue(CalogT *calog, uint64_t targetId, MessageT *message); +static CalogContextT *contextAtIndex(CalogT *calog, int64_t index); static int32_t contextPostRelease(CalogT *calog, uint64_t targetId, CalogFnT *callable); +static void contextReclaimCallables(CalogContextT *context); static void contextReply(CalogT *calog, MessageT *message, int32_t status, CalogValueT *result); static void contextRequestShutdown(CalogContextT *context); static int32_t contextSendBlocking(CalogT *calog, uint64_t targetId, MessageT *message, CalogValueT *result); static bool dispatchCommon(CalogContextT *context, MessageT *message); -static void enqueueRaw(CalogContextT *context, MessageT *message); +static bool enqueueRaw(CalogContextT *context, MessageT *message); static int32_t hookGrow(void **array, int64_t count, int64_t *cap, size_t elemSize); static void hostDispatch(CalogT *calog, MessageT *message); static uint64_t idCompose(int64_t index, uint32_t generation); @@ -327,9 +338,13 @@ static void actorReleaseCallable(CalogFnT *callable) { return; } if (contextPostRelease(runtime, owner, callable) != calogOkE) { - // Owner unreachable -- a quiescence violation (design.md sec 9, deferred). - // Best-effort finalize inline. - calogFnFinalize(callable); + // The release could not be marshalled: the owner is unreachable (a quiescence violation -- + // design.md sec 9), or the message itself would not allocate while the owner is very much + // alive. We are NOT on the owner's thread either way, so finalizing normally could run the + // adapter's engine op (luaL_unref, JS_FreeValue, ...) against a live interpreter from the + // wrong thread. Free the memory and deliberately leak the engine handle instead: it dies + // with its interpreter, and a leak beats corrupting a VM that is still running. + calogFnFinalizeForeign(callable); } } @@ -358,12 +373,47 @@ static int32_t actorRoute(CalogT *calog, CalogEntryT *entry, CalogValueT *args, void calogActorShutdown(CalogT *calog) { - int64_t index; + // How long to park between checks while waiting out a close that is already in flight (0.5 ms, + // the same tick the host pump uses). It is a rendezvous that almost never happens. + struct timespec closeWait = { 0, 500000 }; + int64_t count; + int64_t index; - for (index = 0; index < calog->ctxCount; index++) { + // Latch the registry closed and read its extent in the same breath. Until now a script was free + // to keep opening contexts (taskSpawn, taskLoad); from here calogContextOpen refuses, so the + // slots below are the complete and final set, ctxSlots cannot move under a realloc, and no + // thread can be started that these loops would miss and then free out from under itself. + pthread_mutex_lock(&calog->ctxMutex); + calog->tearingDown = true; + count = calog->ctxCount; + pthread_mutex_unlock(&calog->ctxMutex); + + // A close that was already running when the latch went up (a script's taskClose) owns that + // context's stop-join-free and unlinks its slot when it is done. Wait those out rather than race + // them -- joining one thread from two is undefined, and freeing a context another thread is + // still inside is worse. The latch refuses any new one, so this settles and cannot grow. + for (;;) { + bool pending; + + pending = false; + pthread_mutex_lock(&calog->ctxMutex); + for (index = 0; index < count; index++) { + if (calog->ctxSlots[index].context != NULL && calog->ctxSlots[index].context->closing) { + pending = true; + break; + } + } + pthread_mutex_unlock(&calog->ctxMutex); + if (!pending) { + break; + } + nanosleep(&closeWait, NULL); + } + + for (index = 0; index < count; index++) { CalogContextT *context; - context = calog->ctxSlots[index].context; - if (context == NULL || !context->started) { + context = contextAtIndex(calog, index); + if (context == NULL) { continue; } contextRequestShutdown(context); @@ -371,25 +421,25 @@ void calogActorShutdown(CalogT *calog) { // Join EVERY thread before freeing ANY context: a context tearing down may release a // callable owned by a sibling (marshalled across contexts), which resolves/posts to that // sibling -- so no sibling may be freed while another's thread still runs. - for (index = 0; index < calog->ctxCount; index++) { + for (index = 0; index < count; index++) { CalogContextT *context; - context = calog->ctxSlots[index].context; - if (context != NULL && context->started) { + context = contextAtIndex(calog, index); + if (context != NULL) { pthread_join(context->thread, NULL); } } // All context threads are stopped. Unregister each under ctxMutex (so any in-flight // registryResolveLocked from the host sees NULL rather than a freed slot), drain any // orphaned release, then free. - for (index = 0; index < calog->ctxCount; index++) { + for (index = 0; index < count; index++) { CalogContextT *context; - context = calog->ctxSlots[index].context; + pthread_mutex_lock(&calog->ctxMutex); + context = calog->ctxSlots[index].context; + calog->ctxSlots[index].context = NULL; + pthread_mutex_unlock(&calog->ctxMutex); if (context == NULL) { continue; } - pthread_mutex_lock(&calog->ctxMutex); - calog->ctxSlots[index].context = NULL; - pthread_mutex_unlock(&calog->ctxMutex); contextDrainQueue(context); contextFreeAllow(context); pthread_mutex_destroy(&context->queueMutex); @@ -404,7 +454,6 @@ void calogActorShutdown(CalogT *calog) { calog->ctxCap = 0; calog->ctxFreeCount = 0; calog->ctxFreeCap = 0; - pthread_mutex_destroy(&calog->ctxMutex); // Drain and free the host context. Only clear the calling thread's currentContext // if it named THIS runtime's host -- the thread may still host other runtimes. @@ -427,6 +476,9 @@ void calogActorShutdown(CalogT *calog) { calog->releaseHook = NULL; calog->errorHandler = NULL; calog->errorUserData = NULL; + // Last: draining the host queue above can finalize a callable, and that path resolves the owner + // under ctxMutex (it finds nothing now that every slot is gone), so the mutex has to outlive it. + pthread_mutex_destroy(&calog->ctxMutex); } @@ -625,6 +677,13 @@ CalogContextT *calogContextOpenLimited(CalogT *broker, const CalogEngineT *engin } pthread_mutex_lock(&broker->ctxMutex); + // The runtime is being destroyed: refuse rather than join a registry that is already being + // walked and freed (a taskSpawn racing calogDestroy would otherwise leave a thread running + // past the teardown that never saw it). + if (broker->tearingDown) { + pthread_mutex_unlock(&broker->ctxMutex); + goto fail; + } if (broker->ctxFreeCount > 0) { broker->ctxFreeCount--; index = broker->ctxFree[broker->ctxFreeCount]; @@ -653,16 +712,19 @@ CalogContextT *calogContextOpenLimited(CalogT *broker, const CalogEngineT *engin context->id = idCompose(index, generation); broker->ctxSlots[index].context = context; broker->ctxSlots[index].generation = generation; - pthread_mutex_unlock(&broker->ctxMutex); - + // Start the thread and publish `started` INSIDE the registry lock. A teardown reads both under + // that same lock, so it can never catch this context half-registered -- either the slot is still + // empty, or it is filled and the thread it names is running. The new thread may need the lock + // itself (its first callable, a context init hook); it simply waits the moment or two until the + // unlock below, and nothing here waits on the new thread, so the two cannot deadlock. if (pthread_create(&context->thread, NULL, threadMain, context) != 0) { - pthread_mutex_lock(&broker->ctxMutex); broker->ctxSlots[index].context = NULL; registryFreePush(broker, index); pthread_mutex_unlock(&broker->ctxMutex); goto fail; } context->started = true; + pthread_mutex_unlock(&broker->ctxMutex); return context; fail: @@ -778,6 +840,54 @@ bool calogCurrentShuttingDown(void) { } +// Latch the runtime aborting so no context runs another line of script (see calog.h). Called from a +// native: the engine raises the error returned here, which unwinds the calling script out of its +// chunk at the call site, and every later call -- from this context or any other -- is refused by +// calogCall/calogFnInvoke, so a script that catches the unwind still cannot do anything. +// Idempotent: latching a latched runtime is a no-op. +// +// It deliberately does NOT retire the calling context. Teardown order is the host's (calogDestroy +// releases the cross-context reference holders -- export/pubsub/timer -- while every context is +// still alive, and only then joins them). A context that closed itself here would destroy its own +// interpreter first, stranding the callables those registries still hold: harmless for a VM that +// frees everything on close, fatal for one that asserts it owns no live objects (QuickJS). The +// scripts are stopped either way -- that is what the latch is for. +int32_t calogAbortAll(CalogT *calog, CalogValueT *result) { + atomic_store(&calog->aborting, true); + return calogFail(result, calogErrAbortE, CALOG_ABORT_MESSAGE); +} + + +// Stop just the CALLING script (see calog.h). Same unwind as calogAbortAll -- the native returns +// this, the engine raises it, the script ends at the call -- but scoped to one context: the runtime +// and every other script keep running. The context also retires, so its thread ends once this eval +// unwinds and the host can reap it. Safe to retire from here because the thread reclaims the engine +// handles it owns on its way out (contextReclaimCallables), so nothing it published outlives its VM. +int32_t calogAbortCurrent(CalogValueT *result) { + if (currentContext == NULL || currentContext->id == CALOG_HOST_ID) { + return calogFail(result, calogErrUnsupportedE, "calogAbortCurrent: no calling script to end"); + } + atomic_store(¤tContext->aborting, true); + calogCurrentRetire(); + return calogFail(result, calogErrAbortE, CALOG_ABORT_MESSAGE); +} + + +// True when the caller must stop running script code: this runtime was latched (calogAbortAll), or +// the calling context ended itself (calogAbortCurrent). One query for both, so the dispatch choke +// points and every engine adapter ask the same question. On the host thread, or on a thread whose +// context belongs to another runtime, only the runtime latch applies. +bool calogAborting(CalogT *calog) { + if (atomic_load(&calog->aborting)) { + return true; + } + if (currentContext == NULL || currentContext->broker != calog) { + return false; + } + return atomic_load(¤tContext->aborting); +} + + // Tear down a single context (quiescence assumed). The thread is stopped and joined, // then the slot is unlinked under the registry lock so no foreign enqueue can reach // the freed queue mutex; the freed index returns to the freelist. @@ -789,6 +899,18 @@ void calogContextClose(CalogContextT *context) { return; } broker = context->broker; + // Exactly one closer may stop, join and free a given context. A script thread reaches this + // through taskClose, which can land while calogDestroy is tearing the runtime down -- and two + // threads joining one pthread is undefined. So claim it here, under the registry lock, or leave + // it to whoever holds the claim: the teardown (tearingDown is its claim on every context, and it + // waits out any close already in flight) or another close already running. + pthread_mutex_lock(&broker->ctxMutex); + if (broker->tearingDown || context->closing) { + pthread_mutex_unlock(&broker->ctxMutex); + return; + } + context->closing = true; + pthread_mutex_unlock(&broker->ctxMutex); if (context->started) { contextRequestShutdown(context); pthread_join(context->thread, NULL); @@ -808,6 +930,21 @@ void calogContextClose(CalogContextT *context) { } +// Read one registry slot with ctxMutex held. Teardown walks the registry from the host thread while +// script threads can still be unlinking contexts of their own, so every slot read has to take the +// lock -- an unlocked walk races an open's realloc of the whole array. calogContextOpen fills a slot +// and starts that context's thread inside this same lock, so a slot found here always names a +// context whose thread is running and is therefore joinable. +static CalogContextT *contextAtIndex(CalogT *calog, int64_t index) { + CalogContextT *context; + + pthread_mutex_lock(&calog->ctxMutex); + context = calog->ctxSlots[index].context; + pthread_mutex_unlock(&calog->ctxMutex); + return context; +} + + static int32_t contextDispatch(CalogT *calog, uint64_t targetId, CalogNativeFnT fn, void *userData, CalogValueT *args, int32_t argCount, CalogValueT *result) { MessageT *call; int32_t status; @@ -944,9 +1081,12 @@ static int32_t contextEnqueue(CalogT *calog, uint64_t targetId, MessageT *messag pthread_mutex_lock(&calog->ctxMutex); target = registryResolveLocked(calog, targetId); if (target != NULL) { - enqueueRaw(target, message); + bool queued; + queued = enqueueRaw(target, message); pthread_mutex_unlock(&calog->ctxMutex); - return calogOkE; + // A resolvable context whose queue has closed is on its way out: dead for our purposes, and + // reported as such rather than accepting a message no one will serve. + return queued ? calogOkE : calogErrDeadE; } // Resolve failed: an in-range index means the slot existed and has since been // freed or recycled (a dead context); out of range means the id named nothing. @@ -1036,6 +1176,67 @@ bool calogContextRegistered(CalogT *runtime, uint64_t ctxId) { } +// List a new callable against the context that owns it, so contextReclaimCallables can release its +// engine handle if that context is torn down while something else still holds it. Locks in the +// runtime's established order (ctxMutex, then the context's queueMutex). An owner that no longer +// resolves needs no entry: its interpreter is already gone, which calogFnFinalize handles. +int32_t calogContextTrackFn(CalogT *runtime, uint64_t ownerCtxId, CalogFnT *fn) { + CalogContextT *owner; + void *buffer; + int64_t capacity; + int32_t status; + + if (runtime == NULL || ownerCtxId == CALOG_HOST_ID) { + return calogOkE; // host-owned: no interpreter for the handle to outlive + } + status = calogOkE; + pthread_mutex_lock(&runtime->ctxMutex); + owner = registryResolveLocked(runtime, ownerCtxId); + if (owner != NULL) { + pthread_mutex_lock(&owner->queueMutex); + buffer = owner->ownedFns; + capacity = owner->ownedCap; + status = calogGrow(&buffer, &capacity, owner->ownedCount + 1, sizeof(CalogFnT *)); + if (status == calogOkE) { + owner->ownedFns = (CalogFnT **)buffer; + owner->ownedCap = capacity; + owner->ownedFns[owner->ownedCount] = fn; + owner->ownedCount++; + } + pthread_mutex_unlock(&owner->queueMutex); + } + pthread_mutex_unlock(&runtime->ctxMutex); + return status; +} + + +// Drop a callable from its owner's list as it is finalized. Order within the list carries no +// meaning, so the hole is filled from the end. A callable the owner already reclaimed is not in any +// list (the sweep took the whole array), which is why this can run for one and simply find nothing. +void calogContextUntrackFn(CalogT *runtime, uint64_t ownerCtxId, CalogFnT *fn) { + CalogContextT *owner; + int64_t index; + + if (runtime == NULL || ownerCtxId == CALOG_HOST_ID) { + return; + } + pthread_mutex_lock(&runtime->ctxMutex); + owner = registryResolveLocked(runtime, ownerCtxId); + if (owner != NULL) { + pthread_mutex_lock(&owner->queueMutex); + for (index = 0; index < owner->ownedCount; index++) { + if (owner->ownedFns[index] == fn) { + owner->ownedCount--; + owner->ownedFns[index] = owner->ownedFns[owner->ownedCount]; + break; + } + } + pthread_mutex_unlock(&owner->queueMutex); + } + pthread_mutex_unlock(&runtime->ctxMutex); +} + + void calogPump(CalogT *calog) { CalogContextT *previous; MessageT *message; @@ -1084,6 +1285,43 @@ void calogSetErrorHandler(CalogT *calog, CalogErrorFnT fn, void *userData) { // The reply tail shared by CALL dispatch: hand (status, result) back to whoever is // blocked on this message -- an external caller's reply box, or a context caller via // a REPLY enqueued onto its queue (in the serving context's runtime). Consumes message. +// Release the engine handle of every callable this context still owns, on the context's OWN thread, +// while its interpreter is still alive. Anything else holding one of these callables -- a pubsub +// subscription, an export, a value handed to another engine, an invoke in flight -- keeps a valid +// (dead) shell to release later; without this the engine handle would outlive the VM that owns it, +// which most VMs quietly tolerate and QuickJS aborts on. See design.md sec 26. +static void contextReclaimCallables(CalogContextT *context) { + CalogFnT **owned; + int64_t count; + int64_t index; + + // Take the whole list first: an engine release can cascade (a VM finalizer dropping another of + // this context's callables), which re-enters untrack, and must not run against a list being + // walked. Retaining each entry over the sweep keeps a cascade from freeing one we have not + // reached yet. + pthread_mutex_lock(&context->queueMutex); + owned = context->ownedFns; + count = context->ownedCount; + context->ownedFns = NULL; + context->ownedCount = 0; + context->ownedCap = 0; + pthread_mutex_unlock(&context->queueMutex); + if (owned == NULL) { + return; + } + for (index = 0; index < count; index++) { + calogFnRetain(owned[index]); + } + for (index = 0; index < count; index++) { + calogFnReclaim(owned[index]); + } + for (index = 0; index < count; index++) { + calogFnRelease(owned[index]); // reclaimed above, so a final drop frees the shell only + } + free(owned); +} + + static void contextReply(CalogT *calog, MessageT *message, int32_t status, CalogValueT *result) { if (message->replyBox != NULL) { ReplyBoxT *box; @@ -1216,8 +1454,14 @@ static bool dispatchCommon(CalogContextT *context, MessageT *message) { } -static void enqueueRaw(CalogContextT *context, MessageT *message) { +// Append to a context's queue, or refuse if that context has stopped serving. Returns false only for +// the refusal; the caller owns the message either way. +static bool enqueueRaw(CalogContextT *context, MessageT *message) { pthread_mutex_lock(&context->queueMutex); + if (context->queueClosed) { + pthread_mutex_unlock(&context->queueMutex); + return false; + } message->next = NULL; if (context->tail != NULL) { context->tail->next = message; @@ -1227,6 +1471,7 @@ static void enqueueRaw(CalogContextT *context, MessageT *message) { context->tail = message; pthread_cond_signal(&context->queueCond); pthread_mutex_unlock(&context->queueMutex); + return true; } @@ -1270,6 +1515,12 @@ static MessageT *messageDequeue(CalogContextT *context) { pthread_cond_wait(&context->queueCond, &context->queueMutex); } if (context->head == NULL) { + // Queue drained and shutting down: this thread is about to leave serveLoop. Close the queue + // in the SAME critical section that decides to stop, so a sender can never slip a message in + // between -- one that nothing would ever serve, leaving a blocking caller waiting on a reply + // that cannot come. From here contextEnqueue reports the context dead, which every caller + // already handles. + context->queueClosed = true; pthread_mutex_unlock(&context->queueMutex); return NULL; } @@ -1489,6 +1740,10 @@ static void *threadMain(void *arg) { context->broker->contextHooks[index].shutdown(context, context->broker->contextHooks[index].userData); } } + // Reclaim the engine handle of every callable this context still owns, while the interpreter is + // alive and this is its own thread. Anything still holding one -- a subscription, an export, a + // value another engine received -- is left a dead shell it can safely release later. + contextReclaimCallables(context); // Mark the interpreter dead BEFORE destroying it, so registryResolveLocked stops handing // this context out: a callable this context owns that is released from now on (e.g. one // published via the export library and dropped after the context unloads) finalizes by diff --git a/src/janet/janetAdapter.c b/src/janet/janetAdapter.c index 147b5ef4..84ca4b1e 100644 --- a/src/janet/janetAdapter.c +++ b/src/janet/janetAdapter.c @@ -34,6 +34,10 @@ #include #include +// Starting size of the buffer that captures what janet_dobytes reports (an error line plus a stack +// trace). It grows as needed, so this only saves a few early reallocations. +#define JANET_ERROR_CAPTURE_CAP 256 + struct CalogJanetT { JanetTable *env; CalogT *broker; @@ -809,12 +813,37 @@ static int32_t janetWrapFunction(CalogJanetT *context, JanetFunction *function, int32_t calogJanetRun(CalogJanetT *context, const char *source) { - Janet out; - int flags; + JanetBuffer *captured; + Janet errKey; + Janet out; + bool aborted; + int32_t status; + int flags; gJanetInterrupted = false; // one-shot per run: a fresh over-cap interrupt may fire again + // Unlike every other engine, Janet reports a failed script itself, from inside janet_dobytes + // (the error line plus a stack trace, through janet_eprintf). That is the right output for a + // script that failed and exactly the wrong output for one that calogExit unwound on purpose, so + // capture it and decide afterwards. janet_eprintf honors the "err" dynamic binding; the binding + // must go in the TOP dyn table, because janet_dobytes prints once janet_continue has returned + // and no fiber is current. A script's own (eprint) still goes straight to stderr -- inside a + // fiber the lookup finds this env's bindings, not the top table. + captured = janet_buffer(JANET_ERROR_CAPTURE_CAP); + errKey = janet_ckeywordv("err"); + // Janet does not mark the top dyn table, so an entry parked there is invisible to the collector: + // root both halves by hand or a collection inside the script frees them under the binding. + janet_gcroot(errKey); + janet_gcroot(janet_wrap_buffer(captured)); + janet_setdyn("err", janet_wrap_buffer(captured)); out = janet_wrap_nil(); flags = janet_dobytes(context->env, (const uint8_t *)source, (int32_t)strlen(source), "calog", &out); + janet_setdyn("err", janet_wrap_nil()); + // calogExit (calogAbortAll) unwinding this script is not a failure: drop what Janet reported and + // hand back success, so the runner neither prints a trace nor retires the context as broken. + // Only a RUNTIME trip can be an abort -- a parse or compile error still reports, mid-teardown or + // not, since no native ran to ask for one. + aborted = ((flags & JANET_DO_ERROR_RUNTIME) != 0) && calogAborting(context->broker); + status = (flags != 0 && !aborted) ? calogErrArgE : calogOkE; if (flags != 0) { // If a sandbox limit tripped -- the allocator interrupted the VM this run (gJanetInterrupted, // which survives a GC that later drops memUsed back under the cap) or the wall-clock deadline @@ -829,9 +858,12 @@ int32_t calogJanetRun(CalogJanetT *context, const char *source) { calogCurrentRetire(); } } - // janet_dobytes already printed the error + a stack trace to stderr (janet_eprintf), so - // do not print it again here; just report failure through the single error channel. - return calogErrArgE; + if (status != calogOkE) { + // Relayed verbatim, so a failing janet script reads exactly as it did before. + fwrite(captured->data, 1, (size_t)captured->count, stderr); + } } - return calogOkE; + janet_gcunroot(janet_wrap_buffer(captured)); + janet_gcunroot(errKey); + return status; } diff --git a/src/js/jsAdapter.c b/src/js/jsAdapter.c index 14e24d15..f2c374e0 100644 --- a/src/js/jsAdapter.c +++ b/src/js/jsAdapter.c @@ -448,7 +448,14 @@ int32_t calogJsRun(CalogJsT *context, const char *source) { if (JS_IsException(val)) { JSValue exc; const char *message; - exc = JS_GetException(ctx); + exc = JS_GetException(ctx); + // calogExit (calogAbortAll) unwound this script deliberately: not a failure, and the + // runtime is already tearing down, so report nothing. + if (calogAborting(context->broker)) { + JS_FreeValue(ctx, exc); + JS_FreeValue(ctx, val); + return calogOkE; + } message = JS_ToCString(ctx, exc); fprintf(stderr, "js eval error: %s\n", message != NULL ? message : "(unknown)"); if (message != NULL) { diff --git a/src/lua/luaAdapter.c b/src/lua/luaAdapter.c index 7fa2f088..31cce8d9 100644 --- a/src/lua/luaAdapter.c +++ b/src/lua/luaAdapter.c @@ -471,6 +471,12 @@ int32_t calogLuaRun(CalogLuaT *context, const char *source) { return calogErrArgE; } if (lua_pcall(L, 0, 0, 0) != LUA_OK) { + // calogExit (calogAbortAll) unwound this script deliberately: not a failure, and the + // runtime is already tearing down, so report nothing. + if (calogAborting(context->broker)) { + lua_pop(L, 1); + return calogOkE; + } fprintf(stderr, "lua run error: %s\n", lua_tostring(L, -1)); lua_pop(L, 1); return calogErrArgE; diff --git a/src/mruby/mrubyAdapter.c b/src/mruby/mrubyAdapter.c index dded7e3e..f3f43757 100644 --- a/src/mruby/mrubyAdapter.c +++ b/src/mruby/mrubyAdapter.c @@ -639,6 +639,14 @@ int32_t calogMrubyRun(CalogMrubyT *context, const char *source) { if (mrb->exc != NULL) { mrb_value exc; mrb_value text; + // calogExit (calogAbortAll) unwound this script deliberately: not a failure, and the runtime + // is already tearing down, so report nothing -- and do not run inspect, which would re-enter + // a VM whose every native call is now refused. + if (calogAborting(context->broker)) { + mrb->exc = NULL; + mrb_gc_arena_restore(mrb, arena); + return calogOkE; + } exc = mrb_obj_value(mrb->exc); mrb->exc = NULL; text = mrb_funcall_argv(mrb, exc, mrb_intern_lit(mrb, "inspect"), 0, NULL); diff --git a/src/mybasic/mybasicAdapter.c b/src/mybasic/mybasicAdapter.c index f41807c9..0f3fca7a 100644 --- a/src/mybasic/mybasicAdapter.c +++ b/src/mybasic/mybasicAdapter.c @@ -1911,6 +1911,11 @@ int32_t calogMyBasicRun(CalogMyBasicT *context, const char *source) { } code = mb_run(context->bas, true); if (code != MB_FUNC_OK) { + // calogExit (calogAbortAll) unwound this script deliberately: not a failure, and the + // runtime is already tearing down, so report nothing. + if (calogAborting(context->broker)) { + return calogOkE; + } mbReportError(context, "run"); return calogErrArgE; } diff --git a/src/s7/s7Adapter.c b/src/s7/s7Adapter.c index b190bf3a..b4e69b66 100644 --- a/src/s7/s7Adapter.c +++ b/src/s7/s7Adapter.c @@ -540,6 +540,11 @@ int32_t calogS7Run(CalogS7T *context, const char *source) { s7_eval_c_string(sc, wrapped); free(wrapped); if (context->errorFlag) { + // calogExit (calogAbortAll) unwound this script deliberately: not a failure, and the + // runtime is already tearing down, so report nothing. + if (calogAborting(context->broker)) { + return calogOkE; + } fprintf(stderr, "s7 error: %s\n", context->errorText); return calogErrArgE; } diff --git a/src/squirrel/squirrelAdapter.c b/src/squirrel/squirrelAdapter.c index 7a5edb9f..b8f21d1a 100644 --- a/src/squirrel/squirrelAdapter.c +++ b/src/squirrel/squirrelAdapter.c @@ -528,6 +528,12 @@ int32_t calogSquirrelRun(CalogSquirrelT *context, const char *source) { } sq_pushroottable(v); if (SQ_FAILED(sq_call(v, 1, SQFalse, SQTrue))) { + // calogExit (calogAbortAll) unwound this script deliberately: not a failure, and the + // runtime is already tearing down, so report nothing. + if (calogAborting(context->broker)) { + sq_settop(v, baseTop); + return calogOkE; + } squirrelReportRuntimeError(v, "squirrel run error"); sq_settop(v, baseTop); return calogErrArgE; diff --git a/src/tcl/tclAdapter.c b/src/tcl/tclAdapter.c index 5e2ebea6..090d750c 100644 --- a/src/tcl/tclAdapter.c +++ b/src/tcl/tclAdapter.c @@ -845,6 +845,11 @@ int32_t calogTclRun(CalogTclT *context, const char *source) { code = Tcl_EvalEx(context->interp, source, -1, TCL_EVAL_GLOBAL); if (code != TCL_OK) { + // calogExit (calogAbortAll) unwound this script deliberately: not a failure, and the + // runtime is already tearing down, so report nothing. + if (calogAborting(context->broker)) { + return calogOkE; + } fprintf(stderr, "tcl error: %s\n", Tcl_GetString(Tcl_GetObjResult(context->interp))); return calogErrArgE; } diff --git a/src/value.c b/src/value.c index fe22aee1..754cdcb8 100644 --- a/src/value.c +++ b/src/value.c @@ -191,6 +191,7 @@ int32_t calogAggSet(CalogAggT *aggregate, CalogValueT *key, CalogValueT *value) int32_t calogFnCreate(CalogFnT **out, CalogT *runtime, CalogNativeFnT fn, void *userData, CalogReleaseFnT release, uint64_t ownerCtxId) { CalogFnT *callable; + int32_t status; *out = NULL; callable = (CalogFnT *)calloc(1, sizeof(*callable)); @@ -204,6 +205,14 @@ int32_t calogFnCreate(CalogFnT **out, CalogT *runtime, CalogNativeFnT fn, void * callable->ownerCtxId = ownerCtxId; atomic_init(&callable->refCount, CALLABLE_INITIAL_REFCOUNT); atomic_init(&callable->alive, true); + // List it against the owning context, which reclaims the engine handle if it is torn down while + // something else still holds this callable (design.md sec 26). No list, no reclaim -- so a + // failure here fails the create rather than handing back a callable that could strand its handle. + status = calogContextTrackFn(runtime, ownerCtxId, callable); + if (status != calogOkE) { + free(callable); + return status; + } *out = callable; return calogOkE; } @@ -229,6 +238,11 @@ void calogFnFinalize(CalogFnT *callable) { // // (Freeing userData directly requires every engine's per-callable struct to be a single // heap allocation; all adapters honor that -- see CalogReleaseFnT in calogInternal.h.) + // + // A callable the owner already RECLAIMED (calogFnReclaim, as its interpreter was torn down) + // has no release hook left, so this frees only the shell -- the engine handle went with the + // interpreter that owned it. + calogContextUntrackFn(callable->runtime, callable->ownerCtxId, callable); if (callable->release != NULL) { if (callable->ownerCtxId == CALOG_HOST_ID || calogContextRegistered(callable->runtime, callable->ownerCtxId)) { callable->release(callable); @@ -240,6 +254,25 @@ void calogFnFinalize(CalogFnT *callable) { } +// The owning context is tearing its interpreter down; this runs on that context's own thread, with +// the interpreter still alive, for every callable the context still owns. Running the engine release +// HERE is what keeps the handle from outliving its VM: whoever else still holds a reference -- a +// pubsub subscription, an export, a value sitting in another engine, an invoke in flight -- finds a +// dead callable and finalizes an empty shell. Marking it dead first makes any such invoke fail +// cleanly instead of reaching into a VM that is going away. +void calogFnReclaim(CalogFnT *callable) { + if (callable == NULL) { + return; + } + calogFnMarkDead(callable); + if (callable->release != NULL) { + callable->release(callable); // frees the engine handle AND the adapter's userData block + callable->release = NULL; + callable->userData = NULL; + } +} + + CalogNativeFnT calogFnNative(const CalogFnT *callable) { return callable->fn; } @@ -247,8 +280,18 @@ CalogNativeFnT calogFnNative(const CalogFnT *callable) { int32_t calogFnInvoke(CalogFnT *callable, CalogValueT *args, int32_t argCount, CalogValueT *result) { calogValueNil(result); + // The owner reclaimed this callable as its interpreter went away (calogFnReclaim), so there is + // nothing left to call. calogErrDeadE is the same answer the routing path gives for a context + // that is gone -- the timer library cancels a timer on exactly that, and pubsub stops counting + // the subscriber as delivered. if (!atomic_load_explicit(&callable->alive, memory_order_acquire)) { - return calogFail(result, calogErrNotFoundE, "callable owner no longer exists"); + return calogFail(result, calogErrDeadE, "callable owner no longer exists"); + } + // The caller has been stopped (calogAbortAll, or this script's own calogAbortCurrent): a script + // function is script code, so refuse it for the same reason calogCall refuses a native -- a timer + // or subscriber callback already queued must not start running a script body afterwards. + if (callable->runtime != NULL && calogAborting(callable->runtime)) { + return calogFail(result, calogErrAbortE, CALOG_ABORT_MESSAGE); } // The owning runtime's actor layer, if present, marshals a foreign-thread invoke // to the owner's thread; inline otherwise (a bare broker has no hook). @@ -259,6 +302,29 @@ int32_t calogFnInvoke(CalogFnT *callable, CalogValueT *args, int32_t argCount, C } +// Finalize a callable WITHOUT running the engine release, for a caller that is not on the owner's +// thread and could not marshal the release there (actorReleaseCallable's fallback). The handle +// INSIDE the interpreter is deliberately left behind -- it is pinned until that interpreter is +// destroyed, which for a long-lived context can be a long time, but leaking beats reaching into a +// live VM from the wrong thread. +// +// userData is freed on exactly the same condition as calogFnFinalize: only when a release hook +// exists. That hook is what makes userData calog's to free (an adapter's single heap block, per the +// CalogReleaseFnT contract). A host-owned callable from calogFnFromNative has NO hook and a userData +// the EMBEDDER owns -- freeing that would corrupt their heap, which is worse than anything this +// function exists to prevent. A callable already reclaimed has both cleared, so it frees the shell. +void calogFnFinalizeForeign(CalogFnT *callable) { + if (callable == NULL) { + return; + } + calogContextUntrackFn(callable->runtime, callable->ownerCtxId, callable); + if (callable->release != NULL) { + free(callable->userData); + } + free(callable); +} + + void calogFnMarkDead(CalogFnT *callable) { if (callable == NULL) { return; diff --git a/src/wren/wrenAdapter.c b/src/wren/wrenAdapter.c index 4d6f193c..f1f31e65 100644 --- a/src/wren/wrenAdapter.c +++ b/src/wren/wrenAdapter.c @@ -566,6 +566,14 @@ int32_t calogWrenRun(CalogWrenT *context, const char *source) { context->errorMsg = NULL; result = wrenInterpret(vm, "main", source); if (result != WREN_RESULT_SUCCESS) { + // calogExit (calogAbortAll) unwound this script deliberately: not a failure, and the + // runtime is already tearing down, so report nothing. Wren reports a compile failure as its + // own result, so a syntax error still gets its diagnostic even mid-teardown. + if (result == WREN_RESULT_RUNTIME_ERROR && calogAborting(context->broker)) { + free(context->errorMsg); + context->errorMsg = NULL; + return calogOkE; + } fprintf(stderr, "wren error: %s\n", context->errorMsg != NULL ? context->errorMsg : "(unknown)"); free(context->errorMsg); context->errorMsg = NULL; diff --git a/tests/testBroker.c b/tests/testBroker.c index ece884e8..2310a25a 100644 --- a/tests/testBroker.c +++ b/tests/testBroker.c @@ -273,7 +273,9 @@ static void testCallableDead(void) { calogFnMarkDead(callable); status = calogFnInvoke(callable, NULL, 0, &result); - CHECK(status == calogErrNotFoundE, "invoke on dead callable returns not-found"); + // calogErrDeadE, the same answer the actor layer gives for a callable whose context is gone -- + // one status for "the owner is gone", which is what the timer library cancels a timer on. + CHECK(status == calogErrDeadE, "invoke on dead callable reports the owner is gone"); CHECK(result.type == calogStringE, "dead invoke leaves an error string"); calogValueFree(&result); diff --git a/tests/testExit.c b/tests/testExit.c new file mode 100644 index 00000000..a297ce7c --- /dev/null +++ b/tests/testExit.c @@ -0,0 +1,238 @@ +// testExit.c -- calogAbortAll: one native stops every script in the runtime. +// +// This is the contract API.md states for the runner's calogExit: it DOES NOT RETURN. A native that +// hands back calogAbortAll's value must unwind the CALLING script out of its chunk right at the call +// site, on every engine -- so each script below calls stopAll() and then spins in an infinite loop. +// The loop is the evidence: if the abort unwound the script it is never entered and the context +// retires within milliseconds, while a native that merely returned would pin that context forever. +// Native calls cannot serve as evidence here, because the latch refuses them either way. +// +// The same run also proves the three properties the abort must have: an aborted script is NOT +// reported to the error handler (it did not fail, and its engine must print nothing either), the +// runtime stays latched afterwards, and every later native call is refused instead of served. + +#define _POSIX_C_SOURCE 200809L + +#include "calog.h" + +#include +#include +#include +#include +#include +#include + +#define CHECK(cond, msg) checkImpl((cond), (msg), __LINE__) + +// s7 interns a small, bounded set of "permanent" strings it never reclaims (an s7 trait, not a +// leak), so suppress exactly that site to keep the leak check meaningful. LSan calls this weak hook. +const char *__lsan_default_suppressions(void); +const char *__lsan_default_suppressions(void) { + return "leak:make_permanent_string\n"; +} + +// A retiring context is joined by its own thread returning, so the wait is short in the passing +// case; the budget only has to cover a loaded machine (4000 * 0.5 ms = 2 s). +#define PUMP_INTERVAL_NS 500000 +#define PUMP_LIMIT 4000 + +// One engine's script: reach mark(), abort, then loop forever if the abort let the script continue. +typedef struct EngineCaseT { + const CalogEngineT *engine; + const char *name; + const char *source; +} EngineCaseT; + +static _Atomic int32_t markCount = 0; +static _Atomic int32_t errorCount = 0; +static int32_t testsRun = 0; +static int32_t testsFailed = 0; + +static void checkEngine(bool condition, const char *engineName, const char *what, int32_t line); +static void checkImpl(bool condition, const char *message, int32_t line); +static int32_t nativeMark(CalogValueT *args, int32_t argCount, CalogValueT *result, void *userData); +static int32_t nativeStopAll(CalogValueT *args, int32_t argCount, CalogValueT *result, void *userData); +static void onError(uint64_t contextId, const char *message, void *userData); +static void runCase(const EngineCaseT *item); +static void testLatchIsPerRuntime(void); + + +static void checkEngine(bool condition, const char *engineName, const char *what, int32_t line) { + char message[160]; + + snprintf(message, sizeof(message), "%s: %s", engineName, what); + checkImpl(condition, message, line); +} + + +static void checkImpl(bool condition, const char *message, int32_t line) { + testsRun++; + if (!condition) { + testsFailed++; + printf("FAIL testExit.c:%d %s\n", line, message); + } +} + + +// Host native: records that the script got this far. Registered like the runner's calogPrint, so the +// script blocks on the host thread for it -- the ordinary path a script reaches C through. +static int32_t nativeMark(CalogValueT *args, int32_t argCount, CalogValueT *result, void *userData) { + (void)args; + (void)argCount; + (void)userData; + atomic_fetch_add(&markCount, 1); + calogValueNil(result); + return calogOkE; +} + + +// Inline native: what the runner's calogExit does, minus recording an exit code. Inline is what +// makes the abort reach THIS script -- an error raised on the host thread could not unwind it. +// +// calogCurrentRetire is the test's probe, not part of what calogExit does. Retirement is serviced +// only AFTER the current eval returns, so "this context's thread exited" is precisely "the eval +// returned" -- which is the property under test, and the one thing a script cannot report itself +// once every native call is refused. A script that kept running would sit in its loop forever and +// its thread would never exit. +static int32_t nativeStopAll(CalogValueT *args, int32_t argCount, CalogValueT *result, void *userData) { + (void)args; + (void)argCount; + (void)userData; + calogValueNil(result); + calogCurrentRetire(); + return calogAbortAll(calogCurrent(), result); +} + + +static void onError(uint64_t contextId, const char *message, void *userData) { + (void)contextId; + (void)userData; + atomic_fetch_add(&errorCount, 1); + printf(" (error handler saw: %s)\n", message != NULL ? message : "(null)"); +} + + +// Run one engine's script on its own runtime (the latch is one-way, so each engine needs a fresh +// one), then check what the abort did. +static void runCase(const EngineCaseT *item) { + CalogT *calog; + CalogContextT *ctx; + CalogValueT result; + struct timespec tick = { 0, PUMP_INTERVAL_NS }; + int32_t index; + bool finished; + + atomic_store(&markCount, 0); + atomic_store(&errorCount, 0); + + calog = calogCreate(); + if (calog == NULL) { + checkEngine(false, item->name, "runtime create failed", __LINE__); + return; + } + calogSetErrorHandler(calog, onError, NULL); + calogRegister(calog, "mark", nativeMark, NULL); + calogRegisterInline(calog, "stopAll", nativeStopAll, NULL); + + ctx = calogContextOpen(calog, item->engine); + if (ctx == NULL) { + checkEngine(false, item->name, "context open failed", __LINE__); + calogDestroy(calog); + return; + } + calogContextEval(ctx, item->source); + finished = false; + for (index = 0; index < PUMP_LIMIT; index++) { + calogPump(calog); + if (calogContextFinished(ctx)) { + finished = true; + break; + } + nanosleep(&tick, NULL); + } + calogPump(calog); + + checkEngine(atomic_load(&markCount) == 1, item->name, "the script ran up to the abort", __LINE__); + checkEngine(finished, item->name, "the abort unwound the script -- the loop after it never ran", __LINE__); + checkEngine(atomic_load(&errorCount) == 0, item->name, "an aborted script is not reported as an error", __LINE__); + checkEngine(calogAborting(calog), item->name, "the runtime stays latched aborting", __LINE__); + + calogValueNil(&result); + checkEngine(calogCall(calog, "mark", NULL, 0, &result) == calogErrAbortE, item->name, "a later native call is refused", __LINE__); + calogValueFree(&result); + checkEngine(atomic_load(&markCount) == 1, item->name, "the refused call never reached the native", __LINE__); + + // A context that did NOT retire is still spinning in that infinite loop, and closing it would + // join a thread that never returns -- hanging the test instead of reporting the failure. Leak it + // (the check above has already failed) and move on to the next engine. + if (finished) { + calogContextClose(ctx); + calogDestroy(calog); + } +} + + +// The latch belongs to ONE runtime: aborting a runtime must not stop scripts in a second one that +// happens to share the process. This also covers calogAbortAll called from the host thread, which +// has no script of its own to unwind. +static void testLatchIsPerRuntime(void) { + CalogT *first; + CalogT *second; + CalogValueT result; + + first = calogCreate(); + second = calogCreate(); + if (first == NULL || second == NULL) { + CHECK(false, "per-runtime latch: runtime create failed"); + calogDestroy(first); + calogDestroy(second); + return; + } + calogRegister(first, "mark", nativeMark, NULL); + calogRegister(second, "mark", nativeMark, NULL); + atomic_store(&markCount, 0); + + calogValueNil(&result); + CHECK(calogAbortAll(first, &result) == calogErrAbortE, "calogAbortAll reports calogErrAbortE"); + calogValueFree(&result); + CHECK(calogAborting(first), "the aborted runtime is latched"); + CHECK(!calogAborting(second), "a second runtime is untouched"); + + calogValueNil(&result); + CHECK(calogCall(first, "mark", NULL, 0, &result) == calogErrAbortE, "the aborted runtime refuses a native"); + calogValueFree(&result); + calogValueNil(&result); + CHECK(calogCall(second, "mark", NULL, 0, &result) == calogOkE, "the second runtime still serves natives"); + calogValueFree(&result); + CHECK(atomic_load(&markCount) == 1, "exactly one of the two calls ran the native"); + + calogDestroy(first); + calogDestroy(second); +} + + +int main(void) { + // Each script: reach the host, abort, then a loop that must never be entered. The loop bodies + // differ only in each language's syntax for "forever". + static const EngineCaseT cases[] = { + { &calogLuaEngine, "lua", "mark()\nstopAll()\nwhile true do end" }, + { &calogJsEngine, "js", "mark(); stopAll(); while (true) {}" }, + { &calogSquirrelEngine, "squirrel", "mark(); stopAll(); while(true){}" }, + { &calogMyBasicEngine, "my-basic", "mark()\nstopAll()\nWHILE 1\nWEND" }, + { &calogBerryEngine, "berry", "mark()\nstopAll()\nwhile true\nend" }, + { &calogS7Engine, "s7", "(begin (mark) (stopAll) (do () (#f)))" }, // s7 reads ONE top-level form + { &calogWrenEngine, "wren", "Calog.call(\"mark\", [])\nCalog.call(\"stopAll\", [])\nwhile (true) {}" }, + { &calogMrubyEngine, "mruby", "mark()\nstopAll()\nwhile true do end" }, + { &calogTclEngine, "tcl", "mark\nstopAll\nwhile {1} {}" }, + { &calogJanetEngine, "janet", "(mark) (stopAll) (var i 0) (while true (set i (+ i 1)))" } + }; + size_t index; + + for (index = 0; index < sizeof(cases) / sizeof(cases[0]); index++) { + runCase(&cases[index]); + } + testLatchIsPerRuntime(); + + printf("testExit: %d checks, %d failed\n", testsRun, testsFailed); + return testsFailed == 0 ? 0 : 1; +} diff --git a/tests/testHooks.c b/tests/testHooks.c index d176d373..c8c877da 100644 --- a/tests/testHooks.c +++ b/tests/testHooks.c @@ -8,6 +8,7 @@ #include "calog.h" #include +#include #include #include #include @@ -24,9 +25,12 @@ static _Atomic int32_t shutdownCount = 0; static _Atomic int32_t firstShutdownSeq = -1; static _Atomic int32_t lastShutdownSeq = -1; static _Atomic int64_t seenUserData = 0; +static _Atomic bool lateOpenSucceeded = false; // a context opened DURING teardown (must not happen) +static _Atomic bool lateEvalAccepted = false; // work queued onto a context that has stopped serving static int32_t testsRun = 0; static int32_t testsFailed = 0; static int32_t userToken = 0x5A5A; +static CalogT *calog = NULL; // file scope: the shutdown hook reaches it too static void checkImpl(bool condition, const char *message, const char *file, int32_t line); static void contextInitHook(CalogContextT *context, void *userData); @@ -53,16 +57,32 @@ static void contextInitHook(CalogContextT *context, void *userData) { static void contextShutdownHook(CalogContextT *context, void *userData) { - int32_t seq; - int32_t expected; + CalogContextT *late; + int32_t seq; + int32_t expected; - (void)context; (void)userData; seq = atomic_fetch_add(&seqCounter, 1); atomic_fetch_add(&shutdownCount, 1); expected = -1; atomic_compare_exchange_strong(&firstShutdownSeq, &expected, seq); atomic_store(&lastShutdownSeq, seq); + // This hook runs on the context's own thread while calogDestroy is walking the registry -- the + // one window where a script (taskSpawn) could still ask for a context the teardown has already + // walked past, then be freed with its thread running. Opening one has to be refused here. + (void)context; + late = calogContextOpen(calog, &calogLuaEngine); + if (late != NULL) { + atomic_store(&lateOpenSucceeded, true); + calogContextClose(late); + } + // Same window, the other half: this hook runs AFTER serveLoop stopped serving and BEFORE the + // interpreter is marked dead, so the context still resolves in the registry. Queueing work here + // used to be accepted and then never served -- a fire-and-forget eval silently dropped, and a + // blocking caller left waiting on a reply that could not come. It must be refused. + if (calogContextEval(context, "local dropped = 1") == calogOkE) { + atomic_store(&lateEvalAccepted, true); + } } @@ -77,7 +97,6 @@ static void destroyBeforeHook(void) { int main(void) { - CalogT *calog; CalogContextT *ctxA; CalogContextT *ctxB; struct timespec ts = { 0, 500000 }; @@ -110,6 +129,8 @@ int main(void) { calogDestroy(calog); // before-hook -> tear down contexts (their shutdown hooks) -> after-hook CHECK(atomic_load(&shutdownCount) == 2, "per-context shutdown hook fired once per context"); + CHECK(!atomic_load(&lateOpenSucceeded), "opening a context during teardown is refused, not registered too late"); + CHECK(!atomic_load(&lateEvalAccepted), "queueing work onto a context that has stopped serving is refused, not orphaned"); CHECK(atomic_load(&destroyBeforeSeq) >= 0, "before-context destroy hook fired"); CHECK(atomic_load(&destroyAfterSeq) >= 0, "after-context destroy hook fired"); CHECK(atomic_load(&destroyBeforeSeq) < atomic_load(&firstShutdownSeq), "before-hook ran before context teardown"); diff --git a/tests/testTask.c b/tests/testTask.c index 25c1f9d5..1579e13b 100644 --- a/tests/testTask.c +++ b/tests/testTask.c @@ -38,6 +38,7 @@ static void testTaskError(void); static void testTaskLoad(void); static void testTaskMutualClose(void); static void testTaskSelfClose(void); +static void testTaskSpawnRacesTeardown(void); static void testTaskSpawn(void); static void writeFile(const char *path, const char *text); @@ -283,6 +284,54 @@ static void writeFile(const char *path, const char *text) { } +// A script spawning tasks while the host tears the runtime down: calogDestroy walks the context +// registry, and a spawn landing in that window used to be able to register a context the walk had +// already passed -- freed, with its thread still running, along with the slot array the spawn had +// just reallocated under the walk's feet. calogContextOpen now refuses once teardown has latched, so +// the spawns simply start failing. Its own runtime, since it destroys it: surviving IS the check +// (under ASan a missed context is a use-after-free, under TSan a data race on the slot table). +static void testTaskSpawnRacesTeardown(void) { + CalogT *racer; + CalogContextT *ctx; + struct timespec ts = { 0, 2000000 }; + int32_t i; + + racer = calogCreate(); + if (racer == NULL) { + CHECK(false, "spawn-vs-teardown: runtime create failed"); + return; + } + calogSetErrorHandler(racer, onError, NULL); + if (calogTaskRegister(racer) != calogOkE) { + CHECK(false, "spawn-vs-teardown: task library register failed"); + calogDestroy(racer); + return; + } + ctx = calogContextOpen(racer, &calogLuaEngine); + if (ctx == NULL) { + CHECK(false, "spawn-vs-teardown: context open failed"); + calogDestroy(racer); + return; + } + // Spawn AND close in the same loop: taskClose reaches calogContextClose from this script's own + // thread, so it races the teardown's own stop-join-free of the very same context. Keep going + // until a spawn is refused (which is what teardown does to it) or the budget runs out; each + // child exits immediately, so the thread count stays bounded. + calogContextEval(ctx, + "for i = 1, 200 do\n" + " local ok, task = pcall(taskSpawn, 'lua', 'taskExit()')\n" + " if not ok then break end\n" + " pcall(taskClose, task)\n" + "end"); + for (i = 0; i < 4; i++) { + calogPump(racer); + nanosleep(&ts, NULL); + } + calogDestroy(racer); // races the spawn loop above + CHECK(true, "a spawn loop racing calogDestroy tears down cleanly"); +} + + int main(void) { calog = calogCreate(); if (calog == NULL) { @@ -309,6 +358,7 @@ int main(void) { testTaskError(); calogDestroy(calog); + testTaskSpawnRacesTeardown(); printf("\n%d checks, %d failed\n", testsRun, testsFailed); fflush(stdout); diff --git a/tests/testTeardown.c b/tests/testTeardown.c new file mode 100644 index 00000000..36e4f758 --- /dev/null +++ b/tests/testTeardown.c @@ -0,0 +1,347 @@ +// testTeardown.c -- a library registry that holds a SCRIPT function must let go of it while the +// owning context is still alive. +// +// A CalogFnT is a handle to a function living inside a VM. If a process-global registry (a pubsub +// subscriber, an exported function, a timer callback) still holds one when that context's +// interpreter is destroyed, the release arrives too late to run the engine's own release hook and +// the script object is never freed inside its VM. That is what the two destroy phases are for: a +// library holding context-owned references registers calogDestroyBeforeContextsE, so calogDestroy +// drains it while every context is still serving its queue. +// +// Most VMs hide the mistake -- they free everything on close -- so this test uses JavaScript on +// purpose: QuickJS asserts it owns no live objects at JS_FreeRuntime and ABORTS the process. Every +// case here therefore checks the same thing in the end: that we are still running afterwards. A +// wrong phase does not fail a check, it kills the binary (and LeakSanitizer catches the milder +// variant, a registry resurrected after its shutdown). + +#define _POSIX_C_SOURCE 200809L + +#include "calog.h" + +#include "calogExport.h" +#include "calogPubsub.h" +#include "calogTimer.h" +#include "calogInternal.h" // calogPubsubShutdown/calogExportShutdown: internal, driven by hand here + +#include +#include +#include +#include +#include + +#define CHECK(cond, msg) checkImpl((cond), (msg), __LINE__) + +#define PUMP_INTERVAL_NS 500000 +#define PUMP_LIMIT 4000 + +static CalogT *calog = NULL; +static _Atomic bool readyFlag = false; +static _Atomic int32_t reportValue = -1; +static _Atomic int32_t errorCount = 0; +static int32_t testsRun = 0; +static int32_t testsFailed = 0; + +static void checkImpl(bool condition, const char *message, int32_t line); +static int32_t nativeEndSelf(CalogValueT *args, int32_t argCount, CalogValueT *result, void *userData); +static int32_t nativeReady(CalogValueT *args, int32_t argCount, CalogValueT *result, void *userData); +static int32_t nativeReport(CalogValueT *args, int32_t argCount, CalogValueT *result, void *userData); +static void onError(uint64_t contextId, const char *message, void *userData); +static void pumpUntilReady(void); +static void startRuntime(void); +static void testCrossEngineValueOutlivesOwner(void); +static void testGuardsAfterShutdown(void); +static void testHeldCallableSurvivesTeardown(const char *what, const char *source); +static void testOwnerDiesBeforeTheRuntime(const char *what, const char *source, bool expectError); + + +static void checkImpl(bool condition, const char *message, int32_t line) { + testsRun++; + if (!condition) { + testsFailed++; + printf("FAIL testTeardown.c:%d %s\n", line, message); + } +} + + +// What the runner's calogEnd does: end THIS script, leaving the runtime and every other script +// running. Inline, so it runs on the calling script's own thread and can unwind it. +static int32_t nativeEndSelf(CalogValueT *args, int32_t argCount, CalogValueT *result, void *userData) { + (void)args; + (void)argCount; + (void)userData; + calogValueNil(result); + return calogAbortCurrent(result); +} + + +static int32_t nativeReady(CalogValueT *args, int32_t argCount, CalogValueT *result, void *userData) { + (void)args; + (void)argCount; + (void)userData; + atomic_store(&readyFlag, true); + calogValueNil(result); + return calogOkE; +} + + +static int32_t nativeReport(CalogValueT *args, int32_t argCount, CalogValueT *result, void *userData) { + (void)userData; + calogValueNil(result); + if (argCount == 1 && args[0].type == calogBoolE) { + atomic_store(&reportValue, args[0].as.b ? 1 : 0); + } + return calogOkE; +} + + +static void onError(uint64_t contextId, const char *message, void *userData) { + (void)contextId; + (void)userData; + atomic_fetch_add(&errorCount, 1); + printf(" (error handler saw: %s)\n", message != NULL ? message : "(null)"); +} + + +static void pumpUntilReady(void) { + struct timespec tick = { 0, PUMP_INTERVAL_NS }; + int32_t index; + + for (index = 0; index < PUMP_LIMIT; index++) { + calogPump(calog); + if (atomic_load(&readyFlag)) { + calogPump(calog); + return; + } + nanosleep(&tick, NULL); + } +} + + +// A runtime with the three libraries that hold script functions, plus the two natives a script +// signals through. +static void startRuntime(void) { + calog = calogCreate(); + if (calog == NULL) { + CHECK(false, "runtime create failed"); + return; + } + calogSetErrorHandler(calog, onError, NULL); + calogRegisterInline(calog, "ready", nativeReady, NULL); + calogRegisterInline(calog, "report", nativeReport, NULL); + calogRegisterInline(calog, "endSelf", nativeEndSelf, NULL); + calogPubsubRegister(calog); + calogExportRegister(calog); + calogTimerRegister(calog); + atomic_store(&readyFlag, false); + atomic_store(&reportValue, -1); + atomic_store(&errorCount, 0); +} + + +// Once a library's registry has been drained, a native that would store a NEW script function in it +// has to fail: re-growing a registry that nothing will ever free again would strand the callable it +// holds (the very leak the phase fix exists to prevent). Reads stay safe, which is what the pubsub +// and export tests already rely on. +static void testGuardsAfterShutdown(void) { + CalogContextT *ctx; + + startRuntime(); + if (calog == NULL) { + return; + } + ctx = calogContextOpen(calog, &calogLuaEngine); + if (ctx == NULL) { + CHECK(false, "guards: context open failed"); + calogDestroy(calog); + return; + } + + calogPubsubShutdown(); + calogExportShutdown(); + + // pcall yields (ok, err), so bind ok first -- report takes exactly one value. + calogContextEval(ctx, "local ok = pcall(psSubscribe, 't', function() end)\n report(ok)\n ready()"); + pumpUntilReady(); + CHECK(atomic_load(&reportValue) == 0, "psSubscribe after the pubsub registry is drained fails cleanly"); + + atomic_store(&readyFlag, false); + atomic_store(&reportValue, -1); + calogContextEval(ctx, "local ok = pcall(calogExport, 'e', function() end)\n report(ok)\n ready()"); + pumpUntilReady(); + CHECK(atomic_load(&reportValue) == 0, "calogExport after the export registry is drained fails cleanly"); + + // Reads stay safe either way: a publish simply finds nobody, and an unknown global still + // resolves to nil through the export hook. + atomic_store(&readyFlag, false); + atomic_store(&reportValue, -1); + calogContextEval(ctx, "report(psPublish('t', 1) == 0 and someUndefinedGlobalName == nil)\n ready()"); + pumpUntilReady(); + CHECK(atomic_load(&reportValue) == 1, "publishing and resolving after shutdown stay safe"); + + calogContextClose(ctx); + calogDestroy(calog); +} + + +// Run source on a JavaScript context, leave whatever it registered in place, and tear the runtime +// down with calogDestroy alone -- the path a real host takes. Surviving IS the check. +static void testHeldCallableSurvivesTeardown(const char *what, const char *source) { + CalogContextT *ctx; + + printf(" teardown case: %s\n", what); + fflush(stdout); + startRuntime(); + if (calog == NULL) { + return; + } + ctx = calogContextOpen(calog, &calogJsEngine); + if (ctx == NULL) { + checkImpl(false, what, __LINE__); + calogDestroy(calog); + return; + } + calogContextEval(ctx, source); + pumpUntilReady(); + checkImpl(atomic_load(&readyFlag), what, __LINE__); + checkImpl(atomic_load(&errorCount) == 0, "the script registered its callbacks without error", __LINE__); + // No calogContextClose and no by-hand library shutdown: calogDestroy must get the order right + // on its own, which is exactly what the destroy-hook phases are for. + calogDestroy(calog); + checkImpl(true, what, __LINE__); +} + + +// The other half of the rule, and the one the destroy phases cannot reach: a context that dies while +// the RUNTIME lives on -- a script that errors out, or one that ends itself with calogEnd -- has to +// reclaim its handles just the same. Its interpreter is destroyed on its own thread the moment it +// stops, long before anyone tears the runtime down. +static void testOwnerDiesBeforeTheRuntime(const char *what, const char *source, bool expectError) { + CalogContextT *ctx; + struct timespec tick = { 0, PUMP_INTERVAL_NS }; + int32_t index; + bool finished; + + printf(" early-death case: %s\n", what); + fflush(stdout); + startRuntime(); + if (calog == NULL) { + return; + } + ctx = calogContextOpen(calog, &calogJsEngine); + if (ctx == NULL) { + checkImpl(false, what, __LINE__); + calogDestroy(calog); + return; + } + calogContextEval(ctx, source); + // Wait for the context's own thread to exit: an errored script is retired by the test's error + // handler doing nothing at all -- the context ends because the script ended it (endSelf) or + // because we close it below. + finished = false; + for (index = 0; index < PUMP_LIMIT; index++) { + calogPump(calog); + if (calogContextFinished(ctx)) { + finished = true; + break; + } + if (expectError && atomic_load(&errorCount) > 0) { + break; + } + nanosleep(&tick, NULL); + } + if (expectError) { + checkImpl(atomic_load(&errorCount) > 0, "the script's failure was reported", __LINE__); + } else { + checkImpl(finished, "endSelf ended the script, so its context can be reaped", __LINE__); + checkImpl(atomic_load(&errorCount) == 0, "ending a script is not reported as a failure", __LINE__); + } + // Closing joins the thread, which is where the interpreter is destroyed -- the moment a handle + // that outlived its VM would take the process down. + calogContextClose(ctx); + calogPump(calog); + checkImpl(true, what, __LINE__); + calogDestroy(calog); +} + + +// A function value that crossed engines: JavaScript hands its own closure to a Lua script, which +// keeps it, and then the JavaScript context dies. No library registry holds the closure -- the other +// VM does -- so only reclaiming on the owner's own thread can save it. +static void testCrossEngineValueOutlivesOwner(void) { + CalogContextT *holder; + CalogContextT *giver; + struct timespec tick = { 0, PUMP_INTERVAL_NS }; + int32_t index; + + printf(" early-death case: a JS closure kept by a Lua script when JS dies\n"); + fflush(stdout); + startRuntime(); + if (calog == NULL) { + return; + } + holder = calogContextOpen(calog, &calogLuaEngine); + giver = calogContextOpen(calog, &calogJsEngine); + if (holder == NULL || giver == NULL) { + checkImpl(false, "cross-engine: context open failed", __LINE__); + calogDestroy(calog); + return; + } + calogContextEval(holder, "held = nil\n calogExport('keep', function(fn) held = fn return 1 end)\n ready()"); + pumpUntilReady(); + checkImpl(atomic_load(&readyFlag), "the Lua holder published its keep() export", __LINE__); + + atomic_store(&readyFlag, false); + calogContextEval(giver, "calogCall('keep', function () { return 42; }); ready(); endSelf();"); + pumpUntilReady(); + for (index = 0; index < PUMP_LIMIT; index++) { + calogPump(calog); + if (calogContextFinished(giver)) { + break; + } + nanosleep(&tick, NULL); + } + checkImpl(calogContextFinished(giver), "the giving context ended itself", __LINE__); + calogContextClose(giver); // joins: the JS interpreter is destroyed here + calogPump(calog); + checkImpl(atomic_load(&errorCount) == 0, "no error while the JS owner went away", __LINE__); + + // The Lua script still holds the (now dead) closure. Invoking it must fail cleanly rather than + // reach into the destroyed VM, and dropping it must not double-free anything. + atomic_store(&readyFlag, false); + atomic_store(&reportValue, -1); + calogContextEval(holder, "local ok = pcall(held)\n report(ok)\n held = nil\n ready()"); + pumpUntilReady(); + checkImpl(atomic_load(&reportValue) == 0, "invoking a callable whose owner is gone fails cleanly", __LINE__); + + calogContextClose(holder); + calogDestroy(calog); +} + + +int main(void) { + testHeldCallableSurvivesTeardown("a JS subscriber still registered at teardown", + "psSubscribe('t', function () { return 1; }); ready();"); + testHeldCallableSurvivesTeardown("a JS export still registered at teardown", + "calogExport('e', function () { return 1; }); ready();"); + testHeldCallableSurvivesTeardown("a JS timer callback still armed at teardown", + "timerEvery(1000, function () {}); ready();"); + testHeldCallableSurvivesTeardown("all three at once, several callbacks each", + "psSubscribe('a', function () {}); psSubscribe('b', function () {});" + "calogExport('x', function () {}); calogExport('y', function () {});" + "timerEvery(1000, function () {}); timerAfter(1000, function () {});" + "ready();"); + testOwnerDiesBeforeTheRuntime("a JS subscriber whose script then errors out", + "psSubscribe('t', function () {}); ready(); throw new Error('boom');", true); + testOwnerDiesBeforeTheRuntime("a JS export whose script then errors out", + "calogExport('e', function () {}); ready(); throw new Error('boom');", true); + testOwnerDiesBeforeTheRuntime("a JS timer callback whose script then errors out", + "timerEvery(1000, function () {}); ready(); throw new Error('boom');", true); + testOwnerDiesBeforeTheRuntime("a JS subscriber whose script ends itself (calogEnd)", + "psSubscribe('t', function () {}); ready(); endSelf();", false); + testCrossEngineValueOutlivesOwner(); + testGuardsAfterShutdown(); + + printf("\n%d checks, %d failed\n", testsRun, testsFailed); + fflush(stdout); + return testsFailed == 0 ? 0 : 1; +} diff --git a/tools/crossBuild.sh b/tools/crossBuild.sh index c7bba977..5973bef0 100755 --- a/tools/crossBuild.sh +++ b/tools/crossBuild.sh @@ -32,10 +32,32 @@ LUASRC=$(ls vendor/lua/src/*.c | grep -vE '/(lua|luac)\.c$' | tr '\n' ' ') ENET_UNIX=$(ls vendor/enet/*.c | grep -vE '/win32\.c$' | tr '\n' ' ') ENET_WIN=$(ls vendor/enet/*.c | grep -vE '/unix\.c$' | tr '\n' ' ') SQSRC=$(ls vendor/squirrel-src/squirrel/*.cpp | tr '\n' ' ') +BERRYSRC=$(ls vendor/berry/src/*.c vendor/berry/default/be_port.c vendor/berry/default/be_modtab.c | tr '\n' ' ') +JANETDEF="-DJANET_NO_NET -DJANET_NO_PROCESSES -DJANET_NO_DYNAMIC_MODULES" MBSRC=$(ls vendor/ourbasic/*.c | tr '\n' ' ') ENETDEF="-D_GNU_SOURCE -DHAS_FCNTL=1 -DHAS_POLL=1 -DHAS_GETADDRINFO=1 -DHAS_GETNAMEINFO=1 -DHAS_GETHOSTBYNAME_R=1 -DHAS_GETHOSTBYADDR_R=1 -DHAS_INET_PTON=1 -DHAS_INET_NTOP=1 -DHAS_MSGHDR_FLAGS=1 -DHAS_SOCKLEN_T=1" -pass=0; fail=0 +pass=0; fail=0; skip=0 + +# Per-target OpenSSL, built by `tools/crossDeps.sh openssl`. calogNet has needed it since +# the tcp transport gained TLS, which is what this script's testNet case links against. Echoes the +# include + archive flags for , or nothing when that target's OpenSSL has not been built -- +# the caller then SKIPS testNet rather than reporting a failure the script cannot fix itself. +# mac wants the repacked BSD-format archives; zig's Mach-O linker cannot read OpenSSL's GNU-format .a. +ossl_flags() { # + local dir="$ROOT/build/cross/$1" + local libs="$dir/openssl-src" + case "$1" in + mac-x64|mac-arm64) libs="$dir/openssl-repack" ;; + esac + [ -f "$libs/libssl.a" ] && [ -f "$libs/libcrypto.a" ] || return 1 + echo "-I$dir/openssl-src/include $libs/libssl.a $libs/libcrypto.a" +} +skip_no_ossl() { # + echo " [$2 SKIP] $1 -- no OpenSSL for this target; run: ./tools/crossDeps.sh $2 openssl" + skip=$((skip+1)) +} + mrun() { # name, then compiler args -- build static musl, then RUN it local name=$1; shift if "$@" -o "$OUT/$name-musl" 2>"$OUT/$name-musl.err"; then @@ -48,6 +70,22 @@ mrun() { # name, then compiler args -- build static musl, then RUN it echo " [musl FAIL] $name (see $OUT/$name-musl.err)"; fail=$((fail+1)) fi } +# Like mrun, but the binary MUST exit 0, and its last line is echoed. For tests whose whole point is +# the RUNTIME result: mrun treats a nonzero exit as a pass ("RAN, nonzero"), which would report a +# broken TLS handshake as success. timeout bounds the one hang shape a server test has. +mrunStrict() { # name, then compiler args -- build static musl, then RUN it; nonzero == FAIL + local name=$1; shift + if "$@" -o "$OUT/$name-musl" 2>"$OUT/$name-musl.err"; then + if timeout 120 "$OUT/$name-musl" >"$OUT/$name-musl.out" 2>&1; then + echo " [musl RUN ok] $name -- $(tail -n 1 "$OUT/$name-musl.out")"; pass=$((pass+1)) + else + echo " [musl RUN FAILED] $name (see $OUT/$name-musl.out)"; fail=$((fail+1)) + fi + else + echo " [musl FAIL] $name (see $OUT/$name-musl.err)"; fail=$((fail+1)) + fi +} + wbuild() { # name, then compiler args -- build a Windows .exe, verify it is a PE local name=$1; shift if "$@" -o "$OUT/$name.exe" 2>"$OUT/$name.exe.err"; then @@ -83,11 +121,74 @@ done WPA="$OUT/libwinpthreads.a" WPINC="-Ivendor/winpthreads/include -DWINPTHREAD_STATIC=1" +# s7 on Windows: s7.c takes its POSIX path (it only sets MS_Windows from _MSC_VER, which clang/mingw +# never defines) and includes , which mingw-w64 does not ship. Generate the same +# minimal shim the full-CLI build uses, rather than depend on that one -- it lives under build/, +# which is untracked, so it cannot be assumed present. +mkdir -p "$OUT/shim/sys" +cat > "$OUT/shim/sys/utsname.h" <<'UTSNAME' +// utsname.h -- cross-build compatibility shim for vendored s7 on mingw-w64 (generated by +// tools/crossBuild.sh). s7.c includes under `#if !MS_Windows`, and it only defines +// MS_Windows from _MSC_VER, so the clang/mingw cross build wrongly takes the POSIX path. This +// satisfies the include without patching the pristine vendored source; it is on s7's include path +// only, so nothing else in calog sees it. +#ifndef CALOG_SHIM_SYS_UTSNAME_H +#define CALOG_SHIM_SYS_UTSNAME_H +#include +struct utsname { + char sysname[65]; + char nodename[65]; + char release[65]; + char version[65]; + char machine[65]; +}; +static inline int uname(struct utsname *u) { + if (u == NULL) { + return -1; + } + memset(u, 0, sizeof(*u)); + strcpy(u->sysname, "Windows"); + strcpy(u->nodename, "localhost"); + strcpy(u->release, "10"); + strcpy(u->version, "10"); + strcpy(u->machine, "x86_64"); + return 0; +} +#endif +UTSNAME +S7SHIM="-I$OUT/shim" + echo "== musl (fully static Linux, built AND run) ==" mrun testEngineLua "$ZIG" cc -target x86_64-linux-musl -static -O2 -pthread -w -Isrc -Isrc/lua -Ilibs -Ivendor/lua/src -DLUA_USE_POSIX $CORE src/lua/luaEngine.c src/lua/luaAdapter.c tests/testEngineLua.c $LUASRC -lm mrun testEngineJs "$ZIG" cc -target x86_64-linux-musl -static -O2 -pthread -w -Isrc -Isrc/js -Ilibs -Ivendor/quickjs -D_GNU_SOURCE $CORE src/js/jsEngine.c src/js/jsAdapter.c tests/testEngineJs.c vendor/quickjs/quickjs.c vendor/quickjs/libregexp.c vendor/quickjs/libunicode.c vendor/quickjs/dtoa.c -lm mrun testEngineMyBasic "$ZIG" cc -target x86_64-linux-musl -static -O2 -pthread -w -Isrc -Isrc/mybasic -Ilibs -Ivendor/ourbasic -DMB_DOUBLE_FLOAT $CORE src/mybasic/mybasicEngine.c src/mybasic/mybasicAdapter.c tests/testEngineMyBasic.c $MBSRC -lm -mrun testNet "$ZIG" cc -target x86_64-linux-musl -static -O2 -pthread -w -Isrc -Isrc/lua -Ilibs -Ivendor/lua/src -Ivendor/enet/include -DLUA_USE_POSIX $ENETDEF $CORE src/lua/luaEngine.c src/lua/luaAdapter.c libs/calogNet.c libs/calogHandle.c tests/testNet.c $LUASRC $ENET_UNIX -lm +if OSSL=$(ossl_flags musl); then + mrun testNet "$ZIG" cc -target x86_64-linux-musl -static -O2 -pthread -w -Isrc -Isrc/lua -Ilibs -Ivendor/lua/src -Ivendor/enet/include -DLUA_USE_POSIX $ENETDEF $CORE src/lua/luaEngine.c src/lua/luaAdapter.c libs/calogNet.c libs/calogHandle.c tests/testNet.c $LUASRC $ENET_UNIX $OSSL -lm + # Real TLS at runtime, so the cross-built OpenSSL is EXERCISED rather than merely link-verified: + # testHttps generates an RSA key + self-signed cert in-process and drives a loopback TLS server. + # No external server and no egress. musl only -- testHttps uses POSIX sockets/pthreads/mkstemp, + # which x86_64-windows-gnu cannot compile, and a mac build would add nothing over testNet's link. + mrunStrict testHttps "$ZIG" cc -target x86_64-linux-musl -static -O2 -pthread -w -Isrc -Isrc/lua -Ilibs -Ivendor/lua/src -DLUA_USE_POSIX $CORE src/lua/luaEngine.c src/lua/luaAdapter.c libs/calogHttp.c tests/testHttps.c $LUASRC $OSSL -lm +else + skip_no_ossl testNet musl + skip_no_ossl testHttps musl +fi +mrun testEngineBerry "$ZIG" cc -target x86_64-linux-musl -static -O2 -pthread -w -Isrc -Isrc/berry -Ilibs -Ivendor/berry/src -Ivendor/berry $CORE src/berry/berryEngine.c src/berry/berryAdapter.c tests/testEngineBerry.c $BERRYSRC -lm +mrun testEngineS7 "$ZIG" cc -target x86_64-linux-musl -static -O2 -pthread -w -Isrc -Isrc/s7 -Ilibs -Ivendor/s7 -D_GNU_SOURCE $CORE src/s7/s7Engine.c src/s7/s7Adapter.c tests/testEngineS7.c vendor/s7/s7.c -lm +mrun testEngineWren "$ZIG" cc -target x86_64-linux-musl -static -O2 -pthread -w -Isrc -Isrc/wren -Ilibs -Ivendor/wren $CORE src/wren/wrenEngine.c src/wren/wrenAdapter.c tests/testEngineWren.c vendor/wren/wren.c -lm +mrun testEngineJanet "$ZIG" cc -target x86_64-linux-musl -static -O2 -pthread -w -Isrc -Isrc/janet -Ilibs -Ivendor/janet $JANETDEF $CORE src/janet/janetEngine.c src/janet/janetAdapter.c tests/testEngineJanet.c vendor/janet/janet.c -lm +# mruby and Tcl are the two engines that need a per-target prebuilt archive rather than vendored C: +# tools/crossDeps.sh mruby / tcl produces them. Skip cleanly when they are not there. +if [ -f "$ROOT/build/cross/musl/mruby/libmruby.a" ]; then + mrun testEngineMruby "$ZIG" cc -target x86_64-linux-musl -static -O2 -pthread -w -Isrc -Isrc/mruby -Ilibs -Ivendor/mruby/include -Ivendor/mruby/build/cross-musl/include -DMRB_USE_DEBUG_HOOK $CORE src/mruby/mrubyEngine.c src/mruby/mrubyAdapter.c tests/testEngineMruby.c "$ROOT/build/cross/musl/mruby/libmruby.a" -lm +else + echo " [musl SKIP] testEngineMruby -- run: ./tools/crossDeps.sh musl mruby"; skip=$((skip+1)) +fi +if [ -f "$ROOT/build/cross/musl/tcl/libtcl9.0.a" ]; then + mrun testEngineTcl "$ZIG" cc -target x86_64-linux-musl -static -O2 -pthread -w -Isrc -Isrc/tcl -Ilibs -Ivendor/tcl/generic -Ivendor/tcl/unix $CORE src/tcl/tclEngine.c src/tcl/tclAdapter.c tests/testEngineTcl.c "$ROOT/build/cross/musl/tcl/libtcl9.0.a" "$ROOT/build/cross/musl/lib/libz.a" -lm +else + echo " [musl SKIP] testEngineTcl -- run: ./tools/crossDeps.sh musl tcl"; skip=$((skip+1)) +fi # Squirrel is C++ (vendored VM); the calog .c files stay C (-x c), the VM is C++ (-x c++). if "$ZIG" c++ -target x86_64-linux-musl -static -O2 -pthread -w -Isrc -Isrc/squirrel -Ilibs -Ivendor/squirrel-src/include -Ivendor/squirrel-src/squirrel -D_SQ64 -DSQUSEDOUBLE \ -x c $CORE src/squirrel/squirrelEngine.c src/squirrel/squirrelAdapter.c tests/testEngineSquirrel.c -x c++ $SQSRC -o "$OUT/testEngineSquirrel-musl" 2>"$OUT/sq-musl.err"; then @@ -96,14 +197,70 @@ else echo " [musl FAIL] testEngineSquirrel"; fail=$((fail+1)); fi echo "== Windows x64 (.exe built against vendored winpthreads; run needs wine) ==" wbuild testEngineLua "$ZIG" cc -target x86_64-windows-gnu -O2 -w -Isrc -Isrc/lua -Ilibs -Ivendor/lua/src $WPINC $CORE src/lua/luaEngine.c src/lua/luaAdapter.c tests/testEngineLua.c $LUASRC "$WPA" -wbuild testNet "$ZIG" cc -target x86_64-windows-gnu -O2 -w -Isrc -Isrc/lua -Ilibs -Ivendor/lua/src -Ivendor/enet/include $WPINC $CORE src/lua/luaEngine.c src/lua/luaAdapter.c libs/calogNet.c libs/calogHandle.c tests/testNet.c $LUASRC $ENET_WIN "$WPA" -lws2_32 -lwinmm +wbuild testEngineBerry "$ZIG" cc -target x86_64-windows-gnu -O2 -w -Isrc -Isrc/berry -Ilibs -Ivendor/berry/src -Ivendor/berry $WPINC $CORE src/berry/berryEngine.c src/berry/berryAdapter.c tests/testEngineBerry.c $BERRYSRC "$WPA" +wbuild testEngineS7 "$ZIG" cc -target x86_64-windows-gnu -O2 -w -Isrc -Isrc/s7 -Ilibs -Ivendor/s7 $S7SHIM -D_GNU_SOURCE $WPINC $CORE src/s7/s7Engine.c src/s7/s7Adapter.c tests/testEngineS7.c vendor/s7/s7.c "$WPA" +wbuild testEngineWren "$ZIG" cc -target x86_64-windows-gnu -O2 -w -Isrc -Isrc/wren -Ilibs -Ivendor/wren $WPINC $CORE src/wren/wrenEngine.c src/wren/wrenAdapter.c tests/testEngineWren.c vendor/wren/wren.c "$WPA" +wbuild testEngineJanet "$ZIG" cc -target x86_64-windows-gnu -O2 -w -Isrc -Isrc/janet -Ilibs -Ivendor/janet $JANETDEF $WPINC $CORE src/janet/janetEngine.c src/janet/janetAdapter.c tests/testEngineJanet.c vendor/janet/janet.c "$WPA" +if [ -f "$ROOT/build/cross/win/tcl/libtcl90.a" ]; then + wbuild testEngineTcl "$ZIG" cc -target x86_64-windows-gnu -O2 -w -Isrc -Isrc/tcl -Ilibs -Ivendor/tcl/generic -Ivendor/tcl/win -DSTATIC_BUILD $WPINC $CORE src/tcl/tclEngine.c src/tcl/tclAdapter.c tests/testEngineTcl.c "$ROOT/build/cross/win/tcl/libtcl90.a" "$WPA" -lws2_32 -lnetapi32 -luserenv -lole32 -loleaut32 -luuid +else + echo " [win SKIP] testEngineTcl -- run: ./tools/crossDeps.sh win tcl"; skip=$((skip+1)) +fi +if [ -f "$ROOT/build/cross/win/mruby/libmruby.a" ]; then + wbuild testEngineMruby "$ZIG" cc -target x86_64-windows-gnu -O2 -w -Isrc -Isrc/mruby -Ilibs -Ivendor/mruby/include -Ivendor/mruby/build/cross-mingw/include -DMRB_USE_DEBUG_HOOK $WPINC $CORE src/mruby/mrubyEngine.c src/mruby/mrubyAdapter.c tests/testEngineMruby.c "$ROOT/build/cross/win/mruby/libmruby.a" "$WPA" +else + echo " [win SKIP] testEngineMruby -- run: ./tools/crossDeps.sh win mruby"; skip=$((skip+1)) +fi +if OSSL=$(ossl_flags win); then + wbuild testNet "$ZIG" cc -target x86_64-windows-gnu -O2 -w -Isrc -Isrc/lua -Ilibs -Ivendor/lua/src -Ivendor/enet/include $WPINC $CORE src/lua/luaEngine.c src/lua/luaAdapter.c libs/calogNet.c libs/calogHandle.c tests/testNet.c $LUASRC $ENET_WIN "$WPA" $OSSL -lws2_32 -lwinmm -lcrypt32 -lbcrypt -ladvapi32 -luser32 +else + skip_no_ossl testNet win +fi + +# Squirrel: C++ VM, so the C sources are forced with -x c and the VM with -x c++, as on musl. +if "$ZIG" c++ -target x86_64-windows-gnu -O2 -w -Isrc -Isrc/squirrel -Ilibs -Ivendor/squirrel-src/include -Ivendor/squirrel-src/squirrel -D_SQ64 -DSQUSEDOUBLE $WPINC \ + -x c $CORE src/squirrel/squirrelEngine.c src/squirrel/squirrelAdapter.c tests/testEngineSquirrel.c -x c++ $SQSRC -x none "$WPA" -o "$OUT/testEngineSquirrel.exe" 2>"$OUT/sq-win.err"; then + if file "$OUT/testEngineSquirrel.exe" | grep -q 'PE32+ executable'; then + echo " [win BUILT] testEngineSquirrel.exe"; pass=$((pass+1)) + else echo " [win ??] testEngineSquirrel.exe (not a PE?)"; fail=$((fail+1)); fi +else echo " [win FAIL] testEngineSquirrel.exe (see $OUT/sq-win.err)"; fail=$((fail+1)); fi echo "== macOS (Mach-O built via zig's libSystem stub -- Intel + Apple Silicon; run needs a Mac) ==" for arch in x86_64 aarch64; do mbuild "$arch" testEngineLua "$ZIG" cc -target $arch-macos -O2 -w -Isrc -Isrc/lua -Ilibs -Ivendor/lua/src -DLUA_USE_MACOSX $CORE src/lua/luaEngine.c src/lua/luaAdapter.c tests/testEngineLua.c $LUASRC - mbuild "$arch" testNet "$ZIG" cc -target $arch-macos -O2 -w -Isrc -Isrc/lua -Ilibs -Ivendor/lua/src -Ivendor/enet/include -DLUA_USE_MACOSX $ENETDEF $CORE src/lua/luaEngine.c src/lua/luaAdapter.c libs/calogNet.c libs/calogHandle.c tests/testNet.c $LUASRC $ENET_UNIX + mbuild "$arch" testEngineBerry "$ZIG" cc -target $arch-macos -O2 -w -Isrc -Isrc/berry -Ilibs -Ivendor/berry/src -Ivendor/berry $CORE src/berry/berryEngine.c src/berry/berryAdapter.c tests/testEngineBerry.c $BERRYSRC + mbuild "$arch" testEngineS7 "$ZIG" cc -target $arch-macos -O2 -w -Isrc -Isrc/s7 -Ilibs -Ivendor/s7 -D_GNU_SOURCE $CORE src/s7/s7Engine.c src/s7/s7Adapter.c tests/testEngineS7.c vendor/s7/s7.c + mbuild "$arch" testEngineWren "$ZIG" cc -target $arch-macos -O2 -w -Isrc -Isrc/wren -Ilibs -Ivendor/wren $CORE src/wren/wrenEngine.c src/wren/wrenAdapter.c tests/testEngineWren.c vendor/wren/wren.c + mbuild "$arch" testEngineJanet "$ZIG" cc -target $arch-macos -O2 -w -Isrc -Isrc/janet -Ilibs -Ivendor/janet $JANETDEF $CORE src/janet/janetEngine.c src/janet/janetAdapter.c tests/testEngineJanet.c vendor/janet/janet.c + MACT=$(echo "$arch" | sed 's/^x86_64$/mac-x64/; s/^aarch64$/mac-arm64/') + if [ -f "$ROOT/build/cross/$MACT/mruby/libmruby.a" ]; then + mbuild "$arch" testEngineMruby "$ZIG" cc -target $arch-macos -O2 -w -Isrc -Isrc/mruby -Ilibs -Ivendor/mruby/include -Ivendor/mruby/build/cross-$MACT/include -DMRB_USE_DEBUG_HOOK $CORE src/mruby/mrubyEngine.c src/mruby/mrubyAdapter.c tests/testEngineMruby.c "$ROOT/build/cross/$MACT/mruby/libmruby.a" + else + echo " [mac SKIP] testEngineMruby ($arch) -- run: ./tools/crossDeps.sh $MACT mruby"; skip=$((skip+1)) + fi + # Tcl links zlib; there is no system libz for a cross target, so use the one build_codecs made. + if [ -f "$ROOT/build/cross/$MACT/tcl/libtcl9.0.a" ] && [ -f "$ROOT/build/cross/$MACT/lib/libz.a" ]; then + mbuild "$arch" testEngineTcl "$ZIG" cc -target $arch-macos -O2 -w -Isrc -Isrc/tcl -Ilibs -Ivendor/tcl/generic -Ivendor/tcl/unix $CORE src/tcl/tclEngine.c src/tcl/tclAdapter.c tests/testEngineTcl.c "$ROOT/build/cross/$MACT/tcl/libtcl9.0.a" "$ROOT/build/cross/$MACT/lib/libz.a" + else + echo " [mac SKIP] testEngineTcl ($arch) -- run: ./tools/crossDeps.sh $MACT tcl codecs"; skip=$((skip+1)) + fi + if "$ZIG" c++ -target $arch-macos -O2 -w -Isrc -Isrc/squirrel -Ilibs -Ivendor/squirrel-src/include -Ivendor/squirrel-src/squirrel -D_SQ64 -DSQUSEDOUBLE \ + -x c $CORE src/squirrel/squirrelEngine.c src/squirrel/squirrelAdapter.c tests/testEngineSquirrel.c -x c++ $SQSRC -o "$OUT/testEngineSquirrel-$arch-macos" 2>"$OUT/sq-$arch-macos.err"; then + if file "$OUT/testEngineSquirrel-$arch-macos" | grep -q 'Mach-O'; then + echo " [mac BUILT] testEngineSquirrel ($arch)"; pass=$((pass+1)) + else echo " [mac ??] testEngineSquirrel ($arch, not Mach-O?)"; fail=$((fail+1)); fi + else echo " [mac FAIL] testEngineSquirrel ($arch, see $OUT/sq-$arch-macos.err)"; fail=$((fail+1)); fi + if OSSL=$(ossl_flags "$MACT"); then + mbuild "$arch" testNet "$ZIG" cc -target $arch-macos -O2 -w -Isrc -Isrc/lua -Ilibs -Ivendor/lua/src -Ivendor/enet/include -DLUA_USE_MACOSX $ENETDEF $CORE src/lua/luaEngine.c src/lua/luaAdapter.c libs/calogNet.c libs/calogHandle.c tests/testNet.c $LUASRC $ENET_UNIX $OSSL + else + skip_no_ossl testNet "$arch" + fi done echo -echo "== cross-build: $pass ok, $fail failed (artifacts in $OUT/) ==" +if [ "$skip" -gt 0 ]; then + echo "== cross-build: $pass ok, $fail failed, $skip skipped (artifacts in $OUT/) ==" +else + echo "== cross-build: $pass ok, $fail failed (artifacts in $OUT/) ==" +fi [ "$fail" -eq 0 ] diff --git a/tools/crossDeps.sh b/tools/crossDeps.sh index 8e8ddb75..89291656 100755 --- a/tools/crossDeps.sh +++ b/tools/crossDeps.sh @@ -7,10 +7,16 @@ # winpthreads). It regenerates the zig compiler wrappers + the CMake toolchain file first, so nothing # under build/cross/ needs to pre-exist. # -# Usage: [ZIG=/path/to/zig] ./tools/crossDeps.sh [dep ...] +# Usage: [ZIG=/path/to/zig] ./tools/crossDeps.sh [dep ...] # dep (optional): build only the named dep(s), e.g. `openssl libssh2`. Default: all, in dep order. # Deps in order: codecs xz openssl libxml2 pcre2 libssh2 mariadb postgres tcl mruby winpthreads libarchive # +# musl builds every dep except winpthreads, which is Windows-only by definition and is dropped +# from the default list for that target. It exists so tools/crossBuild.sh can cross-test the whole +# engine matrix statically (tcl and mruby need per-target archives; calogNet needs OpenSSL since +# the tcp transport gained TLS). Note this is the CROSS route from a glibc host; the other musl +# path is `make static` in an Alpine container, where the native toolchain is already musl. +# # Requirements: zig (https://ziglang.org/download/), cmake, perl (OpenSSL), and a POSIX make/rake env. set -eu cd "$(dirname "$0")/.." @@ -27,7 +33,8 @@ case "$T" in win) TRIPLE=x86_64-windows-gnu; SYSNAME=Windows; SYSPROC=x86_64 ;; mac-x64) TRIPLE=x86_64-macos; SYSNAME=Darwin; SYSPROC=x86_64 ;; mac-arm64) TRIPLE=aarch64-macos; SYSNAME=Darwin; SYSPROC=aarch64 ;; - *) echo "usage: $0 [dep ...]" >&2; exit 1 ;; + musl) TRIPLE=x86_64-linux-musl; SYSNAME=Linux; SYSPROC=x86_64 ;; + *) echo "usage: $0 [dep ...]" >&2; exit 1 ;; esac shift @@ -203,6 +210,7 @@ build_openssl() { win) OSSL_TARGET="mingw64" ;; mac-x64) OSSL_TARGET="darwin64-x86_64-cc" ;; mac-arm64) OSSL_TARGET="darwin64-arm64-cc" ;; + musl) OSSL_TARGET="linux-x86_64" ;; *) echo "build_openssl: unknown target $T" >&2; return 1 ;; esac @@ -450,6 +458,14 @@ build_tcl() { CFG="$SRC/configure" LIB="libtcl90.a" HOSTTRIPLE="x86_64-w64-mingw32" + elif [ "$T" = "musl" ]; then + # musl: the SIMPLEST arm. musl is still Unix, so this is the same unix/configure the native + # Linux build uses (Makefile TCLLIB rule) -- no win/ and no macosx/ sources are involved, so + # the trimmed vendor/tcl tree is sufficient. Only the toolchain and --host differ. + SRC="$R/vendor/tcl/unix" + CFG="$SRC/configure" + LIB="libtcl9.0.a" + HOSTTRIPLE="x86_64-linux-musl" else # mac-x64 / mac-arm64: unix/configure, driven out-of-tree from $O/tcl. # GOTCHA: like win/, the trimmed vendor/tcl also lacks the macosx/ subdir, @@ -488,6 +504,29 @@ build_tcl() { RANLIB="$RANLIB" \ RC="$WINRC" \ > cfg.log 2>&1 || { echo "build_tcl: configure failed (see $BUILD/cfg.log)" >&2; exit 1; } + elif [ "$T" = "musl" ]; then + # musl: no tcl_cv_sys_version override -- the target IS Linux, so configure's `uname -r` + # probe on this Linux host reports the right system. + # GOTCHA 1: configure sees the HOST's and selects Tcl's epoll notifier, whose + # tclEpollNotfy.c includes -- a BSD/glibc header musl does not ship, so the + # build dies there. Tell configure the epoll headers are absent and Tcl falls back to its + # portable select() notifier (tclUnixNotfy.c), which musl supports fully. + # GOTCHA 2: zlib. With no zlib visible, configure sets TCL_WITH_INTERNAL_ZLIB but the unix + # Makefile only puts compat/zlib on the include path of the zlib objects themselves, so + # tclEvent.c then fails on a missing . Point it at the SAME vendored zlib the rest + # of calog cross-links (build_codecs, which runs before tcl in the dep order) and the + # normal system-zlib path is taken instead -- header from vendor/zlib, archive from $O/lib. + "$CFG" \ + --host="$HOSTTRIPLE" \ + --disable-shared \ + ac_cv_header_sys_epoll_h=no \ + ac_cv_header_sys_eventfd_h=no \ + CPPFLAGS="-I$R/vendor/zlib" \ + LDFLAGS="-L$O/lib" \ + CC="$CC" \ + AR="$AR" \ + RANLIB="$RANLIB" \ + > cfg.log 2>&1 || { echo "build_tcl: configure failed (see $BUILD/cfg.log)" >&2; exit 1; } else # mac: unix/configure via --host with an Apple-Darwin triple. # tcl_cv_sys_version=Darwin-24.0.0 is REQUIRED: it short-circuits the @@ -554,6 +593,7 @@ build_mruby() { win) buildname=cross-mingw; hosttriple=x86_64-w64-mingw32 ;; mac-x64) buildname=cross-mac-x64; hosttriple=x86_64-apple-darwin ;; mac-arm64) buildname=cross-mac-arm64; hosttriple=aarch64-apple-darwin ;; + musl) buildname=cross-musl; hosttriple=x86_64-linux-musl ;; esac # Windows target links executables with .exe; also gates for_windows? source paths in gems. @@ -592,6 +632,12 @@ MRuby::CrossBuild.new('$buildname') do |conf| conf.cc.command = "$CC" conf.cc.flags << '-O2' conf.cc.defines << 'MRB_INT64' + # Per-instruction VM hook (mrb_state.code_fetch_hook), which calog's adapter installs to enforce a + # per-context wall-clock budget. Off by default in mruby, and it CHANGES THE mrb_state LAYOUT, so + # it has to match src/mruby/build_config.rb and the -DMRB_USE_DEBUG_HOOK the adapter compile in + # cross{Win,Mac}Full.sh passes. Without it the adapter does not compile against this library. + # Host/mrbc does not need it: it affects the VM, not the bytecode mrbc emits (unlike MRB_INT64). + conf.cc.defines << 'MRB_USE_DEBUG_HOOK' conf.linker.command = "$CC" @@ -752,6 +798,15 @@ build_libarchive() { >"$O/la-cfg.log" 2>&1 cmake --build "$O/libarchive" --target archive_static -j >"$O/la-build.log" 2>&1 echo " [lib] libarchive.a (codecs only; win xar via XmlLite/CNG) ($T)" + elif [ "$T" = musl ]; then + # musl has its own libarchive route -- tools/crossArchive.sh, which handles the iconv gotcha + # for it (libarchive's FIND_PATH(iconv.h) otherwise returns the HOST's glibc /usr/include and + # breaks the musl compile; see PORTING.md). Refuse rather than fall through to the macOS + # branch below, which would build a musl archive with an Apple SDK iconv.h and a Darwin .tbd + # link stub, and reference the mac-only openssl-repack archives. It reported success when it + # did that, which is exactly why this guard is explicit. + echo " libarchive ($T): use tools/crossArchive.sh for musl (see PORTING.md)" >&2 + return 1 else # macOS: libarchive gates libxml2 (hence xar) on a working iconv, but zig's SDK-less libSystem # stub exports no iconv symbols, so the detection link-test fails and xar silently stubs out. @@ -798,6 +853,16 @@ build_libarchive() { # Dispatch: build the requested deps (or all, in dependency order). # =================================================================================================== ALL="codecs xz openssl libxml2 pcre2 libssh2 mariadb postgres tcl mruby winpthreads libarchive" +# winpthreads is POSIX threads FOR WINDOWS: meaningless on any other target, so it is not in the +# default list for them (asking for it by name still runs, and still fails, which is honest). +if [ "$T" != win ]; then + ALL="codecs xz openssl libxml2 pcre2 libssh2 mariadb postgres tcl mruby libarchive" +fi +# musl: libarchive is tools/crossArchive.sh's job (see build_libarchive), and winpthreads is +# Windows-only. Asking for either by name still runs, and still explains itself. +if [ "$T" = musl ]; then + ALL="codecs xz openssl libxml2 pcre2 libssh2 mariadb postgres tcl mruby" +fi DEPS=${*:-$ALL} # Tolerant dispatch: build every requested dep, collect failures, and report at the end (so one # broken dep -- e.g. tcl when vendor/tcl lacks win//macosx/ -- does not abort the rest). The `if` diff --git a/tools/crossMacFull.sh b/tools/crossMacFull.sh index c29aa94a..1bf58614 100755 --- a/tools/crossMacFull.sh +++ b/tools/crossMacFull.sh @@ -155,7 +155,9 @@ build_arch(){ cc berryAdapter src/berry/berryAdapter.c -Ivendor/berry/src -Ivendor/berry cc s7Adapter src/s7/s7Adapter.c -Ivendor/s7 cc wrenAdapter src/wren/wrenAdapter.c -Ivendor/wren - cc mrubyAdapter src/mruby/mrubyAdapter.c -Ivendor/mruby/include -Ivendor/mruby/build/cross-$A/include + # -DMRB_USE_DEBUG_HOOK must match the cross libmruby.a (tools/crossDeps.sh sets the same define): + # it adds mrb_state.code_fetch_hook, so the adapter and the library have to agree on the layout. + cc mrubyAdapter src/mruby/mrubyAdapter.c -DMRB_USE_DEBUG_HOOK -Ivendor/mruby/include -Ivendor/mruby/build/cross-$A/include cc tclAdapter src/tcl/tclAdapter.c -Ivendor/tcl/generic -DSTATIC_BUILD cc janetAdapter src/janet/janetAdapter.c -Ivendor/janet cc calogMain src/calogMain.c $ENGDEF diff --git a/tools/crossWinFull.sh b/tools/crossWinFull.sh index 1a6d1d86..c7f5cf03 100644 --- a/tools/crossWinFull.sh +++ b/tools/crossWinFull.sh @@ -76,7 +76,9 @@ cc mybasicAdapter src/mybasic/mybasicAdapter.c $BASE -Ivendor/ourbasic -DMB_DOUB cc berryAdapter src/berry/berryAdapter.c $BASE -Ivendor/berry/src -Ivendor/berry cc s7Adapter src/s7/s7Adapter.c $BASE -Ivendor/s7 cc wrenAdapter src/wren/wrenAdapter.c $BASE -Ivendor/wren -cc mrubyAdapter src/mruby/mrubyAdapter.c $BASE -Ivendor/mruby/include -Ivendor/mruby/build/cross-mingw/include +# -DMRB_USE_DEBUG_HOOK must match the cross libmruby.a (tools/crossDeps.sh sets the same define): +# it adds mrb_state.code_fetch_hook, so the adapter and the library have to agree on the layout. +cc mrubyAdapter src/mruby/mrubyAdapter.c $BASE -DMRB_USE_DEBUG_HOOK -Ivendor/mruby/include -Ivendor/mruby/build/cross-mingw/include cc tclAdapter src/tcl/tclAdapter.c $BASE -Ivendor/tcl/generic -DSTATIC_BUILD cc janetAdapter src/janet/janetAdapter.c $BASE -Ivendor/janet cc calogMain src/calogMain.c $BASE $ENGDEF diff --git a/vendor/ourbasic/CHANGELOG b/vendor/ourbasic/CHANGELOG index d02b607d..4c2a5f0b 100644 --- a/vendor/ourbasic/CHANGELOG +++ b/vendor/ourbasic/CHANGELOG @@ -51,3 +51,21 @@ Verified: full calog `make test` (28 binaries, 0 failed) plus the my-basic engin suite (testMyBasic / testEngineMyBasic / testPolyglot / testLoad / testTask), ASan/UBSan-clean, with def and lambda callbacks firing via timer/pubsub/export and cross-engine (a Lua task calling a my-basic def). + +Portability -- number classification does not trust endptr alone + - `_get_symbol_type` (and VAL / INPUT) decided "this symbol is a number" from + strtoll/strtod alone reaching the string terminator. That test is not portable: + musl's strtoll ADVANCES endptr past leading whitespace and a sign even when no + conversion happens, where glibc leaves endptr at the start. On musl this made + "+" and "-" -- the operators -- and "\n" -- the statement separator -- each + classify as the integer 0, so every expression containing an operator failed to + parse and every numeric assignment failed to run, both with "Operator expected". + A new `_conv_matched_nothing(start, end)` rejects a consumed span that is nothing + but whitespace and sign; a span that consumed anything else (digits, "inf", + "nan") classifies exactly as before, on every libc, and an empty span is left + alone so VAL("")/INPUT with an empty line keep their existing behavior. + Reproduced with a pure my-basic program (no calog code): `x = 1` returned + MB_FUNC_ERR on musl and MB_FUNC_OK on glibc. + +Verified: testEngineMyBasic 20/20 on a fully static musl build via zig (was 13 of 20 +failing), unchanged 20/20 natively, and calog `make test` 943 checks / 0 failed. diff --git a/vendor/ourbasic/ourBasic.c b/vendor/ourbasic/ourBasic.c index ae65b2f1..9b8fe364 100644 --- a/vendor/ourbasic/ourBasic.c +++ b/vendor/ourbasic/ourBasic.c @@ -1541,6 +1541,7 @@ static int _cut_symbol(mb_interpreter_t* s, int pos, unsigned short row, unsigne static int _append_symbol(mb_interpreter_t* s, char* sym, bool_t* delsym, int pos, unsigned short row, unsigned short col); static int _create_symbol(mb_interpreter_t* s, _ls_node_t* l, char* sym, _object_t** obj, _ls_node_t*** asgn, bool_t* delsym); static _data_e _get_symbol_type(mb_interpreter_t* s, char* sym, _raw_t* value); +static bool_t _conv_matched_nothing(const char* start, const char* end); static int _parse_char(mb_interpreter_t* s, const char* str, int n, int pos, unsigned short row, unsigned short col); static void _set_error_pos(mb_interpreter_t* s, int pos, unsigned short row, unsigned short col); static char* _prev_import(mb_interpreter_t* s, char* lf, int* pos, unsigned short* row, unsigned short* col); @@ -3629,6 +3630,14 @@ static char _get_priority(mb_func_t op1, mb_func_t op2) { idx1 = _get_priority_index(op1); idx2 = _get_priority_index(op2); + /* [calog fork] _get_priority_index returns -1 for an operator that is not in its table, and the + * assert guarding this lookup only checks the UPPER bound -- so _PRECEDE_TABLE[-1][...] is an + * out-of-bounds read. It is masked today because mb_assert is a live assert() here, but an + * embedder building with NDEBUG compiles that away and reads out of bounds silently. Report the + * table's own "cannot operate" marker instead; the caller already turns ' ' into a clean + * SE_RN_FAILED_TO_OPERATE script error, which is the right answer for an unknown operator. */ + if(idx1 < 0 || idx2 < 0) + return ' '; mb_assert(idx1 < countof(_PRECEDE_TABLE) && idx2 < countof(_PRECEDE_TABLE[0])); result = _PRECEDE_TABLE[idx1][idx2]; @@ -5549,6 +5558,28 @@ _exit: } /* Get the type of a syntax symbol */ +/* [calog fork] Companion to the "endptr reached the terminator" test that classifies a symbol as a + * number. That test alone is not portable: musl's strtoll ADVANCES endptr past leading whitespace and + * a sign even when no conversion happens, while glibc leaves it at the start. So on musl "+" and "-" + * -- the operators -- and "\n" -- the statement separator -- were every one of them reported as the + * integer 0: every expression containing an operator failed to parse, and every numeric assignment + * failed to run, both with "Operator expected". A span of nothing but whitespace and sign is never a + * number, so that is what to reject. Anything that consumed something else (digits, "inf", "nan") is + * classified exactly as before, on every libc, and an empty span is left alone -- that is the classic + * "no conversion" the callers already handle as they always have. */ +static bool_t _conv_matched_nothing(const char* start, const char* end) { + const char* p = 0; + + if(end <= start) + return false; + for(p = start; p < end; p++) { + if(!(*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r' || *p == '\f' || *p == '\v' || *p == '+' || *p == '-')) + return false; + } + + return true; +} + static _data_e _get_symbol_type(mb_interpreter_t* s, char* sym, _raw_t* value) { _data_e result = _DT_NIL; union { real_t float_point; int_t integer; _object_t* obj; _raw_t any; } tmp; @@ -5570,7 +5601,7 @@ static _data_e _get_symbol_type(mb_interpreter_t* s, char* sym, _raw_t* value) { /* int_t */ tmp.integer = (int_t)mb_strtol(sym, &conv_suc, 0); - if(*conv_suc == _ZERO_CHAR) { + if(*conv_suc == _ZERO_CHAR && !_conv_matched_nothing(sym, conv_suc)) { memcpy(*value, tmp.any, sizeof(_raw_t)); result = _DT_INT; @@ -5579,7 +5610,7 @@ static _data_e _get_symbol_type(mb_interpreter_t* s, char* sym, _raw_t* value) { } /* real_t */ tmp.float_point = (real_t)mb_strtod(sym, &conv_suc); - if(*conv_suc == _ZERO_CHAR) { + if(*conv_suc == _ZERO_CHAR && !_conv_matched_nothing(sym, conv_suc)) { memcpy(*value, tmp.any, sizeof(_raw_t)); result = _DT_REAL; @@ -18143,14 +18174,14 @@ static int _std_val(mb_interpreter_t* s, void** l) { switch(arg.type) { case MB_DT_STRING: ret.value.integer = (int_t)mb_strtol(arg.value.string, &conv_suc, 0); - if(*conv_suc == _ZERO_CHAR) { + if(*conv_suc == _ZERO_CHAR && !_conv_matched_nothing(arg.value.string, conv_suc)) { ret.type = MB_DT_INT; mb_check(mb_push_value(s, l, ret)); goto _exit; } ret.value.float_point = (real_t)mb_strtod(arg.value.string, &conv_suc); - if(*conv_suc == _ZERO_CHAR) { + if(*conv_suc == _ZERO_CHAR && !_conv_matched_nothing(arg.value.string, conv_suc)) { ret.type = MB_DT_REAL; mb_check(mb_push_value(s, l, ret)); @@ -18699,14 +18730,14 @@ static int _std_input(mb_interpreter_t* s, void** l) { _get_inputer(s)(s, pmt, line, sizeof(line)); obj->data.variable->data->type = _DT_INT; obj->data.variable->data->data.integer = (int_t)mb_strtol(line, &conv_suc, 0); - if(*conv_suc == _ZERO_CHAR) { + if(*conv_suc == _ZERO_CHAR && !_conv_matched_nothing(line, conv_suc)) { #if MB_PRINT_INPUT_CONTENT _get_printer(s)(s, MB_INT_FMT "\n", obj->data.variable->data->data.integer); #endif /* MB_PRINT_INPUT_CONTENT */ } else { obj->data.variable->data->type = _DT_REAL; obj->data.variable->data->data.float_point = (real_t)mb_strtod(line, &conv_suc); - if(*conv_suc == _ZERO_CHAR) { + if(*conv_suc == _ZERO_CHAR && !_conv_matched_nothing(line, conv_suc)) { #if MB_PRINT_INPUT_CONTENT _get_printer(s)(s, MB_REAL_FMT "\n", obj->data.variable->data->data.float_point); #endif /* MB_PRINT_INPUT_CONTENT */