// site-sync apply lock (lib/siteLock.js): one apply or rollback per site at a
// time across PROCESSES, and recovery of the plans a stopped process left in
// flight. In-process checks of the lock itself, then real worker processes: they
// race different plans at once, and one is SIGKILLed mid-apply.
//
//   ./test/run.sh applyLock.js     (SQLite)
//   ./test/runPg.sh applyLock.js   (Postgres, plus the per-tenant check)
//
// SS_LOCK_LIB may point the lib modules at a pre-lock copy to prove the process
// checks fail there; only those run then (the in-process checks need the lock).

const childProcess = require("child_process");
const path         = require("path");

const { getState } = require("@saltcorn/data/db/state");
const Plugin = require("@saltcorn/data/models/plugin");

const { db, ok, section, step, summary, installPlugin, bootstrapPlugin, mkRes, mkAdminReq, routeOf } = require("./harness");

const PROOF    = !!process.env.SS_LOCK_LIB;
const LIB      = process.env.SS_LOCK_LIB || path.resolve(__dirname, "..", "lib");
const capture  = require(path.join(LIB, "capture"));
const planLib  = require(path.join(LIB, "plan"));
const applyLib = require(path.join(LIB, "apply"));
const rollback = require(path.join(LIB, "rollback"));
const ops      = require(path.join(LIB, "operations"));
const env      = require(path.join(LIB, "env"));
const siteLock = PROOF ? null : require(path.join(LIB, "siteLock"));
const routes   = PROOF ? null : require(path.join(LIB, "routes"));
const { physicalColumns }  = require(path.join(LIB, "common", "registeredTable"));
const { APPLY_LOCK_TABLE } = require(path.resolve(__dirname, "..", "lib", "constants"));

const WORKER_FLAG    = "--worker";
const WORKERS        = 4;
// Every worker has booted by then, so all start together.
const START_DELAY_MS = 8000;
// How long a worker's install_pack is held up: the window the others race into.
const SLOW_MS        = 2000;
// A lease short enough to outlive several times within a check.
const SHORT_LEASE_MS = 400;
// How far this process's clock is put ahead to show it is not the lease clock.
const CLOCK_SKEW_MS  = 60 * 60 * 1000;
const STALLED        = "STALLED";
const RESULT         = "RESULT ";
// A refusal by the lock: naming the holder, or (SQLite under load) the database
// staying busy while the holder writes. Either way nothing was changed.
const BUSY           = /is running on this site|the site database is busy/;
const BASE_SITE_NAME = "apply-lock-base";
const TENANT         = "locktenant";


const sleep = (ms) => {
    return new Promise((resolve) => setTimeout(resolve, ms));
};


const outcome = async (fn) => {
    try {
        return { value: await fn() };
    } catch (e) {
        return { error: e };
    }
};


const plansRef = () => `${db.getTenantSchemaPrefix()}_ss_plans`;


// What LEASE_MS passing does to a lock whose holder stopped renewing it.
const ageLease = async () => {
    await db.query(`UPDATE ${db.getTenantSchemaPrefix()}${APPLY_LOCK_TABLE} SET expires_ms = 0`);
};


const planStatus = async (planId) => {
    return (await planLib.loadPlan(planId)).status;
};


const setStatus = async (planId, status, claimedFrom) => {
    await db.query(`UPDATE ${plansRef()} SET status = $1, claimed_from = $2 WHERE plan_id = $3`, [status, claimedFrom, planId]);
};


const dropPlans = async (ids) => {
    for (const id of ids) {
        await db.query(`DELETE FROM ${plansRef()} WHERE plan_id = $1`, [id]);
    }
};


// A plan, previewed against this site as it is now, that only renames the site.
const variantPlan = async (siteName) => {
    const desired = JSON.parse(JSON.stringify((await capture.buildDesiredState()).pack));
    desired.config.site_name = siteName;
    const built = await planLib.buildPlan({
        peerRow: null, env: await env.getEnv(), desired, desiredHash: capture.hashPack(desired),
        sourceEnvId: "src", sourceRows: {}
    });
    return built.plan_id;
};


// ---------------- worker processes ----------------

// A worker: boot like a server process, hold up (or, for "stall", never finish)
// install_pack, then at startAt run one apply or rollback through the admin
// operations and print its outcome.
const worker = async () => {
    const [task, planId, startAt, slowMs] = process.argv.slice(process.argv.indexOf(WORKER_FLAG) + 1);
    await Plugin.loadAllPlugins();
    const packMod = require("@saltcorn/admin-models/models/pack");
    const realInstall = packMod.install_pack;
    packMod.install_pack = async (...args) => {
        if (task === "stall") {
            process.stdout.write(STALLED + "\n");
            return await new Promise(() => {});
        }
        await sleep(Number(slowMs));
        return await realInstall(...args);
    };
    await sleep(Math.max(0, Number(startAt) - Date.now()));
    const r = await outcome(() => (task === "rollback" ? ops.rollbackPlan(planId, {}) : ops.applyPlan(planId, {})));
    process.stdout.write(RESULT + JSON.stringify(r.error ? { ok: false, error: r.error.message } : { ok: true, warnings: r.value.warnings }) + "\n");
    process.exit(0);
};


// Start a worker: { child, done (resolves { code, signal, out }), waitFor(text) }.
// waitFor rejects if the worker exits without printing text.
const startWorker = (args) => {
    let out = "";
    let exited = null;
    const waiters = [];
    const child = childProcess.spawn(process.execPath, [__filename, WORKER_FLAG].concat(args), { env: process.env, stdio: ["ignore", "pipe", "inherit"] });
    child.stdout.on("data", (d) => {
        out += d;
        for (const w of waiters.filter((x) => out.includes(x.text))) {
            w.resolve();
        }
    });
    const done = new Promise((resolve) => {
        child.on("exit", (code, signal) => {
            exited = { code, signal, out };
            for (const w of waiters) {
                w.reject(new Error(`worker exited (${code} ${signal || ""}) without printing ${w.text}: ${out.trim()}`));
            }
            resolve(exited);
        });
    });
    const waitFor = (text) => new Promise((resolve, reject) => {
        if (out.includes(text)) {
            resolve();
        } else if (exited) {
            reject(new Error(`worker exited without printing ${text}: ${out.trim()}`));
        } else {
            waiters.push({ text, resolve, reject });
        }
    });
    return { child, done, waitFor };
};


const resultOf = (run) => {
    const line = run.out.split("\n").find((l) => l.startsWith(RESULT));
    return line ? JSON.parse(line.slice(RESULT.length)) : { ok: false, error: `no result (exit ${run.code} ${run.signal || ""})` };
};


// Reload this process's cached Saltcorn state after workers changed the database.
const resync = async () => {
    await getState().refresh(true);
};


// WORKERS workers each run `task` at the same moment, worker i on planIds[i].
const raceWorkers = async (task, planIds) => {
    const startAt = Date.now() + START_DELAY_MS;
    const runs = await Promise.all(planIds.map((id) => startWorker([task, id, String(startAt), String(SLOW_MS)]).done));
    await resync();
    return runs.map(resultOf);
};


const outcomesText = (results) => JSON.stringify(results.map((r) => (r.ok ? "ok" : r.error)));


// ---------------- in-process checks ----------------

const schemaChecks = async () => {
    section("schema");
    await step("the lock table and _ss_plans.claimed_from exist", async () => {
        const cols = await physicalColumns(APPLY_LOCK_TABLE);
        ok("lock table columns", ["lock_name", "holder", "operation", "plan_id", "acquired_at", "expires_ms"].every((c) => cols.has(c)), [...cols].join(","));
        ok("_ss_plans.claimed_from", (await physicalColumns("_ss_plans")).has("claimed_from"));
    });
    await step("upgrade: a pre-0.0.4 _ss_plans gains claimed_from, even with workers booting together", async () => {
        await db.query(`ALTER TABLE ${plansRef()} DROP COLUMN claimed_from`);
        ok("column dropped (as created by 0.0.3)", !(await physicalColumns("_ss_plans")).has("claimed_from"));
        const { createAllTables } = require(path.join(LIB, "schema"));
        const r = await outcome(() => Promise.all([createAllTables(), createAllTables()]));
        ok("two concurrent onLoad schema passes both succeed", !r.error, r.error && r.error.message);
        ok("claimed_from added back", (await physicalColumns("_ss_plans")).has("claimed_from"));
    });
};


const lockChecks = async () => {
    section("the lock");
    await step("acquire, refuse a second holder, release", async () => {
        const a = await siteLock.acquire({ operation: "apply", planId: "plan-a" });
        const st = await siteLock.lockState();
        ok("held: live, naming the operation and plan", !!st && st.live && st.operation === "apply" && st.plan_id === "plan-a");
        const b = await outcome(() => siteLock.acquire({ operation: "rollback", planId: "plan-b" }));
        ok("a second acquire is refused with LockBusyError", b.error instanceof siteLock.LockBusyError, b.error && b.error.message);
        ok("the refusal names what is running", !!b.error && /another apply is running on this site \(plan plan-a,/.test(b.error.message), b.error && b.error.message);
        await a.release();
        ok("released: free", (await siteLock.lockState()) === null);
        const c = await siteLock.acquire({ operation: "apply", planId: "plan-c" });
        ok("acquirable again", (await siteLock.lockState()).plan_id === "plan-c");
        await c.release();
    });
    await step("a held lease is renewed: it outlives its length several times", async () => {
        const a = await siteLock.acquire({ operation: "apply", planId: "renewed", leaseMs: SHORT_LEASE_MS });
        await sleep(SHORT_LEASE_MS * 3);
        const b = await outcome(() => siteLock.acquire({ operation: "apply", planId: "thief" }));
        ok("still held after three lease lengths", b.error instanceof siteLock.LockBusyError);
        ok("the holder still holds it", await a.renew());
        await a.release();
    });
    await step("a lapsed lease is taken over, and its old holder finds out", async () => {
        const a = await siteLock.acquire({ operation: "apply", planId: "stalled" });
        await ageLease();
        ok("lapsed: no longer live", (await siteLock.lockState()).live === false);
        const b = await siteLock.acquire({ operation: "rollback", planId: "taker" });
        ok("taken over", (await siteLock.lockState()).plan_id === "taker");
        const lost = await outcome(() => a.assertHeld());
        ok("the old holder's checkpoint throws LockLostError", lost.error instanceof siteLock.LockLostError);
        ok("and it does not renew any more", (await a.renew()) === false);
        await a.release();
        ok("its release leaves the new holder's lock alone", (await siteLock.lockState()).plan_id === "taker");
        await b.release();
    });
    await step("lease times come from the database clock, not this server's", async () => {
        const a = await siteLock.acquire({ operation: "apply", planId: "clock-holder" });
        const realNow = Date.now;
        Date.now = () => realNow() + CLOCK_SKEW_MS;
        try {
            ok("a server whose clock runs an hour fast still sees the lease live", (await siteLock.lockState()).live === true);
            const b = await outcome(() => siteLock.acquire({ operation: "apply", planId: "fast-clock" }));
            ok("and cannot take it over", b.error instanceof siteLock.LockBusyError);
        } finally {
            Date.now = realNow;
        }
        await a.release();
    });
    await step("withSiteLock: a nested step reuses the lease; released on success and failure", async () => {
        await siteLock.withSiteLock({ operation: "apply", planId: "outer" }, async (lease) => {
            const nested = await siteLock.withSiteLock({ operation: "apply", planId: "inner", lease }, async (inner) => ({ holder: (await siteLock.lockState()).plan_id, recovered: inner.recovered.length }));
            ok("a nested step runs under the outer lease", nested.holder === "outer" && nested.recovered === 0);
        });
        ok("released after success", (await siteLock.lockState()) === null);
        const r = await outcome(() => siteLock.withSiteLock({ operation: "apply", planId: "boom" }, async () => {
            throw new Error("boom");
        }));
        ok("the failure propagates", !!r.error && r.error.message === "boom");
        ok("released after a failure", (await siteLock.lockState()) === null);
    });
};


const recoveryChecks = async () => {
    section("recovery of plans a stopped holder left in flight");
    await step("claimPlanStatus records the status it claimed from", async () => {
        const id = await variantPlan("claim");
        await planLib.claimPlanStatus(id, ["pending"], "applying");
        const p = await planLib.loadPlan(id);
        ok("applying, claimed_from pending", p.status === "applying" && p.claimed_from === "pending");
        await dropPlans([id]);
    });
    await step("acquiring the lock returns each in-flight plan to the status it was claimed from", async () => {
        const ids = [];
        for (const tag of ["a", "b", "c", "d"]) {
            ids.push(await variantPlan(`recover-${tag}`));
        }
        await setStatus(ids[0], "applying", "rolled_back");
        await setStatus(ids[1], "rolling_back", "pending");
        await setStatus(ids[2], "applying", null);
        await setStatus(ids[3], "rolling_back", null);
        const lease = await siteLock.acquire({ operation: "apply", planId: "recoverer" });
        ok("all four recovered", lease.recovered.length === 4 && ids.every((id) => lease.recovered.some((r) => r.plan_id === id)), JSON.stringify(lease.recovered));
        ok("an apply claimed from rolled_back is rolled_back again", (await planStatus(ids[0])) === "rolled_back");
        ok("a rollback claimed from pending is pending again", (await planStatus(ids[1])) === "pending");
        ok("a 0.0.3 apply (no claimed_from) is pending again", (await planStatus(ids[2])) === "pending");
        ok("a 0.0.3 rollback (no claimed_from) is applied again", (await planStatus(ids[3])) === "applied");
        const warnings = siteLock.recoveryWarnings(lease);
        ok("one warning per recovered plan, naming it", warnings.length === 4 && ids.every((id) => warnings.some((w) => w.includes(id))), warnings.join(" | "));
        await lease.release();
        await dropPlans(ids);
    });
};


const standingChecks = async () => {
    section("a plan left in flight is reported (read-only)");
    await step("running while its lease is live, interrupted once nobody holds the lock", async () => {
        const id = await variantPlan("standing");
        const lease = await siteLock.acquire({ operation: "apply", planId: id });
        await planLib.claimPlanStatus(id, ["pending"], "applying");
        const running = await ops.status();
        ok("status: running, naming the plan", running.applyLock.state === "running" && running.applyLock.holder.plan_id === id);
        ok("get_plan: in flight, not interrupted", (await ops.inspectPlan(id)).standing.interrupted === false);
        await lease.release();

        const st = await ops.status();
        ok("status: interrupted, listing the plan", st.applyLock.state === "interrupted" && st.applyLock.plans.some((p) => p.plan_id === id && p.status === "applying"));
        const standing = (await ops.inspectPlan(id)).standing;
        ok("get_plan: interrupted, and appliable once recovered (pending, site unchanged)", standing.interrupted === true && standing.can_apply === true && standing.stale === false, JSON.stringify(standing));
        const listed = (await ops.listPlans({})).find((p) => p.plan_id === id);
        ok("list_plans agrees", !!listed && listed.interrupted === true && listed.status === "applying");

        const res = mkRes();
        await routeOf(routes.routes, "get", "/admin/site-sync/")(mkAdminReq(), res);
        const html = JSON.stringify(res._wrap || {});
        ok("the dashboard shows an interrupted card naming the plan", html.includes("Interrupted apply or rollback") && html.includes(id));
        ok("reading changed nothing", (await planStatus(id)) === "applying");

        const recoverer = await siteLock.acquire({ operation: "apply", planId: "recoverer" });
        ok("the next acquire recovers it", recoverer.recovered.some((r) => r.plan_id === id) && (await planStatus(id)) === "pending");
        await recoverer.release();
        ok("status: idle", (await ops.status()).applyLock.state === "idle");
        await dropPlans([id]);
    });
};


const lostLeaseChecks = async () => {
    section("a holder that loses the lock stops, leaving the plan to the new holder");
    await step("lock taken over during install_pack: the apply stops at its next checkpoint", async () => {
        const id = await variantPlan("lost");
        const packMod = require("@saltcorn/admin-models/models/pack");
        const realInstall = packMod.install_pack;
        let thief = null;
        packMod.install_pack = async (...args) => {
            packMod.install_pack = realInstall;
            await ageLease();
            thief = await siteLock.acquire({ operation: "apply", planId: "thief" });
            return await realInstall(...args);
        };
        let r;
        try {
            r = await outcome(() => applyLib.applyPlan(id, {}));
        } finally {
            packMod.install_pack = realInstall;
        }
        ok("the apply stopped with LockLostError", r.error instanceof siteLock.LockLostError, r.error ? r.error.message : "it finished");
        ok("the error names the plan", !!r.error && r.error.planId === id);
        ok("the new holder recovered the plan", !!thief && thief.recovered.some((x) => x.plan_id === id));
        ok("its status is what the new holder made it (not overwritten)", (await planStatus(id)) === "pending");
        ok("the new holder still holds the lock", (await siteLock.lockState()).plan_id === "thief");
        await thief.release();
        const rb = await rollback.rollbackApply(id, {});
        ok("the half-applied plan rolls back from its restore point", rb.rolledBack === true && getState().getConfig("site_name") === BASE_SITE_NAME);
    });
};


const opsChecks = async () => {
    section("the admin operations while the lock is held");
    await step("apply and rollback are refused cleanly, then work once it is free", async () => {
        const id = await variantPlan("ops");
        const lease = await siteLock.acquire({ operation: "rollback", planId: "busy-holder" });
        const a = await outcome(() => ops.applyPlan(id, {}));
        ok("apply: OpError naming the running rollback", a.error instanceof ops.OpError && /another rollback is running on this site \(plan busy-holder,/.test(a.error.message), a.error && a.error.message);
        ok("no restore-point offer (nothing changed)", !!a.error && a.error.extra === undefined);
        const b = await outcome(() => ops.rollbackPlan(id, {}));
        ok("rollback: refused the same way", b.error instanceof ops.OpError && BUSY.test(b.error.message));
        ok("the plan was not touched", (await planStatus(id)) === "pending");
        await lease.release();
        const c = await ops.applyPlan(id, {});
        ok("applies once the lock is free", c.applied === true && getState().getConfig("site_name") === "ops");
        await ops.rollbackPlan(id, {});
        ok("and rolls back", getState().getConfig("site_name") === BASE_SITE_NAME);
    });
};


// ---------------- process checks ----------------

const raceChecks = async () => {
    section(`${WORKERS} processes apply ${WORKERS} different plans at the same moment`);
    await step("exactly one runs; the others are refused before changing anything", async () => {
        const ids = [];
        for (let i = 0; i < WORKERS; i++) {
            ids.push(await variantPlan(`race-${i}`));
        }
        const results = await raceWorkers("apply", ids);
        ok("one apply ran", results.filter((r) => r.ok).length === 1, outcomesText(results));
        ok("every other one was refused by the lock", results.filter((r) => !r.ok && BUSY.test(r.error)).length === WORKERS - 1, outcomesText(results));
        const statuses = [];
        for (const id of ids) {
            statuses.push(await planStatus(id));
        }
        ok("one plan applied, the rest still pending", statuses.filter((s) => s === "applied").length === 1 && statuses.filter((s) => s === "pending").length === WORKERS - 1, statuses.join(","));
        if (!PROOF) {
            ok("the lock is free again", (await siteLock.lockState()) === null);
        }
        for (const id of ids.filter((x, i) => statuses[i] === "applied")) {
            await rollback.rollbackApply(id, {});
        }
        await dropPlans(ids);
    });
};


const crashChecks = async () => {
    section("a process killed mid-apply");
    await step("its plan is recovered once its lease lapses, by exactly one of several racing rollbacks", async () => {
        const before = (await capture.buildDesiredState()).hash;
        const x = await variantPlan("crash-x");
        const y = await variantPlan("crash-y");
        const w = startWorker(["stall", x, "0", "0"]);
        await w.waitFor(STALLED);
        w.child.kill("SIGKILL");
        const killed = await w.done;
        ok("the worker was killed inside install_pack", killed.signal === "SIGKILL");
        ok("its plan is left applying", (await planStatus(x)) === "applying");
        if (!PROOF) {
            const st = await siteLock.lockState();
            ok("its lock is still held under a live lease", !!st && st.live && st.plan_id === x);
            const busy = await outcome(() => ops.applyPlan(y, {}));
            ok("until the lease lapses other applies are refused, naming it", !!busy.error && busy.error.message.includes(`another apply is running on this site (plan ${x},`), busy.error && busy.error.message);
            ok("the dashboard cannot tell yet: running", (await ops.status()).applyLock.state === "running");
            await ageLease();
            const lapsed = await ops.status();
            ok("lease lapsed: the dashboard reports it interrupted, with its restore point", lapsed.applyLock.state === "interrupted" && lapsed.applyLock.plans.some((p) => p.plan_id === x && p.has_restore_point));
            const standing = (await ops.inspectPlan(x)).standing;
            ok("get_plan: interrupted, can roll back", standing.interrupted === true && standing.can_rollback === true, JSON.stringify(standing));
        }
        const results = await raceWorkers("rollback", Array(WORKERS).fill(x));
        ok("exactly one rollback ran", results.filter((r) => r.ok).length === 1, outcomesText(results));
        ok("the others were refused by the lock", results.filter((r) => !r.ok && BUSY.test(r.error)).length === WORKERS - 1, outcomesText(results));
        const winner = results.find((r) => r.ok);
        ok("the rollback reports the recovery", !!winner && winner.warnings.some((m) => m.includes(x) && /left applying by a process that stopped/.test(m)), winner && JSON.stringify(winner.warnings));
        ok("the plan is rolled back", (await planStatus(x)) === "rolled_back");
        ok("this site is as it was before the killed apply", (await capture.buildDesiredState()).hash === before);
        if (!PROOF) {
            ok("nothing in flight, lock free", (await ops.status()).applyLock.state === "idle" && (await siteLock.lockState()) === null);
        }
        const applied = await outcome(() => ops.applyPlan(y, {}));
        ok("the plan the kill blocked now applies", !applied.error && applied.value.applied === true, applied.error && applied.error.message);
        if (!applied.error) {
            await ops.rollbackPlan(y, {});
        }
        await dropPlans([x, y]);
    });
};


// ---------------- tenants (Postgres) ----------------

const tenantChecks = async () => {
    if (db.isSQLite) {
        return;
    }
    section("tenants have their own locks (Postgres)");
    const { add_tenant } = require("@saltcorn/data/db/state");
    const { create_tenant, switchToTenant, insertTenant } = require("@saltcorn/admin-models/models/tenant");
    await step("a held root lock does not block a tenant; the tenant's lease renews in its own schema", async () => {
        add_tenant(TENANT);
        await switchToTenant(await insertTenant(TENANT, `admin@${TENANT}.local`, "apply lock tenant check"), `http://${TENANT}.example.com/`);
        await create_tenant({ t: TENANT, loadAndSaveNewPlugin() {}, plugin_loader() {} });
        const rootLease = await siteLock.acquire({ operation: "apply", planId: "root-holder" });
        await db.runWithTenant(TENANT, async () => {
            await Plugin.loadAllPlugins();
            await db.runWithTenant(db.connectObj.default_schema, () => getState().setConfig("tenants_unsafe_plugins", true));
            await installPlugin();
            ok("running in the tenant schema", db.getTenantSchema() === TENANT);
            ok("the tenant's lock is free", (await siteLock.lockState()) === null);
            const id = await variantPlan("tenant-plan");
            const r = await ops.applyPlan(id, {});
            ok("the tenant applies while the root lock is held", r.applied === true);
            const tenantLease = await siteLock.acquire({ operation: "apply", planId: "tenant-holder", leaseMs: SHORT_LEASE_MS });
            await sleep(SHORT_LEASE_MS * 3);
            const st = await siteLock.lockState();
            ok("the tenant's short lease was renewed in the tenant schema", !!st && st.live && st.plan_id === "tenant-holder");
            await db.runWithTenant(db.connectObj.default_schema, async () => {
                ok("the root sees only its own holder", (await siteLock.lockState()).plan_id === "root-holder");
            });
            await tenantLease.release();
        });
        await rootLease.release();
    });
};


const main = async () => {
    const where = db.isSQLite ? `sqlite ${process.env.SQLITE_FILEPATH}` : `postgres ${db.connectObj.database} schema ${db.getTenantSchema()}`;
    console.log(`DB = ${where}  LIB = ${LIB}${PROOF ? "  (proof mode: process checks only)" : ""}`);
    await bootstrapPlugin();
    await getState().setConfig("site_name", BASE_SITE_NAME);
    if (!PROOF) {
        await schemaChecks();
        await lockChecks();
        await recoveryChecks();
        await standingChecks();
        await lostLeaseChecks();
        await opsChecks();
    }
    await raceChecks();
    await crashChecks();
    if (!PROOF) {
        await tenantChecks();
    }
    process.exit(summary() === 0 ? 0 : 1);
};


if (process.argv.includes(WORKER_FLAG)) {
    worker();
} else {
    // main() ends with process.exit; an event loop that drains first means a check
    // waited on something that can no longer happen -- a failure, never a pass.
    process.on("beforeExit", () => {
        console.log("\nFAILURES: the run ended before finishing its checks");
        process.exitCode = 1;
    });
    main();
}