sc-site-sync/lib/apply.js

// Apply a persisted plan so this instance converges to the source's desired state.
//
// The upsert (create/update) half is delegated to Saltcorn's own install_pack:
// its two-pass FK-aware ordering (bare tables -> refresh -> fields incl. FKs ->
// constraints -> views -> triggers -> pages -> page_groups) is exactly what we
// want. install_pack is ADDITIVE-only; the surgical delete half lives in
// deletes.js.
//
// Mandatory pre-processing before handing a pack to install_pack (install_pack
// mutates its argument and has non-idempotent appenders):
//   * structuredClone the pack (install_pack sorts pack.tables, rewrites min_role,
//     reassigns role.id in place -- must not corrupt the persisted plan).
//   * name = undefined (skip the non-idempotent installed_packs append).
//   * plugins = [] (site-sync never installs plugin config; parity is a pre-flight
//     warning only, and plugin_pack carries the secret deploy_private_key).
//   * event_logs / model_instances = [] (EventLog.create is append-only).
//   * config re-intersected with SYNCABLE_CONFIG_KEYS (never trust the sender;
//     install_pack's setConfig clobbers unconditionally).
//   * tags are installed in a SECOND, tags-only install_pack call, and only the
//     plan's created/updated tags: an updated tag is deleted just before it is
//     re-created (install_pack's existing-tag path appends entries without
//     dedup), and only after the main install succeeded, so a failure there can
//     never leave the target without its tags; unchanged tags are not touched.
//
// Safety flow: take the site-wide apply lock (lib/siteLock.js: one apply or
// rollback per site at a time, across workers and servers), claim the plan
// atomically, re-check the precondition hash (refuse if the target drifted since
// preview -- optimistic concurrency), take a metadata snapshot + restore point,
// apply (checking before every mutating phase that the lock is still held), then
// one non-silent getState().refresh() so sibling cluster workers converge. On
// failure the plan's status is restored and the error carries `planId`, so the
// caller can offer the rollback of a partially applied plan.

const Table = require("@saltcorn/data/models/table");

const capture  = require("./capture");
const planLib  = require("./plan");
const siteLock = require("./siteLock");
const deletes  = require("./deletes");
const rowsLib = require("./rows");
const layoutFiles = require("./layoutFiles");
const { getEnv } = require("./env");
const { PLAN_STATUS } = require("./constants");

// The statuses a plan can be applied from (an applied plan only with opts.force).
const APPLY_FROM = [PLAN_STATUS.PENDING, PLAN_STATUS.ROLLED_BACK];


// Build the install_pack input: a deep clone reduced to the upsert-safe surface.
// Tags are left out (see installTags).
const buildUpsertPack = (desired) => {
    const pack = structuredClone(desired);
    pack.plugins         = [];
    pack.event_logs      = [];
    pack.model_instances = [];
    pack.tags            = [];
    pack.config          = capture.pickSyncableConfig(pack.config);
    return pack;
};


// Install the plan's created/updated tags with a tags-only install_pack call.
const installTags = async (plan) => {
    const tags = plan.entities.tags.create.concat(plan.entities.tags.update.map((u) => u.to));
    if (tags.length === 0) {
        return;
    }
    const Tag = require("@saltcorn/data/models/tag");
    for (const u of plan.entities.tags.update) {
        const existing = await Tag.findOne({ name: u.key });
        if (existing) {
            await existing.delete();
        }
    }
    const tagPack = { plugins: [], event_logs: [], model_instances: [] };
    for (const k of capture.PACK_KINDS.filter((kind) => kind !== "code_pages")) {
        tagPack[k] = [];
    }
    tagPack.tags = structuredClone(tags);
    await require("@saltcorn/admin-models/models/pack").install_pack(tagPack, undefined, async () => {}, false);
};


// install_pack's View.update never changes a view's table binding, so re-point
// every updated view whose desired table differs from its current one.
const fixViewTables = async (plan) => {
    const View = require("@saltcorn/data/models/view");
    for (const u of plan.entities.views.update) {
        const cur = View.findOne({ name: u.key });
        const tbl = u.to.table ? Table.findOne({ name: u.to.table }) : null;
        if (!cur || (u.to.table && !tbl)) {
            continue;
        }
        const wantId = tbl ? tbl.id : null;
        if ((cur.table_id || null) !== wantId) {
            await View.update({ table_id: wantId }, cur.id);
        }
    }
};


// Build a content-hash file fetcher for the row/layout file layers: a signed,
// response-verified GET of the file by hash from the plan's source peer.
// `opts.fileFetcher` (a test override, e.g. an in-process resolver) wins. Returns
// null when there is no peer to fetch from (files are then skipped with a warning).
const buildFileFetcher = async (loaded, env, opts) => {
    if (typeof opts.fileFetcher === "function") {
        return opts.fileFetcher;
    }
    if (loaded.peer_id === null || loaded.peer_id === undefined) {
        return null;
    }
    const peerRow = await require("./peers").findPeer(loaded.peer_id);
    if (!peerRow) {
        return null;
    }
    const { peerBytes, API_PATHS } = require("./transport");
    return async (hash) => peerBytes(peerRow, env.env_id, API_PATHS.file(hash));
};


// Flatten every non-fatal problem of an apply into human-readable warnings.
const collectApplyWarnings = (parts) => {
    const out = parts.notes.slice();
    for (const f of parts.renames.failed) {
        out.push(`rename "${f.from}" -> "${f.to}" not applied (${f.error}); handled as create + gated drop`);
    }
    for (const s of parts.deletes.skipped || []) {
        out.push(`${s.kind} "${s.name}" not deleted: ${s.reason}`);
    }
    return out.concat(parts.fileWarnings, parts.rows.warnings || []);
};


// The apply proper, run while this caller holds the site lock (lease) and the
// plan's APPLYING claim.
const runApply = async (planId, loaded, opts, lease) => {
    const env = await getEnv();
    const notes = [];

    // Optimistic concurrency: refuse if the target changed since the preview.
    const before = await capture.buildDesiredState();
    if (!opts.force && before.hash !== loaded.precondition_hash) {
        throw new Error("target state changed since this plan was previewed; re-preview the plan before applying");
    }

    // Defense in depth: re-sanitize the desired pack on the mutating side.
    const desired = capture.sanitizeDesired(loaded.desired);
    const gateOpts = {
        onDelete:         env.on_delete,
        prunable:         opts.prunable || [],
        confirmDataDrops: !!opts.confirmDataDrops
    };

    // Saltcorn's own metadata snapshot (best-effort; reported when unavailable).
    let snapshotTaken = false;
    try {
        await require("@saltcorn/admin-models/models/snapshot").take_if_changed(`site-sync pre-apply ${planId}`);
        snapshotTaken = true;
    } catch (e) {
        notes.push(`no Saltcorn snapshot taken: ${e.message}`);
    }

    // site-sync restore point: pre-apply metadata pack + the rows this apply can
    // change or destroy, so it can be one-click reverted. Skipped when we are
    // ourselves executing a rollback. A failure only disables rollback (reported).
    if (!opts.skipRestorePoint) {
        try {
            await require("./rollback").saveRestorePoint(planId, before.pack, loaded.plan, gateOpts);
        } catch (e) {
            notes.push(`no restore point saved (rollback unavailable for this apply): ${e.message}`);
        }
    }

    const doStructural = opts.applyDeletes !== false;

    // ---- rename pre-pass (BEFORE install_pack) ----
    await lease.assertHeld();
    // Data-preserving Table.rename (confirmed renames only) so install_pack then
    // UPDATES the renamed table instead of creating a fresh empty one.
    let renameResult = { renamed: [], failed: [] };
    if (doStructural) {
        renameResult = await deletes.applyRenames(loaded.plan, { confirmRenames: opts.confirmRenames });
    }

    // A content-hash file fetcher for the source peer, reused by both the layout-file
    // resolution below and the reference-row sync later.
    const fileFetcher = await buildFileFetcher(loaded, env, opts);
    const fileWarnings = [];

    // ---- upsert half (create + update) ----
    const upsertPack = buildUpsertPack(desired);
    // Resolve files referenced inside page layouts / view configs / the site logo to
    // LOCAL file values BEFORE install_pack writes those layouts (otherwise the
    // placeholders would be persisted verbatim). resolvePlaceholders dedups against
    // the target's own layout files, and fileIndexSeed (rollback) lets pre-apply
    // files (on disk but no longer referenced) resolve without a fetch.
    try {
        await layoutFiles.resolvePlaceholders(upsertPack, {
            fileFetcher,
            fileIndexSeed: opts.fileIndexSeed,
            warnings:      fileWarnings
        });
    } catch (e) {
        fileWarnings.push(`layout-file resolution failed: ${e.message}`);
    }
    await lease.assertHeld();
    await require("@saltcorn/admin-models/models/pack").install_pack(upsertPack, undefined, async () => {}, false);
    await fixViewTables(loaded.plan);
    await installTags(loaded.plan);

    // ---- delete half (mirror pruning) ----
    let deleteResult = { applied: false, deleted: 0, skipped: [] };
    if (doStructural) {
        await lease.assertHeld();
        deleteResult = await deletes.applyDeletes(loaded.plan, Object.assign({ renamed: renameResult.renamed }, gateOpts));
    }

    // ---- reference-row sync (AFTER install_pack so managed schema exists) ----
    let rowResult = { applied: false, perTable: {}, warnings: [] };
    const manifest = loaded.plan.sourceRows;
    if (manifest && Object.keys(manifest).length > 0 && opts.applyRows !== false) {
        await lease.assertHeld();
        rowResult = await rowsLib.applyRowSync(manifest, { fileFetcher });
        rowResult.applied = true;
    }

    // Cluster convergence: one non-silent refresh so sibling workers pick up the
    // new catalog (install_pack refreshes with noSignal=true internally).
    try {
        const { getState } = require("@saltcorn/data/db/state");
        await getState().refresh();
    } catch (e) {
        notes.push(`cluster refresh failed (other workers may serve stale metadata until restart): ${e.message}`);
    }

    return {
        applied:  true,
        plan_id:  planId,
        snapshotTaken,
        summary:  loaded.plan.summary,
        renames:  renameResult,
        deletes:  deleteResult,
        rows:     rowResult,
        warnings: collectApplyWarnings({ notes, renames: renameResult, deletes: deleteResult, fileWarnings, rows: rowResult })
    };
};


// Apply a persisted plan under the site lock (siteLock.LockBusyError while another
// apply or rollback runs; opts.lease = run under a lock the caller holds already).
// The plan is claimed atomically first (pending or rolled_back -> applying; also
// applied with opts.force). On success the plan is `applied` and the warnings
// first name any abandoned plan acquiring the lock recovered; on failure its prior
// status is restored and the error is rethrown with `planId` set.
const applyPlan = async (planId, opts = {}) => {
    return await siteLock.withSiteLock({ operation: "apply", planId, lease: opts.lease }, async (lease) => {
        const loaded = await planLib.loadPlan(planId);
        if (!loaded) {
            throw new Error(`unknown plan ${planId}`);
        }
        const startable = APPLY_FROM.concat(opts.force ? [PLAN_STATUS.APPLIED] : []);
        if (!(await planLib.claimPlanStatus(planId, startable, PLAN_STATUS.APPLYING))) {
            throw new Error(`plan ${planId} cannot be applied: it is ${loaded.status} (re-preview to apply the source again)`);
        }
        const result = await siteLock.finishClaimed(lease, planId, { prior: loaded.status, done: PLAN_STATUS.APPLIED }, () => runApply(planId, loaded, opts, lease));
        result.warnings = siteLock.recoveryWarnings(lease).concat(result.warnings);
        return result;
    });
};


module.exports = {
    APPLY_FROM,
    applyPlan
};