sc-site-sync/lib/rows.js

// Reference-data (row) sync: the opt-in user/starter/managed layer.
//
//   user    (default) -- rows belong to the target; sync never touches them.
//   starter           -- rows are SEEDED once into a fresh table, then the target
//                        owns them (no later update/delete).
//   managed           -- the source is canonical: rows are mirrored (insert new,
//                        update changed, delete rows the source no longer has),
//                        matched by the stable _ss_row_uuid.
//
// Modes are stored per table in _ss_table_modes. The users table (and any
// ROW_SYNC_LOCKED_TABLES) is hard-locked to `user`. Both the produce side (which
// tables a source ships) and the apply side (which tables a target accepts) gate
// on the LOCAL mode, and the apply side re-enforces the lock -- never trusting the
// sender. FK columns are carried as the referenced row's uuid and resolved back to
// a local id in a second pass so cross-table ordering does not strand FKs.
//
// Re-applying an unchanged source is a no-op: a managed row is only written when
// its resolved values differ from the local row (updateRow fires Update triggers
// and writes history on versioned tables, so blind rewrites are not harmless).

const { isEqual } = require("lodash");

const db    = require("@saltcorn/data/db");
const Table = require("@saltcorn/data/models/table");
const Field = require("@saltcorn/data/models/field");
const File  = require("@saltcorn/data/models/file");

const rowId = require("./rowIdentity");
const { PORTABLE_UUID_KEY, isFileKey, fileFieldOf, rowToPortable, portableToRow } = require("./rowPayload");
const { sha256Hex } = require("./common/hash");
const { DATA_MODES, ADMIN_ROLE_ID, ADMIN_USER_ID, isInfraName, isLockedTable, isSyncedMode } = require("./constants");
const layoutFiles = require("./layoutFiles");

// Content hash of the file a File-field value points at (or null if missing);
// stat-cached, so re-syncing unchanged files does not re-read their bytes.
const hashOfFieldValue = layoutFiles.localFileHash;

const { withoutPk } = rowId;

// A table a target will consider for row sync: synced mode, not infra/locked.
const isRowSyncCandidate = (name, mode) => isSyncedMode(mode) && !isLockedTable(name) && !isInfraName(name);


// Resolve ONE File placeholder to a LOCAL File-field value: keep the existing file
// when its content already matches (idempotent re-sync), else fetch the bytes by
// hash via opts.fileFetcher, verify integrity, and File.from_contents it into this
// tenant's store. On any failure the existing value is KEPT (a transient fetch
// error must not blank a working reference) and a warning is pushed.
const materializeFile = async (spec, tbl, field, opts, existingValue) => {
    const keep = existingValue === undefined ? null : existingValue;
    if (!spec || !spec.hash) {
        return null;
    }
    if (existingValue && (await hashOfFieldValue(existingValue)) === spec.hash) {
        return existingValue;
    }
    if (typeof opts.fileFetcher !== "function") {
        opts.warnings.push(`no file transport available; ${tbl.name}.${field} not updated`);
        return keep;
    }
    let bytes;
    try {
        bytes = await opts.fileFetcher(spec.hash);
    } catch (e) {
        opts.warnings.push(`fetch of ${tbl.name}.${field} (hash ${layoutFiles.shortHash(spec.hash)}) failed: ${e.message}`);
        return keep;
    }
    if (sha256Hex(bytes) !== spec.hash) {
        opts.warnings.push(`content hash mismatch for ${tbl.name}.${field}; refusing`);
        return keep;
    }
    try {
        // File ownership is not portable across two independent user tables, so a
        // synced reference file is owned by the admin user by design; min_role_read
        // (the access gate that IS meaningful) is carried.
        const file = await File.from_contents(spec.filename || "file", spec.mime || "application/octet-stream", bytes, ADMIN_USER_ID, spec.min_role_read || ADMIN_ROLE_ID, "/");
        return file.field_value;
    } catch (e) {
        opts.warnings.push(`could not store ${tbl.name}.${field}: ${e.message}`);
        return keep;
    }
};


// Resolve a portable row's "<field>__file" placeholders (all of them, or only the
// fields in `onlyFields`) to a { <field>: fieldValue } patch to merge into the row.
const materializeFiles = async (portable, tbl, opts, existingRow, onlyFields) => {
    const patch = {};
    for (const key of Object.keys(portable).filter(isFileKey)) {
        const field = fileFieldOf(key);
        if (onlyFields && !onlyFields.has(field)) {
            continue;
        }
        patch[field] = await materializeFile(portable[key], tbl, field, opts, existingRow ? existingRow[field] : undefined);
    }
    return patch;
};


// Would applying portable `desired` (already FK-resolved, File placeholders
// excluded) change `existing`? Values are compared in their JSON form, which is how
// they travel (Date -> ISO string, booleans/JSON as parsed by getRows).
const toJsonValue = (v) => (v === undefined || v === null ? null : JSON.parse(JSON.stringify(v)));
const rowDiffers = (existing, desired) => {
    return Object.keys(desired).some((k) => !isEqual(toJsonValue(existing[k]), toJsonValue(desired[k])));
};


// Does any "<field>__file" placeholder point at different content than `existing`?
const filesDiffer = async (portable, existing) => {
    for (const key of Object.keys(portable).filter(isFileKey)) {
        const spec = portable[key];
        const have = existing[fileFieldOf(key)];
        const want = spec && spec.hash ? spec.hash : null;
        if (want !== (have ? await hashOfFieldValue(have) : null)) {
            return true;
        }
    }
    return false;
};


// FK resolution with a per-table uuid->pk cache, kept current as rows are inserted
// (so the insert fixpoint does not issue a lookup query per row per round).
const makeRefResolver = () => {
    const maps = new Map();
    const mapOf = async (tableName) => {
        if (!maps.has(tableName)) {
            maps.set(tableName, await rowId.localUuidMap(tableName));
        }
        return maps.get(tableName);
    };
    return {
        mapOf,
        resolve: async (tableName, uuid) => {
            const pk = (await mapOf(tableName)).get(uuid);
            return pk === undefined ? null : pk;
        },
        add: async (tableName, uuid, pk) => {
            (await mapOf(tableName)).set(uuid, pk);
        }
    };
};


// --- source-side file serving (content-addressed) ---
// hash -> File-field value, cached per tenant with a short TTL. Content-addressing
// makes the cache safe (a given hash always maps to identical bytes). The TTL bounds
// two things: (1) an authenticated peer spamming UNKNOWN hashes can force at most one
// index rebuild per window (a miss does NOT rebuild if the cache is fresh), and (2)
// staleness after a managed file changes -- the entry is refreshed within the window.
const fileIndexCache = {};
const FILE_INDEX_TTL_MS = 5000;

const buildFileIndex = async () => {
    const modes = await modeMap();
    const map = new Map();
    for (const [name, mode] of Object.entries(modes)) {
        if (!isRowSyncCandidate(name, mode)) {
            continue;
        }
        const tbl = Table.findOne({ name });
        if (!tbl) {
            continue;
        }
        const fileFields = tbl.getFields().filter((f) => f.type === "File");
        if (fileFields.length === 0) {
            continue;
        }
        const rowsData = await tbl.getRows({}, {});
        for (const r of rowsData) {
            for (const f of fileFields) {
                const fv = r[f.name];
                if (!fv) {
                    continue;
                }
                const h = await hashOfFieldValue(fv);
                if (h) {
                    map.set(h, fv);
                }
            }
        }
    }
    // Also index files referenced inside page layouts / view configs / the site logo.
    await layoutFiles.indexLocalFiles(map);
    return map;
};

const resolveFileByHash = async (hash) => {
    const schema = db.getTenantSchema();
    const entry = fileIndexCache[schema];
    const fresh = entry && (Date.now() - entry.builtAt) < FILE_INDEX_TTL_MS;
    // Rebuild only when there is no cache, or the hash is missing AND the cache is
    // stale. An unknown hash against a fresh cache returns null WITHOUT a rebuild,
    // so unknown-hash spam cannot force a scan on every request.
    if (!entry || (!entry.map.has(hash) && !fresh)) {
        fileIndexCache[schema] = { map: await buildFileIndex(), builtAt: Date.now() };
    }
    return fileIndexCache[schema].map.get(hash) || null;
};


// --- data-mode storage (_ss_table_modes) ---

const modeRow = async (tableName) => {
    return await db.selectMaybeOne("_ss_table_modes", { table_name: tableName });
};


const dataModeOf = async (tableName) => {
    const row = await modeRow(tableName);
    return (row && row.data_mode) || DATA_MODES.USER;
};


// name -> stored mode row ({ table_name, data_mode, starter_shipped_at, ... }) for
// every table that has one.
const modeRecords = async () => {
    const rows = await db.select("_ss_table_modes", {});
    return new Map(rows.map((r) => [r.table_name, r]));
};


// name -> mode for every non-user table (used to decide FK-target eligibility).
const modeMap = async () => {
    const m = {};
    for (const [name, r] of await modeRecords()) {
        m[name] = r.data_mode;
    }
    return m;
};


const setDataMode = async (tableName, mode) => {
    if (!Object.values(DATA_MODES).includes(mode)) {
        throw new Error(`invalid data mode: ${mode} (one of ${Object.values(DATA_MODES).join(", ")})`);
    }
    rowId.assertManagedModeAllowed(tableName, mode);
    if (!Table.findOne({ name: tableName })) {
        throw new Error(`no such table ${tableName}`);
    }
    const now = new Date().toISOString();
    const existing = await modeRow(tableName);
    if (existing) {
        await db.updateWhere("_ss_table_modes", { data_mode: mode, updated_at: now }, { table_name: tableName });
    } else {
        await db.insert("_ss_table_modes", { table_name: tableName, data_mode: mode, updated_at: now, starter_shipped_at: null }, { noid: true });
    }
    if (isSyncedMode(mode)) {
        await rowId.ensureManagedSchema(tableName);
    }
};


const markStarterShipped = async (tableName) => {
    await db.updateWhere("_ss_table_modes", { starter_shipped_at: new Date().toISOString() }, { table_name: tableName });
};


// Forget that a starter table was seeded (rollback of the apply that seeded it), so
// the next apply seeds it again.
const clearStarterShipped = async (tableName) => {
    await db.updateWhere("_ss_table_modes", { starter_shipped_at: null }, { table_name: tableName });
};


// Carry a table's data mode across a Table.rename (modes are keyed by name).
const renameModeRow = async (fromName, toName) => {
    if (!(await modeRow(fromName))) {
        return;
    }
    await db.deleteWhere("_ss_table_modes", { table_name: toName });
    await db.updateWhere("_ss_table_modes", { table_name: toName, updated_at: new Date().toISOString() }, { table_name: fromName });
};


// --- produce side: the row manifest a source ships ---

// The present managed/starter tables a source ships (from a modeMap), and a
// warning for each synced table that no longer exists.
const shippedTables = (modes) => {
    const candidates = Object.entries(modes)
        .filter(([name, mode]) => isRowSyncCandidate(name, mode))
        .map(([name]) => name);
    const names = candidates.filter((name) => Table.findOne({ name }));
    const warnings = candidates
        .filter((name) => !names.includes(name))
        .map((name) => `"${name}" is ${modes[name]} but no longer exists (renamed or dropped?); its rows are not shipped -- set the data mode on the current table`);
    return { names, warnings };
};


// { counts: { tableName: { mode, rows } }, warnings } for the tables
// buildRowManifest would ship -- counted, not read, and without its identity
// backfill, so it changes nothing.
const shippedRowCounts = async () => {
    const modes = await modeMap();
    const { names, warnings } = shippedTables(modes);
    const counts = {};
    for (const name of names) {
        counts[name] = { mode: modes[name], rows: await Table.findOne({ name }).countRows() };
    }
    return { counts, warnings };
};


// { tableName: { mode, rows: [portableRow] } } for every locally managed/starter
// table. Rows are read with no user (full system read). A PRE-PASS ensures every
// synced table has _ss_row_uuid backfilled BEFORE any row is read, so translating a
// row's FK to the referenced row's uuid never sees a not-yet-backfilled target
// (which would null the FK just because of table iteration order).
const buildRowManifest = async () => {
    const modes = await modeMap();
    const modeOf = (t) => modes[t] || DATA_MODES.USER;
    const { names: syncedNames, warnings } = shippedTables(modes);

    for (const name of syncedNames) {
        await rowId.ensureManagedSchema(name);
    }

    const manifest = {};
    for (const name of syncedNames) {
        const fresh = Table.findOne({ name });
        const rows = await fresh.getRows({}, {});
        const portableRows = [];
        for (const r of rows) {
            const { portable, warnings: w } = await rowToPortable(r, fresh, modeOf);
            portableRows.push(portable);
            warnings.push(...w);
        }
        manifest[name] = { mode: modes[name], rows: portableRows };
    }
    return { manifest, warnings };
};


// Which manifest tables this target will actually sync, and why the rest are skipped.
const eligibleTables = async (manifest) => {
    const eligible = [];
    const skipped  = [];
    const warnings = [];
    for (const name of Object.keys(manifest || {})) {
        if (isLockedTable(name) || isInfraName(name)) {
            skipped.push({ name, reason: "locked system / infrastructure table" });
            continue;
        }
        const mode = await dataModeOf(name);
        if (!isSyncedMode(mode)) {
            skipped.push({ name, reason: "not managed/starter on this target" });
            continue;
        }
        if (!Table.findOne({ name })) {
            // The mode says managed/starter but the table isn't present locally (e.g.
            // it wasn't created yet, or was dropped). Skip rather than crash.
            skipped.push({ name, reason: "table not present locally" });
            continue;
        }
        eligible.push({ name, mode, rows: identifiedRows(name, manifest[name].rows || [], warnings) });
    }
    return { eligible, skipped, warnings };
};


// The manifest rows that carry an identity, first occurrence per uuid. A row with
// no (or a repeated) uuid could only be inserted again on every apply, so it is
// skipped with a warning.
const identifiedRows = (name, rows, warnings) => {
    const seen = new Set();
    const out  = [];
    for (const p of rows) {
        const uuid = p[PORTABLE_UUID_KEY];
        if (!uuid || seen.has(uuid)) {
            warnings.push(`a "${name}" row without a unique identity was skipped`);
            continue;
        }
        seen.add(uuid);
        out.push(p);
    }
    return out;
};


// --- apply side ---

const requiredFileFields = (tbl) => new Set(tbl.getFields().filter((f) => f.type === "File" && f.required).map((f) => f.name));


// INSERT FIXPOINT: insert rows whose uuid has no local id yet. A row with an
// unresolved REQUIRED FK fails tryInsertRow this round and is retried in later
// rounds once its parent exists (parent->child ordering to arbitrary depth); the
// loop ends when a round inserts nothing, and every row still pending is reported.
// Files are normally materialized only in the converge pass (once the row exists);
// a REQUIRED File field cannot be inserted empty, so it is materialized here ONCE
// per row (cached across rounds, so retries never write another copy).
// Returns name -> Set(uuid) of the rows inserted.
const insertMissingRows = async (active, resolver, fileOpts, result) => {
    const pending   = new Map();
    const inserted  = new Map();
    const lastError = new Map();
    const filePatch = new Map();
    for (const t of active) {
        const local = await resolver.mapOf(t.name);
        pending.set(t.name, t.rows.filter((p) => !local.has(p[PORTABLE_UUID_KEY])));
        inserted.set(t.name, new Set());
    }
    let progress = true;
    while (progress) {
        progress = false;
        for (const t of active) {
            const tbl = Table.findOne({ name: t.name });
            const requiredFiles = requiredFileFields(tbl);
            const left = [];
            for (const portable of pending.get(t.name)) {
                const uuid = portable[PORTABLE_UUID_KEY];
                const { row } = await portableToRow(portable, tbl, resolver.resolve);
                if (requiredFiles.size > 0) {
                    if (!filePatch.has(uuid)) {
                        filePatch.set(uuid, await materializeFiles(portable, tbl, fileOpts, null, requiredFiles));
                    }
                    Object.assign(row, filePatch.get(uuid));
                }
                const r = await tbl.tryInsertRow(withoutPk(row, tbl.pk_name));
                if (r.success === undefined) {
                    lastError.set(uuid, r.error);
                    left.push(portable);
                    continue;
                }
                await resolver.add(t.name, uuid, r.success);
                inserted.get(t.name).add(uuid);
                result.perTable[t.name].inserted += 1;
                progress = true;
            }
            pending.set(t.name, left);
        }
    }
    for (const [name, left] of pending) {
        for (const p of left) {
            result.warnings.push(`"${name}" row ${p[PORTABLE_UUID_KEY]} could not be inserted: ${lastError.get(p[PORTABLE_UUID_KEY])}`);
        }
    }
    return inserted;
};


// CONVERGE pass: now that all rows exist, resolve FKs, materialize files, and write
// a row only when its resolved values differ from the local row. Managed rows
// always converge; starter rows only in the run that seeds them (afterwards the
// target owns them). One getRows per table supplies the comparison baseline.
const convergeRows = async (active, resolver, fileOpts, inserted, result) => {
    for (const t of active) {
        const tbl   = Table.findOne({ name: t.name });
        const pk    = tbl.pk_name;
        const local = await resolver.mapOf(t.name);
        const fresh = inserted.get(t.name);
        const existingByPk = new Map((await tbl.getRows({}, {})).map((r) => [r[pk], r]));
        for (const portable of t.rows) {
            const uuid     = portable[PORTABLE_UUID_KEY];
            const existing = existingByPk.get(local.get(uuid));
            if (!existing || (t.mode === DATA_MODES.STARTER && !fresh.has(uuid))) {
                continue;
            }
            const { row, warnings } = await portableToRow(portable, tbl, resolver.resolve);
            result.warnings.push(...warnings);
            Object.assign(row, await materializeFiles(portable, tbl, fileOpts, existing));
            if (!rowDiffers(existing, row)) {
                continue;
            }
            const err = await tbl.updateRow(withoutPk(row, pk), existing[pk]);
            if (typeof err === "string") {
                result.warnings.push(`"${t.name}" row ${uuid} was not updated: ${err}`);
                continue;
            }
            if (!fresh.has(uuid)) {
                result.perTable[t.name].updated += 1;
            }
        }
    }
};


// Tables OUTSIDE `pruning` whose FK columns reference `tableName`. A managed row
// they still point at is never pruned: a RESTRICT FK would abort the apply, and a
// CASCADE / SET NULL FK would silently change user-mode data.
const protectingRefs = async (tableName, pruning) => {
    const refs = [];
    for (const f of await Field.find({ reftable_name: tableName })) {
        const refTbl = Table.findOne({ id: f.table_id });
        if (refTbl && !pruning.has(refTbl.name)) {
            refs.push({ tbl: refTbl, field: f.name });
        }
    }
    return refs;
};


const referencingTable = async (refs, pk) => {
    for (const r of refs) {
        if ((await r.tbl.countRows({ [r.field]: pk })) > 0) {
            return r.tbl.name;
        }
    }
    return null;
};


// PRUNE: for each target { name, keep: Set(uuid) }, delete the identified local
// rows whose uuid is not kept -- children-first, retrying failed deletes until a
// round makes no progress (resolves FK chains between the pruned tables, including
// self-references). Each failure is isolated and reported -- a prune error never
// aborts the caller. Returns name -> number of rows deleted. Shared by the managed
// row sync and rollback (removing rows an apply inserted).
const pruneRowsNotIn = async (targets, warnings) => {
    const order   = rowId.parentFirstOrder(targets.map((t) => t.name)).reverse();
    const pruning = new Set(order);
    const deleted = new Map(order.map((n) => [n, 0]));
    let pending   = [];
    for (const name of order) {
        const keep = targets.find((t) => t.name === name).keep;
        const tbl  = Table.findOne({ name });
        const refs = await protectingRefs(name, pruning);
        for (const [uuid, pk] of await rowId.localUuidMap(name)) {
            if (keep.has(uuid)) {
                continue;
            }
            const holder = await referencingTable(refs, pk);
            if (holder) {
                warnings.push(`"${name}" row ${uuid} is still referenced by "${holder}"; not removed`);
                continue;
            }
            pending.push({ tbl, pk, uuid, error: null });
        }
    }
    let progress = true;
    while (pending.length > 0 && progress) {
        progress = false;
        const round = pending;
        pending = [];
        for (const p of round) {
            try {
                await p.tbl.deleteRows({ [p.tbl.pk_name]: p.pk });
                deleted.set(p.tbl.name, deleted.get(p.tbl.name) + 1);
                progress = true;
            } catch (e) {
                p.error = e.message;
                pending.push(p);
            }
        }
    }
    for (const p of pending) {
        warnings.push(`"${p.tbl.name}" row ${p.uuid} could not be removed: ${p.error}`);
    }
    return deleted;
};


const applyRowSync = async (manifest, opts = {}) => {
    const { eligible, skipped, warnings } = await eligibleTables(manifest);
    const result = { perTable: {}, skipped, warnings };

    for (const t of eligible) {
        // Never trust the sender: re-enforce the lock + local mode gate.
        rowId.assertRowSyncAllowed(t.name, t.mode);
        await rowId.ensureManagedSchema(t.name);
    }

    // Which tables are actively synced this run (starters already shipped are inert).
    const active = [];
    for (const t of eligible) {
        const info = await modeRow(t.name);
        if (t.mode === DATA_MODES.STARTER && info && info.starter_shipped_at) {
            result.perTable[t.name] = { mode: t.mode, inserted: 0, updated: 0, deleted: 0, note: "starter already shipped" };
            continue;
        }
        result.perTable[t.name] = { mode: t.mode, inserted: 0, updated: 0, deleted: 0 };
        active.push(t);
    }
    const fileOpts = { fileFetcher: opts.fileFetcher, warnings: result.warnings };
    const resolver = makeRefResolver();

    const inserted = await insertMissingRows(active, resolver, fileOpts, result);
    await convergeRows(active, resolver, fileOpts, inserted, result);
    const managed = eligible.filter((t) => t.mode === DATA_MODES.MANAGED);
    const pruned  = await pruneRowsNotIn(managed.map((t) => ({ name: t.name, keep: new Set(t.rows.map((p) => p[PORTABLE_UUID_KEY])) })), result.warnings);
    for (const [name, n] of pruned) {
        result.perTable[name].deleted = n;
    }

    // A starter is marked shipped (never touched again) only once EVERY one of its
    // rows is present; an incomplete seed is retried on the next apply.
    for (const t of active.filter((x) => x.mode === DATA_MODES.STARTER)) {
        const local = await resolver.mapOf(t.name);
        const missing = t.rows.filter((p) => !local.has(p[PORTABLE_UUID_KEY])).length;
        if (missing === 0) {
            await markStarterShipped(t.name);
        } else {
            result.warnings.push(`starter "${t.name}": ${missing} row(s) not seeded; the seed is retried on the next apply`);
        }
    }

    return result;
};


// Preview: per-table insert/update/delete counts, computed exactly as apply would
// decide them but WITHOUT mutating (no uuid backfill: rows that have no identity
// yet are backfilled at apply time and then pruned, so they count as deletes).
const previewRowSync = async (manifest) => {
    const { eligible, skipped, warnings } = await eligibleTables(manifest);
    const perTable = {};
    const resolver = makeRefResolver();
    for (const t of eligible) {
        const info = await modeRow(t.name);
        if (t.mode === DATA_MODES.STARTER && info && info.starter_shipped_at) {
            perTable[t.name] = { mode: t.mode, inserts: 0, updates: 0, deletes: 0 };
            continue;
        }
        const tbl   = Table.findOne({ name: t.name });
        const local = await resolver.mapOf(t.name);
        const managed = t.mode === DATA_MODES.MANAGED;
        const existingByPk = managed ? new Map((await tbl.getRows({}, {})).map((r) => [r[tbl.pk_name], r])) : null;
        let inserts = 0;
        let updates = 0;
        let deletes = 0;
        for (const portable of t.rows) {
            const localId = local.get(portable[PORTABLE_UUID_KEY]);
            if (localId === undefined) {
                inserts += 1;
            } else if (managed) {
                const existing = existingByPk.get(localId);
                const { row } = await portableToRow(portable, tbl, resolver.resolve);
                if (rowDiffers(existing, row) || (await filesDiffer(portable, existing))) {
                    updates += 1;
                }
            }
        }
        if (managed) {
            const sourceUuids = new Set(t.rows.map((p) => p[PORTABLE_UUID_KEY]));
            deletes = [...local.keys()].filter((u) => !sourceUuids.has(u)).length + (existingByPk.size - local.size);
        }
        perTable[t.name] = { mode: t.mode, inserts, updates, deletes };
    }
    return { perTable, skipped, warnings };
};


module.exports = {
    dataModeOf,
    modeRecords,
    modeMap,
    setDataMode,
    clearStarterShipped,
    renameModeRow,
    buildRowManifest,
    shippedRowCounts,
    applyRowSync,
    previewRowSync,
    pruneRowsNotIn,
    rowDiffers,
    resolveFileByHash
};