diff --git a/index.js b/index.js index ccd7679..516e240 100644 --- a/index.js +++ b/index.js @@ -1,10 +1,13 @@ // dev-deploy: Saltcorn plugin for migrating metadata changes across // Dev/Test/Prod environments via an ops journal with stable UUIDs. -const { PLUGIN_NAME, PLUGIN_VERSION } = require("./lib/constants"); +const db = require("@saltcorn/data/db"); + +const { PLUGIN_NAME, PLUGIN_VERSION, DATA_MODES } = require("./lib/constants"); const { createAllTables } = require("./lib/schema"); const { getEnv, initEnvIfMissing, markBootstrapped } = require("./lib/env"); const { backfillAll } = require("./lib/entityIds"); +const { ensureManagedSchema } = require("./lib/rowIdentity"); const { installAllWraps } = require("./lib/wrap"); const { routes } = require("./lib/routes"); @@ -32,19 +35,67 @@ const ensureCsrfBypass = async () => { }; +// One-time migration: managed/starter tables created by an older plugin version +// carry a RAW _dd_row_uuid column (added via ALTER, not registered in _sc_fields). +// Such a column is dumped by backup but absent from the restored table's field +// list, which fails the row import and rolls back the whole table (app-data +// loss). Adopt every existing managed table's column into _sc_fields so it +// round-trips. ensureManagedSchema is idempotent: a no-op once registered. +const migrateRowUuidFields = async () => { + const schema = db.getTenantSchemaPrefix(); + let rows; + try { + rows = await db.query( + `SELECT e.current_name AS name + FROM ${schema}_dd_table_modes m + JOIN ${schema}_dd_entity_ids e ON e.uuid = m.table_uuid + WHERE e.kind = 'table' + AND m.data_mode IN ($1, $2)`, + [DATA_MODES.MANAGED, DATA_MODES.STARTER] + ); + } catch (e) { + return 0; + } + let adopted = 0; + for (const r of rows.rows) { + try { + const res = await ensureManagedSchema(r.name); + if (res && res.adopted) { + adopted++; + } + } catch (e) { + console.error(`[${PLUGIN_NAME}] row-uuid field migration failed for ${r.name}:`, e.message); + } + } + return adopted; +}; + + const onLoad = async (cfg) => { try { await createAllTables(); const env = await initEnvIfMissing(); + // Reconcile the entity-id cache on EVERY load, not just first bootstrap. + // _dd_entity_ids is a raw, un-backed-up cache; after a restore it is empty + // while the restored env reports bootstrapped, so a gated backfill would + // leave the map empty. backfillAll is idempotent (ensureUuid keys on + // (kind,current_id) and derives deterministic UUIDs from name), so running + // it always rebuilds the map against the restored host's live catalog. + const counts = await backfillAll(); + const total = Object.values(counts).reduce((a, b) => a + b, 0); if (!env.bootstrapped_at) { - const counts = await backfillAll(); - const total = Object.values(counts).reduce((a, b) => a + b, 0); await markBootstrapped(env.env_id); log(`v${PLUGIN_VERSION} bootstrapped env_id=${env.env_id} backfilled ${total} entities ${JSON.stringify(counts)}`); + } else if (total > 0) { + log(`v${PLUGIN_VERSION} loaded env_id=${env.env_id}; reconciled entity-id cache (+${total})`); } else { log(`v${PLUGIN_VERSION} loaded env_id=${env.env_id}`); } installAllWraps(); + const adopted = await migrateRowUuidFields(); + if (adopted > 0) { + log(`migrated ${adopted} managed table(s) to a registered _dd_row_uuid field`); + } await ensureCsrfBypass(); } catch (err) { // eslint-disable-next-line no-console diff --git a/lib/configStore.js b/lib/configStore.js new file mode 100644 index 0000000..7c89d43 --- /dev/null +++ b/lib/configStore.js @@ -0,0 +1,49 @@ +// Durable per-tenant key/value storage in Saltcorn's _sc_config, used for the +// dev-deploy ENV IDENTITY (which is created during onLoad). +// +// Why _sc_config and not a registered table or the plugin config blob: +// - It survives backup: backup_config dumps every non-fixed _sc_config key, +// and restore_config restores them BEFORE install_pack runs the plugin's +// onLoad -- so the restored identity is already present when onLoad reads it +// (no onLoad-creates-then-restore-imports duplicate, which a registered +// table suffers because restore_tables runs AFTER onLoad). +// - It does NOT write the plugin's own _sc_plugins row (Plugin.upsert during +// onLoad cascades plugin re-loading and duplicates rows -- the reason the +// config-blob approach was abandoned). setConfig writes _sc_config only. +// +// Reads go DIRECT to _sc_config: getState().getConfig only surfaces keys that +// are in Saltcorn's known configTypes schema, so a custom plugin key is invisible +// through it even though it is stored and backed up. Writes go through setConfig +// (correct jsonb/text encoding on both backends; value wrapped as {v: ...}). + +const db = require("@saltcorn/data/db"); + + +const readKey = async (key) => { + const row = await db.selectMaybeOne("_sc_config", { key: key }); + if (!row || row.value === null || row.value === undefined) { + return undefined; + } + let v = row.value; + if (typeof v === "string") { + try { + v = JSON.parse(v); + } catch (e) { + return undefined; + } + } + // Saltcorn wraps config values as { v: }. + return v && typeof v === "object" && "v" in v ? v.v : v; +}; + + +const writeKey = async (key, value) => { + const { getState } = require("@saltcorn/data/db/state"); + await getState().setConfig(key, value); +}; + + +module.exports = { + readKey, + writeKey +}; diff --git a/lib/entityIds.js b/lib/entityIds.js index 4323919..eca056b 100644 --- a/lib/entityIds.js +++ b/lib/entityIds.js @@ -83,6 +83,21 @@ const ensureUuid = async (kind, currentId, currentName, parentUuid, canonical) = return existing.uuid; } const uuid = deterministicUuid(kind, canonical || currentName); + // RECONCILE: the deterministic UUID (derived from the entity's name) is the + // stable identity. If a row already carries it but at a different current_id, + // the integer id was reassigned (e.g. table recreated, or a restore) -- re- + // point the stable UUID to the new id rather than inserting a duplicate (which + // would violate the uuid PK). No (kind,current_id) row exists here (checked + // above), so the update cannot violate UNIQUE(kind,current_id). + const stale = await db.selectMaybeOne("_dd_entity_ids", { uuid: uuid }); + if (stale) { + await db.updateWhere( + "_dd_entity_ids", + { current_id: currentId, current_name: currentName, parent_uuid: parentUuid || null }, + { uuid: uuid } + ); + return uuid; + } const row = { uuid: uuid, kind: kind, @@ -134,10 +149,20 @@ const removeEntityRow = async (kind, currentId) => { // Backfill helpers: each returns the count of new UUIDs inserted. +// dev-deploy's own registered bookkeeping tables (_dd_peers, _dd_table_modes) +// and saltcorn-idp's (_idp_*) appear in Table.find now that they are registered, +// but they are plugin INFRASTRUCTURE, not user data -- never track, sync, or show +// them as managed tables. +const isInfraTable = (name) => name.startsWith("_dd_") || name.startsWith("_idp_"); + + const backfillTables = async () => { const tables = await Table.find({}, { cached: false }); let added = 0; for (const t of tables) { + if (isInfraTable(t.name)) { + continue; + } const before = await lookupByCurrent(ENTITY_KINDS.TABLE, t.id); await ensureUuid(ENTITY_KINDS.TABLE, t.id, t.name, null, t.name); if (!before) { @@ -152,12 +177,20 @@ const backfillFields = async () => { const tables = await Table.find({}, { cached: false }); let added = 0; for (const t of tables) { + if (isInfraTable(t.name)) { + continue; + } const tableUuidRow = await lookupByCurrent(ENTITY_KINDS.TABLE, t.id); if (!tableUuidRow) { continue; } const fields = t.getFields ? t.getFields() : (await Field.find({ table_id: t.id })); for (const f of fields) { + // _dd_row_uuid is dev-deploy's per-environment row-identity column -- + // never track/sync it (each environment manages its own). + if (f.name.startsWith("_dd_")) { + continue; + } const before = await lookupByCurrent(ENTITY_KINDS.FIELD, f.id); await ensureUuid(ENTITY_KINDS.FIELD, f.id, f.name, tableUuidRow.uuid, `${t.name}.${f.name}`); if (!before) { @@ -318,6 +351,12 @@ const backfillWorkflowSteps = async () => { const backfillAll = async () => { + // Remove any stale tracking of plugin infrastructure: registered _dd_*/_idp_* + // tables and the per-environment _dd_row_uuid column should never be in the + // entity-id map (older builds tracked _dd_row_uuid once it became a field). + const schema = db.getTenantSchemaPrefix(); + await db.query(`DELETE FROM ${schema}_dd_entity_ids WHERE current_name LIKE '_dd_%' OR current_name LIKE '_idp_%'`).catch(() => {}); + const counts = {}; counts.tables = await backfillTables(); counts.fields = await backfillFields(); diff --git a/lib/env.js b/lib/env.js index 4827454..62f665b 100644 --- a/lib/env.js +++ b/lib/env.js @@ -1,34 +1,25 @@ // This Saltcorn instance's dev-deploy identity (env_id, label, policies). -// Stored as a singleton row in _dd_env. - -const db = require("@saltcorn/data/db"); +// +// Stored in _sc_config (key "dev_deploy_env") rather than a _dd_env table. +// _sc_config survives backup AND is restored BEFORE the plugin's onLoad runs, so +// a restored instance keeps its env_id and onLoad does not create a duplicate +// (see lib/configStore.js). Reads are DB-fresh (no in-process cache needed). const { randomUuid } = require("./ids"); const { DESTRUCTIVE_POLICY } = require("./constants"); +const { readKey, writeKey } = require("./configStore"); -// The env identity is schema-scoped, so a single process serving multiple -// tenants must NOT share one cached row across them. Key the cache by tenant -// schema (db.getTenantSchema()), not a module-level singleton. -const cachedEnvByTenant = new Map(); - -const tenantKey = () => (db.getTenantSchema ? db.getTenantSchema() : "public"); +const ENV_KEY = "dev_deploy_env"; const getEnv = async () => { - const key = tenantKey(); - if (cachedEnvByTenant.has(key)) { - return cachedEnvByTenant.get(key); - } - const rows = await db.select("_dd_env", {}); - const env = rows.length > 0 ? rows[0] : null; - cachedEnvByTenant.set(key, env); - return env; + return (await readKey(ENV_KEY)) || null; }; const refreshEnvCache = async () => { - cachedEnvByTenant.delete(tenantKey()); + // No in-process cache anymore; reads are always DB-fresh. Kept for callers. return await getEnv(); }; @@ -38,27 +29,24 @@ const initEnvIfMissing = async () => { if (existing) { return existing; } - const now = new Date().toISOString(); - const row = { + const env = { env_id: randomUuid(), env_label: null, on_destructive_op: DESTRUCTIVE_POLICY.CONFIRM, require_tls: 0, - created_at: now, + created_at: new Date().toISOString(), bootstrapped_at: null }; - await db.insert("_dd_env", row, { noid: true }); - cachedEnvByTenant.set(tenantKey(), row); - return row; + await writeKey(ENV_KEY, env); + return env; }; const markBootstrapped = async (envId) => { - const now = new Date().toISOString(); - await db.updateWhere("_dd_env", { bootstrapped_at: now }, { env_id: envId }); - const cached = cachedEnvByTenant.get(tenantKey()); - if (cached && cached.env_id === envId) { - cached.bootstrapped_at = now; + const env = await getEnv(); + if (env && env.env_id === envId) { + env.bootstrapped_at = new Date().toISOString(); + await writeKey(ENV_KEY, env); } }; diff --git a/lib/peers.js b/lib/peers.js index 5d7cd46..0642461 100644 --- a/lib/peers.js +++ b/lib/peers.js @@ -85,7 +85,13 @@ const addPeer = async ({ envId, label, baseUrl, requireTls, existingSecret }) => } const secret = existingSecret || randomSecret(); const sealed = seal(secret); + // _dd_peers is now a registered Saltcorn table with an auto `id` PK; peer_id + // is a regular unique column, so we assign it ourselves (serial-style max+1) + // to keep peer_id stable + collision-free (anchors reference it by value). + const nextRs = await db.query(`SELECT COALESCE(MAX(peer_id),0)+1 AS next FROM ${db.getTenantSchemaPrefix()}_dd_peers`); + const peerId = nextRs.rows[0].next; const row = { + peer_id: peerId, env_id: envId, label: label || null, base_url: baseUrl, @@ -96,9 +102,7 @@ const addPeer = async ({ envId, label, baseUrl, requireTls, existingSecret }) => created_at: new Date().toISOString(), last_seen_at: null }; - // noid: _dd_peers' PK is peer_id (serial on PG), not "id"; without this - // Saltcorn's default RETURNING id makes the insert fail on Postgres with - // 'column "id" does not exist'. The peer_id is auto-assigned; re-select below. + // noid: let Saltcorn assign the `id` PK; we supply peer_id explicitly. await db.insert("_dd_peers", row, { noid: true }); const fresh = await findPeerByEnvId(envId); return { peer: rowToPeer(fresh), secret: secret }; diff --git a/lib/rowIdentity.js b/lib/rowIdentity.js index eb0656b..3aee691 100644 --- a/lib/rowIdentity.js +++ b/lib/rowIdentity.js @@ -1,15 +1,34 @@ -// Hidden _dd_row_uuid column infrastructure for managed/starter tables. +// _dd_row_uuid column infrastructure for managed/starter tables. // -// When an admin marks a user table as managed or starter, this module -// adds a TEXT column _dd_row_uuid to the underlying SQL table via raw ALTER -// (NOT registered in _sc_fields, so Saltcorn's table builder doesn't show -// it). Existing rows are backfilled with random UUIDs. From that point, -// the wrap layer reads/writes _dd_row_uuid as the cross-environment identity -// for each row. +// When an admin marks a user table as managed or starter, this module adds a +// _dd_row_uuid column to the underlying table as a REGISTERED Saltcorn Field +// (in _sc_fields) -- NOT a raw ALTER outside the catalog. Registration is what +// makes the column survive Saltcorn backup/restore/snapshots: backup dumps a +// row's physical columns (SELECT * / row_to_json), so a raw, unregistered +// column lands in the dump but the restored table -- recreated from the pack's +// field list -- has no such column, and the row insert then fails and rolls +// back the whole table (real app-data loss). A registered Field round-trips: +// install_pack recreates the column before the rows are imported. +// +// The field is created with hidden:true so it stays out of the edit/show form +// builders (it is still visible in the raw table builder field list -- Saltcorn +// has no system/hidden-from-catalog flag, an accepted trade-off for backup +// safety). Existing rows are backfilled with random UUIDs. From that point, the +// wrap layer reads/writes _dd_row_uuid as the cross-environment identity for +// each row. +// +// Creating/deleting this field goes through Field.create / field.delete, which +// the dev-deploy wraps (lib/wrap.js) journal. We run those inside runSuppressed +// so the plugin's own row-identity column is never journaled or propagated to +// peers -- every environment manages its own _dd_row_uuid independently. const crypto = require("crypto"); -const db = require("@saltcorn/data/db"); +const db = require("@saltcorn/data/db"); +const Table = require("@saltcorn/data/models/table"); +const Field = require("@saltcorn/data/models/field"); + +const { runSuppressed } = require("./context"); const COLUMN_NAME = "_dd_row_uuid"; @@ -21,6 +40,11 @@ const tableSqlRef = (tableName) => { }; +const refreshCatalog = async () => { + await require("@saltcorn/data/db/state").getState().refresh_tables(true); +}; + + const columnExists = async (tableName) => { if (db.isSQLite) { const rs = await db.query(`PRAGMA table_info("${db.sqlsanitize(tableName)}")`); @@ -29,10 +53,7 @@ const columnExists = async (tableName) => { // Check the tenant's OWN schema -- the same one tableSqlRef()/ALTER target. // current_schema() is NOT reliable: Saltcorn qualifies queries with // getTenantSchemaPrefix() rather than SET search_path, so current_schema() is - // "public" even inside a tenant. Using it made this falsely report the column - // missing on every call after the first, and the explicitly-qualified ALTER - // then failed with 'column "_dd_row_uuid" already exists' (breaking apply, - // which calls ensureManagedSchema once per set_table_mode + per insert_row). + // "public" even inside a tenant. const rs = await db.query( `SELECT 1 FROM information_schema.columns WHERE table_schema = $1 @@ -44,12 +65,39 @@ const columnExists = async (tableName) => { }; -const ensureManagedSchema = async (tableName) => { - if (await columnExists(tableName)) { - return { added: false }; - } - await db.query(`ALTER TABLE ${tableSqlRef(tableName)} ADD COLUMN ${COLUMN_NAME} TEXT`); - // Backfill existing rows with random UUIDs. +// Is _dd_row_uuid already a registered Saltcorn field on this table? Checked via +// _sc_fields directly so we do not depend on a possibly-stale in-memory catalog. +const fieldIsRegistered = async (tableId) => { + const row = await db.selectMaybeOne("_sc_fields", { table_id: tableId, name: COLUMN_NAME }); + return row || null; +}; + + +// Adopt a pre-existing raw _dd_row_uuid column (added by an older plugin version +// via raw ALTER) into _sc_fields WITHOUT re-issuing ADD COLUMN. Field.create +// always emits the ADD COLUMN DDL, which would fail with "column already +// exists", so for the legacy case we insert the field metadata directly. This +// mirrors the _sc_fields shape Field.create writes (field.ts). +const adoptExistingColumn = async (table) => { + await db.insert("_sc_fields", { + table_id: table.id, + name: COLUMN_NAME, + label: "dev-deploy row id", + type: "String", + required: false, + is_unique: false, + attributes: { hidden: true }, + calculated: false, + expression: null, + stored: false, + description: "Managed by dev-deploy: stable cross-environment row identity." + }); + await refreshCatalog(); +}; + + +// Backfill any rows missing a _dd_row_uuid with a fresh random UUID. +const backfillNulls = async (tableName) => { const rs = await db.query(`SELECT id FROM ${tableSqlRef(tableName)} WHERE ${COLUMN_NAME} IS NULL`); let backfilled = 0; for (const r of rs.rows) { @@ -59,23 +107,78 @@ const ensureManagedSchema = async (tableName) => { ); backfilled++; } - // Index for fast lookups by uuid. + return backfilled; +}; + + +const ensureIndex = async (tableName) => { await db.query( `CREATE INDEX IF NOT EXISTS "${db.sqlsanitize(tableName)}_dd_row_uuid_idx" ON ${tableSqlRef(tableName)} (${COLUMN_NAME})` ).catch(() => {}); - return { added: true, backfilled: backfilled }; }; +// Ensure the managed table has a registered _dd_row_uuid field. Idempotent: +// - already registered -> no-op +// - legacy raw column present -> adopt it into _sc_fields (no DDL) +// - neither -> Field.create (adds column + registers) +// Then backfill UUIDs and ensure the lookup index. +const ensureManagedSchema = async (tableName) => { + const table = Table.findOne({ name: tableName }); + if (!table) { + throw new Error(`ensureManagedSchema: table ${tableName} not in Saltcorn catalog`); + } + if (await fieldIsRegistered(table.id)) { + return { added: false }; + } + + const legacyColumn = await columnExists(tableName); + await runSuppressed(async () => { + if (legacyColumn) { + await adoptExistingColumn(table); + } else { + await Field.create({ + table: table, + name: COLUMN_NAME, + label: "dev-deploy row id", + type: "String", + required: false, + is_unique: false, + attributes: { hidden: true }, + description: "Managed by dev-deploy: stable cross-environment row identity." + }); + } + }); + + const backfilled = await backfillNulls(tableName); + await ensureIndex(tableName); + return { added: true, adopted: !!legacyColumn, backfilled: backfilled }; +}; + + +// Remove the _dd_row_uuid field/column when a table reverts to 'user' mode. +// Prefers the Field model (drops column + _sc_fields entry); falls back to a raw +// DROP COLUMN for a legacy unregistered column. const dropManagedSchema = async (tableName) => { - if (!(await columnExists(tableName))) { + const table = Table.findOne({ name: tableName }); + if (!table) { return { dropped: false }; } - // SQLite 3.35+ and Postgres both support DROP COLUMN. If the SQLite is - // older it will throw — surface the error. - await db.query(`ALTER TABLE ${tableSqlRef(tableName)} DROP COLUMN ${COLUMN_NAME}`); - return { dropped: true }; + const registered = await fieldIsRegistered(table.id); + if (registered) { + const field = await Field.findOne({ id: registered.id }); + if (field) { + await runSuppressed(() => field.delete()); + return { dropped: true }; + } + } + if (await columnExists(tableName)) { + // SQLite 3.35+ and Postgres both support DROP COLUMN. + await db.query(`ALTER TABLE ${tableSqlRef(tableName)} DROP COLUMN ${COLUMN_NAME}`); + return { dropped: true }; + } + return { dropped: false }; }; diff --git a/lib/schema.js b/lib/schema.js index 906ad1e..071ab7e 100644 --- a/lib/schema.js +++ b/lib/schema.js @@ -1,53 +1,131 @@ -// DDL for dev-deploy's six plugin tables. +// Schema setup for dev-deploy's plugin tables. // -// Saltcorn supports SQLite and PostgreSQL. We use a portable subset: -// - TEXT for all strings, JSON payloads, ISO 8601 timestamps -// - INTEGER for booleans (0/1) and surrogate ids -// - No JSONB; payloads are TEXT containing JSON, parsed/stringified at the -// application layer +// Two storage strategies, chosen by WHEN the state is written, so each survives +// Saltcorn backup/restore without fighting the plugin lifecycle: // -// Tables are created idempotently in onLoad. Re-running is safe. +// * ENV IDENTITY -> _sc_config (lib/env.js + lib/configStore.js). It is created +// during onLoad; _sc_config is restored BEFORE onLoad, so a restored host +// keeps its env_id and onLoad does not create a duplicate. +// +// * PEERS + TABLE MODES -> REGISTERED Saltcorn tables (auto `id` PK + the old +// logical keys kept as unique fields). They are written only at runtime (not +// during onLoad/restore), so the catalog-recreate-on-restore path is clean, +// and being in _sc_tables they ride backup. Existing db.select/insert code +// keeps working unchanged (the logical keys are now ordinary columns). +// +// * JOURNAL (_dd_ops), ENTITY-ID CACHE (_dd_entity_ids), SYNC WATERMARKS +// (_dd_anchors) stay RAW: they are a log / a rebuildable cache / re-syncable +// watermarks, so regenerating them on a restored host is acceptable, and the +// journal must NOT be registered (dev-deploy's wraps would journal the +// restore's own entity-creation into it). -const db = require("@saltcorn/data/db"); +const db = require("@saltcorn/data/db"); +const Table = require("@saltcorn/data/models/table"); +const Field = require("@saltcorn/data/models/field"); + +const { runSuppressed } = require("./context"); +const { readKey, writeKey } = require("./configStore"); -const createDdEnv = async () => { - const schema = db.getTenantSchemaPrefix(); - await db.query(` - CREATE TABLE IF NOT EXISTS ${schema}_dd_env ( - env_id TEXT PRIMARY KEY, - env_label TEXT, - on_destructive_op TEXT NOT NULL DEFAULT 'confirm', - require_tls INTEGER NOT NULL DEFAULT 0, - created_at TEXT NOT NULL, - bootstrapped_at TEXT - ) - `); +const ENV_KEY = "dev_deploy_env"; + +// Field definitions (excluding the auto `id` PK) for the registered tables. +const PEERS_FIELDS = [ + { name: "peer_id", type: "Integer", is_unique: true }, + { name: "env_id", type: "String", is_unique: true, required: true }, + { name: "label", type: "String" }, + { name: "base_url", type: "String", required: true }, + { name: "peer_secret_ciphertext", type: "String" }, + { name: "peer_secret_iv", type: "String" }, + { name: "peer_secret_tag", type: "String" }, + { name: "require_tls", type: "Integer" }, + { name: "created_at", type: "String", required: true }, + { name: "last_seen_at", type: "String" } +]; + +const TABLE_MODES_FIELDS = [ + { name: "table_uuid", type: "String", is_unique: true, required: true }, + { name: "data_mode", type: "String", required: true }, + { name: "updated_at", type: "String", required: true }, + { name: "starter_shipped_at", type: "String" } +]; + + +const rawTableExists = async (name) => { + if (db.isSQLite) { + const rs = await db.query(`SELECT name FROM sqlite_master WHERE type='table' AND name='${name}'`); + return rs.rows.length > 0; + } + const rs = await db.query( + `SELECT 1 FROM information_schema.tables WHERE table_schema='${db.getTenantSchema()}' AND table_name='${name}'` + ); + return rs.rows.length > 0; }; -const createDdPeers = async () => { - // Secret components stored as hex TEXT rather than BLOB: Saltcorn's - // SQLite insert layer JSON-stringifies any object value, which would - // mangle Buffer columns. Hex is portable and survives that path intact. +// Make `name` a registered Saltcorn table with an auto `id` PK plus `fieldDefs`. +// Idempotent: a no-op once registered. If a legacy RAW table exists, its rows are +// migrated into the registered table (logical-key columns preserved). Table.create +// + Field.create run under runSuppressed so dev-deploy's own wraps don't journal +// the plugin's bookkeeping tables. +const registerTable = async (name, fieldDefs) => { + if (Table.findOne({ name })) { + return; + } const schema = db.getTenantSchemaPrefix(); - // Portable auto-increment PK: sqlite "integer primary key" auto-assigns - // rowids; postgres uses "serial" (AUTOINCREMENT is sqlite-only syntax). - const serial = db.isSQLite ? "integer" : "serial"; - await db.query(` - CREATE TABLE IF NOT EXISTS ${schema}_dd_peers ( - peer_id ${serial} PRIMARY KEY, - env_id TEXT NOT NULL UNIQUE, - label TEXT, - base_url TEXT NOT NULL, - peer_secret_ciphertext TEXT, - peer_secret_iv TEXT, - peer_secret_tag TEXT, - require_tls INTEGER, - created_at TEXT NOT NULL, - last_seen_at TEXT - ) - `); + let rows = []; + if (await rawTableExists(name)) { + rows = (await db.query(`SELECT * FROM ${schema}"${name}"`)).rows; + await db.query(`DROP TABLE ${schema}"${name}"`); + } + await runSuppressed(async () => { + const t = await Table.create(name, { min_role_read: 1, min_role_write: 1 }); + for (const f of fieldDefs) { + await Field.create({ + table: t, + name: f.name, + label: f.label || f.name, + type: f.type, + is_unique: !!f.is_unique, + required: !!f.required, + attributes: f.attributes || {} + }); + } + }); + for (const r of rows) { + const row = {}; + for (const f of fieldDefs) { + if (r[f.name] !== undefined && r[f.name] !== null) { + row[f.name] = r[f.name]; + } + } + await db.insert(name, row, { noid: true }); + } +}; + + +// Move a legacy _dd_env table into the _sc_config identity key, then drop it. +const migrateEnvToConfig = async () => { + if (await readKey(ENV_KEY)) { + return; + } + if (!(await rawTableExists("_dd_env"))) { + return; + } + const schema = db.getTenantSchemaPrefix(); + const rows = (await db.query(`SELECT * FROM ${schema}_dd_env`)).rows; + if (rows.length > 0) { + const e = rows[0]; + await writeKey(ENV_KEY, { + env_id: e.env_id, + env_label: e.env_label, + on_destructive_op: e.on_destructive_op, + require_tls: e.require_tls, + created_at: e.created_at, + bootstrapped_at: e.bootstrapped_at + }); + } + await db.query(`DROP TABLE IF EXISTS ${schema}_dd_env`); }; @@ -94,10 +172,7 @@ const createDdOps = async () => { await db.query(`CREATE INDEX IF NOT EXISTS _dd_ops_correlation ON ${schema}_dd_ops (correlation_id)`); await db.query(`CREATE INDEX IF NOT EXISTS _dd_ops_status ON ${schema}_dd_ops (status) WHERE status = 'conflict'`).catch(() => {}); - // Idempotent migration for instances installed before conflict_with_op_id - // existed. On PG a failed statement poisons the surrounding transaction, so a - // bare ALTER caught in JS is not enough -- use ADD COLUMN IF NOT EXISTS there; - // sqlite lacks that clause, so run the bare ALTER and swallow the error. + // Idempotent migration for instances installed before conflict_with_op_id existed. if (db.isSQLite) { try { await db.query(`ALTER TABLE ${schema}_dd_ops ADD COLUMN conflict_with_op_id TEXT`); @@ -124,35 +199,16 @@ const createDdAnchors = async () => { }; -const createDdTableModes = async () => { - const schema = db.getTenantSchemaPrefix(); - await db.query(` - CREATE TABLE IF NOT EXISTS ${schema}_dd_table_modes ( - table_uuid TEXT PRIMARY KEY, - data_mode TEXT NOT NULL, - updated_at TEXT NOT NULL, - starter_shipped_at TEXT - ) - `); - // Idempotent migration for older installs that lack starter_shipped_at. - // (See createDdOps: PG needs IF NOT EXISTS or a failed ALTER poisons the txn.) - if (db.isSQLite) { - try { - await db.query(`ALTER TABLE ${schema}_dd_table_modes ADD COLUMN starter_shipped_at TEXT`); - } catch (e) { /* column exists */ } - } else { - await db.query(`ALTER TABLE ${schema}_dd_table_modes ADD COLUMN IF NOT EXISTS starter_shipped_at TEXT`); - } -}; - - const createAllTables = async () => { - await createDdEnv(); - await createDdPeers(); + // Identity -> _sc_config (migrate any legacy table first). + await migrateEnvToConfig(); + // Backed-up state -> registered Saltcorn tables. + await registerTable("_dd_peers", PEERS_FIELDS); + await registerTable("_dd_table_modes", TABLE_MODES_FIELDS); + // Regenerable -> raw tables. await createDdEntityIds(); await createDdOps(); await createDdAnchors(); - await createDdTableModes(); }; diff --git a/lib/wrap.js b/lib/wrap.js index 5fed1ee..f97d77d 100644 --- a/lib/wrap.js +++ b/lib/wrap.js @@ -54,6 +54,17 @@ const { } = require("./rowPayload"); +// Plugin infrastructure tables (dev-deploy's own registered tables + saltcorn-idp's, +// when co-installed) must never be tracked/journaled/synced as user data. +const isInfraTableName = (name) => !!name && (name.startsWith("_dd_") || name.startsWith("_idp_")); + +const isInfraTableId = (tableId) => { + if (!tableId) return false; + const t = Table.findOne({ id: tableId }); + return t ? isInfraTableName(t.name) : false; +}; + + const snapshotInstance = (inst, keys) => { const out = {}; for (const k of keys) { @@ -171,6 +182,9 @@ const wrapTable = () => { wrap(Table, "create", ENTITY_KINDS.TABLE, "create", { after: async ({ result }) => { if (!result || !result.id) return null; + // Never track plugin infrastructure tables (dev-deploy's own + saltcorn-idp's + // registered tables, when co-installed) -- they are not user data to sync. + if (isInfraTableName(result.name)) return null; const uuid = await assignNewUuid(ENTITY_KINDS.TABLE, result.id, result.name, null); return { entityUuid: uuid, @@ -234,6 +248,10 @@ const wrapField = () => { wrap(Field, "create", ENTITY_KINDS.FIELD, "create", { after: async ({ args, result }) => { if (!result || !result.id) return null; + // dev-deploy's own per-environment row-identity column, and any field + // on a plugin-infrastructure table, are never journaled. + if (result.name && result.name.startsWith("_dd_")) return null; + if (isInfraTableId(result.table_id)) return null; let parentUuid = null; if (result.table_id) { const t = await lookupByCurrent(ENTITY_KINDS.TABLE, result.table_id); @@ -581,6 +599,8 @@ const wrapTableConstraint = () => { // multiple module-resolution contexts when sc-exec uses createRequire). const already = await lookupByCurrent(ENTITY_KINDS.CONSTRAINT, result.id); if (already) return null; + // Skip constraints on plugin-infrastructure tables. + if (isInfraTableId(result.table_id)) return null; let parentUuid = null; if (result.table_id) { const t = await lookupByCurrent(ENTITY_KINDS.TABLE, result.table_id); diff --git a/test/e2e.js b/test/e2e.js index adf7606..d853bc8 100644 --- a/test/e2e.js +++ b/test/e2e.js @@ -63,7 +63,9 @@ const resetInstanceDb = (dbPath) => { for (const t of userTables) { if (t) sql(dbPath, `DROP TABLE IF EXISTS "${t}"`); } - sql(dbPath, "DELETE FROM _sc_tables WHERE name != 'users'; DELETE FROM _sc_fields WHERE table_id NOT IN (SELECT id FROM _sc_tables);"); + // Spare the registered dev-deploy bookkeeping tables (_dd_peers, + // _dd_table_modes) -- they are plugin infrastructure, not per-test user data. + sql(dbPath, "DELETE FROM _sc_tables WHERE name != 'users' AND name NOT LIKE '\\_dd\\_%' ESCAPE '\\'; DELETE FROM _sc_fields WHERE table_id NOT IN (SELECT id FROM _sc_tables);"); }; @@ -208,9 +210,11 @@ const main = async () => { console.log(` test env_id: ${testEnv}`); section("schema"); - await test("six _dd_* tables exist on main", async () => { + await test("dev-deploy _dd_* tables exist on main", async () => { const tables = sqlRows(MAIN.db, "SELECT name FROM sqlite_master WHERE type='table' AND name LIKE '_dd_%' ORDER BY name").join(","); - assert.equal(tables, "_dd_anchors,_dd_entity_ids,_dd_env,_dd_ops,_dd_peers,_dd_table_modes"); + // env identity moved to _sc_config; peers + table_modes are now registered + // Saltcorn tables (still physical _dd_* tables, plus _sc_tables entries). + assert.equal(tables, "_dd_anchors,_dd_entity_ids,_dd_ops,_dd_peers,_dd_table_modes"); }); await test("_dd_ops has conflict_with_op_id column", async () => { const cols = sqlRows(MAIN.db, "PRAGMA table_info(_dd_ops)").map((r) => r.split("|")[1]);