AI_Slop/sc-dev-deploy
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fixes

Browse source
This commit is contained in:
Scott Duensing 2026-06-17 17:37:16 -05:00
parent d95798e895
commit 168bff28b8
9 changed files with 447 additions and 133 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

57
index.js
View file

@ -1,10 +1,13 @@
// dev-deploy: Saltcorn plugin for migrating metadata changes across
// Dev/Test/Prod environments via an ops journal with stable UUIDs.
const { PLUGIN_NAME, PLUGIN_VERSION } = require("./lib/constants");
const db = require("@saltcorn/data/db");
const { PLUGIN_NAME, PLUGIN_VERSION, DATA_MODES } = require("./lib/constants");
const { createAllTables } = require("./lib/schema");
const { getEnv, initEnvIfMissing, markBootstrapped } = require("./lib/env");
const { backfillAll } = require("./lib/entityIds");
const { ensureManagedSchema } = require("./lib/rowIdentity");
const { installAllWraps } = require("./lib/wrap");
const { routes } = require("./lib/routes");
@ -32,19 +35,67 @@ const ensureCsrfBypass = async () => {
};
// One-time migration: managed/starter tables created by an older plugin version
// carry a RAW _dd_row_uuid column (added via ALTER, not registered in _sc_fields).
// Such a column is dumped by backup but absent from the restored table's field
// list, which fails the row import and rolls back the whole table (app-data
// loss). Adopt every existing managed table's column into _sc_fields so it
// round-trips. ensureManagedSchema is idempotent: a no-op once registered.
const migrateRowUuidFields = async () => {
const schema = db.getTenantSchemaPrefix();
let rows;
try {
rows = await db.query(
`SELECT e.current_name AS name
FROM ${schema}_dd_table_modes m
JOIN ${schema}_dd_entity_ids e ON e.uuid = m.table_uuid
WHERE e.kind = 'table'
AND m.data_mode IN ($1, $2)`,
[DATA_MODES.MANAGED, DATA_MODES.STARTER]
);
} catch (e) {
return 0;
}
let adopted = 0;
for (const r of rows.rows) {
try {
const res = await ensureManagedSchema(r.name);
if (res && res.adopted) {
adopted++;
}
} catch (e) {
console.error(`[${PLUGIN_NAME}] row-uuid field migration failed for ${r.name}:`, e.message);
}
}
return adopted;
};
const onLoad = async (cfg) => {
try {
await createAllTables();
const env = await initEnvIfMissing();
// Reconcile the entity-id cache on EVERY load, not just first bootstrap.
// _dd_entity_ids is a raw, un-backed-up cache; after a restore it is empty
// while the restored env reports bootstrapped, so a gated backfill would
// leave the map empty. backfillAll is idempotent (ensureUuid keys on
// (kind,current_id) and derives deterministic UUIDs from name), so running
// it always rebuilds the map against the restored host's live catalog.
const counts = await backfillAll();
const total = Object.values(counts).reduce((a, b) => a + b, 0);
if (!env.bootstrapped_at) {
const counts = await backfillAll();
const total = Object.values(counts).reduce((a, b) => a + b, 0);
await markBootstrapped(env.env_id);
log(`v${PLUGIN_VERSION} bootstrapped env_id=${env.env_id} backfilled ${total} entities ${JSON.stringify(counts)}`);
} else if (total > 0) {
log(`v${PLUGIN_VERSION} loaded env_id=${env.env_id}; reconciled entity-id cache (+${total})`);
} else {
log(`v${PLUGIN_VERSION} loaded env_id=${env.env_id}`);
}
installAllWraps();
const adopted = await migrateRowUuidFields();
if (adopted > 0) {
log(`migrated ${adopted} managed table(s) to a registered _dd_row_uuid field`);
}
await ensureCsrfBypass();
} catch (err) {
// eslint-disable-next-line no-console

49
lib/configStore.js Normal file
View file

@ -0,0 +1,49 @@
// Durable per-tenant key/value storage in Saltcorn's _sc_config, used for the
// dev-deploy ENV IDENTITY (which is created during onLoad).
//
// Why _sc_config and not a registered table or the plugin config blob:
// - It survives backup: backup_config dumps every non-fixed _sc_config key,
// and restore_config restores them BEFORE install_pack runs the plugin's
// onLoad -- so the restored identity is already present when onLoad reads it
// (no onLoad-creates-then-restore-imports duplicate, which a registered
// table suffers because restore_tables runs AFTER onLoad).
// - It does NOT write the plugin's own _sc_plugins row (Plugin.upsert during
// onLoad cascades plugin re-loading and duplicates rows -- the reason the
// config-blob approach was abandoned). setConfig writes _sc_config only.
//
// Reads go DIRECT to _sc_config: getState().getConfig only surfaces keys that
// are in Saltcorn's known configTypes schema, so a custom plugin key is invisible
// through it even though it is stored and backed up. Writes go through setConfig
// (correct jsonb/text encoding on both backends; value wrapped as {v: ...}).
const db = require("@saltcorn/data/db");
const readKey = async (key) => {
const row = await db.selectMaybeOne("_sc_config", { key: key });
if (!row || row.value === null || row.value === undefined) {
return undefined;
}
let v = row.value;
if (typeof v === "string") {
try {
v = JSON.parse(v);
} catch (e) {
return undefined;
}
}
// Saltcorn wraps config values as { v: <value> }.
return v && typeof v === "object" && "v" in v ? v.v : v;
};
const writeKey = async (key, value) => {
const { getState } = require("@saltcorn/data/db/state");
await getState().setConfig(key, value);
};
module.exports = {
readKey,
writeKey
};

39
lib/entityIds.js
View file

@ -83,6 +83,21 @@ const ensureUuid = async (kind, currentId, currentName, parentUuid, canonical) =
return existing.uuid;
}
const uuid = deterministicUuid(kind, canonical || currentName);
// RECONCILE: the deterministic UUID (derived from the entity's name) is the
// stable identity. If a row already carries it but at a different current_id,
// the integer id was reassigned (e.g. table recreated, or a restore) -- re-
// point the stable UUID to the new id rather than inserting a duplicate (which
// would violate the uuid PK). No (kind,current_id) row exists here (checked
// above), so the update cannot violate UNIQUE(kind,current_id).
const stale = await db.selectMaybeOne("_dd_entity_ids", { uuid: uuid });
if (stale) {
await db.updateWhere(
"_dd_entity_ids",
{ current_id: currentId, current_name: currentName, parent_uuid: parentUuid || null },
{ uuid: uuid }
);
return uuid;
}
const row = {
uuid: uuid,
kind: kind,
@ -134,10 +149,20 @@ const removeEntityRow = async (kind, currentId) => {
// Backfill helpers: each returns the count of new UUIDs inserted.
// dev-deploy's own registered bookkeeping tables (_dd_peers, _dd_table_modes)
// and saltcorn-idp's (_idp_*) appear in Table.find now that they are registered,
// but they are plugin INFRASTRUCTURE, not user data -- never track, sync, or show
// them as managed tables.
const isInfraTable = (name) => name.startsWith("_dd_") || name.startsWith("_idp_");
const backfillTables = async () => {
const tables = await Table.find({}, { cached: false });
let added = 0;
for (const t of tables) {
if (isInfraTable(t.name)) {
continue;
}
const before = await lookupByCurrent(ENTITY_KINDS.TABLE, t.id);
await ensureUuid(ENTITY_KINDS.TABLE, t.id, t.name, null, t.name);
if (!before) {
@ -152,12 +177,20 @@ const backfillFields = async () => {
const tables = await Table.find({}, { cached: false });
let added = 0;
for (const t of tables) {
if (isInfraTable(t.name)) {
continue;
}
const tableUuidRow = await lookupByCurrent(ENTITY_KINDS.TABLE, t.id);
if (!tableUuidRow) {
continue;
}
const fields = t.getFields ? t.getFields() : (await Field.find({ table_id: t.id }));
for (const f of fields) {
// _dd_row_uuid is dev-deploy's per-environment row-identity column --
// never track/sync it (each environment manages its own).
if (f.name.startsWith("_dd_")) {
continue;
}
const before = await lookupByCurrent(ENTITY_KINDS.FIELD, f.id);
await ensureUuid(ENTITY_KINDS.FIELD, f.id, f.name, tableUuidRow.uuid, `${t.name}.${f.name}`);
if (!before) {
@ -318,6 +351,12 @@ const backfillWorkflowSteps = async () => {
const backfillAll = async () => {
// Remove any stale tracking of plugin infrastructure: registered _dd_*/_idp_*
// tables and the per-environment _dd_row_uuid column should never be in the
// entity-id map (older builds tracked _dd_row_uuid once it became a field).
const schema = db.getTenantSchemaPrefix();
await db.query(`DELETE FROM ${schema}_dd_entity_ids WHERE current_name LIKE '_dd_%' OR current_name LIKE '_idp_%'`).catch(() => {});
const counts = {};
counts.tables = await backfillTables();
counts.fields = await backfillFields();

46
lib/env.js
View file

@ -1,34 +1,25 @@
// This Saltcorn instance's dev-deploy identity (env_id, label, policies).
// Stored as a singleton row in _dd_env.
const db = require("@saltcorn/data/db");
//
// Stored in _sc_config (key "dev_deploy_env") rather than a _dd_env table.
// _sc_config survives backup AND is restored BEFORE the plugin's onLoad runs, so
// a restored instance keeps its env_id and onLoad does not create a duplicate
// (see lib/configStore.js). Reads are DB-fresh (no in-process cache needed).
const { randomUuid } = require("./ids");
const { DESTRUCTIVE_POLICY } = require("./constants");
const { readKey, writeKey } = require("./configStore");
// The env identity is schema-scoped, so a single process serving multiple
// tenants must NOT share one cached row across them. Key the cache by tenant
// schema (db.getTenantSchema()), not a module-level singleton.
const cachedEnvByTenant = new Map();
const tenantKey = () => (db.getTenantSchema ? db.getTenantSchema() : "public");
const ENV_KEY = "dev_deploy_env";
const getEnv = async () => {
const key = tenantKey();
if (cachedEnvByTenant.has(key)) {
return cachedEnvByTenant.get(key);
}
const rows = await db.select("_dd_env", {});
const env = rows.length > 0 ? rows[0] : null;
cachedEnvByTenant.set(key, env);
return env;
return (await readKey(ENV_KEY)) || null;
};
const refreshEnvCache = async () => {
cachedEnvByTenant.delete(tenantKey());
// No in-process cache anymore; reads are always DB-fresh. Kept for callers.
return await getEnv();
};
@ -38,27 +29,24 @@ const initEnvIfMissing = async () => {
if (existing) {
return existing;
}
const now = new Date().toISOString();
const row = {
const env = {
env_id: randomUuid(),
env_label: null,
on_destructive_op: DESTRUCTIVE_POLICY.CONFIRM,
require_tls: 0,
created_at: now,
created_at: new Date().toISOString(),
bootstrapped_at: null
};
await db.insert("_dd_env", row, { noid: true });
cachedEnvByTenant.set(tenantKey(), row);
return row;
await writeKey(ENV_KEY, env);
return env;
};
const markBootstrapped = async (envId) => {
const now = new Date().toISOString();
await db.updateWhere("_dd_env", { bootstrapped_at: now }, { env_id: envId });
const cached = cachedEnvByTenant.get(tenantKey());
if (cached && cached.env_id === envId) {
cached.bootstrapped_at = now;
const env = await getEnv();
if (env && env.env_id === envId) {
env.bootstrapped_at = new Date().toISOString();
await writeKey(ENV_KEY, env);
}
};

10
lib/peers.js
View file

@ -85,7 +85,13 @@ const addPeer = async ({ envId, label, baseUrl, requireTls, existingSecret }) =>
}
const secret = existingSecret || randomSecret();
const sealed = seal(secret);
// _dd_peers is now a registered Saltcorn table with an auto `id` PK; peer_id
// is a regular unique column, so we assign it ourselves (serial-style max+1)
// to keep peer_id stable + collision-free (anchors reference it by value).
const nextRs = await db.query(`SELECT COALESCE(MAX(peer_id),0)+1 AS next FROM ${db.getTenantSchemaPrefix()}_dd_peers`);
const peerId = nextRs.rows[0].next;
const row = {
peer_id: peerId,
env_id: envId,
label: label || null,
base_url: baseUrl,
@ -96,9 +102,7 @@ const addPeer = async ({ envId, label, baseUrl, requireTls, existingSecret }) =>
created_at: new Date().toISOString(),
last_seen_at: null
};
// noid: _dd_peers' PK is peer_id (serial on PG), not "id"; without this
// Saltcorn's default RETURNING id makes the insert fail on Postgres with
// 'column "id" does not exist'. The peer_id is auto-assigned; re-select below.
// noid: let Saltcorn assign the `id` PK; we supply peer_id explicitly.
await db.insert("_dd_peers", row, { noid: true });
const fresh = await findPeerByEnvId(envId);
return { peer: rowToPeer(fresh), secret: secret };

153
lib/rowIdentity.js
View file

@ -1,15 +1,34 @@
// Hidden _dd_row_uuid column infrastructure for managed/starter tables.
// _dd_row_uuid column infrastructure for managed/starter tables.
//
// When an admin marks a user table as managed or starter, this module
// adds a TEXT column _dd_row_uuid to the underlying SQL table via raw ALTER
// (NOT registered in _sc_fields, so Saltcorn's table builder doesn't show
// it). Existing rows are backfilled with random UUIDs. From that point,
// the wrap layer reads/writes _dd_row_uuid as the cross-environment identity
// for each row.
// When an admin marks a user table as managed or starter, this module adds a
// _dd_row_uuid column to the underlying table as a REGISTERED Saltcorn Field
// (in _sc_fields) -- NOT a raw ALTER outside the catalog. Registration is what
// makes the column survive Saltcorn backup/restore/snapshots: backup dumps a
// row's physical columns (SELECT * / row_to_json), so a raw, unregistered
// column lands in the dump but the restored table -- recreated from the pack's
// field list -- has no such column, and the row insert then fails and rolls
// back the whole table (real app-data loss). A registered Field round-trips:
// install_pack recreates the column before the rows are imported.
//
// The field is created with hidden:true so it stays out of the edit/show form
// builders (it is still visible in the raw table builder field list -- Saltcorn
// has no system/hidden-from-catalog flag, an accepted trade-off for backup
// safety). Existing rows are backfilled with random UUIDs. From that point, the
// wrap layer reads/writes _dd_row_uuid as the cross-environment identity for
// each row.
//
// Creating/deleting this field goes through Field.create / field.delete, which
// the dev-deploy wraps (lib/wrap.js) journal. We run those inside runSuppressed
// so the plugin's own row-identity column is never journaled or propagated to
// peers -- every environment manages its own _dd_row_uuid independently.
const crypto = require("crypto");
const db = require("@saltcorn/data/db");
const db = require("@saltcorn/data/db");
const Table = require("@saltcorn/data/models/table");
const Field = require("@saltcorn/data/models/field");
const { runSuppressed } = require("./context");
const COLUMN_NAME = "_dd_row_uuid";
@ -21,6 +40,11 @@ const tableSqlRef = (tableName) => {
};
const refreshCatalog = async () => {
await require("@saltcorn/data/db/state").getState().refresh_tables(true);
};
const columnExists = async (tableName) => {
if (db.isSQLite) {
const rs = await db.query(`PRAGMA table_info("${db.sqlsanitize(tableName)}")`);
@ -29,10 +53,7 @@ const columnExists = async (tableName) => {
// Check the tenant's OWN schema -- the same one tableSqlRef()/ALTER target.
// current_schema() is NOT reliable: Saltcorn qualifies queries with
// getTenantSchemaPrefix() rather than SET search_path, so current_schema() is
// "public" even inside a tenant. Using it made this falsely report the column
// missing on every call after the first, and the explicitly-qualified ALTER
// then failed with 'column "_dd_row_uuid" already exists' (breaking apply,
// which calls ensureManagedSchema once per set_table_mode + per insert_row).
// "public" even inside a tenant.
const rs = await db.query(
`SELECT 1 FROM information_schema.columns
WHERE table_schema = $1
@ -44,12 +65,39 @@ const columnExists = async (tableName) => {
};
const ensureManagedSchema = async (tableName) => {
if (await columnExists(tableName)) {
return { added: false };
}
await db.query(`ALTER TABLE ${tableSqlRef(tableName)} ADD COLUMN ${COLUMN_NAME} TEXT`);
// Backfill existing rows with random UUIDs.
// Is _dd_row_uuid already a registered Saltcorn field on this table? Checked via
// _sc_fields directly so we do not depend on a possibly-stale in-memory catalog.
const fieldIsRegistered = async (tableId) => {
const row = await db.selectMaybeOne("_sc_fields", { table_id: tableId, name: COLUMN_NAME });
return row || null;
};
// Adopt a pre-existing raw _dd_row_uuid column (added by an older plugin version
// via raw ALTER) into _sc_fields WITHOUT re-issuing ADD COLUMN. Field.create
// always emits the ADD COLUMN DDL, which would fail with "column already
// exists", so for the legacy case we insert the field metadata directly. This
// mirrors the _sc_fields shape Field.create writes (field.ts).
const adoptExistingColumn = async (table) => {
await db.insert("_sc_fields", {
table_id: table.id,
name: COLUMN_NAME,
label: "dev-deploy row id",
type: "String",
required: false,
is_unique: false,
attributes: { hidden: true },
calculated: false,
expression: null,
stored: false,
description: "Managed by dev-deploy: stable cross-environment row identity."
});
await refreshCatalog();
};
// Backfill any rows missing a _dd_row_uuid with a fresh random UUID.
const backfillNulls = async (tableName) => {
const rs = await db.query(`SELECT id FROM ${tableSqlRef(tableName)} WHERE ${COLUMN_NAME} IS NULL`);
let backfilled = 0;
for (const r of rs.rows) {
@ -59,23 +107,78 @@ const ensureManagedSchema = async (tableName) => {
);
backfilled++;
}
// Index for fast lookups by uuid.
return backfilled;
};
const ensureIndex = async (tableName) => {
await db.query(
`CREATE INDEX IF NOT EXISTS "${db.sqlsanitize(tableName)}_dd_row_uuid_idx"
ON ${tableSqlRef(tableName)} (${COLUMN_NAME})`
).catch(() => {});
return { added: true, backfilled: backfilled };
};
// Ensure the managed table has a registered _dd_row_uuid field. Idempotent:
// - already registered -> no-op
// - legacy raw column present -> adopt it into _sc_fields (no DDL)
// - neither -> Field.create (adds column + registers)
// Then backfill UUIDs and ensure the lookup index.
const ensureManagedSchema = async (tableName) => {
const table = Table.findOne({ name: tableName });
if (!table) {
throw new Error(`ensureManagedSchema: table ${tableName} not in Saltcorn catalog`);
}
if (await fieldIsRegistered(table.id)) {
return { added: false };
}
const legacyColumn = await columnExists(tableName);
await runSuppressed(async () => {
if (legacyColumn) {
await adoptExistingColumn(table);
} else {
await Field.create({
table: table,
name: COLUMN_NAME,
label: "dev-deploy row id",
type: "String",
required: false,
is_unique: false,
attributes: { hidden: true },
description: "Managed by dev-deploy: stable cross-environment row identity."
});
}
});
const backfilled = await backfillNulls(tableName);
await ensureIndex(tableName);
return { added: true, adopted: !!legacyColumn, backfilled: backfilled };
};
// Remove the _dd_row_uuid field/column when a table reverts to 'user' mode.
// Prefers the Field model (drops column + _sc_fields entry); falls back to a raw
// DROP COLUMN for a legacy unregistered column.
const dropManagedSchema = async (tableName) => {
if (!(await columnExists(tableName))) {
const table = Table.findOne({ name: tableName });
if (!table) {
return { dropped: false };
}
// SQLite 3.35+ and Postgres both support DROP COLUMN. If the SQLite is
// older it will throw — surface the error.
await db.query(`ALTER TABLE ${tableSqlRef(tableName)} DROP COLUMN ${COLUMN_NAME}`);
return { dropped: true };
const registered = await fieldIsRegistered(table.id);
if (registered) {
const field = await Field.findOne({ id: registered.id });
if (field) {
await runSuppressed(() => field.delete());
return { dropped: true };
}
}
if (await columnExists(tableName)) {
// SQLite 3.35+ and Postgres both support DROP COLUMN.
await db.query(`ALTER TABLE ${tableSqlRef(tableName)} DROP COLUMN ${COLUMN_NAME}`);
return { dropped: true };
}
return { dropped: false };
};

196
lib/schema.js
View file

@ -1,53 +1,131 @@
// DDL for dev-deploy's six plugin tables.
// Schema setup for dev-deploy's plugin tables.
//
// Saltcorn supports SQLite and PostgreSQL. We use a portable subset:
// - TEXT for all strings, JSON payloads, ISO 8601 timestamps
// - INTEGER for booleans (0/1) and surrogate ids
// - No JSONB; payloads are TEXT containing JSON, parsed/stringified at the
// application layer
// Two storage strategies, chosen by WHEN the state is written, so each survives
// Saltcorn backup/restore without fighting the plugin lifecycle:
//
// Tables are created idempotently in onLoad. Re-running is safe.
// * ENV IDENTITY -> _sc_config (lib/env.js + lib/configStore.js). It is created
// during onLoad; _sc_config is restored BEFORE onLoad, so a restored host
// keeps its env_id and onLoad does not create a duplicate.
//
// * PEERS + TABLE MODES -> REGISTERED Saltcorn tables (auto `id` PK + the old
// logical keys kept as unique fields). They are written only at runtime (not
// during onLoad/restore), so the catalog-recreate-on-restore path is clean,
// and being in _sc_tables they ride backup. Existing db.select/insert code
// keeps working unchanged (the logical keys are now ordinary columns).
//
// * JOURNAL (_dd_ops), ENTITY-ID CACHE (_dd_entity_ids), SYNC WATERMARKS
// (_dd_anchors) stay RAW: they are a log / a rebuildable cache / re-syncable
// watermarks, so regenerating them on a restored host is acceptable, and the
// journal must NOT be registered (dev-deploy's wraps would journal the
// restore's own entity-creation into it).
const db = require("@saltcorn/data/db");
const db = require("@saltcorn/data/db");
const Table = require("@saltcorn/data/models/table");
const Field = require("@saltcorn/data/models/field");
const { runSuppressed } = require("./context");
const { readKey, writeKey } = require("./configStore");
const createDdEnv = async () => {
const schema = db.getTenantSchemaPrefix();
await db.query(`
CREATE TABLE IF NOT EXISTS ${schema}_dd_env (
env_id TEXT PRIMARY KEY,
env_label TEXT,
on_destructive_op TEXT NOT NULL DEFAULT 'confirm',
require_tls INTEGER NOT NULL DEFAULT 0,
created_at TEXT NOT NULL,
bootstrapped_at TEXT
)
`);
const ENV_KEY = "dev_deploy_env";
// Field definitions (excluding the auto `id` PK) for the registered tables.
const PEERS_FIELDS = [
{ name: "peer_id", type: "Integer", is_unique: true },
{ name: "env_id", type: "String", is_unique: true, required: true },
{ name: "label", type: "String" },
{ name: "base_url", type: "String", required: true },
{ name: "peer_secret_ciphertext", type: "String" },
{ name: "peer_secret_iv", type: "String" },
{ name: "peer_secret_tag", type: "String" },
{ name: "require_tls", type: "Integer" },
{ name: "created_at", type: "String", required: true },
{ name: "last_seen_at", type: "String" }
];
const TABLE_MODES_FIELDS = [
{ name: "table_uuid", type: "String", is_unique: true, required: true },
{ name: "data_mode", type: "String", required: true },
{ name: "updated_at", type: "String", required: true },
{ name: "starter_shipped_at", type: "String" }
];
const rawTableExists = async (name) => {
if (db.isSQLite) {
const rs = await db.query(`SELECT name FROM sqlite_master WHERE type='table' AND name='${name}'`);
return rs.rows.length > 0;
}
const rs = await db.query(
`SELECT 1 FROM information_schema.tables WHERE table_schema='${db.getTenantSchema()}' AND table_name='${name}'`
);
return rs.rows.length > 0;
};
const createDdPeers = async () => {
// Secret components stored as hex TEXT rather than BLOB: Saltcorn's
// SQLite insert layer JSON-stringifies any object value, which would
// mangle Buffer columns. Hex is portable and survives that path intact.
// Make `name` a registered Saltcorn table with an auto `id` PK plus `fieldDefs`.
// Idempotent: a no-op once registered. If a legacy RAW table exists, its rows are
// migrated into the registered table (logical-key columns preserved). Table.create
// + Field.create run under runSuppressed so dev-deploy's own wraps don't journal
// the plugin's bookkeeping tables.
const registerTable = async (name, fieldDefs) => {
if (Table.findOne({ name })) {
return;
}
const schema = db.getTenantSchemaPrefix();
// Portable auto-increment PK: sqlite "integer primary key" auto-assigns
// rowids; postgres uses "serial" (AUTOINCREMENT is sqlite-only syntax).
const serial = db.isSQLite ? "integer" : "serial";
await db.query(`
CREATE TABLE IF NOT EXISTS ${schema}_dd_peers (
peer_id ${serial} PRIMARY KEY,
env_id TEXT NOT NULL UNIQUE,
label TEXT,
base_url TEXT NOT NULL,
peer_secret_ciphertext TEXT,
peer_secret_iv TEXT,
peer_secret_tag TEXT,
require_tls INTEGER,
created_at TEXT NOT NULL,
last_seen_at TEXT
)
`);
let rows = [];
if (await rawTableExists(name)) {
rows = (await db.query(`SELECT * FROM ${schema}"${name}"`)).rows;
await db.query(`DROP TABLE ${schema}"${name}"`);
}
await runSuppressed(async () => {
const t = await Table.create(name, { min_role_read: 1, min_role_write: 1 });
for (const f of fieldDefs) {
await Field.create({
table: t,
name: f.name,
label: f.label || f.name,
type: f.type,
is_unique: !!f.is_unique,
required: !!f.required,
attributes: f.attributes || {}
});
}
});
for (const r of rows) {
const row = {};
for (const f of fieldDefs) {
if (r[f.name] !== undefined && r[f.name] !== null) {
row[f.name] = r[f.name];
}
}
await db.insert(name, row, { noid: true });
}
};
// Move a legacy _dd_env table into the _sc_config identity key, then drop it.
const migrateEnvToConfig = async () => {
if (await readKey(ENV_KEY)) {
return;
}
if (!(await rawTableExists("_dd_env"))) {
return;
}
const schema = db.getTenantSchemaPrefix();
const rows = (await db.query(`SELECT * FROM ${schema}_dd_env`)).rows;
if (rows.length > 0) {
const e = rows[0];
await writeKey(ENV_KEY, {
env_id: e.env_id,
env_label: e.env_label,
on_destructive_op: e.on_destructive_op,
require_tls: e.require_tls,
created_at: e.created_at,
bootstrapped_at: e.bootstrapped_at
});
}
await db.query(`DROP TABLE IF EXISTS ${schema}_dd_env`);
};
@ -94,10 +172,7 @@ const createDdOps = async () => {
await db.query(`CREATE INDEX IF NOT EXISTS _dd_ops_correlation ON ${schema}_dd_ops (correlation_id)`);
await db.query(`CREATE INDEX IF NOT EXISTS _dd_ops_status ON ${schema}_dd_ops (status) WHERE status = 'conflict'`).catch(() => {});
// Idempotent migration for instances installed before conflict_with_op_id
// existed. On PG a failed statement poisons the surrounding transaction, so a
// bare ALTER caught in JS is not enough -- use ADD COLUMN IF NOT EXISTS there;
// sqlite lacks that clause, so run the bare ALTER and swallow the error.
// Idempotent migration for instances installed before conflict_with_op_id existed.
if (db.isSQLite) {
try {
await db.query(`ALTER TABLE ${schema}_dd_ops ADD COLUMN conflict_with_op_id TEXT`);
@ -124,35 +199,16 @@ const createDdAnchors = async () => {
};
const createDdTableModes = async () => {
const schema = db.getTenantSchemaPrefix();
await db.query(`
CREATE TABLE IF NOT EXISTS ${schema}_dd_table_modes (
table_uuid TEXT PRIMARY KEY,
data_mode TEXT NOT NULL,
updated_at TEXT NOT NULL,
starter_shipped_at TEXT
)
`);
// Idempotent migration for older installs that lack starter_shipped_at.
// (See createDdOps: PG needs IF NOT EXISTS or a failed ALTER poisons the txn.)
if (db.isSQLite) {
try {
await db.query(`ALTER TABLE ${schema}_dd_table_modes ADD COLUMN starter_shipped_at TEXT`);
} catch (e) { /* column exists */ }
} else {
await db.query(`ALTER TABLE ${schema}_dd_table_modes ADD COLUMN IF NOT EXISTS starter_shipped_at TEXT`);
}
};
const createAllTables = async () => {
await createDdEnv();
await createDdPeers();
// Identity -> _sc_config (migrate any legacy table first).
await migrateEnvToConfig();
// Backed-up state -> registered Saltcorn tables.
await registerTable("_dd_peers", PEERS_FIELDS);
await registerTable("_dd_table_modes", TABLE_MODES_FIELDS);
// Regenerable -> raw tables.
await createDdEntityIds();
await createDdOps();
await createDdAnchors();
await createDdTableModes();
};

20
lib/wrap.js
View file

@ -54,6 +54,17 @@ const {
} = require("./rowPayload");
// Plugin infrastructure tables (dev-deploy's own registered tables + saltcorn-idp's,
// when co-installed) must never be tracked/journaled/synced as user data.
const isInfraTableName = (name) => !!name && (name.startsWith("_dd_") || name.startsWith("_idp_"));
const isInfraTableId = (tableId) => {
if (!tableId) return false;
const t = Table.findOne({ id: tableId });
return t ? isInfraTableName(t.name) : false;
};
const snapshotInstance = (inst, keys) => {
const out = {};
for (const k of keys) {
@ -171,6 +182,9 @@ const wrapTable = () => {
wrap(Table, "create", ENTITY_KINDS.TABLE, "create", {
after: async ({ result }) => {
if (!result || !result.id) return null;
// Never track plugin infrastructure tables (dev-deploy's own + saltcorn-idp's
// registered tables, when co-installed) -- they are not user data to sync.
if (isInfraTableName(result.name)) return null;
const uuid = await assignNewUuid(ENTITY_KINDS.TABLE, result.id, result.name, null);
return {
entityUuid: uuid,
@ -234,6 +248,10 @@ const wrapField = () => {
wrap(Field, "create", ENTITY_KINDS.FIELD, "create", {
after: async ({ args, result }) => {
if (!result || !result.id) return null;
// dev-deploy's own per-environment row-identity column, and any field
// on a plugin-infrastructure table, are never journaled.
if (result.name && result.name.startsWith("_dd_")) return null;
if (isInfraTableId(result.table_id)) return null;
let parentUuid = null;
if (result.table_id) {
const t = await lookupByCurrent(ENTITY_KINDS.TABLE, result.table_id);
@ -581,6 +599,8 @@ const wrapTableConstraint = () => {
// multiple module-resolution contexts when sc-exec uses createRequire).
const already = await lookupByCurrent(ENTITY_KINDS.CONSTRAINT, result.id);
if (already) return null;
// Skip constraints on plugin-infrastructure tables.
if (isInfraTableId(result.table_id)) return null;
let parentUuid = null;
if (result.table_id) {
const t = await lookupByCurrent(ENTITY_KINDS.TABLE, result.table_id);

10
test/e2e.js
View file

@ -63,7 +63,9 @@ const resetInstanceDb = (dbPath) => {
for (const t of userTables) {
if (t) sql(dbPath, `DROP TABLE IF EXISTS "${t}"`);
}
sql(dbPath, "DELETE FROM _sc_tables WHERE name != 'users'; DELETE FROM _sc_fields WHERE table_id NOT IN (SELECT id FROM _sc_tables);");
// Spare the registered dev-deploy bookkeeping tables (_dd_peers,
// _dd_table_modes) -- they are plugin infrastructure, not per-test user data.
sql(dbPath, "DELETE FROM _sc_tables WHERE name != 'users' AND name NOT LIKE '\\_dd\\_%' ESCAPE '\\'; DELETE FROM _sc_fields WHERE table_id NOT IN (SELECT id FROM _sc_tables);");
};
@ -208,9 +210,11 @@ const main = async () => {
console.log(` test env_id: ${testEnv}`);
section("schema");
await test("six _dd_* tables exist on main", async () => {
await test("dev-deploy _dd_* tables exist on main", async () => {
const tables = sqlRows(MAIN.db, "SELECT name FROM sqlite_master WHERE type='table' AND name LIKE '_dd_%' ORDER BY name").join(",");
assert.equal(tables, "_dd_anchors,_dd_entity_ids,_dd_env,_dd_ops,_dd_peers,_dd_table_modes");
// env identity moved to _sc_config; peers + table_modes are now registered
// Saltcorn tables (still physical _dd_* tables, plus _sc_tables entries).
assert.equal(tables, "_dd_anchors,_dd_entity_ids,_dd_ops,_dd_peers,_dd_table_modes");
});
await test("_dd_ops has conflict_with_op_id column", async () => {
const cols = sqlRows(MAIN.db, "PRAGMA table_info(_dd_ops)").map((r) => r.split("|")[1]);