Fixes

index.js

@@ -1,10 +1,13 @@
 // dev-deploy: Saltcorn plugin for migrating metadata changes across
 // Dev/Test/Prod environments via an ops journal with stable UUIDs.
 
-const { PLUGIN_NAME, PLUGIN_VERSION } = require("./lib/constants");
+const db = require("@saltcorn/data/db");
+
+const { PLUGIN_NAME, PLUGIN_VERSION, DATA_MODES } = require("./lib/constants");
 const { createAllTables }             = require("./lib/schema");
 const { getEnv, initEnvIfMissing, markBootstrapped } = require("./lib/env");
 const { backfillAll }                 = require("./lib/entityIds");
+const { ensureManagedSchema }         = require("./lib/rowIdentity");
 const { installAllWraps }             = require("./lib/wrap");
 const { routes }                      = require("./lib/routes");
 
@@ -32,19 +35,67 @@ const ensureCsrfBypass = async () => {
 };
 
 
+// One-time migration: managed/starter tables created by an older plugin version
+// carry a RAW _dd_row_uuid column (added via ALTER, not registered in _sc_fields).
+// Such a column is dumped by backup but absent from the restored table's field
+// list, which fails the row import and rolls back the whole table (app-data
+// loss). Adopt every existing managed table's column into _sc_fields so it
+// round-trips. ensureManagedSchema is idempotent: a no-op once registered.
+const migrateRowUuidFields = async () => {
+    const schema = db.getTenantSchemaPrefix();
+    let rows;
+    try {
+        rows = await db.query(
+            `SELECT e.current_name AS name
+               FROM ${schema}_dd_table_modes m
+               JOIN ${schema}_dd_entity_ids e ON e.uuid = m.table_uuid
+              WHERE e.kind = 'table'
+                AND m.data_mode IN ($1, $2)`,
+            [DATA_MODES.MANAGED, DATA_MODES.STARTER]
+        );
+    } catch (e) {
+        return 0;
+    }
+    let adopted = 0;
+    for (const r of rows.rows) {
+        try {
+            const res = await ensureManagedSchema(r.name);
+            if (res && res.adopted) {
+                adopted++;
+            }
+        } catch (e) {
+            console.error(`[${PLUGIN_NAME}] row-uuid field migration failed for ${r.name}:`, e.message);
+        }
+    }
+    return adopted;
+};
+
+
 const onLoad = async (cfg) => {
     try {
         await createAllTables();
         const env = await initEnvIfMissing();
+        // Reconcile the entity-id cache on EVERY load, not just first bootstrap.
+        // _dd_entity_ids is a raw, un-backed-up cache; after a restore it is empty
+        // while the restored env reports bootstrapped, so a gated backfill would
+        // leave the map empty. backfillAll is idempotent (ensureUuid keys on
+        // (kind,current_id) and derives deterministic UUIDs from name), so running
+        // it always rebuilds the map against the restored host's live catalog.
+        const counts = await backfillAll();
+        const total = Object.values(counts).reduce((a, b) => a + b, 0);
         if (!env.bootstrapped_at) {
-            const counts = await backfillAll();
-            const total = Object.values(counts).reduce((a, b) => a + b, 0);
             await markBootstrapped(env.env_id);
             log(`v${PLUGIN_VERSION} bootstrapped env_id=${env.env_id} backfilled ${total} entities ${JSON.stringify(counts)}`);
+        } else if (total > 0) {
+            log(`v${PLUGIN_VERSION} loaded env_id=${env.env_id}; reconciled entity-id cache (+${total})`);
         } else {
             log(`v${PLUGIN_VERSION} loaded env_id=${env.env_id}`);
         }
         installAllWraps();
+        const adopted = await migrateRowUuidFields();
+        if (adopted > 0) {
+            log(`migrated ${adopted} managed table(s) to a registered _dd_row_uuid field`);
+        }
         await ensureCsrfBypass();
     } catch (err) {
         // eslint-disable-next-line no-console

lib/configStore.js

@@ -0,0 +1,49 @@
+// Durable per-tenant key/value storage in Saltcorn's _sc_config, used for the
+// dev-deploy ENV IDENTITY (which is created during onLoad).
+//
+// Why _sc_config and not a registered table or the plugin config blob:
+//   - It survives backup: backup_config dumps every non-fixed _sc_config key,
+//     and restore_config restores them BEFORE install_pack runs the plugin's
+//     onLoad -- so the restored identity is already present when onLoad reads it
+//     (no onLoad-creates-then-restore-imports duplicate, which a registered
+//     table suffers because restore_tables runs AFTER onLoad).
+//   - It does NOT write the plugin's own _sc_plugins row (Plugin.upsert during
+//     onLoad cascades plugin re-loading and duplicates rows -- the reason the
+//     config-blob approach was abandoned). setConfig writes _sc_config only.
+//
+// Reads go DIRECT to _sc_config: getState().getConfig only surfaces keys that
+// are in Saltcorn's known configTypes schema, so a custom plugin key is invisible
+// through it even though it is stored and backed up. Writes go through setConfig
+// (correct jsonb/text encoding on both backends; value wrapped as {v: ...}).
+
+const db = require("@saltcorn/data/db");
+
+
+const readKey = async (key) => {
+    const row = await db.selectMaybeOne("_sc_config", { key: key });
+    if (!row || row.value === null || row.value === undefined) {
+        return undefined;
+    }
+    let v = row.value;
+    if (typeof v === "string") {
+        try {
+            v = JSON.parse(v);
+        } catch (e) {
+            return undefined;
+        }
+    }
+    // Saltcorn wraps config values as { v: <value> }.
+    return v && typeof v === "object" && "v" in v ? v.v : v;
+};
+
+
+const writeKey = async (key, value) => {
+    const { getState } = require("@saltcorn/data/db/state");
+    await getState().setConfig(key, value);
+};
+
+
+module.exports = {
+    readKey,
+    writeKey
+};

lib/entityIds.js

@@ -83,6 +83,21 @@ const ensureUuid = async (kind, currentId, currentName, parentUuid, canonical) =
         return existing.uuid;
     }
     const uuid = deterministicUuid(kind, canonical || currentName);
+    // RECONCILE: the deterministic UUID (derived from the entity's name) is the
+    // stable identity. If a row already carries it but at a different current_id,
+    // the integer id was reassigned (e.g. table recreated, or a restore) -- re-
+    // point the stable UUID to the new id rather than inserting a duplicate (which
+    // would violate the uuid PK). No (kind,current_id) row exists here (checked
+    // above), so the update cannot violate UNIQUE(kind,current_id).
+    const stale = await db.selectMaybeOne("_dd_entity_ids", { uuid: uuid });
+    if (stale) {
+        await db.updateWhere(
+            "_dd_entity_ids",
+            { current_id: currentId, current_name: currentName, parent_uuid: parentUuid || null },
+            { uuid: uuid }
+        );
+        return uuid;
+    }
     const row = {
         uuid:         uuid,
         kind:         kind,
@@ -134,10 +149,20 @@ const removeEntityRow = async (kind, currentId) => {
 
 // Backfill helpers: each returns the count of new UUIDs inserted.
 
+// dev-deploy's own registered bookkeeping tables (_dd_peers, _dd_table_modes)
+// and saltcorn-idp's (_idp_*) appear in Table.find now that they are registered,
+// but they are plugin INFRASTRUCTURE, not user data -- never track, sync, or show
+// them as managed tables.
+const isInfraTable = (name) => name.startsWith("_dd_") || name.startsWith("_idp_");
+
+
 const backfillTables = async () => {
     const tables = await Table.find({}, { cached: false });
     let added = 0;
     for (const t of tables) {
+        if (isInfraTable(t.name)) {
+            continue;
+        }
         const before = await lookupByCurrent(ENTITY_KINDS.TABLE, t.id);
         await ensureUuid(ENTITY_KINDS.TABLE, t.id, t.name, null, t.name);
         if (!before) {
@@ -152,12 +177,20 @@ const backfillFields = async () => {
     const tables  = await Table.find({}, { cached: false });
     let added = 0;
     for (const t of tables) {
+        if (isInfraTable(t.name)) {
+            continue;
+        }
         const tableUuidRow = await lookupByCurrent(ENTITY_KINDS.TABLE, t.id);
         if (!tableUuidRow) {
             continue;
         }
         const fields = t.getFields ? t.getFields() : (await Field.find({ table_id: t.id }));
         for (const f of fields) {
+            // _dd_row_uuid is dev-deploy's per-environment row-identity column --
+            // never track/sync it (each environment manages its own).
+            if (f.name.startsWith("_dd_")) {
+                continue;
+            }
             const before = await lookupByCurrent(ENTITY_KINDS.FIELD, f.id);
             await ensureUuid(ENTITY_KINDS.FIELD, f.id, f.name, tableUuidRow.uuid, `${t.name}.${f.name}`);
             if (!before) {
@@ -318,6 +351,12 @@ const backfillWorkflowSteps = async () => {
 
 
 const backfillAll = async () => {
+    // Remove any stale tracking of plugin infrastructure: registered _dd_*/_idp_*
+    // tables and the per-environment _dd_row_uuid column should never be in the
+    // entity-id map (older builds tracked _dd_row_uuid once it became a field).
+    const schema = db.getTenantSchemaPrefix();
+    await db.query(`DELETE FROM ${schema}_dd_entity_ids WHERE current_name LIKE '_dd_%' OR current_name LIKE '_idp_%'`).catch(() => {});
+
     const counts = {};
     counts.tables      = await backfillTables();
     counts.fields      = await backfillFields();

lib/env.js

@@ -1,34 +1,25 @@
 // This Saltcorn instance's dev-deploy identity (env_id, label, policies).
-// Stored as a singleton row in _dd_env.
+//
-
+// Stored in _sc_config (key "dev_deploy_env") rather than a _dd_env table.
-const db = require("@saltcorn/data/db");
+// _sc_config survives backup AND is restored BEFORE the plugin's onLoad runs, so
+// a restored instance keeps its env_id and onLoad does not create a duplicate
+// (see lib/configStore.js). Reads are DB-fresh (no in-process cache needed).
 
 const { randomUuid }         = require("./ids");
 const { DESTRUCTIVE_POLICY } = require("./constants");
+const { readKey, writeKey }  = require("./configStore");
 
 
-// The env identity is schema-scoped, so a single process serving multiple
+const ENV_KEY = "dev_deploy_env";
-// tenants must NOT share one cached row across them. Key the cache by tenant
-// schema (db.getTenantSchema()), not a module-level singleton.
-const cachedEnvByTenant = new Map();
-
-const tenantKey = () => (db.getTenantSchema ? db.getTenantSchema() : "public");
 
 
 const getEnv = async () => {
-    const key = tenantKey();
+    return (await readKey(ENV_KEY)) || null;
-    if (cachedEnvByTenant.has(key)) {
-        return cachedEnvByTenant.get(key);
-    }
-    const rows = await db.select("_dd_env", {});
-    const env  = rows.length > 0 ? rows[0] : null;
-    cachedEnvByTenant.set(key, env);
-    return env;
 };
 
 
 const refreshEnvCache = async () => {
-    cachedEnvByTenant.delete(tenantKey());
+    // No in-process cache anymore; reads are always DB-fresh. Kept for callers.
     return await getEnv();
 };
 
@@ -38,27 +29,24 @@ const initEnvIfMissing = async () => {
     if (existing) {
         return existing;
     }
-    const now = new Date().toISOString();
+    const env = {
-    const row = {
         env_id:            randomUuid(),
         env_label:         null,
         on_destructive_op: DESTRUCTIVE_POLICY.CONFIRM,
         require_tls:       0,
-        created_at:        now,
+        created_at:        new Date().toISOString(),
         bootstrapped_at:   null
     };
-    await db.insert("_dd_env", row, { noid: true });
+    await writeKey(ENV_KEY, env);
-    cachedEnvByTenant.set(tenantKey(), row);
+    return env;
-    return row;
 };
 
 
 const markBootstrapped = async (envId) => {
-    const now = new Date().toISOString();
+    const env = await getEnv();
-    await db.updateWhere("_dd_env", { bootstrapped_at: now }, { env_id: envId });
+    if (env && env.env_id === envId) {
-    const cached = cachedEnvByTenant.get(tenantKey());
+        env.bootstrapped_at = new Date().toISOString();
-    if (cached && cached.env_id === envId) {
+        await writeKey(ENV_KEY, env);
-        cached.bootstrapped_at = now;
     }
 };
 

lib/peers.js

@@ -85,7 +85,13 @@ const addPeer = async ({ envId, label, baseUrl, requireTls, existingSecret }) =>
     }
     const secret = existingSecret || randomSecret();
     const sealed = seal(secret);
+    // _dd_peers is now a registered Saltcorn table with an auto `id` PK; peer_id
+    // is a regular unique column, so we assign it ourselves (serial-style max+1)
+    // to keep peer_id stable + collision-free (anchors reference it by value).
+    const nextRs = await db.query(`SELECT COALESCE(MAX(peer_id),0)+1 AS next FROM ${db.getTenantSchemaPrefix()}_dd_peers`);
+    const peerId = nextRs.rows[0].next;
     const row = {
+        peer_id:                peerId,
         env_id:                 envId,
         label:                  label || null,
         base_url:               baseUrl,
@@ -96,9 +102,7 @@ const addPeer = async ({ envId, label, baseUrl, requireTls, existingSecret }) =>
         created_at:             new Date().toISOString(),
         last_seen_at:           null
     };
-    // noid: _dd_peers' PK is peer_id (serial on PG), not "id"; without this
+    // noid: let Saltcorn assign the `id` PK; we supply peer_id explicitly.
-    // Saltcorn's default RETURNING id makes the insert fail on Postgres with
-    // 'column "id" does not exist'. The peer_id is auto-assigned; re-select below.
     await db.insert("_dd_peers", row, { noid: true });
     const fresh = await findPeerByEnvId(envId);
     return { peer: rowToPeer(fresh), secret: secret };

lib/rowIdentity.js

@@ -1,15 +1,34 @@
-// Hidden _dd_row_uuid column infrastructure for managed/starter tables.
+// _dd_row_uuid column infrastructure for managed/starter tables.
 //
-// When an admin marks a user table as managed or starter, this module
+// When an admin marks a user table as managed or starter, this module adds a
-// adds a TEXT column _dd_row_uuid to the underlying SQL table via raw ALTER
+// _dd_row_uuid column to the underlying table as a REGISTERED Saltcorn Field
-// (NOT registered in _sc_fields, so Saltcorn's table builder doesn't show
+// (in _sc_fields) -- NOT a raw ALTER outside the catalog. Registration is what
-// it). Existing rows are backfilled with random UUIDs. From that point,
+// makes the column survive Saltcorn backup/restore/snapshots: backup dumps a
-// the wrap layer reads/writes _dd_row_uuid as the cross-environment identity
+// row's physical columns (SELECT * / row_to_json), so a raw, unregistered
-// for each row.
+// column lands in the dump but the restored table -- recreated from the pack's
+// field list -- has no such column, and the row insert then fails and rolls
+// back the whole table (real app-data loss). A registered Field round-trips:
+// install_pack recreates the column before the rows are imported.
+//
+// The field is created with hidden:true so it stays out of the edit/show form
+// builders (it is still visible in the raw table builder field list -- Saltcorn
+// has no system/hidden-from-catalog flag, an accepted trade-off for backup
+// safety). Existing rows are backfilled with random UUIDs. From that point, the
+// wrap layer reads/writes _dd_row_uuid as the cross-environment identity for
+// each row.
+//
+// Creating/deleting this field goes through Field.create / field.delete, which
+// the dev-deploy wraps (lib/wrap.js) journal. We run those inside runSuppressed
+// so the plugin's own row-identity column is never journaled or propagated to
+// peers -- every environment manages its own _dd_row_uuid independently.
 
 const crypto = require("crypto");
 
-const db = require("@saltcorn/data/db");
+const db    = require("@saltcorn/data/db");
+const Table = require("@saltcorn/data/models/table");
+const Field = require("@saltcorn/data/models/field");
+
+const { runSuppressed } = require("./context");
 
 
 const COLUMN_NAME = "_dd_row_uuid";
@@ -21,6 +40,11 @@ const tableSqlRef = (tableName) => {
 };
 
 
+const refreshCatalog = async () => {
+    await require("@saltcorn/data/db/state").getState().refresh_tables(true);
+};
+
+
 const columnExists = async (tableName) => {
     if (db.isSQLite) {
         const rs = await db.query(`PRAGMA table_info("${db.sqlsanitize(tableName)}")`);
@@ -29,10 +53,7 @@ const columnExists = async (tableName) => {
     // Check the tenant's OWN schema -- the same one tableSqlRef()/ALTER target.
     // current_schema() is NOT reliable: Saltcorn qualifies queries with
     // getTenantSchemaPrefix() rather than SET search_path, so current_schema() is
-    // "public" even inside a tenant. Using it made this falsely report the column
+    // "public" even inside a tenant.
-    // missing on every call after the first, and the explicitly-qualified ALTER
-    // then failed with 'column "_dd_row_uuid" already exists' (breaking apply,
-    // which calls ensureManagedSchema once per set_table_mode + per insert_row).
     const rs = await db.query(
         `SELECT 1 FROM information_schema.columns
           WHERE table_schema = $1
@@ -44,12 +65,39 @@ const columnExists = async (tableName) => {
 };
 
 
-const ensureManagedSchema = async (tableName) => {
+// Is _dd_row_uuid already a registered Saltcorn field on this table? Checked via
-    if (await columnExists(tableName)) {
+// _sc_fields directly so we do not depend on a possibly-stale in-memory catalog.
-        return { added: false };
+const fieldIsRegistered = async (tableId) => {
-    }
+    const row = await db.selectMaybeOne("_sc_fields", { table_id: tableId, name: COLUMN_NAME });
-    await db.query(`ALTER TABLE ${tableSqlRef(tableName)} ADD COLUMN ${COLUMN_NAME} TEXT`);
+    return row || null;
-    // Backfill existing rows with random UUIDs.
+};
+
+
+// Adopt a pre-existing raw _dd_row_uuid column (added by an older plugin version
+// via raw ALTER) into _sc_fields WITHOUT re-issuing ADD COLUMN. Field.create
+// always emits the ADD COLUMN DDL, which would fail with "column already
+// exists", so for the legacy case we insert the field metadata directly. This
+// mirrors the _sc_fields shape Field.create writes (field.ts).
+const adoptExistingColumn = async (table) => {
+    await db.insert("_sc_fields", {
+        table_id:    table.id,
+        name:        COLUMN_NAME,
+        label:       "dev-deploy row id",
+        type:        "String",
+        required:    false,
+        is_unique:   false,
+        attributes:  { hidden: true },
+        calculated:  false,
+        expression:  null,
+        stored:      false,
+        description: "Managed by dev-deploy: stable cross-environment row identity."
+    });
+    await refreshCatalog();
+};
+
+
+// Backfill any rows missing a _dd_row_uuid with a fresh random UUID.
+const backfillNulls = async (tableName) => {
     const rs = await db.query(`SELECT id FROM ${tableSqlRef(tableName)} WHERE ${COLUMN_NAME} IS NULL`);
     let backfilled = 0;
     for (const r of rs.rows) {
@@ -59,23 +107,78 @@ const ensureManagedSchema = async (tableName) => {
         );
         backfilled++;
     }
-    // Index for fast lookups by uuid.
+    return backfilled;
+};
+
+
+const ensureIndex = async (tableName) => {
     await db.query(
         `CREATE INDEX IF NOT EXISTS "${db.sqlsanitize(tableName)}_dd_row_uuid_idx"
            ON ${tableSqlRef(tableName)} (${COLUMN_NAME})`
     ).catch(() => {});
-    return { added: true, backfilled: backfilled };
 };
 
 
+// Ensure the managed table has a registered _dd_row_uuid field. Idempotent:
+//   - already registered          -> no-op
+//   - legacy raw column present    -> adopt it into _sc_fields (no DDL)
+//   - neither                      -> Field.create (adds column + registers)
+// Then backfill UUIDs and ensure the lookup index.
+const ensureManagedSchema = async (tableName) => {
+    const table = Table.findOne({ name: tableName });
+    if (!table) {
+        throw new Error(`ensureManagedSchema: table ${tableName} not in Saltcorn catalog`);
+    }
+    if (await fieldIsRegistered(table.id)) {
+        return { added: false };
+    }
+
+    const legacyColumn = await columnExists(tableName);
+    await runSuppressed(async () => {
+        if (legacyColumn) {
+            await adoptExistingColumn(table);
+        } else {
+            await Field.create({
+                table:       table,
+                name:        COLUMN_NAME,
+                label:       "dev-deploy row id",
+                type:        "String",
+                required:    false,
+                is_unique:   false,
+                attributes:  { hidden: true },
+                description: "Managed by dev-deploy: stable cross-environment row identity."
+            });
+        }
+    });
+
+    const backfilled = await backfillNulls(tableName);
+    await ensureIndex(tableName);
+    return { added: true, adopted: !!legacyColumn, backfilled: backfilled };
+};
+
+
+// Remove the _dd_row_uuid field/column when a table reverts to 'user' mode.
+// Prefers the Field model (drops column + _sc_fields entry); falls back to a raw
+// DROP COLUMN for a legacy unregistered column.
 const dropManagedSchema = async (tableName) => {
-    if (!(await columnExists(tableName))) {
+    const table = Table.findOne({ name: tableName });
+    if (!table) {
         return { dropped: false };
     }
-    // SQLite 3.35+ and Postgres both support DROP COLUMN. If the SQLite is
+    const registered = await fieldIsRegistered(table.id);
-    // older it will throw — surface the error.
+    if (registered) {
-    await db.query(`ALTER TABLE ${tableSqlRef(tableName)} DROP COLUMN ${COLUMN_NAME}`);
+        const field = await Field.findOne({ id: registered.id });
-    return { dropped: true };
+        if (field) {
+            await runSuppressed(() => field.delete());
+            return { dropped: true };
+        }
+    }
+    if (await columnExists(tableName)) {
+        // SQLite 3.35+ and Postgres both support DROP COLUMN.
+        await db.query(`ALTER TABLE ${tableSqlRef(tableName)} DROP COLUMN ${COLUMN_NAME}`);
+        return { dropped: true };
+    }
+    return { dropped: false };
 };
 
 

lib/schema.js

@@ -1,53 +1,131 @@
-// DDL for dev-deploy's six plugin tables.
+// Schema setup for dev-deploy's plugin tables.
 //
-// Saltcorn supports SQLite and PostgreSQL. We use a portable subset:
+// Two storage strategies, chosen by WHEN the state is written, so each survives
-//   - TEXT for all strings, JSON payloads, ISO 8601 timestamps
+// Saltcorn backup/restore without fighting the plugin lifecycle:
-//   - INTEGER for booleans (0/1) and surrogate ids
-//   - No JSONB; payloads are TEXT containing JSON, parsed/stringified at the
-//     application layer
 //
-// Tables are created idempotently in onLoad. Re-running is safe.
+//   * ENV IDENTITY -> _sc_config (lib/env.js + lib/configStore.js). It is created
+//     during onLoad; _sc_config is restored BEFORE onLoad, so a restored host
+//     keeps its env_id and onLoad does not create a duplicate.
+//
+//   * PEERS + TABLE MODES -> REGISTERED Saltcorn tables (auto `id` PK + the old
+//     logical keys kept as unique fields). They are written only at runtime (not
+//     during onLoad/restore), so the catalog-recreate-on-restore path is clean,
+//     and being in _sc_tables they ride backup. Existing db.select/insert code
+//     keeps working unchanged (the logical keys are now ordinary columns).
+//
+//   * JOURNAL (_dd_ops), ENTITY-ID CACHE (_dd_entity_ids), SYNC WATERMARKS
+//     (_dd_anchors) stay RAW: they are a log / a rebuildable cache / re-syncable
+//     watermarks, so regenerating them on a restored host is acceptable, and the
+//     journal must NOT be registered (dev-deploy's wraps would journal the
+//     restore's own entity-creation into it).
 
-const db = require("@saltcorn/data/db");
+const db    = require("@saltcorn/data/db");
+const Table = require("@saltcorn/data/models/table");
+const Field = require("@saltcorn/data/models/field");
+
+const { runSuppressed }     = require("./context");
+const { readKey, writeKey } = require("./configStore");
 
 
-const createDdEnv = async () => {
+const ENV_KEY = "dev_deploy_env";
-    const schema = db.getTenantSchemaPrefix();
+
-    await db.query(`
+// Field definitions (excluding the auto `id` PK) for the registered tables.
-        CREATE TABLE IF NOT EXISTS ${schema}_dd_env (
+const PEERS_FIELDS = [
-            env_id            TEXT PRIMARY KEY,
+    { name: "peer_id",                type: "Integer", is_unique: true },
-            env_label         TEXT,
+    { name: "env_id",                 type: "String",  is_unique: true, required: true },
-            on_destructive_op TEXT NOT NULL DEFAULT 'confirm',
+    { name: "label",                  type: "String" },
-            require_tls       INTEGER NOT NULL DEFAULT 0,
+    { name: "base_url",               type: "String",  required: true },
-            created_at        TEXT NOT NULL,
+    { name: "peer_secret_ciphertext", type: "String" },
-            bootstrapped_at   TEXT
+    { name: "peer_secret_iv",         type: "String" },
-        )
+    { name: "peer_secret_tag",        type: "String" },
-    `);
+    { name: "require_tls",            type: "Integer" },
+    { name: "created_at",             type: "String",  required: true },
+    { name: "last_seen_at",           type: "String" }
+];
+
+const TABLE_MODES_FIELDS = [
+    { name: "table_uuid",         type: "String", is_unique: true, required: true },
+    { name: "data_mode",          type: "String", required: true },
+    { name: "updated_at",         type: "String", required: true },
+    { name: "starter_shipped_at", type: "String" }
+];
+
+
+const rawTableExists = async (name) => {
+    if (db.isSQLite) {
+        const rs = await db.query(`SELECT name FROM sqlite_master WHERE type='table' AND name='${name}'`);
+        return rs.rows.length > 0;
+    }
+    const rs = await db.query(
+        `SELECT 1 FROM information_schema.tables WHERE table_schema='${db.getTenantSchema()}' AND table_name='${name}'`
+    );
+    return rs.rows.length > 0;
 };
 
 
-const createDdPeers = async () => {
+// Make `name` a registered Saltcorn table with an auto `id` PK plus `fieldDefs`.
-    // Secret components stored as hex TEXT rather than BLOB: Saltcorn's
+// Idempotent: a no-op once registered. If a legacy RAW table exists, its rows are
-    // SQLite insert layer JSON-stringifies any object value, which would
+// migrated into the registered table (logical-key columns preserved). Table.create
-    // mangle Buffer columns. Hex is portable and survives that path intact.
+// + Field.create run under runSuppressed so dev-deploy's own wraps don't journal
+// the plugin's bookkeeping tables.
+const registerTable = async (name, fieldDefs) => {
+    if (Table.findOne({ name })) {
+        return;
+    }
     const schema = db.getTenantSchemaPrefix();
-    // Portable auto-increment PK: sqlite "integer primary key" auto-assigns
+    let rows = [];
-    // rowids; postgres uses "serial" (AUTOINCREMENT is sqlite-only syntax).
+    if (await rawTableExists(name)) {
-    const serial = db.isSQLite ? "integer" : "serial";
+        rows = (await db.query(`SELECT * FROM ${schema}"${name}"`)).rows;
-    await db.query(`
+        await db.query(`DROP TABLE ${schema}"${name}"`);
-        CREATE TABLE IF NOT EXISTS ${schema}_dd_peers (
+    }
-            peer_id                ${serial} PRIMARY KEY,
+    await runSuppressed(async () => {
-            env_id                 TEXT NOT NULL UNIQUE,
+        const t = await Table.create(name, { min_role_read: 1, min_role_write: 1 });
-            label                  TEXT,
+        for (const f of fieldDefs) {
-            base_url               TEXT NOT NULL,
+            await Field.create({
-            peer_secret_ciphertext TEXT,
+                table:      t,
-            peer_secret_iv         TEXT,
+                name:       f.name,
-            peer_secret_tag        TEXT,
+                label:      f.label || f.name,
-            require_tls            INTEGER,
+                type:       f.type,
-            created_at             TEXT NOT NULL,
+                is_unique:  !!f.is_unique,
-            last_seen_at           TEXT
+                required:   !!f.required,
-        )
+                attributes: f.attributes || {}
-    `);
+            });
+        }
+    });
+    for (const r of rows) {
+        const row = {};
+        for (const f of fieldDefs) {
+            if (r[f.name] !== undefined && r[f.name] !== null) {
+                row[f.name] = r[f.name];
+            }
+        }
+        await db.insert(name, row, { noid: true });
+    }
+};
+
+
+// Move a legacy _dd_env table into the _sc_config identity key, then drop it.
+const migrateEnvToConfig = async () => {
+    if (await readKey(ENV_KEY)) {
+        return;
+    }
+    if (!(await rawTableExists("_dd_env"))) {
+        return;
+    }
+    const schema = db.getTenantSchemaPrefix();
+    const rows = (await db.query(`SELECT * FROM ${schema}_dd_env`)).rows;
+    if (rows.length > 0) {
+        const e = rows[0];
+        await writeKey(ENV_KEY, {
+            env_id:            e.env_id,
+            env_label:         e.env_label,
+            on_destructive_op: e.on_destructive_op,
+            require_tls:       e.require_tls,
+            created_at:        e.created_at,
+            bootstrapped_at:   e.bootstrapped_at
+        });
+    }
+    await db.query(`DROP TABLE IF EXISTS ${schema}_dd_env`);
 };
 
 
@@ -94,10 +172,7 @@ const createDdOps = async () => {
     await db.query(`CREATE INDEX IF NOT EXISTS _dd_ops_correlation ON ${schema}_dd_ops (correlation_id)`);
     await db.query(`CREATE INDEX IF NOT EXISTS _dd_ops_status     ON ${schema}_dd_ops (status) WHERE status = 'conflict'`).catch(() => {});
 
-    // Idempotent migration for instances installed before conflict_with_op_id
+    // Idempotent migration for instances installed before conflict_with_op_id existed.
-    // existed. On PG a failed statement poisons the surrounding transaction, so a
-    // bare ALTER caught in JS is not enough -- use ADD COLUMN IF NOT EXISTS there;
-    // sqlite lacks that clause, so run the bare ALTER and swallow the error.
     if (db.isSQLite) {
         try {
             await db.query(`ALTER TABLE ${schema}_dd_ops ADD COLUMN conflict_with_op_id TEXT`);
@@ -124,35 +199,16 @@ const createDdAnchors = async () => {
 };
 
 
-const createDdTableModes = async () => {
-    const schema = db.getTenantSchemaPrefix();
-    await db.query(`
-        CREATE TABLE IF NOT EXISTS ${schema}_dd_table_modes (
-            table_uuid         TEXT PRIMARY KEY,
-            data_mode          TEXT NOT NULL,
-            updated_at         TEXT NOT NULL,
-            starter_shipped_at TEXT
-        )
-    `);
-    // Idempotent migration for older installs that lack starter_shipped_at.
-    // (See createDdOps: PG needs IF NOT EXISTS or a failed ALTER poisons the txn.)
-    if (db.isSQLite) {
-        try {
-            await db.query(`ALTER TABLE ${schema}_dd_table_modes ADD COLUMN starter_shipped_at TEXT`);
-        } catch (e) { /* column exists */ }
-    } else {
-        await db.query(`ALTER TABLE ${schema}_dd_table_modes ADD COLUMN IF NOT EXISTS starter_shipped_at TEXT`);
-    }
-};
-
-
 const createAllTables = async () => {
-    await createDdEnv();
+    // Identity -> _sc_config (migrate any legacy table first).
-    await createDdPeers();
+    await migrateEnvToConfig();
+    // Backed-up state -> registered Saltcorn tables.
+    await registerTable("_dd_peers", PEERS_FIELDS);
+    await registerTable("_dd_table_modes", TABLE_MODES_FIELDS);
+    // Regenerable -> raw tables.
     await createDdEntityIds();
     await createDdOps();
     await createDdAnchors();
-    await createDdTableModes();
 };
 
 

lib/wrap.js

@@ -54,6 +54,17 @@ const {
 } = require("./rowPayload");
 
 
+// Plugin infrastructure tables (dev-deploy's own registered tables + saltcorn-idp's,
+// when co-installed) must never be tracked/journaled/synced as user data.
+const isInfraTableName = (name) => !!name && (name.startsWith("_dd_") || name.startsWith("_idp_"));
+
+const isInfraTableId = (tableId) => {
+    if (!tableId) return false;
+    const t = Table.findOne({ id: tableId });
+    return t ? isInfraTableName(t.name) : false;
+};
+
+
 const snapshotInstance = (inst, keys) => {
     const out = {};
     for (const k of keys) {
@@ -171,6 +182,9 @@ const wrapTable = () => {
     wrap(Table, "create", ENTITY_KINDS.TABLE, "create", {
         after: async ({ result }) => {
             if (!result || !result.id) return null;
+            // Never track plugin infrastructure tables (dev-deploy's own + saltcorn-idp's
+            // registered tables, when co-installed) -- they are not user data to sync.
+            if (isInfraTableName(result.name)) return null;
             const uuid = await assignNewUuid(ENTITY_KINDS.TABLE, result.id, result.name, null);
             return {
                 entityUuid: uuid,
@@ -234,6 +248,10 @@ const wrapField = () => {
     wrap(Field, "create", ENTITY_KINDS.FIELD, "create", {
         after: async ({ args, result }) => {
             if (!result || !result.id) return null;
+            // dev-deploy's own per-environment row-identity column, and any field
+            // on a plugin-infrastructure table, are never journaled.
+            if (result.name && result.name.startsWith("_dd_")) return null;
+            if (isInfraTableId(result.table_id)) return null;
             let parentUuid = null;
             if (result.table_id) {
                 const t = await lookupByCurrent(ENTITY_KINDS.TABLE, result.table_id);
@@ -581,6 +599,8 @@ const wrapTableConstraint = () => {
             // multiple module-resolution contexts when sc-exec uses createRequire).
             const already = await lookupByCurrent(ENTITY_KINDS.CONSTRAINT, result.id);
             if (already) return null;
+            // Skip constraints on plugin-infrastructure tables.
+            if (isInfraTableId(result.table_id)) return null;
             let parentUuid = null;
             if (result.table_id) {
                 const t = await lookupByCurrent(ENTITY_KINDS.TABLE, result.table_id);

test/e2e.js

@@ -63,7 +63,9 @@ const resetInstanceDb = (dbPath) => {
     for (const t of userTables) {
         if (t) sql(dbPath, `DROP TABLE IF EXISTS "${t}"`);
     }
-    sql(dbPath, "DELETE FROM _sc_tables WHERE name != 'users'; DELETE FROM _sc_fields WHERE table_id NOT IN (SELECT id FROM _sc_tables);");
+    // Spare the registered dev-deploy bookkeeping tables (_dd_peers,
+    // _dd_table_modes) -- they are plugin infrastructure, not per-test user data.
+    sql(dbPath, "DELETE FROM _sc_tables WHERE name != 'users' AND name NOT LIKE '\\_dd\\_%' ESCAPE '\\'; DELETE FROM _sc_fields WHERE table_id NOT IN (SELECT id FROM _sc_tables);");
 };
 
 
@@ -208,9 +210,11 @@ const main = async () => {
     console.log(`  test env_id: ${testEnv}`);
 
     section("schema");
-    await test("six _dd_* tables exist on main", async () => {
+    await test("dev-deploy _dd_* tables exist on main", async () => {
         const tables = sqlRows(MAIN.db, "SELECT name FROM sqlite_master WHERE type='table' AND name LIKE '_dd_%' ORDER BY name").join(",");
-        assert.equal(tables, "_dd_anchors,_dd_entity_ids,_dd_env,_dd_ops,_dd_peers,_dd_table_modes");
+        // env identity moved to _sc_config; peers + table_modes are now registered
+        // Saltcorn tables (still physical _dd_* tables, plus _sc_tables entries).
+        assert.equal(tables, "_dd_anchors,_dd_entity_ids,_dd_ops,_dd_peers,_dd_table_modes");
     });
     await test("_dd_ops has conflict_with_op_id column", async () => {
         const cols = sqlRows(MAIN.db, "PRAGMA table_info(_dd_ops)").map((r) => r.split("|")[1]);