commit 4d62c45b8d48a4d979547067c61fce39178e7e24 Author: Scott Duensing Date: Sun May 17 17:31:49 2026 -0500 Initial commit diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..7b32bec --- /dev/null +++ b/.gitattributes @@ -0,0 +1,44 @@ +# Default line-ending handling for text files. +* text=auto + +# Git LFS. Run `git lfs install` once per clone to activate the filters below. +# Nothing in the project currently requires LFS, but these patterns catch +# common binary blobs so anyone dropping one into the tree gets it tracked +# correctly without remembering to update this file. + +# Archives +*.zip filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tar.gz filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.7z filter=lfs diff=lfs merge=lfs -text + +# Databases / snapshots +*.sqlite filter=lfs diff=lfs merge=lfs -text +*.sqlite3 filter=lfs diff=lfs merge=lfs -text +*.db filter=lfs diff=lfs merge=lfs -text + +# Images +*.png filter=lfs diff=lfs merge=lfs -text +*.jpg filter=lfs diff=lfs merge=lfs -text +*.jpeg filter=lfs diff=lfs merge=lfs -text +*.gif filter=lfs diff=lfs merge=lfs -text +*.webp filter=lfs diff=lfs merge=lfs -text +*.ico filter=lfs diff=lfs merge=lfs -text + +# Documents +*.pdf filter=lfs diff=lfs merge=lfs -text + +# Fonts +*.ttf filter=lfs diff=lfs merge=lfs -text +*.otf filter=lfs diff=lfs merge=lfs -text +*.woff filter=lfs diff=lfs merge=lfs -text +*.woff2 filter=lfs diff=lfs merge=lfs -text + +# Audio / video +*.mp3 filter=lfs diff=lfs merge=lfs -text +*.mp4 filter=lfs diff=lfs merge=lfs -text +*.wav filter=lfs diff=lfs merge=lfs -text +*.webm filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d5894e0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,21 @@ +# Upstream Saltcorn checkout. It has its own .git and remote; manage it via +# `git -C saltcorn ...` or re-clone via ./installSaltcorn.sh. +/saltcorn/ + +# Per-instance runtime state. Contains SQLite DB, uploaded files, and an +# env.sh with a generated session secret -- none of which should be shared. +# Recreate via installSaltcorn.sh. +/.dev-state/ +/.dev-state-test/ + +# npm output (in case dev-deploy or future siblings ever take deps). +node_modules/ + +# Editor / OS junk. +.DS_Store +Thumbs.db +*.swp +*.swo +*~ +.idea/ +.vscode/ diff --git a/dev-deploy/index.js b/dev-deploy/index.js new file mode 100644 index 0000000..ccd7679 --- /dev/null +++ b/dev-deploy/index.js @@ -0,0 +1,62 @@ +// dev-deploy: Saltcorn plugin for migrating metadata changes across +// Dev/Test/Prod environments via an ops journal with stable UUIDs. + +const { PLUGIN_NAME, PLUGIN_VERSION } = require("./lib/constants"); +const { createAllTables } = require("./lib/schema"); +const { getEnv, initEnvIfMissing, markBootstrapped } = require("./lib/env"); +const { backfillAll } = require("./lib/entityIds"); +const { installAllWraps } = require("./lib/wrap"); +const { routes } = require("./lib/routes"); + + +const log = (msg) => { + // eslint-disable-next-line no-console + console.log(`[${PLUGIN_NAME}] ${msg}`); +}; + + +const ensureCsrfBypass = async () => { + try { + const { getState } = require("@saltcorn/data/db/state"); + const current = getState().getConfig("disable_csrf_routes", ""); + const want = "/dev-deploy/api/"; + const entries = current.split(",").map((s) => s.trim()).filter(Boolean); + if (!entries.includes(want)) { + entries.push(want); + await getState().setConfig("disable_csrf_routes", entries.join(",")); + } + } catch (e) { + // eslint-disable-next-line no-console + console.error(`[${PLUGIN_NAME}] failed to register csrf bypass:`, e); + } +}; + + +const onLoad = async (cfg) => { + try { + await createAllTables(); + const env = await initEnvIfMissing(); + if (!env.bootstrapped_at) { + const counts = await backfillAll(); + const total = Object.values(counts).reduce((a, b) => a + b, 0); + await markBootstrapped(env.env_id); + log(`v${PLUGIN_VERSION} bootstrapped env_id=${env.env_id} backfilled ${total} entities ${JSON.stringify(counts)}`); + } else { + log(`v${PLUGIN_VERSION} loaded env_id=${env.env_id}`); + } + installAllWraps(); + await ensureCsrfBypass(); + } catch (err) { + // eslint-disable-next-line no-console + console.error(`[${PLUGIN_NAME}] onLoad failed:`, err); + throw err; + } +}; + + +module.exports = { + sc_plugin_api_version: 1, + plugin_name: PLUGIN_NAME, + onLoad: onLoad, + routes: routes +}; diff --git a/dev-deploy/lib/apply.js b/dev-deploy/lib/apply.js new file mode 100644 index 0000000..6fee2db --- /dev/null +++ b/dev-deploy/lib/apply.js @@ -0,0 +1,1135 @@ +// Apply handlers — replay an op from a peer onto this instance. +// +// Each handler: +// - Resolves the op's entity_uuid (and any parent_uuid) to this instance's +// local integer id via _dd_entity_ids. +// - Invokes the Saltcorn model method to reproduce the change. +// - Updates _dd_entity_ids (insert / rename / remove) accordingly. +// +// While a handler runs, the CRUD wraps are suppressed (no journal writes, no +// auto-UUID assignment) — we manage those side effects ourselves so the +// foreign op_id and entity_uuid are preserved across instances. + +const db = require("@saltcorn/data/db"); + +const Table = require("@saltcorn/data/models/table"); +const Field = require("@saltcorn/data/models/field"); +const View = require("@saltcorn/data/models/view"); +const Page = require("@saltcorn/data/models/page"); +const Trigger = require("@saltcorn/data/models/trigger"); +const Role = require("@saltcorn/data/models/role"); +const Library = require("@saltcorn/data/models/library"); +const Tag = require("@saltcorn/data/models/tag"); +const TableConstraint = require("@saltcorn/data/models/table_constraints"); +const File = require("@saltcorn/data/models/file"); +const PageGroup = require("@saltcorn/data/models/page_group"); +const PageGroupMember = require("@saltcorn/data/models/page_group_member"); +const WorkflowStep = require("@saltcorn/data/models/workflow_step"); + +const { + lookupByUuid, + adoptUuid, + updateName, + removeEntityRow, + lookupByCurrent, + constraintDisplayName +} = require("./entityIds"); +const { runSuppressed } = require("./context"); +const { ENTITY_KINDS, fileLocationToId } = require("./constants"); +const { refreshState } = require("./state"); +const { sha256Buffer, writeFileBytes, toAbsolutePath } = require("./files"); +const { signedFetchBinary } = require("./transport"); +const peers = require("./peers"); +const { getEnv } = require("./env"); +const { fromPlaceholders } = require("./payloadRefs"); +const { + ensureManagedSchema, + setRowUuid, + findIdByRowUuid, + COLUMN_NAME: ROW_UUID_COL +} = require("./rowIdentity"); +const { portableToRow } = require("./rowPayload"); + + +// Strip surrogate keys that don't translate across instances. Returns a new +// object with all non-id properties. +const stripSurrogateKeys = (obj, extraIdKeys) => { + const drop = new Set(["id", "table_id", "view_id", "page_id", "role_id_for_create", ...(extraIdKeys || [])]); + const out = {}; + for (const k of Object.keys(obj || {})) { + if (drop.has(k)) continue; + out[k] = obj[k]; + } + return out; +}; + + +const requireLocalEntity = async (uuid, kind) => { + const m = await lookupByUuid(uuid); + if (!m) { + throw new Error(`local entity for uuid=${uuid} (kind=${kind}) not found`); + } + return m; +}; + + +// ---------------- Tables ---------------- + +const applyCreateTable = async ({ op, payload }) => { + const after = payload.after || {}; + if (!after.name) { + throw new Error("create_table missing after.name"); + } + const existing = await lookupByUuid(op.entity_uuid); + if (existing) { + return { status: "noop", reason: "uuid already mapped" }; + } + const opts = stripSurrogateKeys(after); + const tableName = opts.name; + delete opts.name; + const t = await Table.create(tableName, opts); + await adoptUuid(op.entity_uuid, ENTITY_KINDS.TABLE, t.id, t.name, null); + return { status: "created", local_id: t.id }; +}; + + +const applyUpdateTable = async ({ op, payload }) => { + const mapping = await requireLocalEntity(op.entity_uuid, ENTITY_KINDS.TABLE); + const t = Table.findOne({ id: mapping.current_id }); + if (!t) { + throw new Error(`table id=${mapping.current_id} not found`); + } + const patch = stripSurrogateKeys(payload.patch || {}); + if (Object.keys(patch).length === 0) { + return { status: "noop", reason: "empty patch" }; + } + await t.update(patch); + if (patch.name !== undefined && patch.name !== mapping.current_name) { + await updateName(ENTITY_KINDS.TABLE, mapping.current_id, patch.name); + } + return { status: "updated" }; +}; + + +const applyDropTable = async ({ op, payload }) => { + const mapping = await lookupByUuid(op.entity_uuid); + if (!mapping) { + return { status: "noop", reason: "already dropped" }; + } + const t = Table.findOne({ id: mapping.current_id }); + if (!t) { + await removeEntityRow(ENTITY_KINDS.TABLE, mapping.current_id); + return { status: "noop", reason: "table not in saltcorn" }; + } + await t.delete(); + await removeEntityRow(ENTITY_KINDS.TABLE, mapping.current_id); + return { status: "dropped" }; +}; + + +// ---------------- Fields ---------------- + +const applyCreateField = async ({ op, payload }) => { + const after = payload.after || {}; + const parentUuid = payload.parent_uuid; + if (!parentUuid) { + throw new Error("create_field missing parent_uuid"); + } + const existing = await lookupByUuid(op.entity_uuid); + if (existing) { + return { status: "noop", reason: "uuid already mapped" }; + } + const parent = await requireLocalEntity(parentUuid, ENTITY_KINDS.TABLE); + const table = Table.findOne({ id: parent.current_id }); + if (!table) { + throw new Error(`parent table id=${parent.current_id} not found`); + } + const cfg = stripSurrogateKeys(after); + cfg.table_id = table.id; + const f = await Field.create(cfg); + await adoptUuid(op.entity_uuid, ENTITY_KINDS.FIELD, f.id, f.name, parentUuid); + return { status: "created", local_id: f.id }; +}; + + +const applyUpdateField = async ({ op, payload }) => { + const mapping = await requireLocalEntity(op.entity_uuid, ENTITY_KINDS.FIELD); + const f = await Field.findOne({ id: mapping.current_id }); + if (!f) { + throw new Error(`field id=${mapping.current_id} not found`); + } + const patch = stripSurrogateKeys(payload.patch || {}); + if (Object.keys(patch).length === 0) { + return { status: "noop", reason: "empty patch" }; + } + await f.update(patch); + if (patch.name !== undefined && patch.name !== mapping.current_name) { + await updateName(ENTITY_KINDS.FIELD, mapping.current_id, patch.name); + } + return { status: "updated" }; +}; + + +const applyDropField = async ({ op, payload }) => { + const mapping = await lookupByUuid(op.entity_uuid); + if (!mapping) { + return { status: "noop", reason: "already dropped" }; + } + const f = await Field.findOne({ id: mapping.current_id }); + if (!f) { + await removeEntityRow(ENTITY_KINDS.FIELD, mapping.current_id); + return { status: "noop" }; + } + await f.delete(); + await removeEntityRow(ENTITY_KINDS.FIELD, mapping.current_id); + return { status: "dropped" }; +}; + + +// ---------------- Views ---------------- + +const applyCreateView = async ({ op, payload }) => { + const after = payload.after || {}; + const parentUuid = payload.parent_uuid; + const existing = await lookupByUuid(op.entity_uuid); + if (existing) { + return { status: "noop", reason: "uuid already mapped" }; + } + const cfg = stripSurrogateKeys(after); + if (parentUuid) { + const parent = await requireLocalEntity(parentUuid, ENTITY_KINDS.TABLE); + cfg.table_id = parent.current_id; + } + const v = await View.create(cfg); + await adoptUuid(op.entity_uuid, ENTITY_KINDS.VIEW, v.id, v.name, parentUuid || null); + return { status: "created", local_id: v.id }; +}; + + +const applyUpdateView = async ({ op, payload }) => { + const mapping = await requireLocalEntity(op.entity_uuid, ENTITY_KINDS.VIEW); + const patch = stripSurrogateKeys(payload.patch || {}); + if (Object.keys(patch).length === 0) { + return { status: "noop", reason: "empty patch" }; + } + await View.update(patch, mapping.current_id); + if (patch.name !== undefined && patch.name !== mapping.current_name) { + await updateName(ENTITY_KINDS.VIEW, mapping.current_id, patch.name); + } + return { status: "updated" }; +}; + + +const applyDropView = async ({ op, payload }) => { + const mapping = await lookupByUuid(op.entity_uuid); + if (!mapping) { + return { status: "noop", reason: "already dropped" }; + } + const v = View.findOne({ id: mapping.current_id }); + if (!v) { + await removeEntityRow(ENTITY_KINDS.VIEW, mapping.current_id); + return { status: "noop" }; + } + await v.delete(); + await removeEntityRow(ENTITY_KINDS.VIEW, mapping.current_id); + return { status: "dropped" }; +}; + + +// ---------------- Pages ---------------- + +const applyCreatePage = async ({ op, payload }) => { + const after = payload.after || {}; + const existing = await lookupByUuid(op.entity_uuid); + if (existing) { + return { status: "noop", reason: "uuid already mapped" }; + } + const cfg = stripSurrogateKeys(after); + const p = await Page.create(cfg); + await adoptUuid(op.entity_uuid, ENTITY_KINDS.PAGE, p.id, p.name, null); + return { status: "created", local_id: p.id }; +}; + + +const applyUpdatePage = async ({ op, payload }) => { + const mapping = await requireLocalEntity(op.entity_uuid, ENTITY_KINDS.PAGE); + const patch = stripSurrogateKeys(payload.patch || {}); + if (Object.keys(patch).length === 0) { + return { status: "noop", reason: "empty patch" }; + } + await Page.update(mapping.current_id, patch); + if (patch.name !== undefined && patch.name !== mapping.current_name) { + await updateName(ENTITY_KINDS.PAGE, mapping.current_id, patch.name); + } + return { status: "updated" }; +}; + + +const applyDropPage = async ({ op, payload }) => { + const mapping = await lookupByUuid(op.entity_uuid); + if (!mapping) { + return { status: "noop", reason: "already dropped" }; + } + const p = Page.findOne({ id: mapping.current_id }); + if (!p) { + await removeEntityRow(ENTITY_KINDS.PAGE, mapping.current_id); + return { status: "noop" }; + } + await p.delete(); + await removeEntityRow(ENTITY_KINDS.PAGE, mapping.current_id); + return { status: "dropped" }; +}; + + +// ---------------- Triggers ---------------- + +const applyCreateTrigger = async ({ op, payload }) => { + const after = payload.after || {}; + const parentUuid = payload.parent_uuid; + const existing = await lookupByUuid(op.entity_uuid); + if (existing) { + return { status: "noop", reason: "uuid already mapped" }; + } + const cfg = stripSurrogateKeys(after); + if (parentUuid) { + const parent = await requireLocalEntity(parentUuid, ENTITY_KINDS.TABLE); + cfg.table_id = parent.current_id; + } + const tr = await Trigger.create(cfg); + await adoptUuid(op.entity_uuid, ENTITY_KINDS.TRIGGER, tr.id, tr.name || `trigger_${tr.id}`, parentUuid || null); + return { status: "created", local_id: tr.id }; +}; + + +const applyUpdateTrigger = async ({ op, payload }) => { + const mapping = await requireLocalEntity(op.entity_uuid, ENTITY_KINDS.TRIGGER); + const patch = stripSurrogateKeys(payload.patch || {}); + if (Object.keys(patch).length === 0) { + return { status: "noop", reason: "empty patch" }; + } + await Trigger.update(mapping.current_id, patch); + if (patch.name !== undefined && patch.name !== mapping.current_name) { + await updateName(ENTITY_KINDS.TRIGGER, mapping.current_id, patch.name); + } + return { status: "updated" }; +}; + + +const applyDropTrigger = async ({ op, payload }) => { + const mapping = await lookupByUuid(op.entity_uuid); + if (!mapping) { + return { status: "noop", reason: "already dropped" }; + } + const tr = Trigger.findOne({ id: mapping.current_id }); + if (!tr) { + await removeEntityRow(ENTITY_KINDS.TRIGGER, mapping.current_id); + return { status: "noop" }; + } + await tr.delete(); + await removeEntityRow(ENTITY_KINDS.TRIGGER, mapping.current_id); + return { status: "dropped" }; +}; + + +// ---------------- Roles ---------------- + +const applyCreateRole = async ({ op, payload }) => { + const after = payload.after || {}; + if (!after.id || !after.role) { + throw new Error("create_role missing after.id or after.role"); + } + const existing = await lookupByUuid(op.entity_uuid); + if (existing) { + return { status: "noop", reason: "uuid already mapped" }; + } + await Role.create({ id: after.id, role: after.role }); + await adoptUuid(op.entity_uuid, ENTITY_KINDS.ROLE, after.id, after.role, null); + return { status: "created", local_id: after.id }; +}; + + +const applyUpdateRole = async ({ op, payload }) => { + const mapping = await requireLocalEntity(op.entity_uuid, ENTITY_KINDS.ROLE); + const r = await Role.findOne({ id: mapping.current_id }); + const patch = payload.patch || {}; + if (Object.keys(patch).length === 0) { + return { status: "noop", reason: "empty patch" }; + } + await r.update(patch); + if (patch.role !== undefined && patch.role !== mapping.current_name) { + await updateName(ENTITY_KINDS.ROLE, mapping.current_id, patch.role); + } + return { status: "updated" }; +}; + + +const applyDropRole = async ({ op, payload }) => { + const mapping = await lookupByUuid(op.entity_uuid); + if (!mapping) { + return { status: "noop" }; + } + const r = await Role.findOne({ id: mapping.current_id }); + if (r) { + await r.delete(); + } + await removeEntityRow(ENTITY_KINDS.ROLE, mapping.current_id); + return { status: "dropped" }; +}; + + +// ---------------- Library ---------------- + +const applyCreateLibrary = async ({ op, payload }) => { + const after = payload.after || {}; + if (!after.name) { + throw new Error("create_library missing after.name"); + } + const existing = await lookupByUuid(op.entity_uuid); + if (existing) { + return { status: "noop", reason: "uuid already mapped" }; + } + const cfg = stripSurrogateKeys(after); + await Library.create(cfg); + const fresh = await Library.findOne({ name: after.name }); + if (!fresh) { + throw new Error("library not found after create"); + } + await adoptUuid(op.entity_uuid, ENTITY_KINDS.LIBRARY, fresh.id, fresh.name, null); + return { status: "created", local_id: fresh.id }; +}; + + +const applyUpdateLibrary = async ({ op, payload }) => { + const mapping = await requireLocalEntity(op.entity_uuid, ENTITY_KINDS.LIBRARY); + const li = await Library.findOne({ id: mapping.current_id }); + const patch = stripSurrogateKeys(payload.patch || {}); + if (Object.keys(patch).length === 0) { + return { status: "noop", reason: "empty patch" }; + } + await li.update(patch); + if (patch.name !== undefined && patch.name !== mapping.current_name) { + await updateName(ENTITY_KINDS.LIBRARY, mapping.current_id, patch.name); + } + return { status: "updated" }; +}; + + +const applyDropLibrary = async ({ op, payload }) => { + const mapping = await lookupByUuid(op.entity_uuid); + if (!mapping) { + return { status: "noop" }; + } + const li = await Library.findOne({ id: mapping.current_id }); + if (li) { + await li.delete(); + } + await removeEntityRow(ENTITY_KINDS.LIBRARY, mapping.current_id); + return { status: "dropped" }; +}; + + +// ---------------- Tags ---------------- + +const applyCreateTag = async ({ op, payload }) => { + const after = payload.after || {}; + const existing = await lookupByUuid(op.entity_uuid); + if (existing) { + return { status: "noop", reason: "uuid already mapped" }; + } + const cfg = stripSurrogateKeys(after); + const tg = await Tag.create(cfg); + await adoptUuid(op.entity_uuid, ENTITY_KINDS.TAG, tg.id, tg.name, null); + return { status: "created", local_id: tg.id }; +}; + + +const applyUpdateTag = async ({ op, payload }) => { + const mapping = await requireLocalEntity(op.entity_uuid, ENTITY_KINDS.TAG); + const tg = await Tag.findOne({ id: mapping.current_id }); + const patch = stripSurrogateKeys(payload.patch || {}); + if (Object.keys(patch).length === 0) { + return { status: "noop", reason: "empty patch" }; + } + await tg.update(patch); + if (patch.name !== undefined && patch.name !== mapping.current_name) { + await updateName(ENTITY_KINDS.TAG, mapping.current_id, patch.name); + } + return { status: "updated" }; +}; + + +const applyDropTag = async ({ op, payload }) => { + const mapping = await lookupByUuid(op.entity_uuid); + if (!mapping) { + return { status: "noop" }; + } + const tg = await Tag.findOne({ id: mapping.current_id }); + if (tg) { + await tg.delete(); + } + await removeEntityRow(ENTITY_KINDS.TAG, mapping.current_id); + return { status: "dropped" }; +}; + + +// ---------------- Files ---------------- + +const applyCreateFile = async ({ op, payload, opts }) => { + const after = payload.after || {}; + const relPath = after.relative_path; + if (!relPath) throw new Error("create_file missing relative_path"); + if (!after.content_hash) throw new Error("create_file missing content_hash"); + + const existing = await lookupByUuid(op.entity_uuid); + if (existing) { + return { status: "noop", reason: "uuid already mapped" }; + } + if (!opts || !opts.peerId) { + throw new Error("file apply requires opts.peerId to fetch binary from"); + } + + const peer = await peers.findPeer(opts.peerId); + if (!peer) throw new Error(`peer ${opts.peerId} not found`); + const secret = await peers.peerSecret(opts.peerId); + const env = await getEnv(); + + const r = await signedFetchBinary({ + baseUrl: peer.base_url, + method: "GET", + path: `/dev-deploy/api/file/${encodeURIComponent(op.entity_uuid)}`, + body: null, + sourceEnvId: env.env_id, + secret: secret + }); + if (!r.ok) { + throw new Error(`binary fetch returned ${r.status}`); + } + const bytes = r.bytes; + const actualHash = sha256Buffer(bytes); + if (actualHash !== after.content_hash) { + throw new Error(`content hash mismatch: expected ${after.content_hash}, got ${actualHash}`); + } + + const absPath = toAbsolutePath(File, db, relPath); + await writeFileBytes(absPath, bytes); + + const localFile = await File.create({ + filename: after.filename, + location: absPath, + uploaded_at: new Date(), + size_kb: after.size_kb, + mime_super: after.mime_super, + mime_sub: after.mime_sub, + min_role_read: after.min_role_read + }); + + const synthId = fileLocationToId(relPath); + await adoptUuid(op.entity_uuid, ENTITY_KINDS.FILE, synthId, relPath, null); + + return { status: "created", location: absPath, bytes: bytes.length }; +}; + + +const applyDropFile = async ({ op, payload }) => { + const mapping = await lookupByUuid(op.entity_uuid); + if (!mapping) { + return { status: "noop", reason: "already dropped" }; + } + const absPath = toAbsolutePath(File, db, mapping.current_name); + const file = await File.findOne({ location: absPath }); + if (file) { + await file.delete(); + } + await removeEntityRow(ENTITY_KINDS.FILE, mapping.current_id); + return { status: "dropped" }; +}; + + +// ---------------- TableConstraints ---------------- + +const applyCreateConstraint = async ({ op, payload }) => { + const after = payload.after || {}; + const parentUuid = payload.parent_uuid; + if (!parentUuid) throw new Error("create_constraint missing parent_uuid"); + if (!after.type) throw new Error("create_constraint missing type"); + const existing = await lookupByUuid(op.entity_uuid); + if (existing) { + return { status: "noop", reason: "uuid already mapped" }; + } + const parent = await requireLocalEntity(parentUuid, "table"); + const cfg = { + table_id: parent.current_id, + type: after.type, + configuration: after.configuration || {} + }; + const result = await TableConstraint.create(cfg); + await adoptUuid(op.entity_uuid, "constraint", result.id, constraintDisplayName(result), parentUuid); + return { status: "created", local_id: result.id }; +}; + + +const applyDropConstraint = async ({ op, payload }) => { + const mapping = await lookupByUuid(op.entity_uuid); + if (!mapping) { + return { status: "noop", reason: "already dropped" }; + } + const con = await TableConstraint.findOne({ id: mapping.current_id }); + if (con) { + await con.delete(); + } + await removeEntityRow("constraint", mapping.current_id); + return { status: "dropped" }; +}; + + +// ---------------- PageGroup ---------------- + +const applyCreatePageGroup = async ({ op, payload }) => { + const after = payload.after || {}; + if (!after.name) throw new Error("create_page_group missing after.name"); + const existing = await lookupByUuid(op.entity_uuid); + if (existing) { + return { status: "noop", reason: "uuid already mapped" }; + } + // Create the group first, empty members. PageGroupMember's constructor + // demands page_group_id which doesn't exist until after the insert. + const cfg = stripSurrogateKeys(after); + cfg.members = []; + const result = await PageGroup.create(cfg); + + // Now resolve each member's page_uuid to local page_id and addMember. + for (const m of payload.members || []) { + if (!m.page_uuid) { + throw new Error("page_group member missing page_uuid"); + } + const local = await lookupByUuid(m.page_uuid); + if (!local) { + throw new Error(`page_group member references unmapped page_uuid=${m.page_uuid}`); + } + await result.addMember({ + page_id: local.current_id, + eligible_formula: m.eligible_formula, + description: m.description, + sequence: m.sequence + }); + } + await adoptUuid(op.entity_uuid, "page_group", result.id, result.name, null); + return { status: "created", local_id: result.id }; +}; + + +const applyUpdatePageGroup = async ({ op, payload }) => { + const mapping = await requireLocalEntity(op.entity_uuid, "page_group"); + const patch = stripSurrogateKeys(payload.patch || {}); + if (Object.keys(patch).length === 0) { + return { status: "noop", reason: "empty patch" }; + } + await PageGroup.update(mapping.current_id, patch); + if (patch.name !== undefined && patch.name !== mapping.current_name) { + await updateName("page_group", mapping.current_id, patch.name); + } + return { status: "updated" }; +}; + + +const applyDropPageGroup = async ({ op, payload }) => { + const mapping = await lookupByUuid(op.entity_uuid); + if (!mapping) return { status: "noop" }; + const pg = PageGroup.findOne({ id: mapping.current_id }); + if (pg) await pg.delete(); + await removeEntityRow("page_group", mapping.current_id); + return { status: "dropped" }; +}; + + +// ---------------- Table rows (managed/starter) ---------------- + +const findLocalTableByUuid = async (tableUuid) => { + const ent = await lookupByUuid(tableUuid); + if (!ent || ent.kind !== "table") return null; + await refreshState(); + return Table.findOne({ id: ent.current_id }); +}; + + +const applyInsertRow = async ({ op, payload }) => { + if (!payload || !payload.table_uuid) throw new Error("insert_row missing table_uuid"); + if (!op.entity_uuid) throw new Error("insert_row missing row_uuid (entity_uuid)"); + const tbl = await findLocalTableByUuid(payload.table_uuid); + if (!tbl) throw new Error(`local table for uuid=${payload.table_uuid} not found`); + await ensureManagedSchema(tbl.name); + // Idempotency: if a row with this uuid already exists, skip + const existing = await findIdByRowUuid(tbl.name, op.entity_uuid); + if (existing) { + return { status: "noop", reason: "row uuid already present" }; + } + const rowData = await portableToRow(payload.after || {}, tbl); + const newId = await tbl.insertRow(rowData); + if (!newId) throw new Error("insertRow returned no id"); + await setRowUuid(tbl.name, newId, op.entity_uuid); + return { status: "inserted", local_id: newId }; +}; + + +const applyUpdateRow = async ({ op, payload }) => { + if (!payload || !payload.table_uuid) throw new Error("update_row missing table_uuid"); + if (!op.entity_uuid) throw new Error("update_row missing row_uuid"); + const tbl = await findLocalTableByUuid(payload.table_uuid); + if (!tbl) throw new Error(`local table for uuid=${payload.table_uuid} not found`); + await ensureManagedSchema(tbl.name); + const localId = await findIdByRowUuid(tbl.name, op.entity_uuid); + if (!localId) { + // The row doesn't exist on target yet — treat as insert + const rowData = await portableToRow(payload.patch || {}, tbl); + const newId = await tbl.insertRow(rowData); + await setRowUuid(tbl.name, newId, op.entity_uuid); + return { status: "inserted_for_update", local_id: newId }; + } + const patch = await portableToRow(payload.patch || {}, tbl); + if (Object.keys(patch).length === 0) { + return { status: "noop", reason: "empty patch" }; + } + await tbl.updateRow(patch, localId); + return { status: "updated", local_id: localId }; +}; + + +const applyDropRow = async ({ op, payload }) => { + if (!payload || !payload.table_uuid) throw new Error("drop_row missing table_uuid"); + if (!op.entity_uuid) throw new Error("drop_row missing row_uuid"); + const tbl = await findLocalTableByUuid(payload.table_uuid); + if (!tbl) return { status: "noop", reason: "table not present locally" }; + const localId = await findIdByRowUuid(tbl.name, op.entity_uuid); + if (!localId) return { status: "noop", reason: "row uuid not present locally" }; + await tbl.deleteRows({ id: localId }); + return { status: "dropped", local_id: localId }; +}; + + +const applySetTableMode = async ({ op, payload }) => { + if (!payload || !payload.table_uuid) throw new Error("set_table_mode missing table_uuid"); + const tbl = await findLocalTableByUuid(payload.table_uuid); + if (!tbl) throw new Error(`local table for uuid=${payload.table_uuid} not found`); + const mode = payload.data_mode || "user"; + if (mode === "managed" || mode === "starter") { + await ensureManagedSchema(tbl.name); + } + const now = new Date().toISOString(); + const existing = await db.selectMaybeOne("_dd_table_modes", { table_uuid: payload.table_uuid }); + if (existing) { + await db.updateWhere("_dd_table_modes", { data_mode: mode, updated_at: now }, { table_uuid: payload.table_uuid }); + } else { + await db.insert("_dd_table_modes", { table_uuid: payload.table_uuid, data_mode: mode, updated_at: now }, { noid: true }); + } + return { status: "set", mode: mode }; +}; + + +// ---------------- Plugin configuration ---------------- + +const applyUpdatePluginConfig = async ({ op, payload }) => { + const Plugin = require("@saltcorn/data/models/plugin"); + if (!payload || !payload.name) { + throw new Error("update_plugin_config missing payload.name"); + } + if (payload.name === "dev-deploy") { + return { status: "noop", reason: "skipping our own plugin" }; + } + const plugin = await Plugin.findOne({ name: payload.name }); + if (!plugin) { + throw new Error(`plugin "${payload.name}" is not installed on this instance`); + } + plugin.configuration = payload.configuration || {}; + await plugin.upsert(); + return { status: "updated", plugin: payload.name }; +}; + + +// ---------------- Config (menu_items etc.) ---------------- + +const applySetConfig = async ({ op, payload }) => { + const { key, value } = payload || {}; + if (!key) throw new Error("set_config missing key"); + const { getState } = require("@saltcorn/data/db/state"); + await getState().setConfig(key, value); + return { status: "set", key: key }; +}; + + +// ---------------- PageGroupMember ---------------- + +const applyCreatePageGroupMember = async ({ op, payload }) => { + const after = payload.after || {}; + const parentUuid = payload.parent_uuid; + if (!parentUuid) throw new Error("create_page_group_member missing parent_uuid"); + if (!after.page_uuid) throw new Error("create_page_group_member missing page_uuid"); + const existing = await lookupByUuid(op.entity_uuid); + if (existing) return { status: "noop", reason: "uuid already mapped" }; + const group = await requireLocalEntity(parentUuid, "page_group"); + const page = await requireLocalEntity(after.page_uuid, "page"); + const pg = PageGroup.findOne({ id: group.current_id }); + if (!pg) throw new Error(`local page_group id=${group.current_id} not found`); + const member = await pg.addMember({ + page_id: page.current_id, + eligible_formula: after.eligible_formula, + description: after.description + }); + await adoptUuid(op.entity_uuid, "page_group_member", member.id, `member_${member.id}`, parentUuid); + return { status: "created", local_id: member.id }; +}; + + +const applyDropPageGroupMember = async ({ op, payload }) => { + const mapping = await lookupByUuid(op.entity_uuid); + if (!mapping) return { status: "noop" }; + await PageGroupMember.delete(mapping.current_id); + await removeEntityRow("page_group_member", mapping.current_id); + return { status: "dropped" }; +}; + + +// ---------------- WorkflowStep ---------------- + +const applyCreateWorkflowStep = async ({ op, payload }) => { + const after = payload.after || {}; + const parentUuid = payload.parent_uuid; + const existing = await lookupByUuid(op.entity_uuid); + if (existing) return { status: "noop", reason: "uuid already mapped" }; + let parentTriggerId = null; + if (parentUuid) { + const parent = await requireLocalEntity(parentUuid, "trigger"); + parentTriggerId = parent.current_id; + } + const cfg = stripSurrogateKeys(after); + if (parentTriggerId) cfg.trigger_id = parentTriggerId; + const result = await WorkflowStep.create(cfg); + // WorkflowStep.create returns the inserted integer id, not an instance. + const id = typeof result === "number" ? result : (result && result.id); + if (!id) throw new Error("WorkflowStep.create returned no id"); + await adoptUuid(op.entity_uuid, "workflow_step", id, after.name || `step_${id}`, parentUuid || null); + return { status: "created", local_id: id }; +}; + + +const applyUpdateWorkflowStep = async ({ op, payload }) => { + const mapping = await requireLocalEntity(op.entity_uuid, "workflow_step"); + const ws = await WorkflowStep.findOne({ id: mapping.current_id }); + if (!ws) throw new Error(`workflow_step id=${mapping.current_id} not found`); + const patch = stripSurrogateKeys(payload.patch || {}); + if (Object.keys(patch).length === 0) { + return { status: "noop", reason: "empty patch" }; + } + await ws.update(patch); + if (patch.name !== undefined && patch.name !== mapping.current_name) { + await updateName("workflow_step", mapping.current_id, patch.name); + } + return { status: "updated" }; +}; + + +const applyDropWorkflowStep = async ({ op, payload }) => { + const mapping = await lookupByUuid(op.entity_uuid); + if (!mapping) return { status: "noop" }; + const ws = await WorkflowStep.findOne({ id: mapping.current_id }); + if (ws) await ws.delete(); + await removeEntityRow("workflow_step", mapping.current_id); + return { status: "dropped" }; +}; + + +// ---------------- Dispatch ---------------- + +const HANDLERS = { + create_table: applyCreateTable, + update_table: applyUpdateTable, + drop_table: applyDropTable, + create_field: applyCreateField, + update_field: applyUpdateField, + drop_field: applyDropField, + create_view: applyCreateView, + update_view: applyUpdateView, + drop_view: applyDropView, + create_page: applyCreatePage, + update_page: applyUpdatePage, + drop_page: applyDropPage, + create_trigger: applyCreateTrigger, + update_trigger: applyUpdateTrigger, + drop_trigger: applyDropTrigger, + create_role: applyCreateRole, + update_role: applyUpdateRole, + drop_role: applyDropRole, + create_library: applyCreateLibrary, + update_library: applyUpdateLibrary, + drop_library: applyDropLibrary, + create_tag: applyCreateTag, + update_tag: applyUpdateTag, + drop_tag: applyDropTag, + create_constraint: applyCreateConstraint, + drop_constraint: applyDropConstraint, + create_file: applyCreateFile, + drop_file: applyDropFile, + create_page_group: applyCreatePageGroup, + update_page_group: applyUpdatePageGroup, + drop_page_group: applyDropPageGroup, + create_page_group_member: applyCreatePageGroupMember, + drop_page_group_member: applyDropPageGroupMember, + create_workflow_step: applyCreateWorkflowStep, + update_workflow_step: applyUpdateWorkflowStep, + drop_workflow_step: applyDropWorkflowStep, + set_config: applySetConfig, + update_plugin_config: applyUpdatePluginConfig, + insert_row: applyInsertRow, + update_row: applyUpdateRow, + drop_row: applyDropRow, + set_table_mode: applySetTableMode +}; + + +// Record an op into our journal with its source-side identity preserved. +// applied_at is set; status may be 'committed' (apply succeeded), 'skipped_cascade' +// (parent in same batch handled it), 'error' (apply failed), or 'conflict' +// (a local op touched the same entity since the last sync with this peer). +const persistOp = async (op, status, applied, extra) => { + const payload = typeof op.payload === "string" ? op.payload : JSON.stringify(op.payload || {}); + const row = { + op_id: op.op_id, + source_env_id: op.source_env_id, + op_type: op.op_type, + entity_kind: op.entity_kind, + entity_uuid: op.entity_uuid, + payload: payload, + parent_op_id: op.parent_op_id, + correlation_id: op.correlation_id, + schema_version: op.schema_version || 1, + created_at: op.created_at, + applied_at: applied ? new Date().toISOString() : null, + status: status, + conflict_with_op_id: (extra && extra.conflict_with_op_id) || null + }; + await db.insert("_dd_ops", row, { noid: true }); +}; + + +// Find the most recent local op on the same entity_uuid that was applied +// after our last inbound sync from this peer. If one exists, this incoming op +// represents concurrent divergent changes -- a conflict. +// +// Returns the local op_id if conflicting, null otherwise. +const findConflictingLocalOp = async (op, opts) => { + if (!op.entity_uuid || !opts.peerId || !opts.myEnvId) return null; + const anchor = await db.selectMaybeOne("_dd_anchors", { peer_id: opts.peerId, direction: "inbound" }); + let cutoff = "1970-01-01T00:00:00.000Z"; + if (anchor) { + const anchorOp = await db.selectMaybeOne("_dd_ops", { op_id: anchor.last_op_id }); + if (anchorOp && anchorOp.applied_at) cutoff = anchorOp.applied_at; + } + const rs = await db.query( + `SELECT op_id FROM _dd_ops + WHERE entity_uuid = $1 + AND source_env_id = $2 + AND applied_at IS NOT NULL + AND applied_at > $3 + AND status NOT IN ('rejected', 'reverted') + ORDER BY applied_at DESC + LIMIT 1`, + [op.entity_uuid, opts.myEnvId, cutoff] + ); + return rs.rows.length > 0 ? rs.rows[0].op_id : null; +}; + + +// Apply a batch of ops in created_at order. Children whose parent_op_id is in +// the same batch are journaled but not re-applied — their parent's apply will +// reproduce the cascade locally. +// +// opts.peerId + opts.myEnvId enable conflict detection: if an incoming op's +// entity_uuid has a local op applied since the last sync with this peer, the +// incoming op is journaled with status='conflict' instead of being applied. +// The admin resolves via /admin/dev-deploy/conflicts. +const applyBatch = async (ops, opts) => { + opts = opts || {}; + await refreshState(); + const sorted = [...ops].sort((a, b) => String(a.created_at).localeCompare(String(b.created_at))); + const opIdSet = new Set(sorted.map((o) => o.op_id)); + const results = []; + + for (const op of sorted) { + const existing = await db.selectMaybeOne("_dd_ops", { op_id: op.op_id }); + if (existing) { + results.push({ op_id: op.op_id, status: "already_applied" }); + continue; + } + if (op.parent_op_id && opIdSet.has(op.parent_op_id)) { + await persistOp(op, "skipped_cascade", true); + results.push({ op_id: op.op_id, status: "skipped_cascade" }); + continue; + } + const conflictWith = await findConflictingLocalOp(op, opts); + if (conflictWith) { + await persistOp(op, "conflict", false, { conflict_with_op_id: conflictWith }); + results.push({ op_id: op.op_id, status: "conflict", conflict_with: conflictWith }); + continue; + } + const handler = HANDLERS[op.op_type]; + if (!handler) { + await persistOp(op, "error", true); + results.push({ op_id: op.op_id, status: "error", error: `no handler for ${op.op_type}` }); + continue; + } + try { + const payload = typeof op.payload === "string" ? JSON.parse(op.payload) : (op.payload || {}); + // Resolve __dd_file_ref:: placeholders to local file paths + // before handing the payload to the model handler. + try { await fromPlaceholders(payload); } catch (e) { /* best-effort */ } + const r = await runSuppressed(() => handler({ op: op, payload: payload, opts: opts })); + await persistOp(op, "committed", true); + results.push({ op_id: op.op_id, status: "applied", detail: r }); + } catch (err) { + // eslint-disable-next-line no-console + console.error(`[dev-deploy] apply ${op.op_type} (op=${op.op_id.slice(0,8)}) failed:`, err && err.stack ? err.stack : err); + await persistOp(op, "error", true); + results.push({ op_id: op.op_id, status: "error", error: err.message }); + } + } + return results; +}; + + +// Resolve a pending conflict. +// action='theirs': apply the conflicting incoming op now (suppressed), +// mark status='committed', clear conflict_with_op_id. +// action='mine' : mark the incoming op status='rejected', clear conflict_with. +// The local op stays as-is. The peer keeps sending this op_id +// on future pulls; we skip-by-id-already-present at the top of +// applyBatch (status check covers 'rejected' too via idempotency). +const resolveConflict = async (opId, action) => { + const op = await db.selectMaybeOne("_dd_ops", { op_id: opId }); + if (!op) throw new Error(`op ${opId} not found`); + if (op.status !== "conflict") throw new Error(`op ${opId} is not in conflict status (status=${op.status})`); + + if (action === "mine") { + await db.updateWhere("_dd_ops", { + status: "rejected", + conflict_with_op_id: null, + applied_at: new Date().toISOString() + }, { op_id: opId }); + return { status: "rejected" }; + } + if (action === "theirs") { + const handler = HANDLERS[op.op_type]; + if (!handler) throw new Error(`no handler for ${op.op_type}`); + await refreshState(); + const payload = typeof op.payload === "string" ? JSON.parse(op.payload) : (op.payload || {}); + const r = await runSuppressed(() => handler({ op: op, payload: payload })); + await db.updateWhere("_dd_ops", { + status: "committed", + conflict_with_op_id: null, + applied_at: new Date().toISOString() + }, { op_id: opId }); + return { status: "applied", detail: r }; + } + throw new Error(`unknown action '${action}', expected 'theirs' or 'mine'`); +}; + + +// Find the entity instance for a conflict op's entity_uuid. +// Returns { instance, kind } or null if the entity isn't present locally. +const findEntityForConflict = async (op) => { + if (!op.op_type || !op.entity_uuid) return null; + const dash = op.op_type.indexOf("_"); + if (dash < 0) return null; + const kind = op.op_type.substring(dash + 1); + const m = await lookupByUuid(op.entity_uuid); + if (!m) return null; + await refreshState(); + const Cls = { + table: require("@saltcorn/data/models/table"), + field: require("@saltcorn/data/models/field"), + view: require("@saltcorn/data/models/view"), + page: require("@saltcorn/data/models/page"), + trigger: require("@saltcorn/data/models/trigger"), + role: require("@saltcorn/data/models/role"), + library: require("@saltcorn/data/models/library"), + tag: require("@saltcorn/data/models/tag") + }[kind]; + if (!Cls) return null; + const inst = await Cls.findOne({ id: m.current_id }); + if (!inst) return null; + return { instance: inst, kind: kind }; +}; + + +// Compute fields where the incoming op's patch diverges from the local entity's +// current state. Returns an array of { field, currentValue, incomingValue }. +// Only meaningful for update_X conflicts; returns [] for other op types. +const conflictFieldDiff = async (incomingOp) => { + if (!incomingOp.op_type || !incomingOp.op_type.startsWith("update_")) { + return { diffs: [], reason: "merge only meaningful for update ops" }; + } + const found = await findEntityForConflict(incomingOp); + if (!found) { + return { diffs: [], reason: "entity not present locally" }; + } + const payload = typeof incomingOp.payload === "string" ? JSON.parse(incomingOp.payload) : (incomingOp.payload || {}); + const patch = payload.patch || {}; + const diffs = []; + for (const [field, incomingValue] of Object.entries(patch)) { + const currentValue = found.instance[field]; + if (JSON.stringify(currentValue) === JSON.stringify(incomingValue)) { + continue; + } + diffs.push({ + field: field, + currentValue: currentValue, + incomingValue: incomingValue + }); + } + return { diffs: diffs, kind: found.kind, instance: found.instance }; +}; + + +// Apply a manual per-field merge. choices is { field: chosenValue } -- only +// fields that should be written are included; if empty, no model update runs. +// The original conflict op is marked status='merged' regardless. +const resolveConflictByMerge = async (opId, choices) => { + const op = await db.selectMaybeOne("_dd_ops", { op_id: opId }); + if (!op) throw new Error(`op ${opId} not found`); + if (op.status !== "conflict") throw new Error(`op ${opId} is not in conflict status (status=${op.status})`); + if (!op.op_type.startsWith("update_")) { + throw new Error(`merge only supported for update_X ops, got ${op.op_type}`); + } + const found = await findEntityForConflict(op); + if (!found) throw new Error(`entity ${op.entity_uuid} not present locally`); + + if (choices && Object.keys(choices).length > 0) { + const View = require("@saltcorn/data/models/view"); + const Page = require("@saltcorn/data/models/page"); + const Trigger = require("@saltcorn/data/models/trigger"); + if (found.kind === "view") { + await View.update(choices, found.instance.id); + } else if (found.kind === "page") { + await Page.update(found.instance.id, choices); + } else if (found.kind === "trigger") { + await Trigger.update(found.instance.id, choices); + } else { + await found.instance.update(choices); + } + } + + await db.updateWhere("_dd_ops", { + status: "merged", + conflict_with_op_id: null, + applied_at: new Date().toISOString() + }, { op_id: opId }); + + return { status: "merged", applied: choices || {} }; +}; + + +module.exports = { + applyBatch, + resolveConflict, + resolveConflictByMerge, + conflictFieldDiff, + HANDLERS +}; diff --git a/dev-deploy/lib/constants.js b/dev-deploy/lib/constants.js new file mode 100644 index 0000000..824c2c8 --- /dev/null +++ b/dev-deploy/lib/constants.js @@ -0,0 +1,65 @@ +// Compile-time constants for the dev-deploy plugin. + +const PLUGIN_NAME = "dev-deploy"; +const PLUGIN_VERSION = "0.0.1"; + +// Namespace UUID for deterministic IDs derived from (kind, name). +// Generated once via crypto.randomUUID() and frozen here forever. +// Two environments that bootstrap from the same metadata population +// will assign identical UUIDs to entities with the same (kind, name). +const ID_NAMESPACE = "8b3a1e0d-4f6c-4d2a-9c5b-7e8f9a0b1c2d"; + +const OP_SCHEMA_VERSION = 1; + +const DATA_MODES = { + MANAGED: "managed", + STARTER: "starter", + USER: "user" +}; + +const DESTRUCTIVE_POLICY = { + AUTO: "auto", + CONFIRM: "confirm", + REFUSE: "refuse" +}; + +// Entity kinds the plugin tracks. Children point at parents via parent_uuid. +const ENTITY_KINDS = { + TABLE: "table", + FIELD: "field", + VIEW: "view", + PAGE: "page", + TRIGGER: "trigger", + ROLE: "role", + LIBRARY: "library", + TAG: "tag", + CONSTRAINT: "constraint", + FILE: "file", + PAGE_GROUP: "page_group", + PAGE_GROUP_MEMBER: "page_group_member", + WORKFLOW_STEP: "workflow_step" +}; + + +// Files don't have a meaningful integer id in current Saltcorn (File.create +// doesn't insert into _sc_files anymore — files are identified by their disk +// location and metadata via xattrs). We derive a stable 31-bit int from the +// location string so files fit the existing UNIQUE(kind, current_id) shape of +// _dd_entity_ids without a schema change. +const fileLocationToId = (location) => { + const crypto = require("crypto"); + const hash = crypto.createHash("sha256").update(String(location)).digest(); + return hash.readUInt32BE(0) & 0x7FFFFFFF; +}; + + +module.exports = { + PLUGIN_NAME, + PLUGIN_VERSION, + ID_NAMESPACE, + OP_SCHEMA_VERSION, + DATA_MODES, + DESTRUCTIVE_POLICY, + ENTITY_KINDS, + fileLocationToId +}; diff --git a/dev-deploy/lib/context.js b/dev-deploy/lib/context.js new file mode 100644 index 0000000..40da110 --- /dev/null +++ b/dev-deploy/lib/context.js @@ -0,0 +1,65 @@ +// Per-async-flow context for the mutation wraps. +// +// AsyncLocalStorage gives every wrap a stack of in-flight op_ids so that ops +// triggered while another op is running (e.g. cascading field deletes inside +// a table delete) can attach themselves to the outer op as parent_op_id, and +// share its correlation_id. + +const { AsyncLocalStorage } = require("async_hooks"); + +const { randomUuid } = require("./ids"); + + +const storage = new AsyncLocalStorage(); + + +const enterOp = async (opId, fn) => { + const current = storage.getStore(); + const correlationId = current ? current.correlationId : randomUuid(); + const parentStack = current ? current.stack : []; + const newStore = { + stack: [...parentStack, opId], + correlationId: correlationId + }; + return await storage.run(newStore, fn); +}; + + +const currentParentOpId = () => { + const store = storage.getStore(); + if (!store || store.stack.length < 2) { + return null; + } + return store.stack[store.stack.length - 2]; +}; + + +const currentCorrelationId = () => { + const store = storage.getStore(); + return store ? store.correlationId : null; +}; + + +// Suppression mode: when applying ingested ops we run the underlying Saltcorn +// model methods but want the CRUD wraps to pass through silently — no journal +// writes, no entity_id assignment. The apply handler manages those side effects +// itself with the source-env's UUID. +const runSuppressed = async (fn) => { + const current = storage.getStore() || { stack: [], correlationId: null }; + return await storage.run({ ...current, suppressed: true }, fn); +}; + + +const isSuppressed = () => { + const s = storage.getStore(); + return !!(s && s.suppressed); +}; + + +module.exports = { + enterOp, + currentParentOpId, + currentCorrelationId, + runSuppressed, + isSuppressed +}; diff --git a/dev-deploy/lib/crypto.js b/dev-deploy/lib/crypto.js new file mode 100644 index 0000000..d19b149 --- /dev/null +++ b/dev-deploy/lib/crypto.js @@ -0,0 +1,150 @@ +// Crypto primitives for dev-deploy peer auth. +// +// seal/open — AES-256-GCM for at-rest encryption of peer secrets. +// The 32-byte KEK is derived once per process via +// HKDF-SHA256 from SALTCORN_SESSION_SECRET. +// sign/verify — HMAC-SHA256 for signed peer-to-peer requests. +// buildCanonical — canonical string format every signed request agrees on. +// randomSecret — 32 random bytes (peer_secret) at pairing time. +// +// Rotating SALTCORN_SESSION_SECRET invalidates all peer pairings (the KEK +// changes, so existing ciphertexts no longer decrypt). Documented behavior. + +const crypto = require("crypto"); + + +const KEK_INFO = "dev-deploy:peer-secrets:aes-gcm-key:v1"; +const HMAC_ALGORITHM = "sha256"; +const GCM_ALGORITHM = "aes-256-gcm"; +const IV_BYTES = 12; +const TAG_BYTES = 16; +const SECRET_BYTES = 32; +const NONCE_BYTES = 16; +const SKEW_TOLERANCE_MS = 5 * 60 * 1000; + + +let cachedKek = null; + + +const getSessionSecret = () => { + const fromEnv = process.env.SALTCORN_SESSION_SECRET; + if (fromEnv && fromEnv.length > 0) { + return fromEnv; + } + // Fallback to Saltcorn state config if available + try { + const { getState } = require("@saltcorn/data/db/state"); + const v = getState().getConfig("session_secret"); + if (v) return v; + } catch (e) { + // ignore + } + throw new Error("dev-deploy: SALTCORN_SESSION_SECRET not available; cannot derive KEK"); +}; + + +const getKek = () => { + if (cachedKek) { + return cachedKek; + } + const sessionSecret = getSessionSecret(); + const salt = Buffer.from(KEK_INFO, "utf8"); + const ikm = Buffer.from(sessionSecret, "utf8"); + cachedKek = crypto.hkdfSync(HMAC_ALGORITHM, ikm, salt, Buffer.from(KEK_INFO, "utf8"), 32); + return cachedKek; +}; + + +const seal = (plaintext) => { + const iv = crypto.randomBytes(IV_BYTES); + const cipher = crypto.createCipheriv(GCM_ALGORITHM, getKek(), iv); + const buf = Buffer.isBuffer(plaintext) ? plaintext : Buffer.from(plaintext); + const ct = Buffer.concat([cipher.update(buf), cipher.final()]); + const tag = cipher.getAuthTag(); + return { ciphertext: ct, iv: iv, tag: tag }; +}; + + +const open = (sealed) => { + const decipher = crypto.createDecipheriv(GCM_ALGORITHM, getKek(), sealed.iv); + decipher.setAuthTag(sealed.tag); + return Buffer.concat([decipher.update(sealed.ciphertext), decipher.final()]); +}; + + +const randomSecret = () => { + return crypto.randomBytes(SECRET_BYTES); +}; + + +const randomNonce = () => { + return crypto.randomBytes(NONCE_BYTES); +}; + + +const sha256Hex = (data) => { + const buf = Buffer.isBuffer(data) ? data : Buffer.from(data || ""); + return crypto.createHash("sha256").update(buf).digest("hex"); +}; + + +const buildCanonical = ({ timestamp, nonce, method, path, body }) => { + const bodyHash = sha256Hex(body || ""); + return [ + String(timestamp), + String(nonce), + String(method).toUpperCase(), + String(path), + bodyHash + ].join("\n"); +}; + + +const sign = (secret, canonical) => { + const mac = crypto.createHmac(HMAC_ALGORITHM, secret); + mac.update(canonical, "utf8"); + return mac.digest("hex"); +}; + + +const verifySignature = (secret, canonical, providedHex) => { + if (!providedHex || typeof providedHex !== "string") { + return false; + } + let expectedBuf; + let providedBuf; + try { + expectedBuf = Buffer.from(sign(secret, canonical), "hex"); + providedBuf = Buffer.from(providedHex, "hex"); + } catch (e) { + return false; + } + if (expectedBuf.length !== providedBuf.length) { + return false; + } + return crypto.timingSafeEqual(expectedBuf, providedBuf); +}; + + +const timestampWithinSkew = (tsString) => { + const ts = Number(tsString); + if (!Number.isFinite(ts)) { + return false; + } + const now = Date.now(); + return Math.abs(now - ts) <= SKEW_TOLERANCE_MS; +}; + + +module.exports = { + seal, + open, + randomSecret, + randomNonce, + buildCanonical, + sign, + verifySignature, + timestampWithinSkew, + sha256Hex, + SKEW_TOLERANCE_MS +}; diff --git a/dev-deploy/lib/entityIds.js b/dev-deploy/lib/entityIds.js new file mode 100644 index 0000000..4323919 --- /dev/null +++ b/dev-deploy/lib/entityIds.js @@ -0,0 +1,349 @@ +// Stable UUIDs for Saltcorn metadata entities. +// +// Saltcorn core identifies entities by integer id and human name. dev-deploy +// needs a stable identity that survives across environments and renames, so we +// maintain a side-table _dd_entity_ids mapping (kind, current_id) -> uuid. +// +// First-run "backfill" assigns deterministic UUIDs (hash of namespace + kind +// + canonical name) so two environments installed from the same pack converge +// on the same UUIDs without a coordination step. + +const db = require("@saltcorn/data/db"); + +const Table = require("@saltcorn/data/models/table"); +const Field = require("@saltcorn/data/models/field"); +const View = require("@saltcorn/data/models/view"); +const Page = require("@saltcorn/data/models/page"); +const Trigger = require("@saltcorn/data/models/trigger"); +const Role = require("@saltcorn/data/models/role"); +const Library = require("@saltcorn/data/models/library"); +const Tag = require("@saltcorn/data/models/tag"); +const TableConstraint = require("@saltcorn/data/models/table_constraints"); +const PageGroup = require("@saltcorn/data/models/page_group"); +const WorkflowStep = require("@saltcorn/data/models/workflow_step"); + +const { deterministicUuid, randomUuid } = require("./ids"); +const { ENTITY_KINDS } = require("./constants"); + + +const assignNewUuid = async (kind, currentId, currentName, parentUuid) => { + const row = { + uuid: randomUuid(), + kind: kind, + current_name: currentName, + current_id: currentId, + parent_uuid: parentUuid || null, + created_at: new Date().toISOString() + }; + await db.insert("_dd_entity_ids", row, { noid: true }); + return row.uuid; +}; + + +const canonicalKey = (kind, entity, parentName) => { + if (kind === ENTITY_KINDS.FIELD) { + return `${parentName || "?"}.${entity.name}`; + } + return entity.name; +}; + + +// Stable fingerprint for a constraint, scoped to its parent table's UUID so +// it's globally unique across the journal. Two instances installed from the +// same base agree on this fingerprint and so derive identical UUIDs. +const constraintFingerprint = (con, parentUuid) => { + const cfg = con.configuration || {}; + let detail; + if (con.type === "Unique" && cfg.fields) { + detail = [...cfg.fields].sort().join(","); + } else if (con.type === "Index") { + detail = cfg.field || ""; + } else if (con.type === "Formula") { + detail = cfg.formula || ""; + } else { + detail = JSON.stringify(cfg); + } + return `${parentUuid || "?"}|${con.type}|${detail}`; +}; + + +// Friendly display name for _dd_entity_ids.current_name. Not used for identity. +const constraintDisplayName = (con) => { + const cfg = con.configuration || {}; + if (con.type === "Unique") return `unique(${(cfg.fields || []).join(",")})`; + if (con.type === "Index") return `index(${cfg.field || ""})`; + if (con.type === "Formula") return `formula`; + return `${con.type}_${con.id || "new"}`; +}; + + +const ensureUuid = async (kind, currentId, currentName, parentUuid, canonical) => { + const existing = await db.selectMaybeOne("_dd_entity_ids", { kind: kind, current_id: currentId }); + if (existing) { + return existing.uuid; + } + const uuid = deterministicUuid(kind, canonical || currentName); + const row = { + uuid: uuid, + kind: kind, + current_name: currentName, + current_id: currentId, + parent_uuid: parentUuid || null, + created_at: new Date().toISOString() + }; + await db.insert("_dd_entity_ids", row, { noid: true }); + return uuid; +}; + + +const lookupByCurrent = async (kind, currentId) => { + return await db.selectMaybeOne("_dd_entity_ids", { kind: kind, current_id: currentId }); +}; + + +const lookupByUuid = async (uuid) => { + return await db.selectMaybeOne("_dd_entity_ids", { uuid: uuid }); +}; + + +// Insert a row with a specific UUID (used by apply.js when ingesting an op +// that created an entity on a peer — we want to preserve the peer's UUID). +const adoptUuid = async (uuid, kind, currentId, currentName, parentUuid) => { + const row = { + uuid: uuid, + kind: kind, + current_name: currentName, + current_id: currentId, + parent_uuid: parentUuid || null, + created_at: new Date().toISOString() + }; + await db.insert("_dd_entity_ids", row, { noid: true }); + return uuid; +}; + + +const updateName = async (kind, currentId, newName) => { + await db.updateWhere("_dd_entity_ids", { current_name: newName }, { kind: kind, current_id: currentId }); +}; + + +const removeEntityRow = async (kind, currentId) => { + await db.deleteWhere("_dd_entity_ids", { kind: kind, current_id: currentId }); +}; + + +// Backfill helpers: each returns the count of new UUIDs inserted. + +const backfillTables = async () => { + const tables = await Table.find({}, { cached: false }); + let added = 0; + for (const t of tables) { + const before = await lookupByCurrent(ENTITY_KINDS.TABLE, t.id); + await ensureUuid(ENTITY_KINDS.TABLE, t.id, t.name, null, t.name); + if (!before) { + added += 1; + } + } + return added; +}; + + +const backfillFields = async () => { + const tables = await Table.find({}, { cached: false }); + let added = 0; + for (const t of tables) { + const tableUuidRow = await lookupByCurrent(ENTITY_KINDS.TABLE, t.id); + if (!tableUuidRow) { + continue; + } + const fields = t.getFields ? t.getFields() : (await Field.find({ table_id: t.id })); + for (const f of fields) { + const before = await lookupByCurrent(ENTITY_KINDS.FIELD, f.id); + await ensureUuid(ENTITY_KINDS.FIELD, f.id, f.name, tableUuidRow.uuid, `${t.name}.${f.name}`); + if (!before) { + added += 1; + } + } + } + return added; +}; + + +const backfillViews = async () => { + const views = await View.find({}); + let added = 0; + for (const v of views) { + const before = await lookupByCurrent(ENTITY_KINDS.VIEW, v.id); + let parentUuid = null; + if (v.table_id) { + const t = await lookupByCurrent(ENTITY_KINDS.TABLE, v.table_id); + parentUuid = t ? t.uuid : null; + } + await ensureUuid(ENTITY_KINDS.VIEW, v.id, v.name, parentUuid, v.name); + if (!before) { + added += 1; + } + } + return added; +}; + + +const backfillPages = async () => { + const pages = await Page.find({}); + let added = 0; + for (const p of pages) { + const before = await lookupByCurrent(ENTITY_KINDS.PAGE, p.id); + await ensureUuid(ENTITY_KINDS.PAGE, p.id, p.name, null, p.name); + if (!before) { + added += 1; + } + } + return added; +}; + + +const backfillTriggers = async () => { + const triggers = Trigger.find({}); + let added = 0; + for (const tr of triggers) { + const before = await lookupByCurrent(ENTITY_KINDS.TRIGGER, tr.id); + let parentUuid = null; + if (tr.table_id) { + const t = await lookupByCurrent(ENTITY_KINDS.TABLE, tr.table_id); + parentUuid = t ? t.uuid : null; + } + await ensureUuid(ENTITY_KINDS.TRIGGER, tr.id, tr.name || `trigger_${tr.id}`, parentUuid, tr.name || `trigger_${tr.id}`); + if (!before) { + added += 1; + } + } + return added; +}; + + +const backfillRoles = async () => { + const roles = await Role.find({}); + let added = 0; + for (const r of roles) { + const before = await lookupByCurrent(ENTITY_KINDS.ROLE, r.id); + await ensureUuid(ENTITY_KINDS.ROLE, r.id, r.role, null, r.role); + if (!before) { + added += 1; + } + } + return added; +}; + + +const backfillLibrary = async () => { + const items = await Library.find({}); + let added = 0; + for (const li of items) { + const before = await lookupByCurrent(ENTITY_KINDS.LIBRARY, li.id); + await ensureUuid(ENTITY_KINDS.LIBRARY, li.id, li.name, null, li.name); + if (!before) { + added += 1; + } + } + return added; +}; + + +const backfillTags = async () => { + const tags = await Tag.find({}); + let added = 0; + for (const tg of tags) { + const before = await lookupByCurrent(ENTITY_KINDS.TAG, tg.id); + await ensureUuid(ENTITY_KINDS.TAG, tg.id, tg.name, null, tg.name); + if (!before) { + added += 1; + } + } + return added; +}; + + +const backfillTableConstraints = async () => { + const cons = await TableConstraint.find({}); + let added = 0; + for (const c of cons) { + const before = await lookupByCurrent(ENTITY_KINDS.CONSTRAINT, c.id); + let parentUuid = null; + if (c.table_id) { + const t = await lookupByCurrent(ENTITY_KINDS.TABLE, c.table_id); + parentUuid = t ? t.uuid : null; + } + const canonical = constraintFingerprint(c, parentUuid); + await ensureUuid(ENTITY_KINDS.CONSTRAINT, c.id, constraintDisplayName(c), parentUuid, canonical); + if (!before) { + added += 1; + } + } + return added; +}; + + +const backfillPageGroups = async () => { + const groups = await PageGroup.find({}); + let added = 0; + for (const g of groups) { + const before = await lookupByCurrent(ENTITY_KINDS.PAGE_GROUP, g.id); + await ensureUuid(ENTITY_KINDS.PAGE_GROUP, g.id, g.name, null, g.name); + if (!before) { + added += 1; + } + } + return added; +}; + + +const backfillWorkflowSteps = async () => { + const steps = await WorkflowStep.find({}); + let added = 0; + for (const s of steps) { + const before = await lookupByCurrent(ENTITY_KINDS.WORKFLOW_STEP, s.id); + let parentUuid = null; + if (s.trigger_id) { + const tr = await lookupByCurrent(ENTITY_KINDS.TRIGGER, s.trigger_id); + parentUuid = tr ? tr.uuid : null; + } + const canonical = `${parentUuid || "?"}|${s.name || s.id}`; + await ensureUuid(ENTITY_KINDS.WORKFLOW_STEP, s.id, s.name || `step_${s.id}`, parentUuid, canonical); + if (!before) { + added += 1; + } + } + return added; +}; + + +const backfillAll = async () => { + const counts = {}; + counts.tables = await backfillTables(); + counts.fields = await backfillFields(); + counts.views = await backfillViews(); + counts.pages = await backfillPages(); + counts.triggers = await backfillTriggers(); + counts.roles = await backfillRoles(); + counts.library = await backfillLibrary(); + counts.tags = await backfillTags(); + counts.constraints = await backfillTableConstraints(); + counts.page_groups = await backfillPageGroups(); + counts.workflow_steps = await backfillWorkflowSteps(); + return counts; +}; + + +module.exports = { + backfillAll, + ensureUuid, + assignNewUuid, + lookupByCurrent, + lookupByUuid, + adoptUuid, + updateName, + removeEntityRow, + canonicalKey, + constraintFingerprint, + constraintDisplayName +}; diff --git a/dev-deploy/lib/env.js b/dev-deploy/lib/env.js new file mode 100644 index 0000000..e4dd271 --- /dev/null +++ b/dev-deploy/lib/env.js @@ -0,0 +1,63 @@ +// This Saltcorn instance's dev-deploy identity (env_id, label, policies). +// Stored as a singleton row in _dd_env. + +const db = require("@saltcorn/data/db"); + +const { randomUuid } = require("./ids"); +const { DESTRUCTIVE_POLICY } = require("./constants"); + + +let cachedEnv = null; + + +const getEnv = async () => { + if (cachedEnv) { + return cachedEnv; + } + const rows = await db.select("_dd_env", {}); + cachedEnv = rows.length > 0 ? rows[0] : null; + return cachedEnv; +}; + + +const refreshEnvCache = async () => { + cachedEnv = null; + return await getEnv(); +}; + + +const initEnvIfMissing = async () => { + const existing = await getEnv(); + if (existing) { + return existing; + } + const now = new Date().toISOString(); + const row = { + env_id: randomUuid(), + env_label: null, + on_destructive_op: DESTRUCTIVE_POLICY.CONFIRM, + require_tls: 0, + created_at: now, + bootstrapped_at: null + }; + await db.insert("_dd_env", row, { noid: true }); + cachedEnv = row; + return row; +}; + + +const markBootstrapped = async (envId) => { + const now = new Date().toISOString(); + await db.updateWhere("_dd_env", { bootstrapped_at: now }, { env_id: envId }); + if (cachedEnv && cachedEnv.env_id === envId) { + cachedEnv.bootstrapped_at = now; + } +}; + + +module.exports = { + getEnv, + initEnvIfMissing, + markBootstrapped, + refreshEnvCache +}; diff --git a/dev-deploy/lib/files.js b/dev-deploy/lib/files.js new file mode 100644 index 0000000..e02a110 --- /dev/null +++ b/dev-deploy/lib/files.js @@ -0,0 +1,60 @@ +// File helpers: content hashing, relative-path normalization, reading bytes. + +const crypto = require("crypto"); +const fs = require("fs"); + + +// SHA-256 of the file's contents, returned as lowercase hex. +const sha256File = async (absPath) => { + return new Promise((resolve, reject) => { + const hash = crypto.createHash("sha256"); + const stream = fs.createReadStream(absPath); + stream.on("data", (chunk) => hash.update(chunk)); + stream.on("end", () => resolve(hash.digest("hex"))); + stream.on("error", reject); + }); +}; + + +const sha256Buffer = (buf) => { + return crypto.createHash("sha256").update(buf).digest("hex"); +}; + + +// Convert Saltcorn's absolute file location to a tenant-relative serve path, +// using File.absPathToServePath. The relative path is what we transport +// between instances (each has a different file_store root). +const toRelativePath = (File, absPath) => { + if (!absPath) return ""; + return File.absPathToServePath(absPath); +}; + + +// Convert a relative serve path back to the absolute path on this instance. +const toAbsolutePath = (File, db, relPath) => { + const path = require("path"); + const tenant = db.getTenantSchema(); + return path.join(db.connectObj.file_store, tenant, relPath); +}; + + +const readFileBytes = async (absPath) => { + return await fs.promises.readFile(absPath); +}; + + +const writeFileBytes = async (absPath, buf) => { + const path = require("path"); + await fs.promises.mkdir(path.dirname(absPath), { recursive: true }); + await fs.promises.writeFile(absPath, buf); +}; + + +module.exports = { + sha256File, + sha256Buffer, + toRelativePath, + toAbsolutePath, + readFileBytes, + writeFileBytes +}; diff --git a/dev-deploy/lib/ids.js b/dev-deploy/lib/ids.js new file mode 100644 index 0000000..d6a6455 --- /dev/null +++ b/dev-deploy/lib/ids.js @@ -0,0 +1,34 @@ +// UUID helpers for dev-deploy. +// +// Two flavors: +// - randomUuid(): a fresh v4 UUID, used for newly-created ops and entities +// - deterministicUuid(kind, name): a stable, repeatable UUID derived from +// (namespace, kind, name) using SHA-256. Used during first-run backfill so +// that two environments installed from the same base produce identical +// UUIDs for the same (kind, name) pair. RFC 4122 v5 shape with namespace. + +const crypto = require("crypto"); + +const { ID_NAMESPACE } = require("./constants"); + + +const deterministicUuid = (kind, name) => { + const input = `${ID_NAMESPACE}|${kind}|${name}`; + const hash = crypto.createHash("sha256").update(input).digest(); + const bytes = Buffer.from(hash.subarray(0, 16)); + bytes[6] = (bytes[6] & 0x0f) | 0x50; + bytes[8] = (bytes[8] & 0x3f) | 0x80; + const hex = bytes.toString("hex"); + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20, 32)}`; +}; + + +const randomUuid = () => { + return crypto.randomUUID(); +}; + + +module.exports = { + deterministicUuid, + randomUuid +}; diff --git a/dev-deploy/lib/ops.js b/dev-deploy/lib/ops.js new file mode 100644 index 0000000..cef1c05 --- /dev/null +++ b/dev-deploy/lib/ops.js @@ -0,0 +1,47 @@ +// Journal append. + +const db = require("@saltcorn/data/db"); + +const { getEnv } = require("./env"); +const { randomUuid } = require("./ids"); +const { OP_SCHEMA_VERSION } = require("./constants"); +const { currentParentOpId, currentCorrelationId } = require("./context"); + + +const recordOp = async (rec) => { + const env = await getEnv(); + const now = new Date().toISOString(); + const row = { + op_id: rec.op_id || randomUuid(), + source_env_id: env.env_id, + op_type: rec.op_type, + entity_kind: rec.entity_kind || null, + entity_uuid: rec.entity_uuid || null, + payload: JSON.stringify(rec.payload || {}), + parent_op_id: rec.parent_op_id || currentParentOpId(), + correlation_id: rec.correlation_id || currentCorrelationId(), + schema_version: OP_SCHEMA_VERSION, + created_at: now, + applied_at: now, + status: "committed" + }; + await db.insert("_dd_ops", row, { noid: true }); + return row.op_id; +}; + + +const recordOpSafely = async (rec) => { + try { + return await recordOp(rec); + } catch (err) { + // eslint-disable-next-line no-console + console.error(`[dev-deploy] failed to record op ${rec.op_type}:`, err); + return null; + } +}; + + +module.exports = { + recordOp, + recordOpSafely +}; diff --git a/dev-deploy/lib/payloadRefs.js b/dev-deploy/lib/payloadRefs.js new file mode 100644 index 0000000..c981a35 --- /dev/null +++ b/dev-deploy/lib/payloadRefs.js @@ -0,0 +1,98 @@ +// Recursive walker for op payloads: translate file references between this +// instance's local form (numeric id or relative path) and a portable +// placeholder ("__dd_file_ref::"). Promote-time wraps call +// toPlaceholders to journal portable payloads; apply-time handlers call +// fromPlaceholders to resolve back to the target's local file path. +// +// Saltcorn's /files/serve route accepts either a numeric file id or a +// relative path, so writing the path back on apply works regardless of which +// form the source used. + +const db = require("@saltcorn/data/db"); + +const { lookupByUuid } = require("./entityIds"); + + +// Common keys in page/view layout JSON that reference a file. +const FILE_REF_KEYS = new Set([ + "fileid", + "file_id", + "bgFileId", + "image_id" +]); + + +const PLACEHOLDER_PREFIX = "__dd_file_ref::"; + + +// Look up a file entity by value (string path or numeric id). Returns the +// entity_ids row or null. +const lookupFileByValue = async (v) => { + if (v === null || v === undefined || v === "") return null; + // Match by current_name (relative path string) + let row = await db.selectMaybeOne("_dd_entity_ids", { kind: "file", current_name: String(v) }); + if (row) return row; + // Also try numeric id, in case the source stored an integer id + const asNum = Number(v); + if (Number.isFinite(asNum)) { + row = await db.selectMaybeOne("_dd_entity_ids", { kind: "file", current_id: asNum }); + if (row) return row; + } + return null; +}; + + +// Walk obj, mutating in place. For each key in FILE_REF_KEYS with a non-empty +// value, replace with the result of `transform(key, value)`. Async-aware. +const transformFileRefs = async (obj, transform) => { + if (Array.isArray(obj)) { + for (let i = 0; i < obj.length; i++) { + if (obj[i] && typeof obj[i] === "object") { + await transformFileRefs(obj[i], transform); + } + } + return; + } + if (!obj || typeof obj !== "object") return; + for (const k of Object.keys(obj)) { + const v = obj[k]; + if (FILE_REF_KEYS.has(k) && v !== null && v !== undefined && v !== "" && typeof v !== "object") { + obj[k] = await transform(k, v); + } else if (v && typeof v === "object") { + await transformFileRefs(v, transform); + } + } +}; + + +// Convert local file refs (id or path) -> portable placeholders. +const toPlaceholders = async (payload) => { + await transformFileRefs(payload, async (k, v) => { + const ent = await lookupFileByValue(v); + if (ent) return PLACEHOLDER_PREFIX + ent.uuid; + return v; + }); + return payload; +}; + + +// Convert portable placeholders -> local file refs (write back as relative +// path, which Saltcorn's /files/serve accepts). +const fromPlaceholders = async (payload) => { + await transformFileRefs(payload, async (k, v) => { + if (typeof v !== "string") return v; + if (!v.startsWith(PLACEHOLDER_PREFIX)) return v; + const uuid = v.substring(PLACEHOLDER_PREFIX.length); + const ent = await lookupByUuid(uuid); + return ent ? ent.current_name : v; + }); + return payload; +}; + + +module.exports = { + toPlaceholders, + fromPlaceholders, + PLACEHOLDER_PREFIX, + FILE_REF_KEYS +}; diff --git a/dev-deploy/lib/peerAuth.js b/dev-deploy/lib/peerAuth.js new file mode 100644 index 0000000..1e17320 --- /dev/null +++ b/dev-deploy/lib/peerAuth.js @@ -0,0 +1,115 @@ +// Verify incoming HMAC-signed peer requests. +// +// Required headers: +// X-DD-Env-Id caller's env UUID (looked up in _dd_peers) +// X-DD-Timestamp ms-since-epoch; rejected if skew > 5 min +// X-DD-Nonce random per-request opaque bytes (replay padding) +// X-DD-Signature hex HMAC-SHA256 over canonical string +// +// On success: req.dd_peer is set (peer row), req.body is the parsed JSON body, +// and the function returns the peer. On failure: a 4xx response is sent and +// null is returned. +// +// Peer requests use Content-Type: application/vnd.dev-deploy+json so Saltcorn's +// express.json() middleware doesn't consume the stream upstream. That lets us +// read the exact raw bytes here and use them in the HMAC -- no JSON-canonical +// fragility, no re-serialization assumptions about whitespace or key order. + +const { + buildCanonical, + verifySignature, + timestampWithinSkew +} = require("./crypto"); +const { + findPeerByEnvId, + peerSecret, + touchPeerLastSeen +} = require("./peers"); + + +const REQUIRED_HEADERS = ["x-dd-env-id", "x-dd-timestamp", "x-dd-nonce", "x-dd-signature"]; + + +const readRawBody = async (req) => { + const chunks = []; + for await (const chunk of req) { + chunks.push(typeof chunk === "string" ? Buffer.from(chunk) : chunk); + } + return Buffer.concat(chunks).toString("utf8"); +}; + + +const requirePeerAuth = async (req, res) => { + for (const h of REQUIRED_HEADERS) { + if (!req.headers[h]) { + res.status(400).json({ error: `missing header ${h}` }); + return null; + } + } + const callerEnvId = req.headers["x-dd-env-id"]; + const timestamp = req.headers["x-dd-timestamp"]; + const nonce = req.headers["x-dd-nonce"]; + const signature = req.headers["x-dd-signature"]; + + if (!timestampWithinSkew(timestamp)) { + res.status(401).json({ error: "timestamp out of skew window" }); + return null; + } + + const peerRow = await findPeerByEnvId(callerEnvId); + if (!peerRow) { + res.status(401).json({ error: "unknown peer env_id" }); + return null; + } + + let secret; + try { + secret = await peerSecret(peerRow.peer_id); + } catch (e) { + res.status(401).json({ error: "peer not provisioned" }); + return null; + } + + // Read raw bytes (empty for GET/HEAD). HMAC covers exactly what arrived. + let bodyRaw = ""; + if (req.method !== "GET" && req.method !== "HEAD") { + try { + bodyRaw = await readRawBody(req); + } catch (e) { + res.status(400).json({ error: "failed to read request body" }); + return null; + } + } + + const fullPath = req.originalUrl || req.url; + const canonical = buildCanonical({ + timestamp: timestamp, + nonce: nonce, + method: req.method, + path: fullPath, + body: bodyRaw + }); + + if (!verifySignature(secret, canonical, signature)) { + res.status(401).json({ error: "bad signature" }); + return null; + } + + if (bodyRaw) { + try { + req.body = JSON.parse(bodyRaw); + } catch (e) { + res.status(400).json({ error: "body is not valid JSON" }); + return null; + } + } + + await touchPeerLastSeen(peerRow.peer_id); + req.dd_peer = peerRow; + return peerRow; +}; + + +module.exports = { + requirePeerAuth +}; diff --git a/dev-deploy/lib/peers.js b/dev-deploy/lib/peers.js new file mode 100644 index 0000000..6afa1a2 --- /dev/null +++ b/dev-deploy/lib/peers.js @@ -0,0 +1,142 @@ +// Peer model: CRUD on _dd_peers. +// +// Peer secrets are stored AES-256-GCM sealed; plaintext only crosses the +// process boundary at pairing time (when the operator copies the secret to +// the other instance's UI) and at HMAC sign/verify time. + +const db = require("@saltcorn/data/db"); + +const { + seal, + open, + randomSecret +} = require("./crypto"); + + +const rowToPeer = (row) => { + if (!row) return null; + return { + peer_id: row.peer_id, + env_id: row.env_id, + label: row.label, + base_url: row.base_url, + require_tls: !!row.require_tls, + created_at: row.created_at, + last_seen_at: row.last_seen_at, + // sealed components kept out of plain accessors -- use peerSecret() + }; +}; + + +const listPeers = async () => { + const rows = await db.select("_dd_peers", {}, { orderBy: "peer_id" }); + return rows.map(rowToPeer); +}; + + +const findPeer = async (peerId) => { + const row = await db.selectMaybeOne("_dd_peers", { peer_id: peerId }); + return rowToPeer(row); +}; + + +const findPeerByEnvId = async (envId) => { + return await db.selectMaybeOne("_dd_peers", { env_id: envId }); +}; + + +// Returns the plaintext 32-byte secret for an existing peer by id. +// Throws if the peer is missing or has no sealed secret. +const peerSecret = async (peerId) => { + const row = await db.selectMaybeOne("_dd_peers", { peer_id: peerId }); + if (!row) { + throw new Error(`peer ${peerId} not found`); + } + if (!row.peer_secret_ciphertext || !row.peer_secret_iv || !row.peer_secret_tag) { + throw new Error(`peer ${peerId} has no sealed secret`); + } + return open({ + ciphertext: Buffer.from(row.peer_secret_ciphertext, "hex"), + iv: Buffer.from(row.peer_secret_iv, "hex"), + tag: Buffer.from(row.peer_secret_tag, "hex") + }); +}; + + +const peerSecretByEnvId = async (envId) => { + const row = await findPeerByEnvId(envId); + if (!row) { + return null; + } + return await peerSecret(row.peer_id); +}; + + +// Create a new peer. If `existingSecret` is provided, use it; otherwise +// generate a fresh one. Returns { peer, secret } where secret is the plaintext +// Buffer -- only available at this single moment. +const addPeer = async ({ envId, label, baseUrl, requireTls, existingSecret }) => { + if (!envId || !baseUrl) { + throw new Error("addPeer requires envId and baseUrl"); + } + const dup = await findPeerByEnvId(envId); + if (dup) { + throw new Error(`peer with env_id ${envId} already exists`); + } + const secret = existingSecret || randomSecret(); + const sealed = seal(secret); + const row = { + env_id: envId, + label: label || null, + base_url: baseUrl, + peer_secret_ciphertext: sealed.ciphertext.toString("hex"), + peer_secret_iv: sealed.iv.toString("hex"), + peer_secret_tag: sealed.tag.toString("hex"), + require_tls: requireTls ? 1 : 0, + created_at: new Date().toISOString(), + last_seen_at: null + }; + await db.insert("_dd_peers", row); + const fresh = await findPeerByEnvId(envId); + return { peer: rowToPeer(fresh), secret: secret }; +}; + + +const rotatePeerSecret = async (peerId) => { + const peer = await findPeer(peerId); + if (!peer) { + throw new Error(`peer ${peerId} not found`); + } + const secret = randomSecret(); + const sealed = seal(secret); + await db.updateWhere("_dd_peers", { + peer_secret_ciphertext: sealed.ciphertext.toString("hex"), + peer_secret_iv: sealed.iv.toString("hex"), + peer_secret_tag: sealed.tag.toString("hex") + }, { peer_id: peerId }); + return { peer: peer, secret: secret }; +}; + + +const deletePeer = async (peerId) => { + await db.deleteWhere("_dd_peers", { peer_id: peerId }); + await db.deleteWhere("_dd_anchors", { peer_id: peerId }); +}; + + +const touchPeerLastSeen = async (peerId) => { + await db.updateWhere("_dd_peers", { last_seen_at: new Date().toISOString() }, { peer_id: peerId }); +}; + + +module.exports = { + listPeers, + findPeer, + findPeerByEnvId, + peerSecret, + peerSecretByEnvId, + addPeer, + rotatePeerSecret, + deletePeer, + touchPeerLastSeen +}; diff --git a/dev-deploy/lib/revert.js b/dev-deploy/lib/revert.js new file mode 100644 index 0000000..695772a --- /dev/null +++ b/dev-deploy/lib/revert.js @@ -0,0 +1,205 @@ +// Revert: append a compensating op for any local op_id. +// +// create_X → drop the entity that was created +// drop_X → recreate the entity from payload.before +// update_X → re-apply payload.before as a new patch +// +// The revert calls the same Saltcorn model methods the admin UI does, so the +// existing CRUD wraps fire and journal the inverse op naturally with this +// instance's env_id. The inverse op then promotes to peers like any other op. +// +// Caveats: +// - Reverting a drop produces a *new* entity with a new random UUID. The +// original UUID is gone for good. Content is restored; identity is fresh. +// - Reverting a field/view/trigger drop requires the parent table to still +// exist locally. parent_uuid was captured by the drop wrap; we resolve it +// to the current local id. + +const Table = require("@saltcorn/data/models/table"); +const Field = require("@saltcorn/data/models/field"); +const View = require("@saltcorn/data/models/view"); +const Page = require("@saltcorn/data/models/page"); +const Trigger = require("@saltcorn/data/models/trigger"); +const Role = require("@saltcorn/data/models/role"); +const Library = require("@saltcorn/data/models/library"); +const Tag = require("@saltcorn/data/models/tag"); + +const db = require("@saltcorn/data/db"); + +const { lookupByUuid } = require("./entityIds"); +const { ENTITY_KINDS } = require("./constants"); + + +const MODELS = { + [ENTITY_KINDS.TABLE]: Table, + [ENTITY_KINDS.FIELD]: Field, + [ENTITY_KINDS.VIEW]: View, + [ENTITY_KINDS.PAGE]: Page, + [ENTITY_KINDS.TRIGGER]: Trigger, + [ENTITY_KINDS.ROLE]: Role, + [ENTITY_KINDS.LIBRARY]: Library, + [ENTITY_KINDS.TAG]: Tag +}; + + +const stripIds = (obj) => { + const out = { ...(obj || {}) }; + for (const k of ["id", "table_id", "view_id", "page_id"]) delete out[k]; + return out; +}; + + +const { refreshState } = require("./state"); + + +const findLocalInstance = async (kind, entityUuid) => { + const m = await lookupByUuid(entityUuid); + if (!m) return null; + const Cls = MODELS[kind]; + if (!Cls || typeof Cls.findOne !== "function") return null; + await refreshState(); + const inst = await Cls.findOne({ id: m.current_id }); + return inst || null; +}; + + +// --- Drop the entity (revert of create_X) --- +const revertCreate = async (kind, op) => { + const inst = await findLocalInstance(kind, op.entity_uuid); + if (!inst) { + return { status: "noop", reason: "entity no longer present" }; + } + await inst.delete(); + return { status: "dropped" }; +}; + + +// --- Recreate from before-snapshot (revert of drop_X) --- +const recreateFromSnapshot = async (kind, payload) => { + const before = stripIds(payload.before || {}); + const parentUuid = payload.parent_uuid; + + let parentLocalId = null; + if (parentUuid) { + const parent = await lookupByUuid(parentUuid); + if (!parent) { + throw new Error(`parent uuid=${parentUuid} not present locally; cannot recreate ${kind}`); + } + parentLocalId = parent.current_id; + } + + switch (kind) { + case ENTITY_KINDS.TABLE: { + if (!before.name) throw new Error("missing before.name"); + const opts = { ...before }; + const name = opts.name; + delete opts.name; + await Table.create(name, opts); + return { status: "recreated" }; + } + case ENTITY_KINDS.FIELD: { + const cfg = { ...before, table_id: parentLocalId }; + await Field.create(cfg); + return { status: "recreated" }; + } + case ENTITY_KINDS.VIEW: { + const cfg = { ...before }; + if (parentLocalId) cfg.table_id = parentLocalId; + await View.create(cfg); + return { status: "recreated" }; + } + case ENTITY_KINDS.PAGE: { + await Page.create(before); + return { status: "recreated" }; + } + case ENTITY_KINDS.TRIGGER: { + const cfg = { ...before }; + if (parentLocalId) cfg.table_id = parentLocalId; + await Trigger.create(cfg); + return { status: "recreated" }; + } + case ENTITY_KINDS.ROLE: { + if (!before.id || !before.role) throw new Error("missing role identity"); + await Role.create({ id: before.id, role: before.role }); + return { status: "recreated" }; + } + case ENTITY_KINDS.LIBRARY: { + await Library.create(before); + return { status: "recreated" }; + } + case ENTITY_KINDS.TAG: { + await Tag.create(before); + return { status: "recreated" }; + } + default: + throw new Error(`unknown kind ${kind}`); + } +}; + + +// --- Re-apply payload.before as a new patch (revert of update_X) --- +const reapplyBefore = async (kind, op, payload) => { + const inst = await findLocalInstance(kind, op.entity_uuid); + if (!inst) { + throw new Error(`entity_uuid=${op.entity_uuid} (${kind}) not present locally`); + } + const patch = stripIds(payload.before || {}); + if (Object.keys(patch).length === 0) { + return { status: "noop", reason: "empty before-snapshot" }; + } + switch (kind) { + case ENTITY_KINDS.TABLE: + case ENTITY_KINDS.FIELD: + case ENTITY_KINDS.ROLE: + case ENTITY_KINDS.LIBRARY: + case ENTITY_KINDS.TAG: + await inst.update(patch); + return { status: "updated" }; + case ENTITY_KINDS.VIEW: + await View.update(patch, inst.id); + return { status: "updated" }; + case ENTITY_KINDS.PAGE: + await Page.update(inst.id, patch); + return { status: "updated" }; + case ENTITY_KINDS.TRIGGER: + await Trigger.update(inst.id, patch); + return { status: "updated" }; + default: + throw new Error(`unknown kind ${kind}`); + } +}; + + +const ACTION_HANDLERS = { + create: revertCreate, + drop: recreateFromSnapshot, + update: reapplyBefore +}; + + +// Look up the original op, parse its payload, dispatch to the inverse action. +// The local wrap journals the inverse as a new op with a fresh op_id. +const revertOp = async (opId) => { + const orig = await db.selectMaybeOne("_dd_ops", { op_id: opId }); + if (!orig) { + throw new Error(`op ${opId} not found`); + } + const [action, kind] = (orig.op_type || "").split("_", 2); + const handler = ACTION_HANDLERS[action]; + if (!handler) { + throw new Error(`no revert handler for action '${action}' (op_type=${orig.op_type})`); + } + if (!MODELS[kind]) { + throw new Error(`no model for kind '${kind}'`); + } + const payload = typeof orig.payload === "string" ? JSON.parse(orig.payload) : (orig.payload || {}); + if (action === "drop") { + return await handler(kind, payload); + } + return await handler(kind, orig, payload); +}; + + +module.exports = { + revertOp +}; diff --git a/dev-deploy/lib/routes.js b/dev-deploy/lib/routes.js new file mode 100644 index 0000000..6689c2a --- /dev/null +++ b/dev-deploy/lib/routes.js @@ -0,0 +1,1060 @@ +// HTTP routes for dev-deploy. +// +// Admin UI (session + admin role): +// GET /admin/dev-deploy/ +// GET /admin/dev-deploy/ops +// GET /admin/dev-deploy/peers +// POST /admin/dev-deploy/peers/add +// POST /admin/dev-deploy/peers/rotate +// POST /admin/dev-deploy/peers/delete +// GET /admin/dev-deploy/plan +// POST /admin/dev-deploy/promote +// +// Machine API (HMAC peer auth): +// GET /dev-deploy/api/journal?since=op_id +// POST /dev-deploy/api/ingest + +const db = require("@saltcorn/data/db"); + +const { PLUGIN_NAME, PLUGIN_VERSION } = require("./constants"); +const { getEnv } = require("./env"); +const peers = require("./peers"); +const { requirePeerAuth } = require("./peerAuth"); +const { signedFetch } = require("./transport"); +const { applyBatch, resolveConflict, resolveConflictByMerge, conflictFieldDiff } = require("./apply"); +const { revertOp } = require("./revert"); +const { DATA_MODES } = require("./constants"); + + +const getInboundAnchor = async (peerId) => { + return await db.selectMaybeOne("_dd_anchors", { peer_id: peerId, direction: "inbound" }); +}; + + +const getOutboundAnchor = async (peerId) => { + return await db.selectMaybeOne("_dd_anchors", { peer_id: peerId, direction: "outbound" }); +}; + + +const upsertAnchor = async (peerId, direction, opId) => { + const now = new Date().toISOString(); + const existing = await db.selectMaybeOne("_dd_anchors", { peer_id: peerId, direction: direction }); + if (existing) { + await db.updateWhere("_dd_anchors", { last_op_id: opId, updated_at: now }, { peer_id: peerId, direction: direction }); + } else { + await db.insert("_dd_anchors", { peer_id: peerId, direction: direction, last_op_id: opId, updated_at: now }, { noid: true }); + } +}; + + +const isAdmin = (req) => !!(req && req.user && req.user.role_id === 1); + + +const escape = (s) => { + if (s === null || s === undefined) return ""; + return String(s) + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """); +}; + + +const csrfField = (req) => { + const t = req.csrfToken ? req.csrfToken() : ""; + return ``; +}; + + +const layout = (title, body, flash) => ` + + +${escape(title)} + + + +${flash || ""} +${body} +`; + + +const flashMsg = (req) => { + const m = req.query.msg; + const e = req.query.err; + if (m) return `
${escape(m)}
`; + if (e) return `
${escape(e)}
`; + return ""; +}; + + +// ---------------- Admin dashboard ---------------- + +const dashboard = async (req, res) => { + if (!isAdmin(req)) { res.status(403).type("text/plain").send("admin only"); return; } + const env = await getEnv(); + const opCount = (await db.query(`SELECT COUNT(*) AS c FROM _dd_ops`)).rows[0].c; + const opsByKind = (await db.query(`SELECT op_type, COUNT(*) AS c FROM _dd_ops GROUP BY op_type ORDER BY op_type`)).rows; + const entCounts = (await db.query(`SELECT kind, COUNT(*) AS c FROM _dd_entity_ids GROUP BY kind ORDER BY kind`)).rows; + const peerList = await peers.listPeers(); + const conflictCount = (await db.query(`SELECT COUNT(*) AS c FROM _dd_ops WHERE status='conflict'`)).rows[0].c; + + const opsByKindRows = opsByKind.length === 0 + ? `No ops recorded yet` + : opsByKind.map((r) => `${escape(r.op_type)}${escape(r.c)}`).join(""); + + const entRows = entCounts.length === 0 + ? `No entities tracked` + : entCounts.map((r) => `${escape(r.kind)}${escape(r.c)}`).join(""); + + const body = ` +

dev-deploy dashboard

+ + + + + + + + + +
Env ID${escape(env ? env.env_id : "?")}
Label${env && env.env_label ? escape(env.env_label) : '(unset)'}
Destructive-op policy${escape(env ? env.on_destructive_op : "?")}
Require TLS (default)${env && env.require_tls ? "yes" : "no"}
Bootstrapped at${escape(env ? env.bootstrapped_at : "")}
Ops recorded${escape(opCount)}
Peers configured${escape(peerList.length)}
Pending conflicts${conflictCount > 0 ? `${escape(conflictCount)}` : "0"}
+

Ops by type

+ ${opsByKindRows}
op_typecount
+

Entities tracked

+ ${entRows}
kindcount
+ `; + res.type("text/html").send(layout("dev-deploy dashboard", body, flashMsg(req))); +}; + + +// ---------------- Ops viewer ---------------- + +const fetchOps = async (limit, since) => { + let sql = `SELECT op_id, source_env_id, op_type, entity_kind, entity_uuid, payload, parent_op_id, correlation_id, schema_version, created_at, applied_at, status FROM _dd_ops`; + const params = []; + if (since) { + const anchor = (await db.query(`SELECT created_at FROM _dd_ops WHERE op_id = $1`, [since])).rows[0]; + if (anchor) { + sql += ` WHERE created_at > $${params.length + 1}`; + params.push(anchor.created_at); + } + } + sql += ` ORDER BY created_at DESC LIMIT $${params.length + 1}`; + params.push(limit || 100); + return (await db.query(sql, params)).rows; +}; + + +const opsView = async (req, res) => { + if (!isAdmin(req)) { res.status(403).type("text/plain").send("admin only"); return; } + const wantJson = (req.headers.accept || "").includes("application/json"); + const limit = Math.min(parseInt(req.query.limit || "100", 10) || 100, 1000); + const since = req.query.since; + const ops = await fetchOps(limit, since); + if (wantJson) { res.json({ ops: ops }); return; } + const rows = ops.length === 0 + ? `Journal is empty` + : ops.map((o) => ` + ${escape(o.op_id.slice(0, 8))} + ${escape(o.op_type)} + ${escape((o.entity_uuid || "").slice(0, 8))} + ${escape((o.parent_op_id || "").slice(0, 8))} + ${escape(o.status)} + ${escape(o.created_at)} + +
+ ${csrfField(req)} + + +
+ + 
${escape(o.payload)}
+ `).join(""); + const body = ` +

Journal

+

Showing up to ${escape(limit)} ops${since ? `, since op ${escape(since.slice(0, 8))}` : ""}. Newest first. Revert appends a compensating op rather than rewriting history.

+ + + ${rows} +
opop_typeentityparentstatuscreatedactionspayload
+ `; + res.type("text/html").send(layout("dev-deploy journal", body, flashMsg(req))); +}; + + +// ---------------- Peers ---------------- + +const peersView = async (req, res) => { + if (!isAdmin(req)) { res.status(403).type("text/plain").send("admin only"); return; } + const env = await getEnv(); + const list = await peers.listPeers(); + const rows = list.length === 0 + ? `No peers yet` + : list.map((p) => ` + ${escape(p.peer_id)} + ${escape(p.label || "(unset)")} + ${escape(p.env_id)} + ${escape(p.base_url)} + ${escape(p.last_seen_at || "never")} + +
+ ${csrfField(req)} + + +
+
+ ${csrfField(req)} + + +
+
+ ${csrfField(req)} + + +
+
+ ${csrfField(req)} + + +
+ + `).join(""); + + const body = ` +

Peers

+

This instance's env_id is ${escape(env ? env.env_id : "?")}. Paste this into the other instance's peer form.

+ + + ${rows} +
idlabelenv_idbase_urllast seenactions
+ +

Add peer

+
+ ${csrfField(req)} +
+ Peer info +

+

+

+

+
+
+ Shared secret +

Leave blank to generate one (shown once after submit). Paste the same secret in the peer's own pairing form.

+

+
+

+
+ `; + res.type("text/html").send(layout("dev-deploy peers", body, flashMsg(req))); +}; + + +const peersAdd = async (req, res) => { + if (!isAdmin(req)) { res.status(403).type("text/plain").send("admin only"); return; } + try { + const envId = (req.body.env_id || "").trim(); + const label = (req.body.label || "").trim() || null; + const baseUrl = (req.body.base_url || "").trim(); + const requireTls = !!req.body.require_tls; + const provided = (req.body.existing_secret || "").trim(); + let existingSecret = null; + if (provided) { + if (!/^[0-9a-fA-F]{64}$/.test(provided)) { + throw new Error("existing_secret must be 64 hex characters"); + } + existingSecret = Buffer.from(provided, "hex"); + } + const { peer, secret } = await peers.addPeer({ envId: envId, label: label, baseUrl: baseUrl, requireTls: requireTls, existingSecret: existingSecret }); + const secretHex = secret.toString("hex"); + const body = ` +

Peer ${escape(peer.label || peer.env_id)} paired

+

Copy this secret into the peer's pairing form (it will not be shown again):

+

${escape(secretHex)}

+

If this is a brand-new pairing, give the peer this side's env_id too.

+

Back to peers

+ `; + res.type("text/html").send(layout("Peer paired", body)); + } catch (err) { + res.redirect("/admin/dev-deploy/peers?err=" + encodeURIComponent(err.message)); + } +}; + + +const peersRotate = async (req, res) => { + if (!isAdmin(req)) { res.status(403).type("text/plain").send("admin only"); return; } + try { + const peerId = parseInt(req.body.peer_id, 10); + const { peer, secret } = await peers.rotatePeerSecret(peerId); + const body = ` +

Peer ${escape(peer.label || peer.env_id)} secret rotated

+

New secret (shown once):

+

${escape(secret.toString("hex"))}

+

Paste this on the other side via Rotate or by re-pairing.

+

Back to peers

+ `; + res.type("text/html").send(layout("Secret rotated", body)); + } catch (err) { + res.redirect("/admin/dev-deploy/peers?err=" + encodeURIComponent(err.message)); + } +}; + + +const peersDelete = async (req, res) => { + if (!isAdmin(req)) { res.status(403).type("text/plain").send("admin only"); return; } + try { + const peerId = parseInt(req.body.peer_id, 10); + await peers.deletePeer(peerId); + res.redirect("/admin/dev-deploy/peers?msg=" + encodeURIComponent("peer deleted")); + } catch (err) { + res.redirect("/admin/dev-deploy/peers?err=" + encodeURIComponent(err.message)); + } +}; + + +// ---------------- Plan + promote ---------------- + +const planView = async (req, res) => { + if (!isAdmin(req)) { res.status(403).type("text/plain").send("admin only"); return; } + const peerIdRaw = req.query.peer; + const peerList = await peers.listPeers(); + if (!peerIdRaw) { + const opts = peerList.length === 0 + ? `` + : peerList.map((p) => ``).join(""); + const body = ` +

Plan

+
+ + +
+ `; + res.type("text/html").send(layout("dev-deploy plan", body, flashMsg(req))); + return; + } + const peerId = parseInt(peerIdRaw, 10); + const peer = await peers.findPeer(peerId); + if (!peer) { res.status(404).send("peer not found"); return; } + const env = await getEnv(); + // Anchor: the last_op_id we sent outbound to this peer (or epoch) + const anchor = await db.selectMaybeOne("_dd_anchors", { peer_id: peerId, direction: "outbound" }); + let sql = `SELECT op_id, op_type, entity_kind, entity_uuid, payload, parent_op_id, status, created_at FROM _dd_ops WHERE source_env_id = $1`; + const params = [env.env_id]; + if (anchor) { + const anchorRow = await db.selectMaybeOne("_dd_ops", { op_id: anchor.last_op_id }); + if (anchorRow) { + sql += ` AND created_at > $${params.length + 1}`; + params.push(anchorRow.created_at); + } + } + sql += ` ORDER BY created_at ASC LIMIT 500`; + const planRows = (await db.query(sql, params)).rows; + const rowsHtml = planRows.length === 0 + ? `No new ops to send` + : planRows.map((o) => ` + ${escape(o.op_id.slice(0, 8))} + ${escape(o.op_type)} + ${escape((o.entity_uuid || "").slice(0, 8))} + ${escape(o.status)} + ${escape(o.created_at)} + `).join(""); + const body = ` +

Plan: promote to ${escape(peer.label || peer.env_id)}

+

Anchor: ${anchor ? `${escape(anchor.last_op_id.slice(0, 8))}` : '(none — will send from epoch)'}

+

Ops that would be sent: ${escape(planRows.length)}

+ + + ${rowsHtml} +
opop_typeentitystatuscreated
+
+ ${csrfField(req)} + +

+
+ `; + res.type("text/html").send(layout("dev-deploy plan", body, flashMsg(req))); +}; + + +const promote = async (req, res) => { + if (!isAdmin(req)) { res.status(403).type("text/plain").send("admin only"); return; } + try { + const peerId = parseInt(req.body.peer_id, 10); + const peer = await peers.findPeer(peerId); + if (!peer) throw new Error(`peer ${peerId} not found`); + const env = await getEnv(); + const anchor = await getOutboundAnchor(peerId); + let sql = `SELECT op_id, source_env_id, op_type, entity_kind, entity_uuid, payload, parent_op_id, correlation_id, schema_version, created_at, status FROM _dd_ops WHERE source_env_id = $1`; + const params = [env.env_id]; + if (anchor) { + const anchorRow = await db.selectMaybeOne("_dd_ops", { op_id: anchor.last_op_id }); + if (anchorRow) { + sql += ` AND created_at > $${params.length + 1}`; + params.push(anchorRow.created_at); + } + } + sql += ` ORDER BY created_at ASC LIMIT 500`; + const ops = (await db.query(sql, params)).rows; + if (ops.length === 0) { + res.redirect("/admin/dev-deploy/peers?msg=" + encodeURIComponent("no ops to promote")); + return; + } + const secret = await peers.peerSecret(peerId); + const r = await signedFetch({ + baseUrl: peer.base_url, + method: "POST", + path: "/dev-deploy/api/ingest", + body: { ops: ops }, + sourceEnvId: env.env_id, + secret: secret + }); + if (!r.ok) { + throw new Error(`peer responded ${r.status}: ${JSON.stringify(r.body)}`); + } + await upsertAnchor(peerId, "outbound", ops[ops.length - 1].op_id); + const applied = (r.body && r.body.results || []).filter((x) => x.status === "applied").length; + const errors = (r.body && r.body.results || []).filter((x) => x.status === "error").length; + let msg = `promoted ${ops.length} ops (${applied} applied, ${errors} errors)`; + const localPlugins = (await db.query(`SELECT name, source, version FROM _sc_plugins ORDER BY name`)).rows; + const warnings = await diffPluginsWithPeer(peer, env, localPlugins); + if (warnings.length > 0) { + msg += " | WARNINGS: " + warnings.join("; "); + } + res.redirect("/admin/dev-deploy/peers?msg=" + encodeURIComponent(msg)); + } catch (err) { + res.redirect("/admin/dev-deploy/peers?err=" + encodeURIComponent(err.message)); + } +}; + + +const pull = async (req, res) => { + if (!isAdmin(req)) { res.status(403).type("text/plain").send("admin only"); return; } + try { + const peerId = parseInt(req.body.peer_id, 10); + const peer = await peers.findPeer(peerId); + if (!peer) throw new Error(`peer ${peerId} not found`); + const env = await getEnv(); + const anchor = await getInboundAnchor(peerId); + const since = anchor ? anchor.last_op_id : null; + const path = since ? `/dev-deploy/api/journal?since=${encodeURIComponent(since)}` : "/dev-deploy/api/journal"; + const secret = await peers.peerSecret(peerId); + const r = await signedFetch({ + baseUrl: peer.base_url, + method: "GET", + path: path, + body: null, + sourceEnvId: env.env_id, + secret: secret + }); + if (!r.ok) { + throw new Error(`peer responded ${r.status}: ${JSON.stringify(r.body)}`); + } + const ops = (r.body && r.body.ops) || []; + if (ops.length === 0) { + res.redirect("/admin/dev-deploy/peers?msg=" + encodeURIComponent("nothing to pull")); + return; + } + const results = await applyBatch(ops, { peerId: peerId, myEnvId: env.env_id }); + const applied = results.filter((x) => x.status === "applied").length; + const errors = results.filter((x) => x.status === "error").length; + const conflicts = results.filter((x) => x.status === "conflict").length; + await upsertAnchor(peerId, "inbound", ops[ops.length - 1].op_id); + let sum = `pulled ${ops.length} ops (${applied} applied, ${errors} errors, ${conflicts} conflicts)`; + const localPlugins = (await db.query(`SELECT name, source, version FROM _sc_plugins ORDER BY name`)).rows; + const warnings = await diffPluginsWithPeer(peer, env, localPlugins); + if (warnings.length > 0) { + sum += " | WARNINGS: " + warnings.join("; "); + } + const dest = conflicts > 0 ? "/admin/dev-deploy/conflicts?msg=" : "/admin/dev-deploy/peers?msg="; + res.redirect(dest + encodeURIComponent(sum)); + } catch (err) { + res.redirect("/admin/dev-deploy/peers?err=" + encodeURIComponent(err.message)); + } +}; + + +// ---------------- Conflicts ---------------- + +const conflictsView = async (req, res) => { + if (!isAdmin(req)) { res.status(403).type("text/plain").send("admin only"); return; } + + const conflicts = (await db.query(` + SELECT i.op_id AS i_op_id, + i.source_env_id AS i_source, + i.op_type AS i_op_type, + i.entity_kind AS i_kind, + i.entity_uuid AS i_uuid, + i.payload AS i_payload, + i.created_at AS i_created, + l.op_id AS l_op_id, + l.op_type AS l_op_type, + l.payload AS l_payload, + l.applied_at AS l_applied + FROM _dd_ops i + LEFT JOIN _dd_ops l ON l.op_id = i.conflict_with_op_id + WHERE i.status = 'conflict' + ORDER BY i.created_at ASC + `)).rows; + + const isMergeable = (c) => + c.i_op_type && c.l_op_type && + c.i_op_type.startsWith("update_") && + c.l_op_type.startsWith("update_") && + c.i_op_type === c.l_op_type; + + const rowsHtml = conflicts.length === 0 + ? `No pending conflicts` + : conflicts.map((c) => ` + + incoming ${escape(c.i_op_id.slice(0, 8))} from ${escape((c.i_source || "").slice(0, 8))}
+ ${escape(c.i_op_type)} on entity ${escape((c.i_uuid || "").slice(0, 8))}
+ created ${escape(c.i_created)} + 
${escape(c.i_payload)}
+ + + ${c.l_op_id ? `local ${escape(c.l_op_id.slice(0, 8))}
+ ${escape(c.l_op_type)}
+ applied ${escape(c.l_applied || "")} + 
${escape(c.l_payload)}
` : '(no local op recorded)'} + + + ${isMergeable(c) ? `

` : ""} +
+ ${csrfField(req)} + + + +
+
+ ${csrfField(req)} + + + +
+ + `).join(""); + + const body = ` +

Pending conflicts

+

A conflict means an incoming op and a local op both touched the same entity since the last sync. The incoming op was NOT applied; pick which version wins.

+
    +
  • Use theirs: applies the incoming op now (overwrites local change). The local op stays in the journal but its effect is overwritten.
    • +
  • Use mine: marks the incoming op as rejected. The local state stands. The peer may re-send the op on future syncs; subsequent pulls will skip it via idempotency.
    • +
+ + + ${rowsHtml} +
TheirsMineAction
+ `; + res.type("text/html").send(layout("dev-deploy conflicts", body, flashMsg(req))); +}; + + +const conflictsResolve = async (req, res) => { + if (!isAdmin(req)) { res.status(403).type("text/plain").send("admin only"); return; } + try { + const opId = (req.body.op_id || "").trim(); + const action = (req.body.action || "").trim(); + if (!opId) throw new Error("op_id required"); + if (!["theirs", "mine"].includes(action)) throw new Error("action must be 'theirs' or 'mine'"); + const r = await resolveConflict(opId, action); + res.redirect("/admin/dev-deploy/conflicts?msg=" + encodeURIComponent(`resolved ${opId.slice(0, 8)} with action=${action}: ${JSON.stringify(r)}`)); + } catch (err) { + res.redirect("/admin/dev-deploy/conflicts?err=" + encodeURIComponent(err.message)); + } +}; + + +const renderValue = (v) => { + if (v === null || v === undefined) { + return '(unset)'; + } + if (typeof v === "object") { + return `
${escape(JSON.stringify(v, null, 2))}
`; + } + return escape(String(v)); +}; + + +const conflictsMergeView = async (req, res) => { + if (!isAdmin(req)) { res.status(403).type("text/plain").send("admin only"); return; } + const opId = (req.query.op_id || "").trim(); + if (!opId) { res.redirect("/admin/dev-deploy/conflicts?err=op_id+required"); return; } + try { + const op = await db.selectMaybeOne("_dd_ops", { op_id: opId }); + if (!op) throw new Error(`op ${opId} not found`); + if (op.status !== "conflict") throw new Error(`op ${opId} is not in conflict status`); + + const diff = await conflictFieldDiff(op); + const diffs = diff.diffs || []; + + const ent = diff.instance + ? `

Entity: ${escape(diff.kind)} ${escape(diff.instance.name || diff.instance.role || diff.instance.id)} (local id ${escape(diff.instance.id)})

` + : `

${escape(diff.reason || "no entity diff available")}

`; + + let formBody; + if (diffs.length === 0) { + formBody = `

No field-level differences detected (current state already matches the incoming op's patch on every field). Applying the merge will just mark this conflict as resolved.

`; + } else { + const rows = diffs.map((d) => ` + ${escape(d.field)} + ${renderValue(d.currentValue)} + ${renderValue(d.incomingValue)} + +
+
+ + + `).join(""); + formBody = ` + + + ${rows} +
fieldcurrent (mine)incoming (theirs)resolution
+

Defaults to keep current for every field — submit as-is for a no-op resolution that just clears the conflict marker.

+ `; + } + + const body = ` +

Merge conflict per field

+ ${ent} +

Incoming op ${escape(op.op_id.slice(0, 8))} from ${escape((op.source_env_id || "").slice(0, 8))}; ${escape(op.op_type)}.

+
+ ${csrfField(req)} + + ${formBody} +

+ + Cancel +

+
+ `; + res.type("text/html").send(layout("dev-deploy merge", body, flashMsg(req))); + } catch (err) { + res.redirect("/admin/dev-deploy/conflicts?err=" + encodeURIComponent(err.message)); + } +}; + + +// Parse number/boolean/null literal strings from a text input. JSON.parse first +// (handles "true", "42", "null"); fall back to the raw string. +const coerce = (s) => { + if (s === undefined || s === null) return s; + try { + const parsed = JSON.parse(s); + if (parsed === null || ["boolean", "number"].includes(typeof parsed)) return parsed; + return s; + } catch (e) { + return s; + } +}; + + +const conflictsMergeApply = async (req, res) => { + if (!isAdmin(req)) { res.status(403).type("text/plain").send("admin only"); return; } + try { + const opId = (req.body.op_id || "").trim(); + if (!opId) throw new Error("op_id required"); + + const op = await db.selectMaybeOne("_dd_ops", { op_id: opId }); + if (!op) throw new Error(`op ${opId} not found`); + if (op.status !== "conflict") throw new Error(`op ${opId} is not in conflict status`); + const payload = typeof op.payload === "string" ? JSON.parse(op.payload) : (op.payload || {}); + const incomingPatch = payload.patch || {}; + + // For each "choice_" entry in the form body, decide what value + // to write -- if any. "current" means don't touch the field. + const choices = {}; + for (const [k, v] of Object.entries(req.body || {})) { + if (!k.startsWith("choice_")) continue; + const field = k.substring("choice_".length); + if (v === "incoming") { + choices[field] = incomingPatch[field]; + } else if (v === "custom") { + const customVal = req.body[`custom_${field}`]; + choices[field] = coerce(customVal); + } + } + + const r = await resolveConflictByMerge(opId, choices); + res.redirect("/admin/dev-deploy/conflicts?msg=" + encodeURIComponent(`merged ${opId.slice(0, 8)}: ${JSON.stringify(r)}`)); + } catch (err) { + res.redirect("/admin/dev-deploy/conflicts?err=" + encodeURIComponent(err.message)); + } +}; + + +// ---------------- Tables (data_mode) ---------------- + +const tablesView = async (req, res) => { + if (!isAdmin(req)) { res.status(403).type("text/plain").send("admin only"); return; } + const rows = (await db.query(` + SELECT e.uuid, e.current_name, e.current_id, + COALESCE(m.data_mode, 'user') AS data_mode, + m.updated_at, + m.starter_shipped_at + FROM _dd_entity_ids e + LEFT JOIN _dd_table_modes m ON m.table_uuid = e.uuid + WHERE e.kind = 'table' + ORDER BY e.current_name + `)).rows; + + const lockedNames = new Set(["users"]); + const modeOpts = [DATA_MODES.USER, DATA_MODES.STARTER, DATA_MODES.MANAGED]; + + const rowsHtml = rows.length === 0 + ? `No tables tracked yet` + : rows.map((r) => { + const locked = lockedNames.has(r.current_name); + const select = locked + ? `user (locked)` + : `
+ ${csrfField(req)} + + + +
`; + const shipped = r.starter_shipped_at ? `
shipped ${escape(r.starter_shipped_at)}` : ""; + return ` + ${escape(r.current_name)} + ${escape(r.uuid.slice(0, 8))} + ${escape(r.current_id)} + ${select}${shipped} + ${escape(r.updated_at || "—")} + `; + }).join(""); + + const body = ` +

Tables — data mode

+

Controls how each table's row content propagates between environments. Choose carefully — switching from user to managed or starter rewrites the table's schema (adds a hidden _dd_row_uuid column) and ships existing rows.

+
    +
  • user (default) — rows belong to the local environment; deploys never touch them. The only safe choice for end-user-entered data.
    • +
  • starter — rows ship to target on first install, then the target owns them; future changes on this side don't propagate. Good for default user roles, sample categories, template data the user expects to customize.
    • +
  • managed — rows always sync from source. Source is canonical; target's edits get overwritten or surface as conflicts. Good for catalogs, lookup tables, anything dev-curated.
    • +
+

The Saltcorn users table is locked to user and cannot be changed.

+ + + ${rowsHtml} +
tableuuidlocal iddata_modeupdated_at
+ `; + res.type("text/html").send(layout("dev-deploy tables", body, flashMsg(req))); +}; + + +const tablesSet = async (req, res) => { + if (!isAdmin(req)) { res.status(403).type("text/plain").send("admin only"); return; } + try { + const tableUuid = (req.body.table_uuid || "").trim(); + const dataMode = (req.body.data_mode || "").trim(); + if (!tableUuid) throw new Error("table_uuid required"); + const allowed = new Set(Object.values(DATA_MODES)); + if (!allowed.has(dataMode)) throw new Error(`data_mode must be one of ${[...allowed].join(", ")}`); + const ent = await db.selectMaybeOne("_dd_entity_ids", { uuid: tableUuid }); + if (!ent || ent.kind !== "table") throw new Error("table not found"); + if (ent.current_name === "users") throw new Error("the users table is locked to data_mode=user"); + + const { ensureManagedSchema, dropManagedSchema, allRowsWithUuid, setRowUuid, newRowUuid, COLUMN_NAME: ROW_UUID_COL } = require("./rowIdentity"); + const { rowToPortable, markStarterShipped } = require("./rowPayload"); + const Table = require("@saltcorn/data/models/table"); + const { randomUuid } = require("./ids"); + const { enterOp } = require("./context"); + const { recordOpSafely } = require("./ops"); + const { refreshState } = require("./state"); + + await refreshState(); + + const prior = await db.selectMaybeOne("_dd_table_modes", { table_uuid: tableUuid }); + const now = new Date().toISOString(); + + // Upsert mode row first. + if (prior) { + await db.updateWhere("_dd_table_modes", { data_mode: dataMode, updated_at: now, starter_shipped_at: null }, { table_uuid: tableUuid }); + } else { + await db.insert("_dd_table_modes", { table_uuid: tableUuid, data_mode: dataMode, updated_at: now, starter_shipped_at: null }, { noid: true }); + } + + // Journal set_table_mode FIRST so target's apply sees mode change before any row ops. + { + const opId = randomUuid(); + await enterOp(opId, async () => { + await recordOpSafely({ + op_id: opId, + op_type: "set_table_mode", + entity_kind: "table_mode", + entity_uuid: tableUuid, + payload: { table_uuid: tableUuid, data_mode: dataMode } + }); + }); + } + + let initialShipped = 0; + if (dataMode === DATA_MODES.MANAGED || dataMode === DATA_MODES.STARTER) { + // Make sure THIS instance has the hidden column + UUIDs on existing rows. + await ensureManagedSchema(ent.current_name); + + // Initial ship: journal an insert_row op for every existing row. + const table = Table.findOne({ id: ent.current_id }); + if (table) { + const rows = await allRowsWithUuid(ent.current_name); + for (const row of rows) { + let rowUuid = row[ROW_UUID_COL]; + if (!rowUuid) { + rowUuid = newRowUuid(); + await setRowUuid(ent.current_name, row.id, rowUuid); + } + const { portable } = await rowToPortable(row, table); + const opId = randomUuid(); + await enterOp(opId, async () => { + await recordOpSafely({ + op_id: opId, + op_type: "insert_row", + entity_kind: "table_row", + entity_uuid: rowUuid, + payload: { table_uuid: tableUuid, after: portable } + }); + }); + initialShipped++; + } + } + // For starter: lock out further row ops. + if (dataMode === DATA_MODES.STARTER) { + await markStarterShipped(tableUuid); + } + } else if (prior && (prior.data_mode === DATA_MODES.MANAGED || prior.data_mode === DATA_MODES.STARTER)) { + // Reverting to user — drop the hidden column for cleanliness. Best-effort. + try { + await dropManagedSchema(ent.current_name); + } catch (e) { + // ignore on older SQLite that doesn't support DROP COLUMN + } + } + + const summary = initialShipped > 0 + ? `set ${ent.current_name} to ${dataMode}; shipped ${initialShipped} rows` + : `set ${ent.current_name} to ${dataMode}`; + res.redirect("/admin/dev-deploy/tables?msg=" + encodeURIComponent(summary)); + } catch (err) { + res.redirect("/admin/dev-deploy/tables?err=" + encodeURIComponent(err.message)); + } +}; + + +// ---------------- Revert ---------------- + +const revertView = async (req, res) => { + if (!isAdmin(req)) { res.status(403).type("text/plain").send("admin only"); return; } + try { + const opId = (req.body.op_id || "").trim(); + if (!opId) throw new Error("op_id required"); + const result = await revertOp(opId); + res.redirect("/admin/dev-deploy/ops?msg=" + encodeURIComponent(`reverted op ${opId.slice(0, 8)}: ${JSON.stringify(result)}`)); + } catch (err) { + res.redirect("/admin/dev-deploy/ops?err=" + encodeURIComponent(err.message)); + } +}; + + +// ---------------- Machine endpoints ---------------- + +const apiJournal = async (req, res) => { + const peer = await requirePeerAuth(req, res); + if (!peer) return; + const since = req.query.since; + const env = await getEnv(); + let sql = `SELECT op_id, source_env_id, op_type, entity_kind, entity_uuid, payload, parent_op_id, correlation_id, schema_version, created_at, status FROM _dd_ops WHERE source_env_id = $1`; + const params = [env.env_id]; + if (since) { + const anchorRow = await db.selectMaybeOne("_dd_ops", { op_id: since }); + if (anchorRow) { + sql += ` AND created_at > $${params.length + 1}`; + params.push(anchorRow.created_at); + } + } + sql += ` ORDER BY created_at ASC LIMIT 1000`; + const ops = (await db.query(sql, params)).rows; + res.json({ source_env_id: env.env_id, ops: ops }); +}; + + +const apiHealth = async (req, res) => { + const peer = await requirePeerAuth(req, res); + if (!peer) return; + const env = await getEnv(); + const plugins = (await db.query(`SELECT name, source, version FROM _sc_plugins ORDER BY name`)).rows; + res.json({ + env_id: env.env_id, + label: env.env_label, + plugins: plugins + }); +}; + + +// Compare local plugin list with peer's. Returns array of human-readable +// warning strings (empty if all match). Best-effort: if the peer's health +// endpoint is unreachable or returns non-200, returns a single "couldn't +// reach peer's health endpoint" warning and lets the caller proceed. +const diffPluginsWithPeer = async (peerRow, env, localPlugins) => { + let r; + try { + const secret = await peers.peerSecret(peerRow.peer_id); + r = await signedFetch({ + baseUrl: peerRow.base_url, + method: "GET", + path: "/dev-deploy/api/health", + body: null, + sourceEnvId: env.env_id, + secret: secret + }); + } catch (e) { + return [`could not check peer plugin list: ${e.message}`]; + } + if (!r.ok || !r.body || !Array.isArray(r.body.plugins)) { + return [`peer's health endpoint returned ${r.status}`]; + } + const localByName = new Map(localPlugins.map((p) => [p.name, p])); + const peerByName = new Map(r.body.plugins.map((p) => [p.name, p])); + const warnings = []; + for (const [name, mine] of localByName) { + const theirs = peerByName.get(name); + if (!theirs) { + warnings.push(`peer missing plugin "${name}"`); + } else if ((mine.version || "") !== (theirs.version || "")) { + warnings.push(`plugin version mismatch on "${name}": local ${mine.version || "?"}, peer ${theirs.version || "?"}`); + } + } + for (const [name, theirs] of peerByName) { + if (!localByName.has(name)) { + warnings.push(`peer has plugin not installed here: "${name}"`); + } + } + return warnings; +}; + + +const apiFile = async (req, res) => { + try { + const peer = await requirePeerAuth(req, res); + if (!peer) return; + const uuid = req.params.uuid; + if (!uuid) { + res.status(400).json({ error: "uuid required" }); + return; + } + const mapping = await db.selectMaybeOne("_dd_entity_ids", { uuid: uuid, kind: "file" }); + if (!mapping) { + res.status(404).json({ error: "file not found", uuid: uuid }); + return; + } + const path = require("path"); + const dbMod = require("@saltcorn/data/db"); + const absPath = path.join(dbMod.connectObj.file_store, dbMod.getTenantSchema(), mapping.current_name); + res.type("application/octet-stream"); + // dotfiles: 'allow' so paths containing .dev-state (etc.) aren't + // silently treated as not-found by Express's default dotfile policy. + res.sendFile(absPath, { dotfiles: "allow" }, (err) => { + if (err && !res.headersSent) { + // eslint-disable-next-line no-console + console.error(`[dev-deploy] sendFile failed for ${absPath}:`, err.message); + res.status(500).json({ error: "failed to read file: " + err.message, path: absPath }); + } + }); + } catch (err) { + // eslint-disable-next-line no-console + console.error(`[dev-deploy] apiFile crashed:`, err && err.stack ? err.stack : err); + if (!res.headersSent) { + res.status(500).json({ error: err.message }); + } + } +}; + + +const apiIngest = async (req, res) => { + const peer = await requirePeerAuth(req, res); + if (!peer) return; + const ops = (req.body && req.body.ops) || []; + if (!Array.isArray(ops)) { + res.status(400).json({ error: "ops must be an array" }); + return; + } + const env = await getEnv(); + const results = await applyBatch(ops, { peerId: peer.peer_id, myEnvId: env.env_id }); + // Advance inbound anchor to the last op_id from the source side + if (ops.length > 0) { + const lastOp = ops[ops.length - 1]; + const now = new Date().toISOString(); + const existing = await db.selectMaybeOne("_dd_anchors", { peer_id: peer.peer_id, direction: "inbound" }); + if (existing) { + await db.updateWhere("_dd_anchors", { last_op_id: lastOp.op_id, updated_at: now }, { peer_id: peer.peer_id, direction: "inbound" }); + } else { + await db.insert("_dd_anchors", { peer_id: peer.peer_id, direction: "inbound", last_op_id: lastOp.op_id, updated_at: now }, { noid: true }); + } + } + res.json({ received: ops.length, results: results }); +}; + + +// ---------------- Route registration ---------------- + +const routes = [ + { url: "/admin/dev-deploy/", method: "get", callback: dashboard }, + { url: "/admin/dev-deploy/ops", method: "get", callback: opsView }, + { url: "/admin/dev-deploy/peers", method: "get", callback: peersView }, + { url: "/admin/dev-deploy/peers/add", method: "post", callback: peersAdd }, + { url: "/admin/dev-deploy/peers/rotate", method: "post", callback: peersRotate }, + { url: "/admin/dev-deploy/peers/delete", method: "post", callback: peersDelete }, + { url: "/admin/dev-deploy/plan", method: "get", callback: planView }, + { url: "/admin/dev-deploy/promote", method: "post", callback: promote }, + { url: "/admin/dev-deploy/pull", method: "post", callback: pull }, + { url: "/admin/dev-deploy/revert", method: "post", callback: revertView }, + { url: "/admin/dev-deploy/tables", method: "get", callback: tablesView }, + { url: "/admin/dev-deploy/tables/set", method: "post", callback: tablesSet }, + { url: "/admin/dev-deploy/conflicts", method: "get", callback: conflictsView }, + { url: "/admin/dev-deploy/conflicts/resolve", method: "post", callback: conflictsResolve }, + { url: "/admin/dev-deploy/conflicts/merge", method: "get", callback: conflictsMergeView }, + { url: "/admin/dev-deploy/conflicts/merge/apply", method: "post", callback: conflictsMergeApply }, + { url: "/dev-deploy/api/journal", method: "get", callback: apiJournal, noCsrf: true }, + { url: "/dev-deploy/api/ingest", method: "post", callback: apiIngest, noCsrf: true }, + { url: "/dev-deploy/api/file/:uuid", method: "get", callback: apiFile, noCsrf: true }, + { url: "/dev-deploy/api/health", method: "get", callback: apiHealth, noCsrf: true } +]; + + +module.exports = { + routes +}; diff --git a/dev-deploy/lib/rowIdentity.js b/dev-deploy/lib/rowIdentity.js new file mode 100644 index 0000000..59c7324 --- /dev/null +++ b/dev-deploy/lib/rowIdentity.js @@ -0,0 +1,124 @@ +// Hidden _dd_row_uuid column infrastructure for managed/starter tables. +// +// When an admin marks a user table as managed or starter, this module +// adds a TEXT column _dd_row_uuid to the underlying SQL table via raw ALTER +// (NOT registered in _sc_fields, so Saltcorn's table builder doesn't show +// it). Existing rows are backfilled with random UUIDs. From that point, +// the wrap layer reads/writes _dd_row_uuid as the cross-environment identity +// for each row. + +const crypto = require("crypto"); + +const db = require("@saltcorn/data/db"); + + +const COLUMN_NAME = "_dd_row_uuid"; + + +const tableSqlRef = (tableName) => { + const schema = db.getTenantSchemaPrefix(); + return `${schema}"${db.sqlsanitize(tableName)}"`; +}; + + +const columnExists = async (tableName) => { + if (db.isSQLite) { + const rs = await db.query(`PRAGMA table_info("${db.sqlsanitize(tableName)}")`); + return rs.rows.some((r) => r.name === COLUMN_NAME); + } + const rs = await db.query( + `SELECT 1 FROM information_schema.columns + WHERE table_schema = current_schema() + AND table_name = $1 + AND column_name = $2`, + [tableName, COLUMN_NAME] + ); + return rs.rows.length > 0; +}; + + +const ensureManagedSchema = async (tableName) => { + if (await columnExists(tableName)) { + return { added: false }; + } + await db.query(`ALTER TABLE ${tableSqlRef(tableName)} ADD COLUMN ${COLUMN_NAME} TEXT`); + // Backfill existing rows with random UUIDs. + const rs = await db.query(`SELECT id FROM ${tableSqlRef(tableName)} WHERE ${COLUMN_NAME} IS NULL`); + let backfilled = 0; + for (const r of rs.rows) { + await db.query( + `UPDATE ${tableSqlRef(tableName)} SET ${COLUMN_NAME} = $1 WHERE id = $2`, + [crypto.randomUUID(), r.id] + ); + backfilled++; + } + // Index for fast lookups by uuid. + await db.query( + `CREATE INDEX IF NOT EXISTS "${db.sqlsanitize(tableName)}_dd_row_uuid_idx" + ON ${tableSqlRef(tableName)} (${COLUMN_NAME})` + ).catch(() => {}); + return { added: true, backfilled: backfilled }; +}; + + +const dropManagedSchema = async (tableName) => { + if (!(await columnExists(tableName))) { + return { dropped: false }; + } + // SQLite 3.35+ and Postgres both support DROP COLUMN. If the SQLite is + // older it will throw — surface the error. + await db.query(`ALTER TABLE ${tableSqlRef(tableName)} DROP COLUMN ${COLUMN_NAME}`); + return { dropped: true }; +}; + + +const getRowUuid = async (tableName, id) => { + const rs = await db.query( + `SELECT ${COLUMN_NAME} AS uuid FROM ${tableSqlRef(tableName)} WHERE id = $1`, + [id] + ); + return rs.rows.length > 0 ? rs.rows[0].uuid : null; +}; + + +const findIdByRowUuid = async (tableName, uuid) => { + const rs = await db.query( + `SELECT id FROM ${tableSqlRef(tableName)} WHERE ${COLUMN_NAME} = $1`, + [uuid] + ); + return rs.rows.length > 0 ? rs.rows[0].id : null; +}; + + +const setRowUuid = async (tableName, id, uuid) => { + await db.query( + `UPDATE ${tableSqlRef(tableName)} SET ${COLUMN_NAME} = $1 WHERE id = $2`, + [uuid, id] + ); +}; + + +const newRowUuid = () => crypto.randomUUID(); + + +// All rows currently in the table, including their _dd_row_uuid. Used during +// the initial managed/starter ship to journal an insert_row op for each. +const allRowsWithUuid = async (tableName) => { + const rs = await db.query( + `SELECT * FROM ${tableSqlRef(tableName)} ORDER BY id ASC` + ); + return rs.rows; +}; + + +module.exports = { + COLUMN_NAME, + columnExists, + ensureManagedSchema, + dropManagedSchema, + getRowUuid, + findIdByRowUuid, + setRowUuid, + newRowUuid, + allRowsWithUuid +}; diff --git a/dev-deploy/lib/rowPayload.js b/dev-deploy/lib/rowPayload.js new file mode 100644 index 0000000..d0aa880 --- /dev/null +++ b/dev-deploy/lib/rowPayload.js @@ -0,0 +1,144 @@ +// Row payload translation: FK ids -> row uuids (on journal) and back (on apply). +// +// For each FK field on the row's table (field.is_fkey), look up the referenced +// row's _dd_row_uuid. Store under "__uuid" to avoid colliding with +// the original field name. On apply, resolve back to the local row id. +// +// FKs to user-mode tables are NULLed on the target, with a warning attached +// to the payload so it surfaces in the admin UI. + +const db = require("@saltcorn/data/db"); + +const { getRowUuid, findIdByRowUuid, COLUMN_NAME } = require("./rowIdentity"); +const { lookupByCurrent } = require("./entityIds"); + + +const UUID_SUFFIX = "__uuid"; + + +// Returns data_mode for the table with given current_id (Saltcorn table id). +// 'user' (default), 'starter', or 'managed'. +const tableModeByCurrentId = async (tableId) => { + const ent = await lookupByCurrent("table", tableId); + if (!ent) return "user"; + const row = await db.selectMaybeOne("_dd_table_modes", { table_uuid: ent.uuid }); + return row ? row.data_mode : "user"; +}; + + +const tableModeByUuid = async (tableUuid) => { + if (!tableUuid) return "user"; + const row = await db.selectMaybeOne("_dd_table_modes", { table_uuid: tableUuid }); + return row ? row.data_mode : "user"; +}; + + +// rowData: a plain row from the table. +// table: the Saltcorn Table instance whose fields define the FK shape. +// Returns { portable: {...with __uuid keys for FKs}, warnings: [...] }. +const rowToPortable = async (rowData, table) => { + const portable = {}; + const warnings = []; + for (const field of table.fields || []) { + if (field.name === COLUMN_NAME || field.name === "id") continue; + const v = rowData[field.name]; + if (field.is_fkey && field.reftable_name && v !== null && v !== undefined) { + const refMode = await tableModeByCurrentId_byName(field.reftable_name); + if (refMode === "managed" || refMode === "starter") { + const refUuid = await getRowUuid(field.reftable_name, v); + if (refUuid) { + portable[field.name + UUID_SUFFIX] = refUuid; + } else { + portable[field.name + UUID_SUFFIX] = null; + warnings.push(`${field.name}: source row references ${field.reftable_name}.id=${v} but it has no _dd_row_uuid yet`); + } + } else { + // FK into a user-mode table — can't translate. Drop and warn. + portable[field.name + UUID_SUFFIX] = null; + warnings.push(`${field.name} → user-mode table ${field.reftable_name}; FK will be null on target`); + } + } else { + portable[field.name] = v; + } + } + return { portable: portable, warnings: warnings }; +}; + + +// portable: payload from a journaled row op. +// table: the Saltcorn Table instance on the local (target) side. +const portableToRow = async (portable, table) => { + const row = {}; + for (const field of table.fields || []) { + if (field.name === COLUMN_NAME || field.name === "id") continue; + const uuidKey = field.name + UUID_SUFFIX; + if (uuidKey in portable) { + const refUuid = portable[uuidKey]; + if (!refUuid) { + row[field.name] = null; + } else if (field.is_fkey && field.reftable_name) { + const localId = await findIdByRowUuid(field.reftable_name, refUuid); + row[field.name] = localId; // may be null if target doesn't have that row yet + } else { + row[field.name] = null; + } + } else if (field.name in portable) { + row[field.name] = portable[field.name]; + } + } + return row; +}; + + +// Helper: lookup mode by table name (via entity_ids). +const tableModeByCurrentId_byName = async (tableName) => { + const Table = require("@saltcorn/data/models/table"); + const t = Table.findOne({ name: tableName }); + if (!t) return "user"; + return await tableModeByCurrentId(t.id); +}; + + +// True if a starter table has already had its initial-ship completed; managed +// tables ignore this (they always journal). +const isStarterShipped = async (tableUuid) => { + const row = await db.selectMaybeOne("_dd_table_modes", { table_uuid: tableUuid }); + return !!(row && row.starter_shipped_at); +}; + + +const markStarterShipped = async (tableUuid) => { + await db.updateWhere("_dd_table_modes", { starter_shipped_at: new Date().toISOString() }, { table_uuid: tableUuid }); +}; + + +// For a given Saltcorn Table id, returns { mode, tableUuid, shouldJournal }. +// shouldJournal is the wrap-level decision: true → record the row op; false → +// pass through silently. +const journalDecision = async (tableId) => { + const { lookupByCurrent } = require("./entityIds"); + const ent = await lookupByCurrent("table", tableId); + if (!ent) return { mode: "user", tableUuid: null, shouldJournal: false }; + const mode = await tableModeByUuid(ent.uuid); + if (mode === "managed") { + return { mode: mode, tableUuid: ent.uuid, shouldJournal: true }; + } + if (mode === "starter") { + const shipped = await isStarterShipped(ent.uuid); + return { mode: mode, tableUuid: ent.uuid, shouldJournal: !shipped }; + } + return { mode: "user", tableUuid: ent.uuid, shouldJournal: false }; +}; + + +module.exports = { + rowToPortable, + portableToRow, + tableModeByCurrentId, + tableModeByUuid, + tableModeByCurrentId_byName, + isStarterShipped, + markStarterShipped, + journalDecision, + UUID_SUFFIX +}; diff --git a/dev-deploy/lib/schema.js b/dev-deploy/lib/schema.js new file mode 100644 index 0000000..74521dd --- /dev/null +++ b/dev-deploy/lib/schema.js @@ -0,0 +1,140 @@ +// DDL for dev-deploy's six plugin tables. +// +// Saltcorn supports SQLite and PostgreSQL. We use a portable subset: +// - TEXT for all strings, JSON payloads, ISO 8601 timestamps +// - INTEGER for booleans (0/1) and surrogate ids +// - No JSONB; payloads are TEXT containing JSON, parsed/stringified at the +// application layer +// +// Tables are created idempotently in onLoad. Re-running is safe. + +const db = require("@saltcorn/data/db"); + + +const createDdEnv = async () => { + await db.query(` + CREATE TABLE IF NOT EXISTS _dd_env ( + env_id TEXT PRIMARY KEY, + env_label TEXT, + on_destructive_op TEXT NOT NULL DEFAULT 'confirm', + require_tls INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL, + bootstrapped_at TEXT + ) + `); +}; + + +const createDdPeers = async () => { + // Secret components stored as hex TEXT rather than BLOB: Saltcorn's + // SQLite insert layer JSON-stringifies any object value, which would + // mangle Buffer columns. Hex is portable and survives that path intact. + await db.query(` + CREATE TABLE IF NOT EXISTS _dd_peers ( + peer_id INTEGER PRIMARY KEY AUTOINCREMENT, + env_id TEXT NOT NULL UNIQUE, + label TEXT, + base_url TEXT NOT NULL, + peer_secret_ciphertext TEXT, + peer_secret_iv TEXT, + peer_secret_tag TEXT, + require_tls INTEGER, + created_at TEXT NOT NULL, + last_seen_at TEXT + ) + `); +}; + + +const createDdEntityIds = async () => { + await db.query(` + CREATE TABLE IF NOT EXISTS _dd_entity_ids ( + uuid TEXT PRIMARY KEY, + kind TEXT NOT NULL, + current_name TEXT NOT NULL, + current_id INTEGER NOT NULL, + parent_uuid TEXT, + created_at TEXT NOT NULL, + UNIQUE (kind, current_id) + ) + `); + await db.query(`CREATE INDEX IF NOT EXISTS _dd_entity_ids_kind_name ON _dd_entity_ids (kind, current_name)`); + await db.query(`CREATE INDEX IF NOT EXISTS _dd_entity_ids_parent ON _dd_entity_ids (parent_uuid)`); +}; + + +const createDdOps = async () => { + await db.query(` + CREATE TABLE IF NOT EXISTS _dd_ops ( + op_id TEXT PRIMARY KEY, + source_env_id TEXT NOT NULL, + op_type TEXT NOT NULL, + entity_kind TEXT, + entity_uuid TEXT, + payload TEXT NOT NULL, + parent_op_id TEXT, + correlation_id TEXT, + schema_version INTEGER NOT NULL DEFAULT 1, + created_at TEXT NOT NULL, + applied_at TEXT, + status TEXT NOT NULL DEFAULT 'committed', + conflict_with_op_id TEXT + ) + `); + await db.query(`CREATE INDEX IF NOT EXISTS _dd_ops_created ON _dd_ops (created_at)`); + await db.query(`CREATE INDEX IF NOT EXISTS _dd_ops_source ON _dd_ops (source_env_id, created_at)`); + await db.query(`CREATE INDEX IF NOT EXISTS _dd_ops_entity ON _dd_ops (entity_uuid)`); + await db.query(`CREATE INDEX IF NOT EXISTS _dd_ops_correlation ON _dd_ops (correlation_id)`); + await db.query(`CREATE INDEX IF NOT EXISTS _dd_ops_status ON _dd_ops (status) WHERE status = 'conflict'`).catch(() => {}); + + // Idempotent migration for instances installed before conflict_with_op_id existed + try { + await db.query(`ALTER TABLE _dd_ops ADD COLUMN conflict_with_op_id TEXT`); + } catch (e) { + // column already exists; ignore + } +}; + + +const createDdAnchors = async () => { + await db.query(` + CREATE TABLE IF NOT EXISTS _dd_anchors ( + peer_id INTEGER NOT NULL, + direction TEXT NOT NULL, + last_op_id TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (peer_id, direction) + ) + `); +}; + + +const createDdTableModes = async () => { + await db.query(` + CREATE TABLE IF NOT EXISTS _dd_table_modes ( + table_uuid TEXT PRIMARY KEY, + data_mode TEXT NOT NULL, + updated_at TEXT NOT NULL, + starter_shipped_at TEXT + ) + `); + // Idempotent migration for older installs that lack starter_shipped_at. + try { + await db.query(`ALTER TABLE _dd_table_modes ADD COLUMN starter_shipped_at TEXT`); + } catch (e) { /* column exists */ } +}; + + +const createAllTables = async () => { + await createDdEnv(); + await createDdPeers(); + await createDdEntityIds(); + await createDdOps(); + await createDdAnchors(); + await createDdTableModes(); +}; + + +module.exports = { + createAllTables +}; diff --git a/dev-deploy/lib/state.js b/dev-deploy/lib/state.js new file mode 100644 index 0000000..d2ce190 --- /dev/null +++ b/dev-deploy/lib/state.js @@ -0,0 +1,27 @@ +// Shared helpers for Saltcorn state-cache interactions. + +// Refresh Saltcorn's in-memory model caches. Mutations done in another process +// (e.g. saltcorn run-js, prior ingest, or a peer's push) are not visible to +// Model.findOne / Model.find until we refresh, because those methods read from +// state.tables/views/pages/triggers arrays populated at startup. +// +// Best-effort: if Saltcorn changes its API and a refresh function disappears, +// callers fall back to whatever's in the cache. +const refreshState = async () => { + try { + const { getState } = require("@saltcorn/data/db/state"); + const s = getState(); + if (s.refresh_tables) await s.refresh_tables(true); + if (s.refresh_views) await s.refresh_views(true); + if (s.refresh_pages) await s.refresh_pages(true); + if (s.refresh_triggers) await s.refresh_triggers(true); + if (s.refresh_page_groups) await s.refresh_page_groups(true); + } catch (e) { + // ignore — best-effort + } +}; + + +module.exports = { + refreshState +}; diff --git a/dev-deploy/lib/transport.js b/dev-deploy/lib/transport.js new file mode 100644 index 0000000..4fc0e29 --- /dev/null +++ b/dev-deploy/lib/transport.js @@ -0,0 +1,98 @@ +// Outbound signed peer requests. +// +// Build a canonical string, sign it with the peer's shared secret, and send +// it via fetch. Body is JSON-serialized once and used for both the HTTP body +// and the signature payload to keep client/server hash agreement trivial. + +const { + buildCanonical, + sign, + randomNonce +} = require("./crypto"); + + +const signedFetch = async ({ baseUrl, method, path, body, sourceEnvId, secret }) => { + const timestamp = String(Date.now()); + const nonce = randomNonce().toString("hex"); + const bodyStr = body ? JSON.stringify(body) : ""; + const canonical = buildCanonical({ + timestamp: timestamp, + nonce: nonce, + method: method, + path: path, + body: bodyStr + }); + const signature = sign(secret, canonical); + + const url = baseUrl.replace(/\/+$/, "") + path; + const headers = { + "X-DD-Env-Id": sourceEnvId, + "X-DD-Timestamp": timestamp, + "X-DD-Nonce": nonce, + "X-DD-Signature": signature + }; + if (bodyStr) { + // Custom Content-Type so Saltcorn's express.json() middleware leaves the + // request stream untouched; the server reads exact bytes for HMAC. + headers["Content-Type"] = "application/vnd.dev-deploy+json"; + } + + const init = { method: method, headers: headers }; + if (bodyStr) { + init.body = bodyStr; + } + + const res = await fetch(url, init); + const text = await res.text(); + let parsed = null; + if (text) { + try { + parsed = JSON.parse(text); + } catch (e) { + parsed = { raw: text }; + } + } + return { status: res.status, ok: res.ok, body: parsed }; +}; + + +// Like signedFetch but returns the response body as a Buffer (raw bytes). +// For binary endpoints like GET /dev-deploy/api/file/:uuid. +const signedFetchBinary = async ({ baseUrl, method, path, body, sourceEnvId, secret }) => { + const timestamp = String(Date.now()); + const nonce = randomNonce().toString("hex"); + const bodyStr = body ? JSON.stringify(body) : ""; + const canonical = buildCanonical({ + timestamp: timestamp, + nonce: nonce, + method: method, + path: path, + body: bodyStr + }); + const signature = sign(secret, canonical); + + const url = baseUrl.replace(/\/+$/, "") + path; + const headers = { + "X-DD-Env-Id": sourceEnvId, + "X-DD-Timestamp": timestamp, + "X-DD-Nonce": nonce, + "X-DD-Signature": signature + }; + if (bodyStr) { + headers["Content-Type"] = "application/vnd.dev-deploy+json"; + } + + const init = { method: method, headers: headers }; + if (bodyStr) { + init.body = bodyStr; + } + const res = await fetch(url, init); + const bytes = Buffer.from(await res.arrayBuffer()); + return { status: res.status, ok: res.ok, bytes: bytes }; +}; + + +module.exports = { + signedFetch, + signedFetchBinary +}; diff --git a/dev-deploy/lib/wrap.js b/dev-deploy/lib/wrap.js new file mode 100644 index 0000000..180f9b9 --- /dev/null +++ b/dev-deploy/lib/wrap.js @@ -0,0 +1,999 @@ +// Monkey-patches Saltcorn metadata model classes to journal every CRUD action. +// +// Each wrap: +// 1. Pre-generates an op_id and enters an AsyncLocalStorage scope so any +// child mutations triggered by the original method (e.g. cascading field +// deletes from a table delete) see this op as their parent. +// 2. Optionally captures pre-state via a "before" hook. +// 3. Invokes the original method. +// 4. On success, runs an "after" hook to compute entity uuid + payload and +// appends the op to _dd_ops. +// +// Failures inside the after-hook are logged but do not throw -- we never want +// the journal to corrupt user-facing operations. Failures inside the original +// method propagate normally (no op is recorded for an aborted mutation). + +const Table = require("@saltcorn/data/models/table"); +const Field = require("@saltcorn/data/models/field"); +const View = require("@saltcorn/data/models/view"); +const Page = require("@saltcorn/data/models/page"); +const Trigger = require("@saltcorn/data/models/trigger"); +const Role = require("@saltcorn/data/models/role"); +const Library = require("@saltcorn/data/models/library"); +const Tag = require("@saltcorn/data/models/tag"); +const TableConstraint = require("@saltcorn/data/models/table_constraints"); +const File = require("@saltcorn/data/models/file"); +const PageGroup = require("@saltcorn/data/models/page_group"); +const PageGroupMember = require("@saltcorn/data/models/page_group_member"); +const WorkflowStep = require("@saltcorn/data/models/workflow_step"); +const Plugin = require("@saltcorn/data/models/plugin"); +const db = require("@saltcorn/data/db"); + +const { randomUuid } = require("./ids"); +const { enterOp, isSuppressed } = require("./context"); +const { recordOpSafely } = require("./ops"); +const { + assignNewUuid, + lookupByCurrent, + updateName, + removeEntityRow, + constraintDisplayName +} = require("./entityIds"); +const { ENTITY_KINDS, fileLocationToId } = require("./constants"); +const { sha256File, toRelativePath } = require("./files"); +const { toPlaceholders } = require("./payloadRefs"); +const { + setRowUuid, + getRowUuid, + newRowUuid, + COLUMN_NAME: ROW_UUID_COL +} = require("./rowIdentity"); +const { + rowToPortable, + journalDecision +} = require("./rowPayload"); + + +const snapshotInstance = (inst, keys) => { + const out = {}; + for (const k of keys) { + if (inst[k] !== undefined) { + out[k] = inst[k]; + } + } + return out; +}; + + +// Shared before/after hooks for instance `delete()` methods. +// Captures the entity's UUID, parent UUID (if any), and snapshot, then removes +// the entity_ids row after the original delete completes so reused integer ids +// don't collide. parent_uuid is preserved so revert can find the parent later. +const standardDropHooks = (kind, keys) => ({ + before: async function () { + const existing = await lookupByCurrent(kind, this.id); + return { + uuid: existing ? existing.uuid : null, + currentId: this.id, + parentUuid: existing ? existing.parent_uuid : null, + snapshot: snapshotInstance(this, keys) + }; + }, + after: async function ({ before }) { + if (!before || !before.uuid) return null; + await removeEntityRow(kind, before.currentId); + return { + entityUuid: before.uuid, + payload: { before: before.snapshot, parent_uuid: before.parentUuid } + }; + } +}); + + +const wrap = (target, method, kind, action, hooks) => { + const original = target[method]; + if (typeof original !== "function") { + // eslint-disable-next-line no-console + console.warn(`[dev-deploy] no method ${kind}.${method} to wrap`); + return; + } + if (original.__ddWrapped) { + return; + } + const wrapped = async function (...args) { + if (isSuppressed()) { + return await original.apply(this, args); + } + const opId = randomUuid(); + const beforeCtx = hooks.before ? await safeCall(hooks.before, this, [args]) : null; + return await enterOp(opId, async () => { + const result = await original.apply(this, args); + const details = await safeCall(hooks.after, this, [{ args: args, result: result, before: beforeCtx }]); + if (details) { + // Translate any local file refs (numeric ids or paths) in the + // payload to portable __dd_file_ref:: placeholders so + // the journal stores cross-environment-stable references. + if (details.payload) { + try { + await toPlaceholders(details.payload); + } catch (e) { + // best-effort; payload still journals as-is + } + } + await recordOpSafely({ + op_id: opId, + op_type: `${action}_${kind}`, + entity_kind: kind, + entity_uuid: details.entityUuid, + payload: details.payload + }); + } + return result; + }); + }; + wrapped.__ddWrapped = true; + wrapped.__ddOriginal = original; + target[method] = wrapped; +}; + + +const safeCall = async (fn, ctx, args) => { + if (!fn) { + return null; + } + try { + return await fn.apply(ctx, args); + } catch (err) { + // eslint-disable-next-line no-console + console.error(`[dev-deploy] hook error:`, err); + return null; + } +}; + + +// Common keys to snapshot for each entity. Skips heavy/derived fields. +const TABLE_KEYS = ["id", "name", "min_role_read", "min_role_write", "versioned", "description", "ownership_field_id", "ownership_formula", "external", "provider_name", "provider_cfg"]; +const FIELD_KEYS = ["id", "name", "label", "type", "table_id", "required", "is_unique", "calculated", "stored", "expression", "reftable_name", "reftype", "refname", "primary_key", "attributes"]; +const VIEW_KEYS = ["id", "name", "viewtemplate", "table_id", "configuration", "min_role", "default_render_page", "exttable_name", "description", "slug"]; +const PAGE_KEYS = ["id", "name", "title", "description", "min_role", "layout", "fixed_states", "menu_label"]; +const TRIGGER_KEYS = ["id", "name", "action", "when_trigger", "table_id", "configuration", "min_role", "description", "channel"]; +const ROLE_KEYS = ["id", "role"]; +const LIBRARY_KEYS = ["id", "name", "icon", "layout"]; +const TAG_KEYS = ["id", "name"]; +const CONSTRAINT_KEYS = ["id", "type", "table_id", "configuration"]; +const PAGE_GROUP_KEYS = ["id", "name", "description", "min_role", "random_allocation"]; +const WORKFLOW_STEP_KEYS = ["id", "name", "trigger_id", "action_name", "next_step", "initial_step", "configuration", "only_if"]; + + +// ----- Table ----- + +const wrapTable = () => { + wrap(Table, "create", ENTITY_KINDS.TABLE, "create", { + after: async ({ result }) => { + if (!result || !result.id) return null; + const uuid = await assignNewUuid(ENTITY_KINDS.TABLE, result.id, result.name, null); + return { + entityUuid: uuid, + payload: { after: snapshotInstance(result, TABLE_KEYS) } + }; + } + }); + + wrap(Table.prototype, "update", ENTITY_KINDS.TABLE, "update", { + before: async function (args) { + const existing = await lookupByCurrent(ENTITY_KINDS.TABLE, this.id); + return { + uuid: existing ? existing.uuid : null, + oldName: existing ? existing.current_name : this.name, + snapshot: snapshotInstance(this, TABLE_KEYS) + }; + }, + after: async function ({ args, before }) { + if (!before || !before.uuid) return null; + const patch = args[0] || {}; + const newName = patch.name !== undefined ? patch.name : before.oldName; + if (newName !== before.oldName) { + await updateName(ENTITY_KINDS.TABLE, this.id, newName); + } + return { + entityUuid: before.uuid, + payload: { + before: before.snapshot, + patch: patch, + after: snapshotInstance(this, TABLE_KEYS) + } + }; + } + }); + + wrap(Table.prototype, "delete", ENTITY_KINDS.TABLE, "drop", { + before: async function (args) { + const existing = await lookupByCurrent(ENTITY_KINDS.TABLE, this.id); + return { + uuid: existing ? existing.uuid : null, + currentId: this.id, + snapshot: snapshotInstance(this, TABLE_KEYS), + onlyForget: !!args[0] + }; + }, + after: async function ({ before }) { + if (!before || !before.uuid) return null; + await removeEntityRow(ENTITY_KINDS.TABLE, before.currentId); + return { + entityUuid: before.uuid, + payload: { before: before.snapshot, only_forget: before.onlyForget } + }; + } + }); +}; + + +// ----- Field ----- + +const wrapField = () => { + wrap(Field, "create", ENTITY_KINDS.FIELD, "create", { + after: async ({ args, result }) => { + if (!result || !result.id) return null; + let parentUuid = null; + if (result.table_id) { + const t = await lookupByCurrent(ENTITY_KINDS.TABLE, result.table_id); + parentUuid = t ? t.uuid : null; + } + const uuid = await assignNewUuid(ENTITY_KINDS.FIELD, result.id, result.name, parentUuid); + return { + entityUuid: uuid, + payload: { + after: snapshotInstance(result, FIELD_KEYS), + parent_uuid: parentUuid + } + }; + } + }); + + wrap(Field.prototype, "update", ENTITY_KINDS.FIELD, "update", { + before: async function (args) { + const existing = await lookupByCurrent(ENTITY_KINDS.FIELD, this.id); + return { + uuid: existing ? existing.uuid : null, + oldName: existing ? existing.current_name : this.name, + snapshot: snapshotInstance(this, FIELD_KEYS) + }; + }, + after: async function ({ args, before }) { + if (!before || !before.uuid) return null; + const patch = args[0] || {}; + const newName = patch.name !== undefined ? patch.name : before.oldName; + if (newName !== before.oldName) { + await updateName(ENTITY_KINDS.FIELD, this.id, newName); + } + return { + entityUuid: before.uuid, + payload: { + before: before.snapshot, + patch: patch + } + }; + } + }); + + wrap(Field.prototype, "delete", ENTITY_KINDS.FIELD, "drop", standardDropHooks(ENTITY_KINDS.FIELD, FIELD_KEYS)); +}; + + +// ----- View ----- + +const wrapView = () => { + wrap(View, "create", ENTITY_KINDS.VIEW, "create", { + after: async ({ result }) => { + if (!result || !result.id) return null; + let parentUuid = null; + if (result.table_id) { + const t = await lookupByCurrent(ENTITY_KINDS.TABLE, result.table_id); + parentUuid = t ? t.uuid : null; + } + const uuid = await assignNewUuid(ENTITY_KINDS.VIEW, result.id, result.name, parentUuid); + return { + entityUuid: uuid, + payload: { after: snapshotInstance(result, VIEW_KEYS), parent_uuid: parentUuid } + }; + } + }); + + // View.update is static (v, id) + wrap(View, "update", ENTITY_KINDS.VIEW, "update", { + before: async ({ args }) => { + const patch = args[0] || {}; + const id = args[1]; + const existing = await lookupByCurrent(ENTITY_KINDS.VIEW, id); + return { + id: id, + uuid: existing ? existing.uuid : null, + oldName: existing ? existing.current_name : null, + patch: patch + }; + }, + after: async ({ before }) => { + if (!before || !before.uuid) return null; + const newName = before.patch.name !== undefined ? before.patch.name : before.oldName; + if (newName !== before.oldName) { + await updateName(ENTITY_KINDS.VIEW, before.id, newName); + } + return { + entityUuid: before.uuid, + payload: { patch: before.patch } + }; + } + }); + + wrap(View.prototype, "delete", ENTITY_KINDS.VIEW, "drop", standardDropHooks(ENTITY_KINDS.VIEW, VIEW_KEYS)); +}; + + +// ----- Page ----- + +const wrapPage = () => { + wrap(Page, "create", ENTITY_KINDS.PAGE, "create", { + after: async ({ result }) => { + if (!result || !result.id) return null; + const uuid = await assignNewUuid(ENTITY_KINDS.PAGE, result.id, result.name, null); + return { entityUuid: uuid, payload: { after: snapshotInstance(result, PAGE_KEYS) } }; + } + }); + + // Page.update is static (id, row) + wrap(Page, "update", ENTITY_KINDS.PAGE, "update", { + before: async ({ args }) => { + const id = args[0]; + const row = args[1] || {}; + const existing = await lookupByCurrent(ENTITY_KINDS.PAGE, id); + return { + id: id, + uuid: existing ? existing.uuid : null, + oldName: existing ? existing.current_name : null, + patch: row + }; + }, + after: async ({ before }) => { + if (!before || !before.uuid) return null; + const newName = before.patch.name !== undefined ? before.patch.name : before.oldName; + if (newName !== before.oldName) { + await updateName(ENTITY_KINDS.PAGE, before.id, newName); + } + return { entityUuid: before.uuid, payload: { patch: before.patch } }; + } + }); + + wrap(Page.prototype, "delete", ENTITY_KINDS.PAGE, "drop", standardDropHooks(ENTITY_KINDS.PAGE, PAGE_KEYS)); +}; + + +// ----- Trigger ----- + +const wrapTrigger = () => { + wrap(Trigger, "create", ENTITY_KINDS.TRIGGER, "create", { + after: async ({ result }) => { + if (!result || !result.id) return null; + let parentUuid = null; + if (result.table_id) { + const t = await lookupByCurrent(ENTITY_KINDS.TABLE, result.table_id); + parentUuid = t ? t.uuid : null; + } + const name = result.name || `trigger_${result.id}`; + const uuid = await assignNewUuid(ENTITY_KINDS.TRIGGER, result.id, name, parentUuid); + return { entityUuid: uuid, payload: { after: snapshotInstance(result, TRIGGER_KEYS), parent_uuid: parentUuid } }; + } + }); + + // Trigger.update is static (id, row) + wrap(Trigger, "update", ENTITY_KINDS.TRIGGER, "update", { + before: async ({ args }) => { + const id = args[0]; + const row = args[1] || {}; + const existing = await lookupByCurrent(ENTITY_KINDS.TRIGGER, id); + return { + id: id, + uuid: existing ? existing.uuid : null, + oldName: existing ? existing.current_name : null, + patch: row + }; + }, + after: async ({ before }) => { + if (!before || !before.uuid) return null; + const newName = before.patch.name !== undefined ? before.patch.name : before.oldName; + if (newName && newName !== before.oldName) { + await updateName(ENTITY_KINDS.TRIGGER, before.id, newName); + } + return { entityUuid: before.uuid, payload: { patch: before.patch } }; + } + }); + + wrap(Trigger.prototype, "delete", ENTITY_KINDS.TRIGGER, "drop", standardDropHooks(ENTITY_KINDS.TRIGGER, TRIGGER_KEYS)); +}; + + +// ----- Role ----- + +const wrapRole = () => { + wrap(Role, "create", ENTITY_KINDS.ROLE, "create", { + after: async ({ args, result }) => { + // Role.create returns inserted row; id is in result. Some return shapes vary. + const id = result && result.id ? result.id : (args[0] && args[0].id); + const role = result && result.role ? result.role : (args[0] && args[0].role); + if (!id || !role) return null; + const uuid = await assignNewUuid(ENTITY_KINDS.ROLE, id, role, null); + return { entityUuid: uuid, payload: { after: { id: id, role: role } } }; + } + }); + + wrap(Role.prototype, "update", ENTITY_KINDS.ROLE, "update", { + before: async function () { + const existing = await lookupByCurrent(ENTITY_KINDS.ROLE, this.id); + return { uuid: existing ? existing.uuid : null, oldName: existing ? existing.current_name : this.role, snapshot: snapshotInstance(this, ROLE_KEYS) }; + }, + after: async function ({ args, before }) { + if (!before || !before.uuid) return null; + const patch = args[0] || {}; + const newName = patch.role !== undefined ? patch.role : before.oldName; + if (newName !== before.oldName) { + await updateName(ENTITY_KINDS.ROLE, this.id, newName); + } + return { entityUuid: before.uuid, payload: { before: before.snapshot, patch: patch } }; + } + }); + + wrap(Role.prototype, "delete", ENTITY_KINDS.ROLE, "drop", standardDropHooks(ENTITY_KINDS.ROLE, ROLE_KEYS)); +}; + + +// ----- Library ----- + +const wrapLibrary = () => { + wrap(Library, "create", ENTITY_KINDS.LIBRARY, "create", { + after: async ({ args, result }) => { + // Library.create returns void; we look up by name from args + const cfg = args[0] || {}; + if (!cfg.name) return null; + const Library = require("@saltcorn/data/models/library"); + const lib = await Library.findOne({ name: cfg.name }); + if (!lib) return null; + const uuid = await assignNewUuid(ENTITY_KINDS.LIBRARY, lib.id, lib.name, null); + return { entityUuid: uuid, payload: { after: snapshotInstance(lib, LIBRARY_KEYS) } }; + } + }); + + wrap(Library.prototype, "update", ENTITY_KINDS.LIBRARY, "update", { + before: async function () { + const existing = await lookupByCurrent(ENTITY_KINDS.LIBRARY, this.id); + return { uuid: existing ? existing.uuid : null, oldName: existing ? existing.current_name : this.name, snapshot: snapshotInstance(this, LIBRARY_KEYS) }; + }, + after: async function ({ args, before }) { + if (!before || !before.uuid) return null; + const patch = args[0] || {}; + const newName = patch.name !== undefined ? patch.name : before.oldName; + if (newName !== before.oldName) { + await updateName(ENTITY_KINDS.LIBRARY, this.id, newName); + } + return { entityUuid: before.uuid, payload: { before: before.snapshot, patch: patch } }; + } + }); + + wrap(Library.prototype, "delete", ENTITY_KINDS.LIBRARY, "drop", standardDropHooks(ENTITY_KINDS.LIBRARY, LIBRARY_KEYS)); +}; + + +// ----- Tag ----- + +const wrapTag = () => { + wrap(Tag, "create", ENTITY_KINDS.TAG, "create", { + after: async ({ result }) => { + if (!result || !result.id) return null; + const uuid = await assignNewUuid(ENTITY_KINDS.TAG, result.id, result.name, null); + return { entityUuid: uuid, payload: { after: snapshotInstance(result, TAG_KEYS) } }; + } + }); + + wrap(Tag.prototype, "update", ENTITY_KINDS.TAG, "update", { + before: async function () { + const existing = await lookupByCurrent(ENTITY_KINDS.TAG, this.id); + return { uuid: existing ? existing.uuid : null, oldName: existing ? existing.current_name : this.name, snapshot: snapshotInstance(this, TAG_KEYS) }; + }, + after: async function ({ args, before }) { + if (!before || !before.uuid) return null; + const patch = args[0] || {}; + const newName = patch.name !== undefined ? patch.name : before.oldName; + if (newName !== before.oldName) { + await updateName(ENTITY_KINDS.TAG, this.id, newName); + } + return { entityUuid: before.uuid, payload: { before: before.snapshot, patch: patch } }; + } + }); + + wrap(Tag.prototype, "delete", ENTITY_KINDS.TAG, "drop", standardDropHooks(ENTITY_KINDS.TAG, TAG_KEYS)); +}; + + +// ----- File ----- + +const FILE_KEYS = ["filename", "location", "mime_super", "mime_sub", "size_kb", "min_role_read"]; + +const wrapFile = () => { + wrap(File, "create", ENTITY_KINDS.FILE, "create", { + after: async ({ result }) => { + if (!result || !result.location) return null; + const relPath = toRelativePath(File, result.location); + const synthId = fileLocationToId(relPath); + let contentHash = null; + try { + contentHash = await sha256File(result.location); + } catch (e) { + // best-effort; some create paths may set up metadata only + } + const uuid = await assignNewUuid(ENTITY_KINDS.FILE, synthId, relPath, null); + return { + entityUuid: uuid, + payload: { + after: { + filename: result.filename, + relative_path: relPath, + mime_super: result.mime_super, + mime_sub: result.mime_sub, + size_kb: result.size_kb, + min_role_read: result.min_role_read, + content_hash: contentHash + } + } + }; + } + }); + + wrap(File.prototype, "delete", ENTITY_KINDS.FILE, "drop", { + before: async function () { + const relPath = toRelativePath(File, this.location); + const synthId = fileLocationToId(relPath); + const existing = await lookupByCurrent(ENTITY_KINDS.FILE, synthId); + return { + uuid: existing ? existing.uuid : null, + currentId: synthId, + relPath: relPath, + parentUuid: null, + snapshot: snapshotInstance(this, FILE_KEYS) + }; + }, + after: async function ({ before }) { + if (!before || !before.uuid) return null; + await removeEntityRow(ENTITY_KINDS.FILE, before.currentId); + return { + entityUuid: before.uuid, + payload: { before: { ...before.snapshot, relative_path: before.relPath } } + }; + } + }); +}; + + +// ----- TableConstraint ----- + +const wrapTableConstraint = () => { + wrap(TableConstraint, "create", ENTITY_KINDS.CONSTRAINT, "create", { + after: async ({ result }) => { + if (!result || !result.id) return null; + // Guard against double-firing (e.g. the wrap may be installed in + // multiple module-resolution contexts when sc-exec uses createRequire). + const already = await lookupByCurrent(ENTITY_KINDS.CONSTRAINT, result.id); + if (already) return null; + let parentUuid = null; + if (result.table_id) { + const t = await lookupByCurrent(ENTITY_KINDS.TABLE, result.table_id); + parentUuid = t ? t.uuid : null; + } + const name = constraintDisplayName(result); + const uuid = await assignNewUuid(ENTITY_KINDS.CONSTRAINT, result.id, name, parentUuid); + return { + entityUuid: uuid, + payload: { after: snapshotInstance(result, CONSTRAINT_KEYS), parent_uuid: parentUuid } + }; + } + }); + + wrap(TableConstraint.prototype, "delete", ENTITY_KINDS.CONSTRAINT, "drop", standardDropHooks(ENTITY_KINDS.CONSTRAINT, CONSTRAINT_KEYS)); +}; + + +// ----- Table row CRUD (managed/starter data_mode) ----- +// +// Wraps Table.prototype.insertRow / updateRow / deleteRows. Each checks the +// table's data_mode (looked up via _dd_table_modes). 'user' tables pass +// through silently. 'managed' tables always journal. 'starter' tables journal +// only during their initial ship (when starter_shipped_at is NULL); subsequent +// row changes pass through. + +const safeJournal = async (opType, entityUuid, payload) => { + try { + const opId = randomUuid(); + await enterOp(opId, async () => { + await recordOpSafely({ + op_id: opId, + op_type: opType, + entity_kind: "table_row", + entity_uuid: entityUuid, + payload: payload + }); + }); + } catch (e) { + // eslint-disable-next-line no-console + console.error(`[dev-deploy] ${opType} journal failed:`, e); + } +}; + + +const wrapTableRows = () => { + // --- insertRow --- + const origInsert = Table.prototype.insertRow; + if (origInsert && !origInsert.__ddWrapped) { + const wrapped = async function (...args) { + if (isSuppressed()) { + return await origInsert.apply(this, args); + } + const decision = await journalDecision(this.id); + if (!decision.shouldJournal) { + return await origInsert.apply(this, args); + } + const v_in = args[0] || {}; + const newId = await origInsert.apply(this, args); + if (!newId) return newId; + + const rowUuid = newRowUuid(); + await setRowUuid(this.name, newId, rowUuid); + + const { portable, warnings } = await rowToPortable({ ...v_in, id: newId }, this); + await safeJournal("insert_row", rowUuid, { + table_uuid: decision.tableUuid, + after: portable, + warnings: warnings.length > 0 ? warnings : undefined + }); + return newId; + }; + wrapped.__ddWrapped = true; + wrapped.__ddOriginal = origInsert; + Table.prototype.insertRow = wrapped; + } + + // --- updateRow --- + const origUpdate = Table.prototype.updateRow; + if (origUpdate && !origUpdate.__ddWrapped) { + const wrapped = async function (v_in, id_in, ...rest) { + if (isSuppressed()) { + return await origUpdate.apply(this, [v_in, id_in, ...rest]); + } + const decision = await journalDecision(this.id); + if (!decision.shouldJournal) { + return await origUpdate.apply(this, [v_in, id_in, ...rest]); + } + const id = typeof id_in === "object" && id_in !== null ? id_in.id : id_in; + const rowUuid = await getRowUuid(this.name, id); + + const result = await origUpdate.apply(this, [v_in, id_in, ...rest]); + + if (rowUuid) { + const { portable } = await rowToPortable({ ...v_in, id: id }, this); + await safeJournal("update_row", rowUuid, { + table_uuid: decision.tableUuid, + patch: portable + }); + } + return result; + }; + wrapped.__ddWrapped = true; + wrapped.__ddOriginal = origUpdate; + Table.prototype.updateRow = wrapped; + } + + // --- deleteRows --- + const origDelete = Table.prototype.deleteRows; + if (origDelete && !origDelete.__ddWrapped) { + const wrapped = async function (where, ...rest) { + if (isSuppressed()) { + return await origDelete.apply(this, [where, ...rest]); + } + const decision = await journalDecision(this.id); + if (!decision.shouldJournal) { + return await origDelete.apply(this, [where, ...rest]); + } + // Capture rows BEFORE deletion so we can journal their uuids. + const rows = await db.select(this.name, where || {}); + const result = await origDelete.apply(this, [where, ...rest]); + for (const row of rows) { + const rowUuid = row[ROW_UUID_COL]; + if (!rowUuid) continue; + const { portable } = await rowToPortable(row, this); + await safeJournal("drop_row", rowUuid, { + table_uuid: decision.tableUuid, + before: portable + }); + } + return result; + }; + wrapped.__ddWrapped = true; + wrapped.__ddOriginal = origDelete; + Table.prototype.deleteRows = wrapped; + } +}; + + +// ----- Plugin configuration ----- + +// Wrap Plugin.prototype.upsert to journal configuration changes. Skip the +// dev-deploy plugin itself (no self-reference). Skip cases where the upsert +// doesn't change the configuration — Saltcorn calls upsert on every plugin +// load, which would otherwise flood the journal. +const wrapPlugin = () => { + wrap(Plugin.prototype, "upsert", "plugin_config", "update", { + before: async function () { + if (!this.name || this.name === "dev-deploy") return null; + let priorConfig = null; + if (this.id) { + const existing = await db.selectMaybeOne("_sc_plugins", { id: this.id }); + if (existing) { + try { + priorConfig = typeof existing.configuration === "string" + ? JSON.parse(existing.configuration) + : (existing.configuration || null); + } catch (e) { + priorConfig = null; + } + } + } + return { name: this.name, priorConfig: priorConfig }; + }, + after: async function ({ before }) { + if (!before) return null; + const newConfig = this.configuration || {}; + if (JSON.stringify(before.priorConfig || {}) === JSON.stringify(newConfig)) { + return null; + } + return { + entityUuid: null, + payload: { name: before.name, configuration: newConfig } + }; + } + }); +}; + + +// ----- Config keys (menu, etc.) ----- + +// Config keys we journal when state.setConfig is called. The op carries the +// full key+value pair; apply replays it on the target via setConfig. +// menu_items references pages/views by NAME (not id), which is naturally +// stable across instances, so no UUID translation is needed. +const TRACKED_CONFIG_KEYS = new Set([ + "menu_items", + "site_name", + "site_logo_id", + "base_url" +]); + + +const wrapSetConfig = () => { + const { getState } = require("@saltcorn/data/db/state"); + const state = getState(); + if (!state || typeof state.setConfig !== "function") return; + if (state.setConfig.__ddWrapped) return; + + const original = state.setConfig.bind(state); + const wrapped = async function (key, value) { + if (isSuppressed()) { + return await original(key, value); + } + const result = await original(key, value); + if (!TRACKED_CONFIG_KEYS.has(key)) { + return result; + } + const opId = randomUuid(); + await enterOp(opId, async () => { + await recordOpSafely({ + op_id: opId, + op_type: "set_config", + entity_kind: "config", + entity_uuid: null, // config keys don't have UUIDs + payload: { key: key, value: value } + }); + }); + return result; + }; + wrapped.__ddWrapped = true; + wrapped.__ddOriginal = original; + state.setConfig = wrapped; +}; + + +// ----- PageGroup ----- + +// Members of a PageGroup reference page_id (integer). For cross-env identity +// we translate page_id <-> page_uuid in payloads. +const translateMembersToPortable = async (members) => { + const out = []; + for (const m of members || []) { + const pageEnt = m.page_id ? await lookupByCurrent(ENTITY_KINDS.PAGE, m.page_id) : null; + out.push({ + page_uuid: pageEnt ? pageEnt.uuid : null, + eligible_formula: m.eligible_formula, + description: m.description, + sequence: m.sequence + }); + } + return out; +}; + + +const wrapPageGroup = () => { + wrap(PageGroup, "create", ENTITY_KINDS.PAGE_GROUP, "create", { + after: async ({ result }) => { + if (!result || !result.id) return null; + const already = await lookupByCurrent(ENTITY_KINDS.PAGE_GROUP, result.id); + if (already) return null; + const uuid = await assignNewUuid(ENTITY_KINDS.PAGE_GROUP, result.id, result.name, null); + const members = await translateMembersToPortable(result.members || []); + return { + entityUuid: uuid, + payload: { after: snapshotInstance(result, PAGE_GROUP_KEYS), members: members } + }; + } + }); + + wrap(PageGroup, "update", ENTITY_KINDS.PAGE_GROUP, "update", { + before: async ({ args }) => { + const id = args[0]; + const row = args[1] || {}; + const existing = await lookupByCurrent(ENTITY_KINDS.PAGE_GROUP, id); + return { + id: id, + uuid: existing ? existing.uuid : null, + oldName: existing ? existing.current_name : null, + patch: row + }; + }, + after: async ({ before }) => { + if (!before || !before.uuid) return null; + const newName = before.patch.name !== undefined ? before.patch.name : before.oldName; + if (newName && newName !== before.oldName) { + await updateName(ENTITY_KINDS.PAGE_GROUP, before.id, newName); + } + return { entityUuid: before.uuid, payload: { patch: before.patch } }; + } + }); + + wrap(PageGroup.prototype, "delete", ENTITY_KINDS.PAGE_GROUP, "drop", standardDropHooks(ENTITY_KINDS.PAGE_GROUP, PAGE_GROUP_KEYS)); +}; + + +// ----- PageGroupMember ----- + +const wrapPageGroupMember = () => { + wrap(PageGroupMember, "create", ENTITY_KINDS.PAGE_GROUP_MEMBER, "create", { + after: async ({ result }) => { + if (!result || !result.id) return null; + const already = await lookupByCurrent(ENTITY_KINDS.PAGE_GROUP_MEMBER, result.id); + if (already) return null; + let groupUuid = null; + if (result.page_group_id) { + const g = await lookupByCurrent(ENTITY_KINDS.PAGE_GROUP, result.page_group_id); + groupUuid = g ? g.uuid : null; + } + let pageUuid = null; + if (result.page_id) { + const p = await lookupByCurrent(ENTITY_KINDS.PAGE, result.page_id); + pageUuid = p ? p.uuid : null; + } + const uuid = await assignNewUuid(ENTITY_KINDS.PAGE_GROUP_MEMBER, result.id, `member_${result.id}`, groupUuid); + return { + entityUuid: uuid, + payload: { + after: { + page_uuid: pageUuid, + eligible_formula: result.eligible_formula, + description: result.description, + sequence: result.sequence + }, + parent_uuid: groupUuid + } + }; + } + }); + + wrap(PageGroupMember, "delete", ENTITY_KINDS.PAGE_GROUP_MEMBER, "drop", { + before: async ({ args }) => { + const id = args[0]; + const existing = await lookupByCurrent(ENTITY_KINDS.PAGE_GROUP_MEMBER, id); + return { + uuid: existing ? existing.uuid : null, + currentId: id, + parentUuid: existing ? existing.parent_uuid : null + }; + }, + after: async ({ before }) => { + if (!before || !before.uuid) return null; + await removeEntityRow(ENTITY_KINDS.PAGE_GROUP_MEMBER, before.currentId); + return { + entityUuid: before.uuid, + payload: { before: { id: before.currentId }, parent_uuid: before.parentUuid } + }; + } + }); +}; + + +// ----- WorkflowStep ----- + +const wrapWorkflowStep = () => { + wrap(WorkflowStep, "create", ENTITY_KINDS.WORKFLOW_STEP, "create", { + after: async ({ args, result }) => { + // WorkflowStep.create returns the inserted integer id (db.insert), + // not an instance. Use the args + id to reconstruct the snapshot. + const id = typeof result === "number" ? result : (result && result.id); + if (!id) return null; + const already = await lookupByCurrent(ENTITY_KINDS.WORKFLOW_STEP, id); + if (already) return null; + const step = await WorkflowStep.findOne({ id: id }); + if (!step) return null; + let parentUuid = null; + if (step.trigger_id) { + const tr = await lookupByCurrent(ENTITY_KINDS.TRIGGER, step.trigger_id); + parentUuid = tr ? tr.uuid : null; + } + const name = step.name || `step_${id}`; + const uuid = await assignNewUuid(ENTITY_KINDS.WORKFLOW_STEP, id, name, parentUuid); + return { + entityUuid: uuid, + payload: { after: snapshotInstance(step, WORKFLOW_STEP_KEYS), parent_uuid: parentUuid } + }; + } + }); + + wrap(WorkflowStep.prototype, "update", ENTITY_KINDS.WORKFLOW_STEP, "update", { + before: async function () { + const existing = await lookupByCurrent(ENTITY_KINDS.WORKFLOW_STEP, this.id); + return { + uuid: existing ? existing.uuid : null, + oldName: existing ? existing.current_name : this.name, + snapshot: snapshotInstance(this, WORKFLOW_STEP_KEYS) + }; + }, + after: async function ({ args, before }) { + if (!before || !before.uuid) return null; + const patch = args[0] || {}; + const newName = patch.name !== undefined ? patch.name : before.oldName; + if (newName && newName !== before.oldName) { + await updateName(ENTITY_KINDS.WORKFLOW_STEP, this.id, newName); + } + return { entityUuid: before.uuid, payload: { before: before.snapshot, patch: patch } }; + } + }); + + wrap(WorkflowStep.prototype, "delete", ENTITY_KINDS.WORKFLOW_STEP, "drop", standardDropHooks(ENTITY_KINDS.WORKFLOW_STEP, WORKFLOW_STEP_KEYS)); +}; + + +const installAllWraps = () => { + wrapTable(); + wrapField(); + wrapView(); + wrapPage(); + wrapTrigger(); + wrapRole(); + wrapLibrary(); + wrapTag(); + wrapTableConstraint(); + wrapFile(); + wrapPageGroup(); + wrapPageGroupMember(); + wrapWorkflowStep(); + wrapSetConfig(); + wrapPlugin(); + wrapTableRows(); +}; + + +module.exports = { + installAllWraps +}; diff --git a/dev-deploy/package.json b/dev-deploy/package.json new file mode 100644 index 0000000..5dc334f --- /dev/null +++ b/dev-deploy/package.json @@ -0,0 +1,21 @@ +{ + "name": "dev-deploy", + "version": "0.0.1", + "description": "Saltcorn plugin: migrate metadata changes (tables, fields, views, pages, triggers, roles, library, tags) across Dev/Test/Prod environments via an ops journal with stable UUIDs and HMAC-authenticated peer endpoints. Detects concurrent-edit conflicts and offers theirs/mine resolution. User-table row data is never touched.", + "main": "index.js", + "scripts": { + "test": "node test/e2e.js" + }, + "keywords": [ + "deploy", + "migration", + "ops-journal", + "hmac", + "metadata-sync" + ], + "engines": { + "node": ">=20" + }, + "author": "Scott Duensing", + "license": "MIT" +} diff --git a/dev-deploy/test/e2e.js b/dev-deploy/test/e2e.js new file mode 100644 index 0000000..62905a4 --- /dev/null +++ b/dev-deploy/test/e2e.js @@ -0,0 +1,980 @@ +// dev-deploy integration test suite. +// +// Prereqs: both saltcorn instances running and reachable, plugin installed + +// bootstrapped, admin@local / AdminP@ss1 present on both. Run from the +// dev-deploy/ directory: +// +// cd ~/claude/saltcorn && ./startServer.sh & ./startServerTest.sh & +// node test/e2e.js +// +// Each test resets only what it needs to. Tests run in order; assertion +// failures are caught and counted -- one failure does not stop the suite. + +const { execFileSync, spawn } = require("node:child_process"); +const assert = require("node:assert/strict"); +const path = require("node:path"); + +const MAIN = { url: "http://localhost:3000", db: "/home/scott/claude/saltcorn/.dev-state/saltcorn.sqlite", env: "/home/scott/claude/saltcorn/.dev-state/env.sh" }; +const TEST = { url: "http://localhost:3001", db: "/home/scott/claude/saltcorn/.dev-state-test/saltcorn.sqlite", env: "/home/scott/claude/saltcorn/.dev-state-test/env.sh" }; + +const MAIN_COOKIES = "/tmp/dd-test-jar-main.txt"; +const TEST_COOKIES = "/tmp/dd-test-jar-test.txt"; + + +// --- helpers --- + +const sql = (dbPath, query) => { + const out = execFileSync("sqlite3", [dbPath, query], { encoding: "utf8" }); + return out.trim(); +}; + + +const sqlRows = (dbPath, query) => { + const out = sql(dbPath, query); + if (!out) return []; + return out.split("\n"); +}; + + +const resetInstanceDb = (dbPath) => { + sql(dbPath, ` + DELETE FROM _dd_ops; + DELETE FROM _dd_peers; + DELETE FROM _dd_anchors; + DELETE FROM _dd_table_modes; + DELETE FROM _sc_table_constraints; + DELETE FROM _sc_workflow_steps; + DELETE FROM _sc_page_group_members; + DELETE FROM _sc_page_groups; + DELETE FROM _sc_pages; + DELETE FROM _sc_triggers; + DELETE FROM _sc_views; + DELETE FROM _sc_config WHERE key IN ('menu_items', 'unrolled_menu_items'); + UPDATE _sc_plugins SET configuration = NULL WHERE name != 'dev-deploy'; + `); + // Strip _dd_entity_ids back to Saltcorn's bootstrap baseline. + sql(dbPath, ` + DELETE FROM _dd_entity_ids WHERE kind NOT IN ('table','field','role'); + DELETE FROM _dd_entity_ids WHERE kind='table' AND current_name != 'users'; + DELETE FROM _dd_entity_ids WHERE kind='field' AND current_name NOT IN ('id','email','role_id'); + DELETE FROM _dd_entity_ids WHERE kind='role' AND current_name NOT IN ('admin','staff','user','public'); + `); + const userTables = sqlRows(dbPath, "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE '\\_%' ESCAPE '\\' AND name != 'users' AND name NOT LIKE 'sqlite_%';"); + for (const t of userTables) { + if (t) sql(dbPath, `DROP TABLE IF EXISTS "${t}"`); + } + sql(dbPath, "DELETE FROM _sc_tables WHERE name != 'users'; DELETE FROM _sc_fields WHERE table_id NOT IN (SELECT id FROM _sc_tables);"); +}; + + +const runJs = (envPath, code) => { + return execFileSync( + "bash", + ["-c", `source ${envPath} && saltcorn run-js --code ${JSON.stringify(code)}`], + { encoding: "utf8" } + ); +}; + + +// scExec is like runJs but uses our test/sc-exec.js shim, giving the JS body +// full require() access -- needed for Field, TableConstraint, File, etc. that +// saltcorn run-js's vm sandbox doesn't expose. Code is piped via stdin to +// dodge shell-escape pitfalls with multi-line strings. +const scExec = (envPath, code) => { + return execFileSync( + "bash", + ["-c", `source ${envPath} && node ${path.join(__dirname, "sc-exec.js")}`], + { encoding: "utf8", input: code } + ); +}; + + +// --- HTTP helpers (browser-style admin auth via cookie jar) --- + +const curl = (args) => { + const out = execFileSync("curl", args, { encoding: "utf8" }); + return out; +}; + + +const extractCsrfFromHtml = (html) => { + const m = html.match(/name="_csrf"[^>]*value="([^"]+)"/); + return m ? m[1] : null; +}; + + +const login = (instance, jar) => { + execFileSync("rm", ["-f", jar]); + const page = curl(["-sS", "-c", jar, `${instance.url}/auth/login`]); + const csrf = extractCsrfFromHtml(page); + if (!csrf) throw new Error(`no csrf on login page for ${instance.url}`); + curl([ + "-sS", "-c", jar, "-b", jar, "-o", "/dev/null", + "-X", "POST", `${instance.url}/auth/login`, + "-d", `email=admin@local&password=AdminP@ss1&_csrf=${csrf}` + ]); +}; + + +const envIdOf = (instance, jar) => { + const html = curl(["-sS", "-b", jar, `${instance.url}/admin/dev-deploy/`]); + const m = html.match(/([0-9a-f-]{36})<\/code>/); + if (!m) throw new Error(`no env_id in dashboard html for ${instance.url}`); + return m[1]; +}; + + +const freshCsrf = (instance, jar, urlPath) => { + const html = curl(["-sS", "-b", jar, `${instance.url}${urlPath}`]); + return extractCsrfFromHtml(html); +}; + + +// Each admin POST handler embeds CSRF in a form on a specific GET page. +// Match the POST path to the page that renders its form. +const CSRF_PAGE_FOR = [ + ["/admin/dev-deploy/peers/add", "/admin/dev-deploy/peers"], + ["/admin/dev-deploy/peers/rotate", "/admin/dev-deploy/peers"], + ["/admin/dev-deploy/peers/delete", "/admin/dev-deploy/peers"], + ["/admin/dev-deploy/promote", "/admin/dev-deploy/peers"], + ["/admin/dev-deploy/pull", "/admin/dev-deploy/peers"], + ["/admin/dev-deploy/conflicts/resolve", "/admin/dev-deploy/conflicts"], + ["/admin/dev-deploy/tables/set", "/admin/dev-deploy/tables"], + ["/admin/dev-deploy/revert", "/admin/dev-deploy/ops"] +]; + + +const adminPost = (instance, jar, urlPath, fields) => { + const mapping = CSRF_PAGE_FOR.find(([p]) => urlPath === p); + if (!mapping) throw new Error(`no CSRF source page mapped for POST ${urlPath}`); + const csrf = freshCsrf(instance, jar, mapping[1]); + if (!csrf) throw new Error(`no CSRF token on ${mapping[1]} for POST ${urlPath}`); + const args = ["-sS", "-b", jar, "-o", "/tmp/dd-test-postbody.html", + "-w", "%{http_code}|%{redirect_url}", + "-X", "POST", `${instance.url}${urlPath}`, + "--data-urlencode", `_csrf=${csrf}`]; + for (const [k, v] of Object.entries(fields)) { + args.push("--data-urlencode", `${k}=${v}`); + } + const out = curl(args); + const [code, redirect] = out.split("|"); + return { status: parseInt(code, 10), redirect: redirect, body: require("node:fs").readFileSync("/tmp/dd-test-postbody.html", "utf8") }; +}; + + +// --- test runner --- + +let passed = 0, failed = 0; +const failures = []; + +const test = async (name, fn) => { + try { + await fn(); + console.log(` PASS ${name}`); + passed++; + } catch (err) { + console.log(` FAIL ${name}`); + console.log(` ${err.message}`); + failures.push({ name, err }); + failed++; + } +}; + + +const section = (name) => { + console.log(`\n[${name}]`); +}; + + +// --- scenarios --- + +const main = async () => { + section("preconditions"); + await test("both servers respond to /auth/login", async () => { + for (const inst of [MAIN, TEST]) { + const out = curl(["-sS", "-o", "/dev/null", "-w", "%{http_code}", `${inst.url}/auth/login`]); + assert.equal(out, "200", `${inst.url} returned ${out}`); + } + }); + + section("reset both instances"); + resetInstanceDb(MAIN.db); + resetInstanceDb(TEST.db); + login(MAIN, MAIN_COOKIES); + login(TEST, TEST_COOKIES); + const mainEnv = envIdOf(MAIN, MAIN_COOKIES); + const testEnv = envIdOf(TEST, TEST_COOKIES); + console.log(` main env_id: ${mainEnv}`); + console.log(` test env_id: ${testEnv}`); + + section("schema"); + await test("six _dd_* tables exist on main", async () => { + const tables = sqlRows(MAIN.db, "SELECT name FROM sqlite_master WHERE type='table' AND name LIKE '_dd_%' ORDER BY name").join(","); + assert.equal(tables, "_dd_anchors,_dd_entity_ids,_dd_env,_dd_ops,_dd_peers,_dd_table_modes"); + }); + await test("_dd_ops has conflict_with_op_id column", async () => { + const cols = sqlRows(MAIN.db, "PRAGMA table_info(_dd_ops)").map((r) => r.split("|")[1]); + assert.ok(cols.includes("conflict_with_op_id"), `cols: ${cols.join(",")}`); + }); + + section("bootstrap identity"); + await test("env_ids are distinct UUIDs", async () => { + assert.notEqual(mainEnv, testEnv); + assert.match(mainEnv, /^[0-9a-f-]{36}$/); + }); + await test("'users' table has identical UUID on both instances (deterministic backfill)", async () => { + const m = sql(MAIN.db, "SELECT uuid FROM _dd_entity_ids WHERE kind='table' AND current_name='users'"); + const t = sql(TEST.db, "SELECT uuid FROM _dd_entity_ids WHERE kind='table' AND current_name='users'"); + assert.equal(m, t, "deterministic UUIDs diverged"); + assert.notEqual(m, ""); + }); + await test("all baseline role UUIDs match across instances", async () => { + const m = sqlRows(MAIN.db, "SELECT current_name||'|'||uuid FROM _dd_entity_ids WHERE kind='role' ORDER BY current_name").join(","); + const t = sqlRows(TEST.db, "SELECT current_name||'|'||uuid FROM _dd_entity_ids WHERE kind='role' ORDER BY current_name").join(","); + assert.equal(m, t); + }); + + section("mutation capture"); + runJs(MAIN.env, 'const t = await Table.create("widgets"); await t.update({ description: "initial" });'); + await test("create_table op is journaled", async () => { + const c = sql(MAIN.db, "SELECT COUNT(*) FROM _dd_ops WHERE op_type='create_table' AND source_env_id='" + mainEnv + "'"); + assert.equal(c, "1"); + }); + await test("update_table op is journaled", async () => { + const c = sql(MAIN.db, "SELECT COUNT(*) FROM _dd_ops WHERE op_type='update_table' AND source_env_id='" + mainEnv + "'"); + assert.ok(parseInt(c, 10) >= 1); + }); + + section("pairing"); + const addRes = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/peers/add", { + env_id: testEnv, + label: "test", + base_url: TEST.url + }); + await test("main /peers/add returns 200 with shared secret", async () => { + assert.equal(addRes.status, 200); + }); + const secretMatch = addRes.body.match(/class="secret">([0-9a-f]{64}) { + assert.equal(pairTest.status, 200); + }); + + section("promote main -> test"); + const peerIdOnMain = sql(MAIN.db, `SELECT peer_id FROM _dd_peers WHERE env_id='${testEnv}'`); + const promoteRes = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/promote", { peer_id: peerIdOnMain }); + await test("promote returns 302 with success message", async () => { + assert.equal(promoteRes.status, 302); + assert.match(promoteRes.redirect, /msg=promoted/); + }); + await test("test instance now has widgets table", async () => { + const r = sql(TEST.db, "SELECT name FROM _sc_tables WHERE name='widgets'"); + assert.equal(r, "widgets"); + }); + await test("widgets UUID matches between main and test", async () => { + const m = sql(MAIN.db, "SELECT uuid FROM _dd_entity_ids WHERE current_name='widgets'"); + const t = sql(TEST.db, "SELECT uuid FROM _dd_entity_ids WHERE current_name='widgets'"); + assert.equal(m, t); + }); + + section("pull test -> main (no conflicts)"); + runJs(TEST.env, 'await Table.create("test_added");'); + const peerIdOnMain2 = sql(MAIN.db, `SELECT peer_id FROM _dd_peers WHERE env_id='${testEnv}'`); + const pullRes = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/pull", { peer_id: peerIdOnMain2 }); + await test("pull returns 302 with success message", async () => { + assert.equal(pullRes.status, 302); + assert.match(pullRes.redirect, /msg=pulled/); + }); + await test("main now has test_added", async () => { + const r = sql(MAIN.db, "SELECT name FROM _sc_tables WHERE name='test_added'"); + assert.equal(r, "test_added"); + }); + await test("test_added UUID matches across instances", async () => { + const m = sql(MAIN.db, "SELECT uuid FROM _dd_entity_ids WHERE current_name='test_added'"); + const t = sql(TEST.db, "SELECT uuid FROM _dd_entity_ids WHERE current_name='test_added'"); + assert.equal(m, t); + }); + + section("conflict detection + resolution"); + runJs(MAIN.env, 'const t = Table.findOne({name: "widgets"}); await t.update({ description: "from MAIN" });'); + runJs(TEST.env, 'const t = Table.findOne({name: "widgets"}); await t.update({ description: "from TEST" });'); + const peerIdForConflict = sql(MAIN.db, `SELECT peer_id FROM _dd_peers WHERE env_id='${testEnv}'`); + const pullConflictRes = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/pull", { peer_id: peerIdForConflict }); + await test("pull with divergent edits returns 1 conflict", async () => { + assert.match(pullConflictRes.redirect, /1%20conflicts/); + }); + await test("conflict op recorded with status=conflict", async () => { + const c = sql(MAIN.db, "SELECT COUNT(*) FROM _dd_ops WHERE status='conflict'"); + assert.equal(c, "1"); + }); + await test("conflict op has conflict_with_op_id set", async () => { + const r = sql(MAIN.db, "SELECT conflict_with_op_id FROM _dd_ops WHERE status='conflict'"); + assert.match(r, /^[0-9a-f-]{36}$/); + }); + await test("widgets description on main is still 'from MAIN' (held op not applied)", async () => { + const r = sql(MAIN.db, "SELECT description FROM _sc_tables WHERE name='widgets'"); + assert.equal(r, "from MAIN"); + }); + + const conflictOp = sql(MAIN.db, "SELECT op_id FROM _dd_ops WHERE status='conflict' LIMIT 1"); + const resolveRes = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/conflicts/resolve", { + op_id: conflictOp, + action: "theirs" + }); + await test("resolve theirs returns 302 with success", async () => { + assert.equal(resolveRes.status, 302); + assert.match(resolveRes.redirect, /msg=resolved/); + }); + await test("widgets description on main is now 'from TEST'", async () => { + const r = sql(MAIN.db, "SELECT description FROM _sc_tables WHERE name='widgets'"); + assert.equal(r, "from TEST"); + }); + await test("resolved op status is committed", async () => { + const r = sql(MAIN.db, `SELECT status FROM _dd_ops WHERE op_id='${conflictOp}'`); + assert.equal(r, "committed"); + }); + await test("pending conflicts is 0", async () => { + const c = sql(MAIN.db, "SELECT COUNT(*) FROM _dd_ops WHERE status='conflict'"); + assert.equal(c, "0"); + }); + + section("conflict merge per-field"); + runJs(MAIN.env, 'const t = Table.findOne({name: "widgets"}); await t.update({ description: "M2" });'); + runJs(TEST.env, 'const t = Table.findOne({name: "widgets"}); await t.update({ description: "T2" });'); + const peerIdForMerge = sql(MAIN.db, `SELECT peer_id FROM _dd_peers WHERE env_id='${testEnv}'`); + adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/pull", { peer_id: peerIdForMerge }); + + await test("a fresh conflict appears for the second divergence", async () => { + const c = sql(MAIN.db, "SELECT COUNT(*) FROM _dd_ops WHERE status='conflict'"); + assert.equal(c, "1"); + }); + + const mergeOp = sql(MAIN.db, "SELECT op_id FROM _dd_ops WHERE status='conflict' LIMIT 1"); + + await test("merge view returns 200 for an update-vs-update conflict", async () => { + const out = curl(["-sS", "-b", MAIN_COOKIES, "-o", "/dev/null", "-w", "%{http_code}", + `${MAIN.url}/admin/dev-deploy/conflicts/merge?op_id=${mergeOp}`]); + assert.equal(out, "200"); + }); + + await test("merge view shows the diverging description field", async () => { + const html = curl(["-sS", "-b", MAIN_COOKIES, + `${MAIN.url}/admin/dev-deploy/conflicts/merge?op_id=${mergeOp}`]); + assert.match(html, /choice_description/); + assert.match(html, /T2/); + }); + + // Fetch the merge page for CSRF then POST a custom value + const mergeHtml = curl(["-sS", "-b", MAIN_COOKIES, + `${MAIN.url}/admin/dev-deploy/conflicts/merge?op_id=${mergeOp}`]); + const mergeCsrf = extractCsrfFromHtml(mergeHtml); + const mergeResp = execFileSync("curl", [ + "-sS", "-b", MAIN_COOKIES, "-o", "/dev/null", + "-w", "%{http_code}|%{redirect_url}", + "-X", "POST", `${MAIN.url}/admin/dev-deploy/conflicts/merge/apply`, + "--data-urlencode", `_csrf=${mergeCsrf}`, + "--data-urlencode", `op_id=${mergeOp}`, + "--data-urlencode", "choice_description=custom", + "--data-urlencode", "custom_description=hybrid value" + ], { encoding: "utf8" }); + const [mergeStatus, mergeRedirect] = mergeResp.split("|"); + + await test("merge apply returns 302 with success message", async () => { + assert.equal(mergeStatus, "302"); + assert.match(mergeRedirect, /msg=merged/); + }); + + await test("widgets description on main is now 'hybrid value'", async () => { + const r = sql(MAIN.db, "SELECT description FROM _sc_tables WHERE name='widgets'"); + assert.equal(r, "hybrid value"); + }); + + await test("merged op has status=merged, conflict_with cleared", async () => { + const r = sql(MAIN.db, `SELECT status||'|'||COALESCE(conflict_with_op_id,'') FROM _dd_ops WHERE op_id='${mergeOp}'`); + assert.equal(r, "merged|"); + }); + + await test("pending conflicts is 0 after merge", async () => { + const c = sql(MAIN.db, "SELECT COUNT(*) FROM _dd_ops WHERE status='conflict'"); + assert.equal(c, "0"); + }); + + section("table constraints"); + // Add a field to widgets and a unique constraint on it. Promote to test. + scExec(MAIN.env, ` + const t = Table.findOne({name: "widgets"}); + await Field.create({ table_id: t.id, name: "sku", label: "SKU", type: "String" }); + const fresh = Table.findOne({name: "widgets"}); + await TableConstraint.create({ table_id: fresh.id, type: "Unique", configuration: { fields: ["sku"] } }); + `); + + await test("create_field op journaled for sku", async () => { + const c = sql(MAIN.db, "SELECT COUNT(*) FROM _dd_ops WHERE op_type='create_field' AND payload LIKE '%sku%'"); + assert.equal(c, "1"); + }); + await test("create_constraint op journaled with type=Unique", async () => { + const c = sql(MAIN.db, "SELECT COUNT(*) FROM _dd_ops WHERE op_type='create_constraint' AND payload LIKE '%Unique%'"); + assert.equal(c, "1"); + }); + + const peerIdForConstraint = sql(MAIN.db, `SELECT peer_id FROM _dd_peers WHERE env_id='${testEnv}'`); + const promoteConstraintRes = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/promote", { peer_id: peerIdForConstraint }); + await test("promote with constraint returns success", async () => { + assert.equal(promoteConstraintRes.status, 302); + assert.match(promoteConstraintRes.redirect, /msg=promoted/); + }); + await test("test now has the sku field", async () => { + const r = sql(TEST.db, "SELECT COUNT(*) FROM _sc_fields WHERE name='sku' AND table_id=(SELECT id FROM _sc_tables WHERE name='widgets')"); + assert.equal(r, "1"); + }); + await test("test now has the unique constraint on sku", async () => { + // _sc_table_constraints uses JSON for configuration; SQLite stores TEXT + const r = sql(TEST.db, "SELECT COUNT(*) FROM _sc_table_constraints WHERE type='Unique' AND configuration LIKE '%sku%'"); + assert.equal(r, "1"); + }); + await test("constraint UUID matches across instances", async () => { + const m = sql(MAIN.db, "SELECT uuid FROM _dd_entity_ids WHERE kind='constraint' AND current_name='unique(sku)'"); + const t = sql(TEST.db, "SELECT uuid FROM _dd_entity_ids WHERE kind='constraint' AND current_name='unique(sku)'"); + assert.equal(m, t); + assert.notEqual(m, ""); + }); + + section("file propagation"); + const FILE_BYTES = "Hello, dev-deploy! This is a test document."; + scExec(MAIN.env, ` + const fs = require("fs"); + const path = require("path"); + const tenant = db.getTenantSchema(); + const absPath = path.join(db.connectObj.file_store, tenant, "test_doc.txt"); + fs.mkdirSync(path.dirname(absPath), { recursive: true }); + fs.writeFileSync(absPath, ${JSON.stringify(FILE_BYTES)}); + await File.create({ + filename: "test_doc.txt", + location: absPath, + uploaded_at: new Date(), + size_kb: 1, + mime_super: "text", + mime_sub: "plain", + min_role_read: 1 + }); + `); + + await test("create_file op journaled with content_hash + relative_path", async () => { + const r = sql(MAIN.db, "SELECT payload FROM _dd_ops WHERE op_type='create_file' LIMIT 1"); + assert.match(r, /content_hash/); + assert.match(r, /test_doc\.txt/); + }); + + await test("file entry in _dd_entity_ids on main", async () => { + const r = sql(MAIN.db, "SELECT current_name FROM _dd_entity_ids WHERE kind='file' LIMIT 1"); + assert.equal(r, "test_doc.txt"); + }); + + const peerIdForFile = sql(MAIN.db, `SELECT peer_id FROM _dd_peers WHERE env_id='${testEnv}'`); + const fileRes = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/promote", { peer_id: peerIdForFile }); + + await test("file promote returns success", async () => { + assert.equal(fileRes.status, 302); + assert.match(fileRes.redirect, /msg=promoted/); + }); + + await test("file exists on test instance with same bytes", async () => { + const fs = require("node:fs"); + const out = fs.readFileSync("/home/scott/claude/saltcorn/.dev-state-test/files/public/test_doc.txt", "utf8"); + assert.equal(out, FILE_BYTES); + }); + + await test("file UUID matches across instances", async () => { + const m = sql(MAIN.db, "SELECT uuid FROM _dd_entity_ids WHERE kind='file' AND current_name='test_doc.txt'"); + const t = sql(TEST.db, "SELECT uuid FROM _dd_entity_ids WHERE kind='file' AND current_name='test_doc.txt'"); + assert.equal(m, t); + assert.notEqual(m, ""); + }); + + section("plugin mismatch warning"); + await test("health endpoint is reachable and HMAC-required", async () => { + const out = curl(["-sS", "-o", "/dev/null", "-w", "%{http_code}", `${TEST.url}/dev-deploy/api/health`]); + assert.equal(out, "400"); // missing HMAC headers + }); + await test("promote includes no plugin warnings when peers match", async () => { + // Both instances have identical plugin lists (base, sbadmin2, dev-deploy) + // so the warning suffix should be absent. + runJs(MAIN.env, 'await Table.create("warning_demo_table");'); + const peerId = sql(MAIN.db, `SELECT peer_id FROM _dd_peers WHERE env_id='${testEnv}'`); + const r = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/promote", { peer_id: peerId }); + assert.equal(r.status, 302); + // Both sides have the same plugins so no WARNINGS suffix should appear. + assert.ok(!/WARNINGS/.test(r.redirect || ""), `unexpected warnings: ${r.redirect}`); + }); + + section("file refs in page layout"); + scExec(MAIN.env, ` + const Page = require("@saltcorn/data/models/page"); + // Pages with a file ref in their layout — fileid as a relative-path + // string (the form Saltcorn produces on a fresh upload). + await Page.create({ + name: "page_with_image", + title: "Page With Image", + description: "Has an image", + min_role: 100, + layout: { type: "image", srctype: "File", fileid: "test_doc.txt", alt: "test" }, + fixed_states: {} + }); + `); + + await test("create_page op replaces fileid with placeholder in journal", async () => { + const r = sql(MAIN.db, "SELECT payload FROM _dd_ops WHERE op_type='create_page' AND payload LIKE '%page_with_image%' LIMIT 1"); + assert.match(r, /__dd_file_ref::/); + // The original string must NOT appear as the raw fileid value + assert.ok(!/"fileid":"test_doc\.txt"/.test(r), `fileid was not translated: ${r}`); + }); + + const peerIdForRefs = sql(MAIN.db, `SELECT peer_id FROM _dd_peers WHERE env_id='${testEnv}'`); + const refsPromote = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/promote", { peer_id: peerIdForRefs }); + await test("page-with-image promote returns success", async () => { + assert.equal(refsPromote.status, 302); + assert.match(refsPromote.redirect, /msg=promoted/); + }); + + await test("page exists on test with fileid resolved back to local path", async () => { + const layout = sql(TEST.db, "SELECT layout FROM _sc_pages WHERE name='page_with_image'"); + // On test, fileid should be the relative path that resolves locally + assert.match(layout, /"fileid":"test_doc\.txt"/); + // Placeholder must NOT leak into the live entity + assert.ok(!/__dd_file_ref::/.test(layout), `placeholder leaked into live page: ${layout}`); + }); + + section("page_group propagation"); + scExec(MAIN.env, ` + const Page = require("@saltcorn/data/models/page"); + const PageGroup = require("@saltcorn/data/models/page_group"); + const page = await Page.create({ + name: "intro_page", + title: "Intro", + description: "Intro page", + min_role: 100, + layout: { type: "blank", contents: "Hello" }, + fixed_states: {} + }); + const pg = await PageGroup.create({ + name: "home_group", + description: "home routing", + min_role: 100, + random_allocation: false, + members: [] + }); + await pg.addMember({ + page_id: page.id, + eligible_formula: "true", + description: "default" + }); + `); + + await test("create_page_group op journaled", async () => { + const r = sql(MAIN.db, "SELECT payload FROM _dd_ops WHERE op_type='create_page_group' LIMIT 1"); + assert.match(r, /home_group/); + }); + await test("create_page_group_member op journaled with page_uuid", async () => { + const r = sql(MAIN.db, "SELECT payload FROM _dd_ops WHERE op_type='create_page_group_member' LIMIT 1"); + assert.match(r, /page_uuid/); + }); + + const peerIdForPG = sql(MAIN.db, `SELECT peer_id FROM _dd_peers WHERE env_id='${testEnv}'`); + const pgPromote = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/promote", { peer_id: peerIdForPG }); + await test("page_group promote returns success", async () => { + assert.equal(pgPromote.status, 302); + assert.match(pgPromote.redirect, /msg=promoted/); + }); + await test("test instance has home_group page_group", async () => { + const r = sql(TEST.db, "SELECT name FROM _sc_page_groups WHERE name='home_group'"); + assert.equal(r, "home_group"); + }); + await test("page_group on test has its member row", async () => { + const r = sql(TEST.db, "SELECT COUNT(*) FROM _sc_page_group_members m JOIN _sc_page_groups g ON m.page_group_id=g.id WHERE g.name='home_group'"); + assert.equal(r, "1"); + }); + await test("page_group UUID matches across instances", async () => { + const m = sql(MAIN.db, "SELECT uuid FROM _dd_entity_ids WHERE kind='page_group' AND current_name='home_group'"); + const t = sql(TEST.db, "SELECT uuid FROM _dd_entity_ids WHERE kind='page_group' AND current_name='home_group'"); + assert.equal(m, t); + assert.notEqual(m, ""); + }); + + section("workflow_step propagation"); + scExec(MAIN.env, ` + const Trigger = require("@saltcorn/data/models/trigger"); + const WorkflowStep = require("@saltcorn/data/models/workflow_step"); + // clean any prior + const oldT = Trigger.findOne({name: "demo_workflow"}); + if (oldT) await oldT.delete(); + const tr = await Trigger.create({ + name: "demo_workflow", + action: "Workflow", + when_trigger: "Never", + min_role: 1, + configuration: {} + }); + await WorkflowStep.create({ + name: "step1", + trigger_id: tr.id, + action_name: "set_context", + next_step: "", + initial_step: true, + configuration: { ctx_values: '{ "hello": "world" }' }, + only_if: "" + }); + `); + + await test("create_workflow_step op journaled", async () => { + const r = sql(MAIN.db, "SELECT COUNT(*) FROM _dd_ops WHERE op_type='create_workflow_step'"); + assert.equal(r, "1"); + }); + + const peerIdForWf = sql(MAIN.db, `SELECT peer_id FROM _dd_peers WHERE env_id='${testEnv}'`); + const wfPromote = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/promote", { peer_id: peerIdForWf }); + await test("workflow_step promote returns success", async () => { + assert.equal(wfPromote.status, 302); + assert.match(wfPromote.redirect, /msg=promoted/); + }); + await test("test has the demo_workflow trigger", async () => { + const r = sql(TEST.db, "SELECT name FROM _sc_triggers WHERE name='demo_workflow'"); + assert.equal(r, "demo_workflow"); + }); + await test("test has the workflow step", async () => { + const r = sql(TEST.db, "SELECT COUNT(*) FROM _sc_workflow_steps s JOIN _sc_triggers t ON s.trigger_id=t.id WHERE t.name='demo_workflow' AND s.name='step1'"); + assert.equal(r, "1"); + }); + await test("workflow_step UUID matches across instances", async () => { + const m = sql(MAIN.db, "SELECT uuid FROM _dd_entity_ids WHERE kind='workflow_step' AND current_name='step1'"); + const t = sql(TEST.db, "SELECT uuid FROM _dd_entity_ids WHERE kind='workflow_step' AND current_name='step1'"); + assert.equal(m, t); + assert.notEqual(m, ""); + }); + + section("config + menu propagation"); + scExec(MAIN.env, ` + const { getState } = require("@saltcorn/data/db/state"); + await getState().setConfig("menu_items", [ + { label: "Widgets", type: "View", viewname: "widget_list" }, + { label: "About", type: "Page", pagename: "about" } + ]); + `); + await test("set_config op journaled for menu_items", async () => { + const r = sql(MAIN.db, "SELECT COUNT(*) FROM _dd_ops WHERE op_type='set_config' AND payload LIKE '%menu_items%'"); + assert.ok(parseInt(r, 10) >= 1, `got ${r} set_config ops`); + }); + const peerIdForMenu = sql(MAIN.db, `SELECT peer_id FROM _dd_peers WHERE env_id='${testEnv}'`); + const menuPromote = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/promote", { peer_id: peerIdForMenu }); + await test("menu promote returns success", async () => { + assert.equal(menuPromote.status, 302); + assert.match(menuPromote.redirect, /msg=promoted/); + }); + await test("test instance has matching menu_items", async () => { + const v = sql(TEST.db, "SELECT value FROM _sc_config WHERE key='menu_items'"); + assert.match(v, /widget_list/); + assert.match(v, /about/); + }); + + section("plugin configuration propagation"); + scExec(MAIN.env, ` + const Plugin = require("@saltcorn/data/models/plugin"); + const p = await Plugin.findOne({ name: "sbadmin2" }); + if (!p) throw new Error("sbadmin2 plugin not found"); + p.configuration = { test_key: "from MAIN", color_scheme: "dark" }; + await p.upsert(); + `); + await test("update_plugin_config op journaled for sbadmin2", async () => { + const r = sql(MAIN.db, "SELECT COUNT(*) FROM _dd_ops WHERE op_type='update_plugin_config' AND payload LIKE '%sbadmin2%'"); + assert.equal(r, "1"); + }); + const peerIdForPC = sql(MAIN.db, `SELECT peer_id FROM _dd_peers WHERE env_id='${testEnv}'`); + const pcPromote = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/promote", { peer_id: peerIdForPC }); + await test("plugin-config promote returns success", async () => { + assert.equal(pcPromote.status, 302); + assert.match(pcPromote.redirect, /msg=promoted/); + }); + await test("test sbadmin2 has updated configuration", async () => { + const cfg = sql(TEST.db, "SELECT configuration FROM _sc_plugins WHERE name='sbadmin2'"); + assert.match(cfg, /from MAIN/); + assert.match(cfg, /color_scheme/); + }); + + section("managed row data propagation"); + + // Create two tables on main: categories + products with FK products.category_id -> categories.id. + // Add some rows. Then mark both managed. Promote. Verify rows on test with same UUIDs. + scExec(MAIN.env, ` + const Table = require("@saltcorn/data/models/table"); + const Field = require("@saltcorn/data/models/field"); + + const cats = await Table.create("dd_categories"); + await Field.create({ table_id: cats.id, name: "name", label: "Name", type: "String" }); + + const prods = await Table.create("dd_products"); + await Field.create({ table_id: prods.id, name: "name", label: "Name", type: "String" }); + await Field.create({ table_id: prods.id, name: "price", label: "Price", type: "Float" }); + await Field.create({ table_id: prods.id, name: "category", label: "Category", type: "Key to dd_categories", reftable_name: "dd_categories", reftype: "Integer" }); + + const cats2 = Table.findOne({name: "dd_categories"}); + const electId = await cats2.insertRow({ name: "Electronics" }); + const apparelId = await cats2.insertRow({ name: "Apparel" }); + + const prods2 = Table.findOne({name: "dd_products"}); + await prods2.insertRow({ name: "Widget", price: 9.99, category: electId }); + await prods2.insertRow({ name: "Gadget", price: 19.95, category: electId }); + await prods2.insertRow({ name: "T-shirt", price: 14.99, category: apparelId }); + `); + + // Mark both tables managed via the admin UI (causes initial ship) + const catsUuid = sql(MAIN.db, "SELECT uuid FROM _dd_entity_ids WHERE kind='table' AND current_name='dd_categories'"); + const prodsUuid = sql(MAIN.db, "SELECT uuid FROM _dd_entity_ids WHERE kind='table' AND current_name='dd_products'"); + + const setCatsManaged = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/tables/set", { table_uuid: catsUuid, data_mode: "managed" }); + const setProdsManaged = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/tables/set", { table_uuid: prodsUuid, data_mode: "managed" }); + + await test("marking dd_categories managed ships 2 rows", async () => { + assert.match(setCatsManaged.redirect, /shipped.{1,3}2.{1,3}rows/); + }); + await test("marking dd_products managed ships 3 rows", async () => { + assert.match(setProdsManaged.redirect, /shipped.{1,3}3.{1,3}rows/); + }); + await test("_dd_row_uuid column exists on dd_categories on main", async () => { + const info = sqlRows(MAIN.db, "PRAGMA table_info(dd_categories);").join(","); + assert.match(info, /_dd_row_uuid/); + }); + await test("insert_row ops journaled for the seed rows", async () => { + const c = sql(MAIN.db, "SELECT COUNT(*) FROM _dd_ops WHERE op_type='insert_row'"); + assert.ok(parseInt(c, 10) >= 5, `expected >=5 insert_row ops, got ${c}`); + }); + + // Promote + const peerIdForRows = sql(MAIN.db, `SELECT peer_id FROM _dd_peers WHERE env_id='${testEnv}'`); + const rowPromote = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/promote", { peer_id: peerIdForRows }); + await test("managed row promote returns success", async () => { + assert.equal(rowPromote.status, 302); + assert.match(rowPromote.redirect, /msg=promoted/); + }); + await test("test has dd_categories rows", async () => { + const r = sql(TEST.db, "SELECT COUNT(*) FROM dd_categories"); + assert.equal(r, "2"); + }); + await test("test has dd_products rows", async () => { + const r = sql(TEST.db, "SELECT COUNT(*) FROM dd_products"); + assert.equal(r, "3"); + }); + await test("a product on test has its FK resolved to a local category id", async () => { + const r = sql(TEST.db, "SELECT p.name || '|' || c.name FROM dd_products p JOIN dd_categories c ON p.category=c.id WHERE p.name='Widget'"); + assert.equal(r, "Widget|Electronics"); + }); + await test("row UUIDs match across instances for a product", async () => { + const m = sql(MAIN.db, "SELECT _dd_row_uuid FROM dd_products WHERE name='Widget'"); + const t = sql(TEST.db, "SELECT _dd_row_uuid FROM dd_products WHERE name='Widget'"); + assert.equal(m, t); + assert.notEqual(m, ""); + }); + + // Update a row on main, promote again, verify on test. + scExec(MAIN.env, ` + const Table = require("@saltcorn/data/models/table"); + const prods = Table.findOne({name: "dd_products"}); + const widget = (await prods.getJoinedRows({ where: { name: "Widget" }}))[0]; + await prods.updateRow({ price: 12.50 }, widget.id); + `); + const rowPromote2 = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/promote", { peer_id: peerIdForRows }); + await test("update_row op promoted", async () => { + assert.match(rowPromote2.redirect, /msg=promoted/); + }); + await test("test sees Widget at new price 12.5", async () => { + const r = sql(TEST.db, "SELECT price FROM dd_products WHERE name='Widget'"); + assert.equal(r, "12.5"); + }); + + // Delete a row on main, promote, verify removed on test. + scExec(MAIN.env, ` + const Table = require("@saltcorn/data/models/table"); + const prods = Table.findOne({name: "dd_products"}); + await prods.deleteRows({ name: "T-shirt" }); + `); + const rowPromote3 = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/promote", { peer_id: peerIdForRows }); + await test("drop_row op promoted", async () => { + assert.match(rowPromote3.redirect, /msg=promoted/); + }); + await test("test no longer has T-shirt row", async () => { + const r = sql(TEST.db, "SELECT COUNT(*) FROM dd_products WHERE name='T-shirt'"); + assert.equal(r, "0"); + }); + + section("starter rows: ship once then detach"); + scExec(MAIN.env, ` + const Table = require("@saltcorn/data/models/table"); + const Field = require("@saltcorn/data/models/field"); + const seed = await Table.create("dd_templates"); + await Field.create({ table_id: seed.id, name: "name", label: "Name", type: "String" }); + const t = Table.findOne({name: "dd_templates"}); + await t.insertRow({ name: "Welcome" }); + await t.insertRow({ name: "Goodbye" }); + `); + const seedUuid = sql(MAIN.db, "SELECT uuid FROM _dd_entity_ids WHERE kind='table' AND current_name='dd_templates'"); + const setSeedStarter = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/tables/set", { table_uuid: seedUuid, data_mode: "starter" }); + await test("marking dd_templates starter ships 2 rows", async () => { + assert.match(setSeedStarter.redirect, /shipped.{1,3}2.{1,3}rows/); + }); + await test("dd_templates marked starter_shipped_at", async () => { + const r = sql(MAIN.db, `SELECT starter_shipped_at IS NOT NULL FROM _dd_table_modes WHERE table_uuid='${seedUuid}'`); + assert.equal(r, "1"); + }); + + // After starter ship, further row ops on main should NOT journal + const beforeCount = sql(MAIN.db, "SELECT COUNT(*) FROM _dd_ops WHERE op_type='insert_row' OR op_type='update_row'"); + scExec(MAIN.env, ` + const Table = require("@saltcorn/data/models/table"); + const t = Table.findOne({name: "dd_templates"}); + await t.insertRow({ name: "Post-ship row" }); + `); + const afterCount = sql(MAIN.db, "SELECT COUNT(*) FROM _dd_ops WHERE op_type='insert_row' OR op_type='update_row'"); + await test("subsequent inserts on starter table do NOT journal", async () => { + assert.equal(afterCount, beforeCount, `journal grew from ${beforeCount} to ${afterCount} on a starter table`); + }); + + const starterPromote = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/promote", { peer_id: peerIdForRows }); + await test("starter promote ships initial rows", async () => { + assert.match(starterPromote.redirect, /msg=promoted/); + }); + await test("test has the 2 initial templates (but not the post-ship row)", async () => { + const c = sql(TEST.db, "SELECT COUNT(*) FROM dd_templates"); + assert.equal(c, "2"); + const has = sql(TEST.db, "SELECT COUNT(*) FROM dd_templates WHERE name='Welcome'"); + assert.equal(has, "1"); + const postship = sql(TEST.db, "SELECT COUNT(*) FROM dd_templates WHERE name='Post-ship row'"); + assert.equal(postship, "0"); + }); + + section("revert"); + runJs(MAIN.env, 'await Table.create("ephemeral");'); + const createOp = sql(MAIN.db, "SELECT op_id FROM _dd_ops WHERE op_type='create_table' AND payload LIKE '%ephemeral%' LIMIT 1"); + const revertRes = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/revert", { op_id: createOp }); + await test("revert returns 302", async () => { + assert.equal(revertRes.status, 302); + }); + await test("ephemeral table is gone after revert", async () => { + const r = sql(MAIN.db, "SELECT COUNT(*) FROM _sc_tables WHERE name='ephemeral'"); + assert.equal(r, "0"); + }); + await test("drop_table op recorded as the revert", async () => { + const r = sql(MAIN.db, "SELECT COUNT(*) FROM _dd_ops WHERE op_type='drop_table' AND payload LIKE '%ephemeral%'"); + assert.ok(parseInt(r, 10) >= 1); + }); + + section("data_mode"); + const widgetsUuid = sql(MAIN.db, "SELECT uuid FROM _dd_entity_ids WHERE current_name='widgets'"); + const setRes = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/tables/set", { + table_uuid: widgetsUuid, + data_mode: "starter" + }); + await test("tables/set redirects with success", async () => { + assert.equal(setRes.status, 302); + assert.match(setRes.redirect, /msg=set/); + }); + await test("_dd_table_modes row inserted with starter", async () => { + const r = sql(MAIN.db, `SELECT data_mode FROM _dd_table_modes WHERE table_uuid='${widgetsUuid}'`); + assert.equal(r, "starter"); + }); + await test("users table is locked: cannot change data_mode", async () => { + const usersUuid = sql(MAIN.db, "SELECT uuid FROM _dd_entity_ids WHERE current_name='users' AND kind='table'"); + const r = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/tables/set", { + table_uuid: usersUuid, + data_mode: "managed" + }); + assert.match(r.redirect || "", /err=/); + const mode = sql(MAIN.db, `SELECT COUNT(*) FROM _dd_table_modes WHERE table_uuid='${usersUuid}'`); + assert.equal(mode, "0", "users table should have no row in _dd_table_modes"); + }); + + section("machine-endpoint security"); + // Test that requests without HMAC are rejected + await test("unsigned POST to ingest returns 400 (missing header)", async () => { + const out = curl([ + "-sS", "-o", "/dev/null", "-w", "%{http_code}", + "-X", "POST", + "-H", "Content-Type: application/vnd.dev-deploy+json", + "--data", '{"ops":[]}', + `${TEST.url}/dev-deploy/api/ingest` + ]); + assert.equal(out, "400"); + }); + await test("ingest with bogus signature returns 401", async () => { + const out = curl([ + "-sS", "-o", "/dev/null", "-w", "%{http_code}", + "-X", "POST", + "-H", "Content-Type: application/vnd.dev-deploy+json", + "-H", `X-DD-Env-Id: ${mainEnv}`, + "-H", `X-DD-Timestamp: ${Date.now()}`, + "-H", "X-DD-Nonce: deadbeefdeadbeefdeadbeefdeadbeef", + "-H", "X-DD-Signature: 0000000000000000000000000000000000000000000000000000000000000000", + "--data", '{"ops":[]}', + `${TEST.url}/dev-deploy/api/ingest` + ]); + assert.equal(out, "401"); + }); + await test("ingest from unknown env_id returns 401", async () => { + const out = curl([ + "-sS", "-o", "/dev/null", "-w", "%{http_code}", + "-X", "POST", + "-H", "Content-Type: application/vnd.dev-deploy+json", + "-H", "X-DD-Env-Id: 00000000-0000-4000-8000-000000000000", + "-H", `X-DD-Timestamp: ${Date.now()}`, + "-H", "X-DD-Nonce: deadbeefdeadbeefdeadbeefdeadbeef", + "-H", "X-DD-Signature: 0000000000000000000000000000000000000000000000000000000000000000", + "--data", '{"ops":[]}', + `${TEST.url}/dev-deploy/api/ingest` + ]); + assert.equal(out, "401"); + }); + await test("ingest with stale timestamp returns 401", async () => { + const stale = String(Date.now() - 10 * 60 * 1000); + const out = curl([ + "-sS", "-o", "/dev/null", "-w", "%{http_code}", + "-X", "POST", + "-H", "Content-Type: application/vnd.dev-deploy+json", + "-H", `X-DD-Env-Id: ${mainEnv}`, + "-H", `X-DD-Timestamp: ${stale}`, + "-H", "X-DD-Nonce: deadbeefdeadbeefdeadbeefdeadbeef", + "-H", "X-DD-Signature: 0000000000000000000000000000000000000000000000000000000000000000", + "--data", '{"ops":[]}', + `${TEST.url}/dev-deploy/api/ingest` + ]); + assert.equal(out, "401"); + }); + + section("admin auth"); + await test("unauthenticated GET /admin/dev-deploy/ returns 403", async () => { + const out = curl(["-sS", "-o", "/dev/null", "-w", "%{http_code}", `${MAIN.url}/admin/dev-deploy/`]); + assert.equal(out, "403"); + }); + + // --- summary --- + console.log(`\n${passed} passed, ${failed} failed`); + if (failed > 0) { + console.log("\nFailures:"); + for (const f of failures) { + console.log(` - ${f.name}`); + console.log(` ${f.err.stack.split("\n").slice(0, 4).join("\n ")}`); + } + process.exit(1); + } + process.exit(0); +}; + + +main().catch((err) => { + console.error("test runner crashed:", err); + process.exit(2); +}); diff --git a/dev-deploy/test/sc-exec.js b/dev-deploy/test/sc-exec.js new file mode 100644 index 0000000..3750a9b --- /dev/null +++ b/dev-deploy/test/sc-exec.js @@ -0,0 +1,57 @@ +// Like `saltcorn run-js` but without the vm sandbox -- the supplied JS body +// runs with full `require()` access so the e2e suite can drive Field / +// TableConstraint / File creation. Invoked from the test runner via: +// +// source && node test/sc-exec.js "" + +const { createRequire } = require("node:module"); +const path = require("node:path"); + +// Resolve @saltcorn/* against the Saltcorn checkout's node_modules. Node +// resolves require from the script's path by default, which doesn't see +// Saltcorn's deps; createRequire reroots the resolution. Layout: +// / +// dev-deploy/test/sc-exec.js (this file) +// saltcorn/packages/saltcorn-data/ +// So: up 2 to project root, then into saltcorn/packages/saltcorn-data/. +const scRequire = createRequire(path.join(__dirname, "..", "..", "saltcorn", "packages", "saltcorn-data", "package.json")); + + +const main = async () => { + // Read code from stdin -- avoids shell-escape gymnastics for multi-line + // bodies (notably \n which bash double-quotes pass through literally). + const code = require("node:fs").readFileSync(0, "utf8"); + if (!code) { + console.error("usage: pipe JS code into sc-exec.js's stdin"); + process.exit(2); + } + + const Plugin = scRequire("@saltcorn/data/models/plugin"); + const { init_multi_tenant } = scRequire("@saltcorn/data/db/state"); + await Plugin.loadAllPlugins(); + await init_multi_tenant(Plugin.loadAllPlugins, undefined, []); + + const Table = scRequire("@saltcorn/data/models/table"); + const Field = scRequire("@saltcorn/data/models/field"); + const View = scRequire("@saltcorn/data/models/view"); + const Page = scRequire("@saltcorn/data/models/page"); + const Trigger = scRequire("@saltcorn/data/models/trigger"); + const TableConstraint = scRequire("@saltcorn/data/models/table_constraints"); + const File = scRequire("@saltcorn/data/models/file"); + const db = scRequire("@saltcorn/data/db"); + + // Build an async closure with the models + a re-rooted require in scope. + // (`new Function` bodies don't inherit require from the script scope.) + const fn = new Function( + "Table", "Field", "View", "Page", "Trigger", "TableConstraint", "File", "db", "require", + `return (async () => { ${code} })();` + ); + await fn(Table, Field, View, Page, Trigger, TableConstraint, File, db, scRequire); + process.exit(0); +}; + + +main().catch((err) => { + console.error(err && err.stack ? err.stack : err); + process.exit(1); +}); diff --git a/devServer.sh b/devServer.sh new file mode 100755 index 0000000..c2a604a --- /dev/null +++ b/devServer.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -e +cd "$(dirname "$0")" +source .dev-state/env.sh + +# See startServer.sh for the rationale. Note: `dev:serve` watches Saltcorn +# repo files but explicitly ignores plugins_folder/, so plugin source edits +# still need an install step to land in localversion/ -- this handles it. +saltcorn install-plugin -d ./dev-deploy 2>&1 | tail -2 || true + +# dev:serve spawns `npm run tsc`, which needs cwd=upstream root. +# Side effect: sessions.sqlite lands in saltcorn/ (not .dev-state/) when using +# dev:serve. Don't run this alongside ./startServer.sh on the MAIN instance. +cd saltcorn +exec saltcorn dev:serve "$@" diff --git a/installSaltcorn.sh b/installSaltcorn.sh new file mode 100755 index 0000000..4d3ce7c --- /dev/null +++ b/installSaltcorn.sh @@ -0,0 +1,150 @@ +#!/usr/bin/env bash +# installSaltcorn.sh -- reproduce this Saltcorn dev environment from scratch. +# +# Layout created: +# / +# saltcorn/ upstream saltcorn checkout (cloned + built) +# .dev-state/ per-instance state (sqlite, files, env.sh, sessions) +# startServer.sh launcher +# +# After this, `cd && ./startServer.sh` boots Saltcorn on +# http://localhost:3000/. +# +# Usage: ./installSaltcorn.sh [destination] (default: ./saltcorn) + +set -euo pipefail + +DEST="${1:-./saltcorn}" +REPO_URL="https://github.com/saltcorn/saltcorn.git" +ADMIN_EMAIL="admin@local" +ADMIN_PASSWORD="AdminP@ss1" + +# --- prerequisites --- + +for cmd in git curl; do + command -v "$cmd" >/dev/null || { echo "$cmd not found in PATH"; exit 1; } +done + +# Use a user-local nvm at ~/.nvm rather than system node, matching this dev +# environment. The .dev-state/env.sh written below sources the same nvm, so +# `saltcorn` keeps working in fresh shells without further setup. +export NVM_DIR="$HOME/.nvm" +NVM_VERSION="v0.40.1" + +if [ ! -s "$NVM_DIR/nvm.sh" ]; then + echo "==> nvm not found at $NVM_DIR -- installing $NVM_VERSION" + curl -fsSL "https://raw.githubusercontent.com/nvm-sh/nvm/${NVM_VERSION}/install.sh" | bash + if [ ! -s "$NVM_DIR/nvm.sh" ]; then + echo "nvm install did not produce $NVM_DIR/nvm.sh" + exit 1 + fi +fi + +# nvm.sh references some unset variables on load; relax 'set -u' while sourcing. +set +u +# shellcheck source=/dev/null +. "$NVM_DIR/nvm.sh" +set -u + +echo "==> Ensuring node 20 is installed and active" +nvm install 20 +nvm use 20 + +NODE_MAJOR=$(node -p 'process.versions.node.split(".")[0]') +if [ "$NODE_MAJOR" -lt 18 ]; then + echo "node >= 18 required, found $(node --version)" + exit 1 +fi + +# Refuse to clobber an existing saltcorn subfolder, but allow $DEST itself to +# pre-exist (it might already contain dev-deploy/ or other user work). +if [ -e "$DEST/saltcorn" ]; then + echo "$DEST/saltcorn already exists; refusing to overwrite" + exit 1 +fi + +# --- clone + build --- + +mkdir -p "$DEST" +DEST_ABS="$(cd "$DEST" && pwd)" + +echo "==> Cloning $REPO_URL into $DEST_ABS/saltcorn" +git clone "$REPO_URL" "$DEST_ABS/saltcorn" + +echo "==> npm install (resolves workspaces; takes several minutes)" +(cd "$DEST_ABS/saltcorn" && npm install) + +echo "==> npm run tsc" +(cd "$DEST_ABS/saltcorn" && npm run tsc) + +# --- .dev-state/ with generated session secret --- + +DEV_DIR="$DEST_ABS/.dev-state" +mkdir -p "$DEV_DIR/files" + +SECRET=$(node -e 'console.log(require("crypto").randomBytes(32).toString("hex"))') + +cat > "$DEV_DIR/env.sh" </saltcorn/. + +__SC_DEV_DIR="\$(cd "\$(dirname "\${BASH_SOURCE[0]}")" && pwd)" +__SC_REPO_DIR="\$(dirname "\$__SC_DEV_DIR")" + +export NVM_DIR="\$HOME/.nvm" +# shellcheck disable=SC1091 +[ -s "\$NVM_DIR/nvm.sh" ] && . "\$NVM_DIR/nvm.sh" + +export SQLITE_FILEPATH="\$__SC_DEV_DIR/saltcorn.sqlite" +export SALTCORN_FILE_STORE="\$__SC_DEV_DIR/files" +export SALTCORN_SESSION_SECRET="$SECRET" + +# Put the in-tree CLI on PATH so \`saltcorn ...\` resolves to this checkout. +case ":\$PATH:" in + *":\$__SC_REPO_DIR/saltcorn/packages/saltcorn-cli/bin:"*) ;; + *) export PATH="\$__SC_REPO_DIR/saltcorn/packages/saltcorn-cli/bin:\$PATH" ;; +esac + +unset __SC_DEV_DIR __SC_REPO_DIR +EOF +chmod +x "$DEV_DIR/env.sh" + +# --- initialize schema and admin user --- + +# shellcheck source=/dev/null +source "$DEV_DIR/env.sh" + +echo "==> Initializing SQLite schema at $SQLITE_FILEPATH" +saltcorn reset-schema -f + +echo "==> Creating admin user $ADMIN_EMAIL" +saltcorn create-user -a -e "$ADMIN_EMAIL" -p "$ADMIN_PASSWORD" + +# --- startServer.sh --- + +cat > "$DEST_ABS/startServer.sh" <<'EOF' +#!/usr/bin/env bash +set -e +cd "$(dirname "$0")" +source .dev-state/env.sh + +# Pick up any local plugin source changes before booting the server. +# install-plugin copies ./dev-deploy into Saltcorn's +# plugins_folder/.../localversion/, so source edits go live on each restart. +# Failures here are non-fatal: the previously-installed version still loads. +saltcorn install-plugin -d ./dev-deploy 2>&1 | tail -2 || true + +# Saltcorn's SQLite session store writes sessions.sqlite at process cwd +# (packages/server/routes/utils.js). Run from inside .dev-state/ so each +# instance gets its own sessions.sqlite alongside its saltcorn.sqlite. +cd .dev-state +exec saltcorn serve "$@" +EOF +chmod +x "$DEST_ABS/startServer.sh" + +echo +echo "Done." +echo " cd $DEST_ABS && ./startServer.sh" +echo " -> http://localhost:3000/ (login: $ADMIN_EMAIL / $ADMIN_PASSWORD)" diff --git a/startServer.sh b/startServer.sh new file mode 100755 index 0000000..7d40146 --- /dev/null +++ b/startServer.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -e +cd "$(dirname "$0")" +source .dev-state/env.sh + +# Pick up any local plugin source changes before booting the server. +# install-plugin copies ./dev-deploy into Saltcorn's +# plugins_folder/.../localversion/, so source edits go live on each restart. +# Failures here are non-fatal: the previously-installed version still loads. +saltcorn install-plugin -d ./dev-deploy 2>&1 | tail -2 || true + +# Saltcorn's SQLite session store writes sessions.sqlite at process cwd +# (packages/server/routes/utils.js). Run from inside .dev-state/ so each +# instance gets its own sessions.sqlite alongside its saltcorn.sqlite. +cd .dev-state +exec saltcorn serve "$@" diff --git a/startServerTest.sh b/startServerTest.sh new file mode 100755 index 0000000..d3bd577 --- /dev/null +++ b/startServerTest.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -e +cd "$(dirname "$0")" +source .dev-state-test/env.sh + +# See startServer.sh for the rationale. +saltcorn install-plugin -d ./dev-deploy 2>&1 | tail -2 || true + +# See startServer.sh for why we cd into the state dir before serving. +cd .dev-state-test +exec saltcorn serve -p "$SALTCORN_PORT" "$@"