From d95798e89514800f6a0b758f46f87b5966ba6356 Mon Sep 17 00:00:00 2001 From: Scott Duensing Date: Mon, 1 Jun 2026 16:43:43 -0500 Subject: [PATCH] Initial commit. --- .gitattributes | 52 ++ .gitignore | 29 + README.md | 227 ++++++ docs/architecture.md | 428 +++++++++++ docs/managed-rows.md | 285 +++++++ docs/multitenancy.md | 222 ++++++ docs/operations.md | 204 +++++ docs/peering.md | 328 +++++++++ docs/testing.md | 114 +++ index.js | 62 ++ lib/apply.js | 1146 +++++++++++++++++++++++++++++ lib/constants.js | 65 ++ lib/context.js | 65 ++ lib/crypto.js | 163 ++++ lib/entityIds.js | 349 +++++++++ lib/env.js | 71 ++ lib/files.js | 60 ++ lib/ids.js | 34 + lib/ops.js | 47 ++ lib/payloadRefs.js | 98 +++ lib/peerAuth.js | 141 ++++ lib/peers.js | 145 ++++ lib/revert.js | 393 ++++++++++ lib/routes.js | 1128 ++++++++++++++++++++++++++++ lib/rowIdentity.js | 131 ++++ lib/rowPayload.js | 144 ++++ lib/schema.js | 161 ++++ lib/state.js | 27 + lib/transport.js | 109 +++ lib/wrap.js | 1012 +++++++++++++++++++++++++ package.json | 21 + scripts/installDevDeployTenant.js | 99 +++ scripts/installDevDeployTenant.sh | 18 + test/e2e.js | 1040 ++++++++++++++++++++++++++ test/managedRowsGate.js | 255 +++++++ test/mixedTopologyGate.js | 267 +++++++ test/mtGate.js | 270 +++++++ test/pullGate.js | 347 +++++++++ test/sc-exec.js | 57 ++ 39 files changed, 9814 insertions(+) create mode 100644 .gitattributes create mode 100644 .gitignore create mode 100644 README.md create mode 100644 docs/architecture.md create mode 100644 docs/managed-rows.md create mode 100644 docs/multitenancy.md create mode 100644 docs/operations.md create mode 100644 docs/peering.md create mode 100644 docs/testing.md create mode 100644 index.js create mode 100644 lib/apply.js create mode 100644 lib/constants.js create mode 100644 lib/context.js create mode 100644 lib/crypto.js create mode 100644 lib/entityIds.js create mode 100644 lib/env.js create mode 100644 lib/files.js create mode 100644 lib/ids.js create mode 100644 lib/ops.js create mode 100644 lib/payloadRefs.js create mode 100644 lib/peerAuth.js create mode 100644 lib/peers.js create mode 100644 lib/revert.js create mode 100644 lib/routes.js create mode 100644 lib/rowIdentity.js create mode 100644 lib/rowPayload.js create mode 100644 lib/schema.js create mode 100644 lib/state.js create mode 100644 lib/transport.js create mode 100644 lib/wrap.js create mode 100644 package.json create mode 100644 scripts/installDevDeployTenant.js create mode 100755 scripts/installDevDeployTenant.sh create mode 100644 test/e2e.js create mode 100644 test/managedRowsGate.js create mode 100644 test/mixedTopologyGate.js create mode 100644 test/mtGate.js create mode 100644 test/pullGate.js create mode 100644 test/sc-exec.js diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..2a74e24 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,52 @@ +# Default line-ending handling for text files: normalize to LF in the repo, +# check out native on each platform. +* text=auto + +# Executable shell scripts must stay LF so the shebang works on Unix even when +# checked out on Windows. +*.sh text eol=lf + +# Collapse the lockfile in diffs / PR reviews and mark it generated. +package-lock.json linguist-generated=true -diff + +# Git LFS. Run `git lfs install` once per clone to activate the filters below. +# This plugin is currently code-only; these patterns are a safety net so any +# binary blob dropped into the tree is tracked correctly without anyone having +# to remember to update this file. + +# Archives +*.zip filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tar.gz filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.7z filter=lfs diff=lfs merge=lfs -text + +# Databases / snapshots +*.sqlite filter=lfs diff=lfs merge=lfs -text +*.sqlite3 filter=lfs diff=lfs merge=lfs -text +*.db filter=lfs diff=lfs merge=lfs -text + +# Images +*.png filter=lfs diff=lfs merge=lfs -text +*.jpg filter=lfs diff=lfs merge=lfs -text +*.jpeg filter=lfs diff=lfs merge=lfs -text +*.gif filter=lfs diff=lfs merge=lfs -text +*.webp filter=lfs diff=lfs merge=lfs -text +*.ico filter=lfs diff=lfs merge=lfs -text + +# Documents +*.pdf filter=lfs diff=lfs merge=lfs -text + +# Fonts +*.ttf filter=lfs diff=lfs merge=lfs -text +*.otf filter=lfs diff=lfs merge=lfs -text +*.woff filter=lfs diff=lfs merge=lfs -text +*.woff2 filter=lfs diff=lfs merge=lfs -text + +# Audio / video +*.mp3 filter=lfs diff=lfs merge=lfs -text +*.mp4 filter=lfs diff=lfs merge=lfs -text +*.wav filter=lfs diff=lfs merge=lfs -text +*.webm filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b0dbfd5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,29 @@ +# Installed npm dependencies. Restored from package.json on install; never +# committed. (This plugin currently has no third-party deps, but the entry is +# here so a future dependency does not get committed by accident.) +node_modules/ + +# npm/yarn diagnostics and `npm pack` output. +npm-debug.log* +yarn-debug.log* +yarn-error.log* +*.tgz + +# Test coverage output. +coverage/ +.nyc_output/ + +# Local environment overrides / secrets. The plugin reads its secrets from the +# host Saltcorn process environment (e.g. the session secret used to derive its +# at-rest key); a local .env must never be committed. +.env +.env.* + +# Editor / OS junk. +.DS_Store +Thumbs.db +*.swp +*.swo +*~ +.idea/ +.vscode/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..0aed965 --- /dev/null +++ b/README.md @@ -0,0 +1,227 @@ +# dev-deploy + +A Saltcorn plugin that migrates **metadata** changes across Dev, Test, and +Prod environments. + +dev-deploy treats your schema as something you promote, not something you +hand-copy. As you edit metadata in one Saltcorn instance, the plugin records +each change as an immutable entry in an append-only **ops journal**. You then +promote those ops to a peer instance (Test, Prod, or another tenant) over an +HMAC-authenticated HTTP endpoint. Each tracked entity carries a **stable UUID** +so the same table/view/page resolves to the same identity on every instance, +even though local integer ids differ. When a peer and the local instance have +both touched the same entity since the last sync, the incoming op is held as a +**conflict** for you to resolve (theirs / mine / per-field merge) rather than +silently overwriting. + +## What it migrates + +The plugin tracks these Saltcorn metadata entity kinds (see +`lib/constants.js`, `ENTITY_KINDS`): + +- `table`, `field`, `constraint` +- `view`, `page`, `page_group`, `page_group_member` +- `trigger`, `workflow_step` +- `role` +- `library` +- `tag` +- `file` + +Children point at their parent via `parent_uuid`, so a field's identity is +anchored to its table, and so on. + +## What it does NOT touch + +**User-table row data is left alone by default.** A table's rows belong to the +local environment and deploys never touch them unless you explicitly change the +table's `data_mode` away from the default `user`: + +- `user` (default) -- rows belong to the local environment; deploys never + touch them. The only safe choice for end-user-entered data. +- `starter` -- rows ship to the target on first install, then the target owns + them; later changes on the source side do not propagate. +- `managed` -- rows always sync from source; the source is canonical and the + target's edits get overwritten or surface as conflicts. + +Switching a table to `managed` or `starter` rewrites that table's schema (it +adds a hidden `_dd_row_uuid` column for stable cross-environment row identity) +and ships the current rows. The Saltcorn `users` table is **locked** to +`data_mode=user` and cannot be changed. See +[docs/managed-rows.md](docs/managed-rows.md). + +## How it works (at a glance) + +1. **Bootstrap.** On first load the plugin creates its six `_dd_*` tables, + assigns this instance an `env_id` (a UUID), and backfills stable UUIDs for + all existing metadata entities. Two environments that bootstrap from the + same metadata population assign identical UUIDs to entities with the same + `(kind, name)`, because UUIDs are derived from a frozen namespace + (`ID_NAMESPACE` in `lib/constants.js`). +2. **Capture.** Saltcorn metadata mutations are wrapped so each change appends + one op to `_dd_ops` (op types like `update_*`, `set_table_mode`, + `insert_row`, etc.). History is append-only. +3. **Promote / pull.** From the admin UI you generate a **plan** (ops not yet + sent to a peer, tracked by a per-peer outbound **anchor**) and **promote** + them, or **pull** a peer's ops inbound. Transport is HMAC-signed. +4. **Apply.** The receiver applies each op, resolving UUIDs to its own local + ids. Concurrent edits are detected and parked as conflicts. +5. **Resolve.** Pending conflicts are resolved from the admin UI as + theirs / mine / per-field merge. +6. **Revert.** Reverting an op appends a compensating op rather than rewriting + history. + +## Features and status + +- **Append-only ops journal** with stable per-entity UUIDs -- done. +- **HMAC-SHA256 authenticated peer transport** (timestamp skew window, nonce, + host-bound canonical) -- done. +- **Plan / promote / pull** with per-peer, per-direction anchors -- done. +- **Concurrent-edit conflict detection** with theirs / mine / per-field + merge -- done. +- **Revert** via compensating ops -- done. +- **Per-table data modes** (`user` / `starter` / `managed`) with hidden + `_dd_row_uuid` row identity -- done. +- **File-entity sync** (files identified by a stable id derived from disk + location) -- done. +- **Multi-tenant**: each tenant schema is its own dev-deploy environment with + its own `env_id` and schema-qualified `_dd_*` tables; HMAC canonical is + host-bound so a request signed for one tenant is rejected against another on + the same server -- done. See [docs/multitenancy.md](docs/multitenancy.md). +- **Plugin-version drift warnings**: promote/pull compare the local and peer + plugin lists and surface mismatches as warnings (non-blocking) -- done. +- **Engines**: Node `>=20`. Saltcorn plugin API version 1. + +## The six `_dd_*` tables + +All six are created idempotently in `onLoad` (`lib/schema.js`) and are +schema-qualified per tenant. Strings, JSON payloads, and ISO 8601 timestamps +are `TEXT`; booleans and surrogate ids are `INTEGER`. + +| Table | Purpose | Key columns | +| --- | --- | --- | +| `_dd_env` | This instance's dev-deploy identity (singleton row). | `env_id` (PK), `env_label`, `on_destructive_op` (default `confirm`), `require_tls`, `created_at`, `bootstrapped_at` | +| `_dd_peers` | Configured peers and their encrypted shared secret. | `peer_id` (PK), `env_id` (UNIQUE), `label`, `base_url`, `peer_secret_ciphertext`, `peer_secret_iv`, `peer_secret_tag`, `require_tls`, `created_at`, `last_seen_at` | +| `_dd_entity_ids` | Maps stable UUID to the local entity. | `uuid` (PK), `kind`, `current_name`, `current_id`, `parent_uuid`, `created_at`, `UNIQUE(kind, current_id)` | +| `_dd_ops` | The append-only ops journal. | `op_id` (PK), `source_env_id`, `op_type`, `entity_kind`, `entity_uuid`, `payload`, `parent_op_id`, `correlation_id`, `schema_version`, `created_at`, `applied_at`, `status` (default `committed`), `conflict_with_op_id` | +| `_dd_anchors` | Per-peer, per-direction sync watermark. | `(peer_id, direction)` (PK), `last_op_id`, `updated_at` | +| `_dd_table_modes` | Per-table `data_mode` and ship state. | `table_uuid` (PK), `data_mode`, `updated_at`, `starter_shipped_at` | + +## HTTP endpoints + +### Admin UI (session, admin role `role_id === 1`) + +| Method | Path | Purpose | +| --- | --- | --- | +| GET | `/admin/dev-deploy/` | Dashboard: env identity, op/entity counts, pending conflicts. | +| GET | `/admin/dev-deploy/ops` | Journal viewer (supports `?limit=`, `?since=op_id`; JSON via `Accept: application/json`). | +| POST | `/admin/dev-deploy/revert` | Append a compensating op to revert an op. | +| GET | `/admin/dev-deploy/peers` | List peers; show this instance's `env_id`; add-peer form. | +| POST | `/admin/dev-deploy/peers/add` | Pair a peer (generates or accepts a 64-hex shared secret). | +| POST | `/admin/dev-deploy/peers/rotate` | Rotate a peer's shared secret. | +| POST | `/admin/dev-deploy/peers/delete` | Delete a peer. | +| GET | `/admin/dev-deploy/plan` | Preview the ops that would be promoted to a chosen peer. | +| POST | `/admin/dev-deploy/promote` | Promote outbound ops to a peer via signed `ingest`. | +| POST | `/admin/dev-deploy/pull` | Pull a peer's ops inbound via signed `journal` and apply them. | +| GET | `/admin/dev-deploy/tables` | List tracked tables and their `data_mode`. | +| POST | `/admin/dev-deploy/tables/set` | Set a table's `data_mode`. | +| GET | `/admin/dev-deploy/conflicts` | List pending conflicts. | +| POST | `/admin/dev-deploy/conflicts/resolve` | Resolve a conflict with `theirs` or `mine`. | +| GET | `/admin/dev-deploy/conflicts/merge` | Per-field merge view for a conflicting `update_*` op. | +| POST | `/admin/dev-deploy/conflicts/merge/apply` | Apply a per-field merge resolution. | + +### Machine API (HMAC peer auth, CSRF-exempt) + +These routes are registered with `noCsrf: true`; `onLoad` also registers +`/dev-deploy/api/` in Saltcorn's `disable_csrf_routes`. + +| Method | Path | Purpose | +| --- | --- | --- | +| GET | `/dev-deploy/api/journal?since=op_id` | Return this env's ops since an op id (max 1000, oldest first). | +| POST | `/dev-deploy/api/ingest` | Apply a batch of ops from a peer; advances the inbound anchor. | +| GET | `/dev-deploy/api/file/:uuid` | Stream a file entity's bytes by UUID. | +| GET | `/dev-deploy/api/health` | Return this env's `env_id`, `label`, and installed plugin list. | + +Every machine-API request must carry these headers (see `lib/peerAuth.js`): + +| Header | Meaning | +| --- | --- | +| `X-DD-Env-Id` | Caller's `env_id`, looked up in `_dd_peers`. | +| `X-DD-Timestamp` | Milliseconds since epoch; rejected if skew > 5 min. | +| `X-DD-Nonce` | Random per-request bytes (replay padding). | +| `X-DD-Signature` | Hex HMAC-SHA256 over the canonical string. | + +The request body uses `Content-Type: application/vnd.dev-deploy+json` so +Saltcorn's `express.json()` does not consume the stream before the HMAC is +verified over the exact raw bytes. The canonical string binds the target host, +so a request signed for one tenant will not verify if replayed against another +tenant on the same server. See [docs/peering.md](docs/peering.md). + +## Quick start + +These steps assume the dual-instance dev layout where the project root holds +your state dirs and the upstream Saltcorn lives in a sibling subfolder, with +`dev-deploy/` next to them. + +1. **Install the plugin into the dev instances.** From the project root: + + ``` + ./reinstallDevDeploy.sh + ``` + + This installs `dev-deploy` into the MAIN (`.dev-state`), TEST + (`.dev-state-test`), and Postgres (`.dev-state-pg`) instances. Because the + plugins folder's `node_modules` symlinks are shared with the sibling + `saltcorn-idp` plugin, re-run `./reinstallIdp.sh` afterward to keep both + installs consistent. + +2. **Run an instance.** From the project root, start MAIN on `:3000`: + + ``` + ./startServer.sh + ``` + + For the second SQLite instance (TEST on `:3001`): + + ``` + ./startServerTest.sh + ``` + + For the multi-tenant Postgres instance (on `:3002`): + + ``` + ./startServerPg.sh + ``` + + On first load the plugin bootstraps: it creates the `_dd_*` tables, assigns + an `env_id`, and backfills entity UUIDs. Open + `http://localhost:3000/admin/dev-deploy/` (as an admin) to reach the + dashboard. + +3. **Run the gates.** The plugin ships an end-to-end harness and four + targeted gates: + + ``` + node test/e2e.js # full end-to-end run (npm test) + node test/mtGate.js # multi-tenant isolation (Postgres) + node test/mixedTopologyGate.js # standalone-to-tenant push peering + host-bound HMAC + node test/managedRowsGate.js # managed-row sync across tenants (Postgres) + node test/pullGate.js # reverse + tenant-to-tenant pull peering (Postgres) + ``` + + The Postgres gates require the `:3002` instance to be running. See + [docs/testing.md](docs/testing.md). + +## Documentation + +| Doc | Covers | +| --- | --- | +| [docs/architecture.md](docs/architecture.md) | The ops journal, stable UUIDs, entity model, and apply pipeline. | +| [docs/peering.md](docs/peering.md) | Pairing, the HMAC transport, anchors, and the machine API. | +| [docs/multitenancy.md](docs/multitenancy.md) | Per-tenant environments, schema-qualified tables, host-bound auth. | +| [docs/managed-rows.md](docs/managed-rows.md) | `user` / `starter` / `managed` data modes and `_dd_row_uuid`. | +| [docs/operations.md](docs/operations.md) | Day-to-day plan / promote / pull, conflict resolution, revert. | +| [docs/testing.md](docs/testing.md) | The e2e harness and the multi-tenant / managed-row gates. | + +## License + +MIT. Author: Scott Duensing. diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..dd01164 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,428 @@ +# dev-deploy architecture + +dev-deploy is a Saltcorn plugin that migrates metadata changes (and, opt-in, row +data) across Dev/Test/Prod environments. It records every metadata mutation as an +append-only journal entry keyed by a stable cross-environment UUID, then replays +those entries onto peer instances over an HMAC-authenticated HTTP transport. + +This document explains the core model: the ops journal, stable entity UUIDs, the +wrap layer that produces ops, the apply layer that consumes them (including +conflict handling), the per-instance environment identity, and the three table +data modes. + +Code references below are `file:line` into the plugin source. + +## Contents + +- [Plugin load sequence](#plugin-load-sequence) +- [The ops journal](#the-ops-journal-_dd_ops) +- [Stable entity UUIDs](#stable-entity-uuids-_dd_entity_ids) +- [The wrap layer](#the-wrap-layer) +- [Apply](#apply) +- [Environment identity](#environment-identity-_dd_env) +- [Data modes](#data-modes-user--starter--managed) +- [Plugin tables](#plugin-tables) +- [HTTP endpoints](#http-endpoints) + +## Plugin load sequence + +`onLoad` (`index.js`) runs on every plugin load and is idempotent: + +1. `createAllTables()` creates the six plugin tables if missing (`index.js`, + `schema.js`). +2. `initEnvIfMissing()` ensures this instance has a singleton identity row in + `_dd_env` (`index.js`, `env.js`). +3. On first load only (when `bootstrapped_at` is NULL), `backfillAll()` assigns + UUIDs to every pre-existing metadata entity, then `markBootstrapped()` stamps + the env so backfill never runs again (`index.js`, `entityIds.js`, + `env.js`). +4. `installAllWraps()` monkey-patches the Saltcorn model classes so subsequent + mutations get journaled (`index.js`, `wrap.js`). +5. `ensureCsrfBypass()` appends `/dev-deploy/api/` to Saltcorn's + `disable_csrf_routes` config so peers can POST to the machine API + (`index.js`). + +The plugin exports `sc_plugin_api_version: 1`, `onLoad`, and `routes` +(`index.js`). + +## The ops journal (`_dd_ops`) + +Every tracked change is one append-only row in `_dd_ops` (`schema.js`). +The journal is never rewritten in place: an undo is itself a new compensating op +(see the Journal viewer note at `routes.js`). + +### Op record shape + +`recordOp` (`ops.js`) builds and inserts a row with these fields: + +| Column | Source | Notes | +| --- | --- | --- | +| `op_id` | `rec.op_id` or a fresh v4 UUID | Primary key (`ops.js`, `ids.js`) | +| `source_env_id` | this instance's `env.env_id` | Which environment authored the op (`ops.js`) | +| `op_type` | `_` | e.g. `create_table`, `update_field`, `drop_view` (`wrap.js`) | +| `entity_kind` | the entity kind | e.g. `table`, `field`; NULL for config ops | +| `entity_uuid` | stable UUID of the touched entity | NULL for `set_config`/`update_plugin_config` (`wrap.js`, `wrap.js`) | +| `payload` | JSON, stored as TEXT | before/after/patch snapshots (`ops.js`) | +| `parent_op_id` | from AsyncLocalStorage | Set for cascaded child ops (`ops.js`, `context.js`) | +| `correlation_id` | from AsyncLocalStorage | Groups all ops in one logical operation (`ops.js`, `context.js`) | +| `schema_version` | `OP_SCHEMA_VERSION` (currently 1) | `ops.js`, `constants.js` | +| `created_at` | ISO 8601 timestamp | Ordering key for sync and apply (`ops.js`) | +| `applied_at` | ISO 8601 timestamp | Set on locally authored ops; on ingest set only if applied (`ops.js`, `apply.js`) | +| `status` | `committed` by default | See status values below (`ops.js`) | +| `conflict_with_op_id` | NULL unless conflicting | The local op_id this incoming op conflicts with (`apply.js`) | + +`payload` is always a JSON string, parsed/stringified at the application layer; +the schema uses portable TEXT/INTEGER types with no JSONB (`schema.js`). +`recordOpSafely` wraps `recordOp` so a journal failure logs but never throws into +the user's mutation (`ops.js`). + +### op_type catalog + +`op_type` is `_`. The set of types is fixed by the apply dispatch +table `HANDLERS` (`apply.js`): + +- Metadata create/update/drop: `create_table`/`update_table`/`drop_table`, + and the same triad for `field`, `view`, `page`, `trigger`, `role`, `library`, + `tag`, `page_group`, `workflow_step` (`apply.js`). +- Create/drop only (no update handler): `constraint`, `file`, + `page_group_member` (`apply.js`, `apply.js`, `apply.js`). +- Config: `set_config` (tracked keys only) and `update_plugin_config` + (`apply.js`, `wrap.js`, `wrap.js`). +- Row data: `insert_row`, `update_row`, `drop_row`, and `set_table_mode` + (`apply.js`). + +### Status values + +`status` is set when an op is recorded or ingested (`apply.js`): + +- `committed` -- authored locally, or ingested and applied successfully. +- `skipped_cascade` -- the op's `parent_op_id` was in the same incoming batch, so + the parent's apply reproduces this child locally (`apply.js`). +- `conflict` -- not applied; a local op touched the same entity since the last + sync. Resolved by the admin (`apply.js`). +- `error` -- apply failed, or no handler exists for the op_type (`apply.js`, + `apply.js`). +- `rejected` -- a conflict the admin resolved with "use mine" (`apply.js`). +- `merged` -- a conflict the admin resolved with a per-field merge + (`apply.js`). +- `reverted` -- excluded from conflict scanning alongside `rejected` + (`apply.js`). + +Indexes on `_dd_ops` cover `created_at`, `(source_env_id, created_at)`, +`entity_uuid`, `correlation_id`, and a partial index on `status='conflict'` +(`schema.js`). + +## Stable entity UUIDs (`_dd_entity_ids`) + +Saltcorn core identifies metadata by integer id and human name, neither of which +is stable across environments. dev-deploy maintains a side table +`_dd_entity_ids` mapping `(kind, current_id) -> uuid` (`entityIds.js`, +`schema.js`). The UNIQUE constraint `(kind, current_id)` enforces one mapping +per local entity (`schema.js`). + +| Column | Meaning | +| --- | --- | +| `uuid` | The stable cross-environment identity (PK) (`schema.js`) | +| `kind` | One of `ENTITY_KINDS` (`constants.js`) | +| `current_name` | Friendly current name; updated on rename, not used for identity (`entityIds.js`) | +| `current_id` | The local Saltcorn integer id | +| `parent_uuid` | Parent entity UUID (e.g. a field's table); preserved so revert can find the parent (`wrap.js`) | +| `created_at` | ISO 8601 timestamp | + +Secondary indexes cover `(kind, current_name)` and `parent_uuid` +(`schema.js`). + +### Two ways a UUID is born + +- **Deterministic (backfill / first run).** `ensureUuid` derives a UUID from + `deterministicUuid(kind, canonical)` -- SHA-256 over + `ID_NAMESPACE | kind | canonicalName` shaped into an RFC 4122 v5-style UUID + (`entityIds.js`, `ids.js`, `constants.js`). Because the namespace is + frozen, two environments installed from the same metadata population converge + on identical UUIDs with no coordination step (`entityIds.js`). The canonical + key is the entity name, except fields use `table.field` and constraints use a + fingerprint scoped to the parent table UUID (`entityIds.js`, + `entityIds.js`). +- **Random (live creation).** When a wrap observes a new entity created after + bootstrap, `assignNewUuid` mints a fresh v4 UUID (`entityIds.js`). On the + receiving side, `adoptUuid` inserts a row with the *source's* UUID so the + identity is preserved across instances (`entityIds.js`). + +`backfillAll` walks every tracked kind in dependency order so parents exist +before children resolve their `parent_uuid`: tables, fields, views, pages, +triggers, roles, library, tags, constraints, page groups, workflow steps +(`entityIds.js`). Each backfill function counts only newly inserted rows so +re-running is a no-op (`entityIds.js`). + +Lifecycle helpers: `lookupByCurrent` / `lookupByUuid` (`entityIds.js`), +`updateName` on rename (`entityIds.js`), and `removeEntityRow` on delete so a +reused local integer id can't collide with a stale mapping (`entityIds.js`). + +## The wrap layer + +`installAllWraps` monkey-patches the Saltcorn model classes so that create, +update, and delete each append an op (`wrap.js`). It wraps Table, Field, +View, Page, Trigger, Role, Library, Tag, TableConstraint, File, PageGroup, +PageGroupMember, WorkflowStep, plus `state.setConfig`, `Plugin.prototype.upsert`, +and the Table row methods (`wrap.js`). + +### The generic wrap + +`wrap(target, method, kind, action, hooks)` (`wrap.js`) replaces a method with +an async wrapper that: + +1. Returns the original immediately if journaling is suppressed -- the apply path + sets this flag (`wrap.js`, `context.js`). +2. Pre-generates an `op_id` and runs an optional `before` hook to capture + pre-state (`wrap.js`). +3. Calls `enterOp(opId, ...)` to push the op_id onto an AsyncLocalStorage stack, + then invokes the original method inside that scope (`wrap.js`, + `context.js`). +4. On success, runs the `after` hook to compute `entityUuid` + `payload`, + translates any local file references to portable + `__dd_file_ref::` placeholders, then calls `recordOpSafely` + (`wrap.js`, `wrap.js`, `wrap.js`). + +Each wrapped method is tagged `__ddWrapped` (with `__ddOriginal` kept) so a +second `installAllWraps` is a no-op (`wrap.js`, `wrap.js`). Hook errors +are caught and logged but do not throw, so the journal can never corrupt a +user-facing operation (`wrap.js`); a failure in the original method +propagates normally and no op is recorded (`wrap.js`). + +### AsyncLocalStorage correlation + +`context.js` holds a per-async-flow store of `{ stack, correlationId, suppressed }` +(`context.js`). `enterOp` pushes the new op_id and inherits the parent's +`correlationId` (or mints one at the top level) (`context.js`). When the +original method triggers nested mutations (e.g. a table delete cascading into +field deletes), each child op reads: + +- `currentParentOpId()` -- the second-from-top of the stack, recorded as + `parent_op_id` (`context.js`). +- `currentCorrelationId()` -- shared across the whole operation + (`context.js`). + +This is how apply later recognizes cascades and skips re-applying children whose +parent is in the same batch (`apply.js`). + +### Payload contents per action + +- **create**: `payload.after` is a snapshot of selected keys; child kinds also + carry `parent_uuid` (`wrap.js`, `wrap.js`). Snapshot key lists per kind + are defined at `wrap.js`. +- **update**: `payload.patch` is the caller's change set; some kinds also include + `before` and `after` snapshots; renames call `updateName` (`wrap.js`, + `wrap.js`). +- **drop**: the shared `standardDropHooks` capture the UUID, `parent_uuid`, and a + `before` snapshot, then `removeEntityRow` after the delete completes + (`wrap.js`). + +### Config and plugin wraps + +`wrapSetConfig` wraps `state.setConfig` but only journals a `set_config` op for a +small allowlist of keys -- `menu_items`, `site_name`, `site_logo_id`, `base_url` +(`wrap.js`, `wrap.js`). `menu_items` references pages/views by name, which +is naturally stable, so no UUID translation is needed (`wrap.js`). + +`wrapPlugin` wraps `Plugin.prototype.upsert` to journal `plugin_config` updates, +skipping the dev-deploy plugin itself and skipping upserts that don't actually +change the configuration (Saltcorn upserts on every plugin load) (`wrap.js`, +`wrap.js`). + +### Row wraps + +`wrapTableRows` wraps `insertRow`/`updateRow`/`deleteRows` on `Table.prototype` +(`wrap.js`). Each consults `journalDecision(this.id)` and passes through +silently unless the table's data mode says to journal (`wrap.js`). Row ops +carry a `table_uuid` plus a row-level UUID as the op's `entity_uuid` +(`wrap.js`). See [Data modes](#data-modes-user--starter--managed) and +[managed-rows.md](./managed-rows.md). + +## Apply + +`apply.js` replays an op authored elsewhere onto this instance. `applyBatch(ops, +opts)` is the entry point used by both `pull` and `apiIngest` (`apply.js`, +`routes.js`, `routes.js`). + +### Batch algorithm + +`applyBatch` sorts the incoming ops by `created_at`, then for each op +(`apply.js`): + +1. **Idempotency** -- if an op with this `op_id` already exists locally, record + `already_applied` and skip (`apply.js`). Every create handler also + re-checks `lookupByUuid(op.entity_uuid)` and returns a `noop` if the UUID is + already mapped (e.g. `apply.js`), giving a second idempotency layer. +2. **Cascade skip** -- if `parent_op_id` is in the same batch, persist + `skipped_cascade` and let the parent's apply reproduce the child + (`apply.js`). +3. **Conflict detection** -- `findConflictingLocalOp` looks for a local op on the + same `entity_uuid` applied since the last inbound sync; if found, persist the + incoming op as `conflict` (not applied) and continue (`apply.js`, + `apply.js`). +4. **Dispatch** -- look up the handler in `HANDLERS`; missing handler -> `error` + (`apply.js`). +5. **Apply suppressed** -- parse the payload, resolve file placeholders, run the + handler inside `runSuppressed(...)` so the inner Saltcorn calls don't + re-journal or auto-assign UUIDs, then persist `committed`; any throw -> `error` + (`apply.js`, `context.js`). + +Each handler resolves the op's `entity_uuid` (and any `parent_uuid`) to the local +integer id, invokes the matching Saltcorn model method, and updates +`_dd_entity_ids` via `adoptUuid` / `updateName` / `removeEntityRow` +(`apply.js`). `stripSurrogateKeys` drops non-portable id columns (`id`, +`table_id`, `view_id`, `page_id`, `role_id_for_create`) from create/patch +payloads before they reach the model (`apply.js`). `persistOp` writes the op +into the local journal preserving its source-side identity (`apply.js`). + +### Idempotency summary + +Apply is safe to re-run: duplicate `op_id` short-circuits at the top of the loop +(`apply.js`); create handlers no-op on an already-mapped UUID +(`apply.js`); drop handlers no-op when the entity or mapping is already gone +(`apply.js`); and `applyInsertRow` no-ops when the row UUID is already present +(`apply.js`). + +### Conflict detection + +`findConflictingLocalOp` (`apply.js`) reads the `inbound` anchor for the peer +to get a cutoff timestamp, then finds the most recent local op (where +`source_env_id` is this env's id) on the same `entity_uuid` with +`applied_at > cutoff`, excluding `rejected` and `reverted` ops. If such an op +exists, the incoming op represents concurrent divergent change and is stored with +`status='conflict'` and `conflict_with_op_id` set to that local op_id. + +### Conflict resolution: theirs / mine / merge + +The admin resolves pending conflicts via the Conflicts UI (`routes.js`): + +- **theirs** -- `resolveConflict(opId, "theirs")` applies the incoming op now + under suppression and marks it `committed`, clearing `conflict_with_op_id` + (`apply.js`). +- **mine** -- `resolveConflict(opId, "mine")` marks the incoming op `rejected` + and leaves local state alone; future pulls skip it by idempotency + (`apply.js`). +- **merge** (update-vs-update only) -- `conflictFieldDiff` computes per-field + differences between the incoming patch and current local state + (`apply.js`); the admin picks current/incoming/custom per field, and + `resolveConflictByMerge` writes only the chosen fields and marks the op + `merged` (`apply.js`). Mergeability is gated to matching `update_` + op types on both sides (`routes.js`). + +## Environment identity (`_dd_env`) + +Each instance has exactly one identity row in `_dd_env`, the singleton selected +by `getEnv` (`env.js`, `schema.js`). + +| Column | Meaning | +| --- | --- | +| `env_id` | This instance's stable UUID; stamped as `source_env_id` on every op (`env.js`) | +| `env_label` | Optional human label (e.g. test, prod) shown in the admin UI | +| `on_destructive_op` | Destructive-op policy: `auto`, `confirm`, or `refuse`; defaults to `confirm` (`constants.js`, `schema.js`) | +| `require_tls` | Default TLS requirement flag (0/1) (`schema.js`) | +| `created_at` | ISO 8601 timestamp | +| `bootstrapped_at` | NULL until first-run backfill completes; gates backfill (`index.js`, `env.js`) | + +`env_id` is created once by `initEnvIfMissing` as a random v4 UUID +(`env.js`). The env cache is keyed per tenant schema, not a module-level +singleton, so a multi-tenant process never shares one env across tenants +(`env.js`). + +The `env_id` is the unit of pairing: an admin copies this instance's `env_id` +into a peer's add-peer form, and it is sent as the source identity on every signed +request (`routes.js`, `routes.js`). + +## Data modes (user / starter / managed) + +Per-table row propagation is governed by a data mode stored in `_dd_table_modes`, +keyed by `table_uuid` (`schema.js`). The Tables admin page sets these +(`routes.js`). The three modes (`constants.js`): + +| Mode | Behavior | +| --- | --- | +| `user` (default) | Rows belong to the local environment; deploys never touch them. The only safe choice for end-user-entered data (`routes.js`). | +| `starter` | Rows ship to the target on first install (initial ship), then the target owns them; later source changes don't propagate (`routes.js`). | +| `managed` | Rows always sync from source; source is canonical, target edits get overwritten or surface as conflicts (`routes.js`). | + +The Saltcorn `users` table is hard-locked to `user` and cannot be changed +(`routes.js`, `routes.js`). + +Switching a table to `managed` or `starter` adds a hidden `_dd_row_uuid` column, +backfills UUIDs onto existing rows, journals a `set_table_mode` op, and then +journals an `insert_row` op per existing row (the initial ship). For `starter`, +`markStarterShipped` then locks out further row ops (`routes.js`, +`routes.js`, `routes.js`). Reverting to `user` best-effort drops the +hidden column (`routes.js`). On the receiving side, `applySetTableMode` +records the mode and ensures the managed schema before any row ops arrive +(`apply.js`). + +For the full row-identity and propagation mechanics (the `_dd_row_uuid` column, +`journalDecision`, portable row payloads, and the binary/file-fetch path), see +[managed-rows.md](./managed-rows.md). + +## Plugin tables + +All six tables are created idempotently by `createAllTables` (`schema.js`) +using portable TEXT/INTEGER types (no JSONB; booleans as 0/1) (`schema.js`). +Names are prefixed with the tenant schema via `db.getTenantSchemaPrefix()`. + +| Table | Purpose | Defined at | +| --- | --- | --- | +| `_dd_env` | Singleton instance identity + policies | `schema.js` | +| `_dd_peers` | Configured peers; HMAC secret stored as hex ciphertext/iv/tag | `schema.js` | +| `_dd_entity_ids` | `(kind, current_id) -> uuid` mapping | `schema.js` | +| `_dd_ops` | Append-only ops journal | `schema.js` | +| `_dd_anchors` | Per-peer per-direction sync cursor (`last_op_id`) | `schema.js` | +| `_dd_table_modes` | Per-table data mode + `starter_shipped_at` | `schema.js` | + +`_dd_peers` stores the shared secret as three hex TEXT columns +(`peer_secret_ciphertext`, `peer_secret_iv`, `peer_secret_tag`) rather than a +BLOB, because Saltcorn's SQLite insert layer would JSON-stringify a Buffer +(`schema.js`). The PK uses `integer` on SQLite and `serial` on Postgres for a +portable auto-increment (`schema.js`). `_dd_anchors` is keyed on +`(peer_id, direction)` where direction is `inbound` or `outbound` +(`schema.js`, `routes.js`). Both `_dd_ops.conflict_with_op_id` and +`_dd_table_modes.starter_shipped_at` are added by idempotent migrations for +older installs (`schema.js`, `schema.js`). + +## HTTP endpoints + +Routes are declared in `routes.js`. Admin UI routes require an admin session +(`role_id === 1`, checked by `isAdmin`, `routes.js`). Machine API routes use +HMAC peer auth via `requirePeerAuth` and are CSRF-exempt (`noCsrf: true`, +registered into `disable_csrf_routes` at load) (`routes.js`, `index.js`). + +### Admin UI (session + admin role) + +| Method | URL | Handler | Purpose | +| --- | --- | --- | --- | +| GET | `/admin/dev-deploy/` | `dashboard` | Env summary, op/entity counts, pending-conflict count (`routes.js`) | +| GET | `/admin/dev-deploy/ops` | `opsView` | Journal viewer; JSON if `Accept: application/json` (`routes.js`) | +| GET | `/admin/dev-deploy/peers` | `peersView` | List peers; add-peer form (`routes.js`) | +| POST | `/admin/dev-deploy/peers/add` | `peersAdd` | Pair a peer; secret shown once (`routes.js`) | +| POST | `/admin/dev-deploy/peers/rotate` | `peersRotate` | Rotate a peer secret (`routes.js`) | +| POST | `/admin/dev-deploy/peers/delete` | `peersDelete` | Delete a peer (`routes.js`) | +| GET | `/admin/dev-deploy/plan` | `planView` | Preview ops to send to a peer (`routes.js`) | +| POST | `/admin/dev-deploy/promote` | `promote` | Push ops since outbound anchor to a peer (`routes.js`) | +| POST | `/admin/dev-deploy/pull` | `pull` | Pull + apply ops since inbound anchor (`routes.js`) | +| POST | `/admin/dev-deploy/revert` | `revertView` | Append a compensating op (`routes.js`) | +| GET | `/admin/dev-deploy/tables` | `tablesView` | View/set per-table data mode (`routes.js`) | +| POST | `/admin/dev-deploy/tables/set` | `tablesSet` | Set a table's data mode (`routes.js`) | +| GET | `/admin/dev-deploy/conflicts` | `conflictsView` | List pending conflicts (`routes.js`) | +| POST | `/admin/dev-deploy/conflicts/resolve` | `conflictsResolve` | Resolve theirs/mine (`routes.js`) | +| GET | `/admin/dev-deploy/conflicts/merge` | `conflictsMergeView` | Per-field merge form (`routes.js`) | +| POST | `/admin/dev-deploy/conflicts/merge/apply` | `conflictsMergeApply` | Apply a per-field merge (`routes.js`) | + +### Machine API (HMAC peer auth, CSRF-exempt) + +| Method | URL | Handler | Purpose | +| --- | --- | --- | --- | +| GET | `/dev-deploy/api/journal` | `apiJournal` | Serve this env's ops since `?since=op_id` (`routes.js`) | +| POST | `/dev-deploy/api/ingest` | `apiIngest` | Receive + apply a batch of ops (`routes.js`) | +| GET | `/dev-deploy/api/file/:uuid` | `apiFile` | Stream a file's bytes for `create_file` apply (`routes.js`) | +| GET | `/dev-deploy/api/health` | `apiHealth` | Report env_id/label and installed plugin list (`routes.js`) | + +Promote and pull advance the per-peer anchor after a successful exchange so the +next sync only carries new ops (`routes.js`, `routes.js`). Both also +compare installed-plugin lists with the peer via `/dev-deploy/api/health` and +surface mismatches as warnings (`routes.js`). diff --git a/docs/managed-rows.md b/docs/managed-rows.md new file mode 100644 index 0000000..b722215 --- /dev/null +++ b/docs/managed-rows.md @@ -0,0 +1,285 @@ +# dev-deploy managed rows (row-data sync) + +By default dev-deploy migrates only metadata (tables, fields, views, pages, +etc.) between environments and never touches the actual rows users entered. The +managed-rows feature is the opt-in extension that also synchronizes row *data* +for tables an admin explicitly marks. It does this with a per-table data mode, +a hidden cross-environment row identity column (`_dd_row_uuid`), row-level ops +in the journal (`insert_row` / `update_row` / `drop_row` / `set_table_mode`), +and UUID-based foreign-key and file translation so a row created on Dev lands +correctly on Test/Prod even though every instance assigns its own integer ids. + +See also: [architecture.md](./architecture.md) for the ops journal, stable +entity UUIDs, the wrap/apply layers, and conflict handling; +[peering.md](./peering.md) for the HMAC transport and sync anchors; +[multitenancy.md](./multitenancy.md) for per-tenant isolation. + +Code references below are `file:line` into the plugin source. + +## Contents + +- [Data modes: user / starter / managed](#data-modes-user--starter--managed) +- [The hidden `_dd_row_uuid` column](#the-hidden-_dd_row_uuid-column) +- [Switching a table's data mode](#switching-a-tables-data-mode) +- [Journaling row changes (the wrap layer)](#journaling-row-changes-the-wrap-layer) +- [Applying row ops (the apply layer)](#applying-row-ops-the-apply-layer) +- [Foreign-key handling](#foreign-key-handling) +- [File propagation](#file-propagation) +- [Tables and columns](#tables-and-columns) +- [Endpoints](#endpoints) + +## Data modes: user / starter / managed + +Each table has a data mode stored in `_dd_table_modes.data_mode`, keyed by the +table's stable UUID. The three valid values are defined in `constants.js` +(`DATA_MODES`): `user`, `starter`, `managed`. A table with no +`_dd_table_modes` row is treated as `user` (the default; +`rowPayload.js`, `rowPayload.js`). + +| Mode | When rows sync | Who owns rows after | Intended use | +| --- | --- | --- | --- | +| `user` (default) | Never. Row CRUD passes through silently and is never journaled. | The local environment. | End-user-entered data; the only safe choice for it. | +| `starter` | Once, at the moment the table is switched to `starter` (the "initial ship"). After that, further row changes do not propagate. | The target, after first install. | Default roles, sample categories, template data the user is expected to customize. | +| `managed` | Always. Every insert/update/delete is journaled and replayed; source is canonical. | The source; target edits get overwritten or surface as conflicts. | Catalogs, lookup tables, anything dev-curated. | + +The per-table decision of whether a given CRUD operation should be journaled is +made by `journalDecision(tableId)` in `rowPayload.js`: + +- `managed` -> always journal (`shouldJournal: true`, `rowPayload.js`). +- `starter` -> journal only while the table has not yet been shipped, i.e. + `_dd_table_modes.starter_shipped_at` is still NULL (`rowPayload.js`, + `isStarterShipped` at `rowPayload.js`). +- `user` (or a table with no entity mapping) -> never journal + (`rowPayload.js`, `rowPayload.js`). + +The `users` table is hard-locked to `user` and cannot be changed +(`routes.js`, `routes.js`). + +## The hidden `_dd_row_uuid` column + +A row's integer `id` is local to one instance and cannot identify the same row +on a peer. The feature therefore adds a hidden TEXT column named `_dd_row_uuid` +(the constant `COLUMN_NAME` in `rowIdentity.js`) to the underlying SQL table +of every `managed` or `starter` table. This column is the row's +cross-environment identity. + +Key properties (`rowIdentity.js`): + +- It is added by raw `ALTER TABLE ... ADD COLUMN _dd_row_uuid TEXT` + (`rowIdentity.js`) and is deliberately NOT registered in Saltcorn's + `_sc_fields`, so Saltcorn's table builder and auto-generated views never + display it (`rowIdentity.js`). +- Existing rows are backfilled with `crypto.randomUUID()` values immediately + after the column is added (`rowIdentity.js`). +- A lookup index `"_dd_row_uuid_idx"` is created on the column + (best-effort; `rowIdentity.js`). +- The SQL table is referenced through `tableSqlRef()`, which schema-qualifies + via `db.getTenantSchemaPrefix()` and `db.sqlsanitize()` (`rowIdentity.js`), + so it works inside any tenant. + +### SQLite PRAGMA vs Postgres information_schema introspection + +`columnExists(tableName)` (`rowIdentity.js`) detects whether the column is +already present, and it branches on the backend: + +- SQLite: `PRAGMA table_info("
")` and checks whether any returned row has + `name === "_dd_row_uuid"` (`rowIdentity.js`). +- Postgres: a query against `information_schema.columns` filtered by + `table_schema = db.getTenantSchema()`, `table_name`, and + `column_name = "_dd_row_uuid"` (`rowIdentity.js`). + +The Postgres path explicitly queries the tenant's own schema rather than relying +on `current_schema()`. The code comment at `rowIdentity.js` documents why: +Saltcorn qualifies queries with `getTenantSchemaPrefix()` instead of +`SET search_path`, so `current_schema()` returns `public` even inside a tenant. +Using it caused `columnExists` to falsely report the column missing on every +call after the first, after which the explicitly-qualified `ALTER` failed with +`column "_dd_row_uuid" already exists` -- breaking apply, which calls +`ensureManagedSchema` once per `set_table_mode` and once per `insert_row`. + +`ensureManagedSchema(tableName)` (`rowIdentity.js`) is the idempotent +entry point: if the column already exists it returns `{ added: false }`; +otherwise it adds the column, backfills, indexes, and returns +`{ added: true, backfilled: }`. `dropManagedSchema(tableName)` +(`rowIdentity.js`) reverses it with `ALTER TABLE ... DROP COLUMN` (relies on +SQLite 3.35+ or Postgres; older SQLite throws and the caller treats the drop as +best-effort, `routes.js`). + +## Switching a table's data mode + +An admin changes a table's mode at the `GET /admin/dev-deploy/tables` page +(`tablesView`, `routes.js`) and submits to +`POST /admin/dev-deploy/tables/set` (`tablesSet`, `routes.js`). The form +warns that switching to managed/starter adds the hidden column and ships current +rows (`routes.js`). + +`tablesSet` (`routes.js`) performs, in order: + +1. Validates `table_uuid` and that `data_mode` is one of `DATA_MODES` + (`routes.js`); rejects the `users` table (`routes.js`). +2. Upserts the `_dd_table_modes` row, resetting `starter_shipped_at` to NULL + (`routes.js`). +3. Journals a `set_table_mode` op FIRST, so a target's apply sees the mode + change before any row ops (`routes.js`, payload + `{ table_uuid, data_mode }`). +4. If the new mode is `managed` or `starter`: calls `ensureManagedSchema` on the + local table (`routes.js`), then does the **initial ship** -- reads every + row via `allRowsWithUuid` (`rowIdentity.js`), assigns a + `_dd_row_uuid` to any row missing one (`routes.js`), converts the row to + its portable form, and journals one `insert_row` op per row with payload + `{ table_uuid, after: portable }` (`routes.js`). +5. If the new mode is `starter`, calls `markStarterShipped` (`routes.js`, + `rowPayload.js`) so subsequent CRUD will not journal. +6. If reverting from a managed/starter mode back to `user`, drops the hidden + column best-effort (`routes.js`). + +`managed` tables intentionally do not set `starter_shipped_at`, so they always +journal going forward. + +## Journaling row changes (the wrap layer) + +Once a table is `managed` (or `starter` and not yet shipped), ongoing row CRUD +is captured by wrapping `Table.prototype.insertRow`, `updateRow`, and +`deleteRows` in `wrapTableRows()` (`wrap.js`). Each wrap: + +- Passes through unchanged if journaling is suppressed (i.e. we are inside an + apply, `wrap.js`) or if `journalDecision` says not to journal + (`wrap.js`). +- Otherwise records a `table_row` op via `safeJournal` (`wrap.js`), which + swallows errors so a journaling failure never breaks the user's write. + +| CRUD method | Op type | Identity used | Payload key | +| --- | --- | --- | --- | +| `insertRow` (`wrap.js`) | `insert_row` | new `_dd_row_uuid` assigned via `setRowUuid` after insert (`wrap.js`) | `after` (portable row, `wrap.js`) | +| `updateRow` (`wrap.js`) | `update_row` | existing `_dd_row_uuid` read via `getRowUuid` (`wrap.js`); skipped if the row has none | `patch` (portable, `wrap.js`) | +| `deleteRows` (`wrap.js`) | `drop_row` | `_dd_row_uuid` read from each row captured BEFORE deletion (`wrap.js`, `wrap.js`) | `before` (portable, `wrap.js`) | + +Every row op payload carries the table's stable UUID as `table_uuid` so the +target can find the corresponding local table. Portable conversion is done by +`rowToPortable` (`rowPayload.js`); see [Foreign-key handling](#foreign-key-handling). + +## Applying row ops (the apply layer) + +On the receiving instance, the row handlers live in `apply.js` and are dispatched +by op type (`apply.js`). They all resolve the table the same +way -- `findLocalTableByUuid` maps `payload.table_uuid` to the local table via +`_dd_entity_ids` (`apply.js`) -- and use `_dd_row_uuid` to map identity: + +| Op | Handler | Behavior | +| --- | --- | --- | +| `insert_row` | `applyInsertRow` (`apply.js`) | `ensureManagedSchema`, then idempotency check via `findIdByRowUuid` (noop if the row UUID already exists, `apply.js`); converts `payload.after` with `portableToRow`, inserts, and stamps the row's `_dd_row_uuid` to the op's `entity_uuid` (`apply.js`). | +| `update_row` | `applyUpdateRow` (`apply.js`) | Finds the local id by row UUID; if absent, treats the update as an insert from `payload.patch` (`apply.js`); otherwise applies the patch via `updateRow` (skips an empty patch, `apply.js`). | +| `drop_row` | `applyDropRow` (`apply.js`) | Noop if the table or the row UUID is not present locally (`apply.js`); otherwise deletes by local id. | +| `set_table_mode` | `applySetTableMode` (`apply.js`) | For `managed`/`starter`, calls `ensureManagedSchema` on the local table (`apply.js`); upserts the local `_dd_table_modes` row (`apply.js`). | + +A row's identity (`_dd_row_uuid`) is carried in the op's `entity_uuid` field, +not in the payload; the apply handlers read `op.entity_uuid` as the row UUID +(`apply.js`, `apply.js`). Because apply runs the model methods inside a +suppressed context (`apply.js`, `runSuppressed`), the row CRUD wraps do not +re-journal these changes. + +## Foreign-key handling + +A row may contain foreign keys whose integer values are meaningless on a peer. +`rowToPortable` (`rowPayload.js`) and `portableToRow` (`rowPayload.js`) +translate FK fields through row UUIDs. For each FK field +(`field.is_fkey && field.reftable_name`), the portable form stores the value +under `"__uuid"` (the `UUID_SUFFIX` constant, `rowPayload.js`) so +it does not collide with the original field name. + +On the source (`rowToPortable`, `rowPayload.js`): + +| Referenced table's mode | Action | +| --- | --- | +| `managed` or `starter` | Look up the referenced row's `_dd_row_uuid` via `getRowUuid` and store it as `__uuid` (`rowPayload.js`). If the referenced row has no UUID yet, store `null` and attach a warning (`rowPayload.js`). | +| `user` | Cannot translate; store `null` and attach a warning that the FK will be null on the target (`rowPayload.js`). | + +On the target (`portableToRow`, `rowPayload.js`): + +- If the portable value is `null`, the local field is set to `null` + (`rowPayload.js`). +- Otherwise, for an FK field, `findIdByRowUuid` resolves the referenced row's + local id by its `_dd_row_uuid` (`rowPayload.js`). This may itself be + `null` if the referenced row has not been applied to the target yet. + +Both directions skip the `id` and `_dd_row_uuid` columns when building the +portable/local row (`rowPayload.js`, `rowPayload.js`). Warnings collected +on the source side ride along on the journaled op payload (`wrap.js`) so they +can surface in the admin UI. + +## File propagation + +Row payloads (and metadata payloads) can reference files. dev-deploy normalizes +file references so they survive transport between instances that have different +file-store roots. + +Reference translation (`payloadRefs.js`): a recursive walker +(`transformFileRefs`, `payloadRefs.js`) visits payload keys named `fileid`, +`file_id`, `bgFileId`, or `image_id` (`FILE_REF_KEYS`, `payloadRefs.js`). + +- On promote, `toPlaceholders` (`payloadRefs.js`) replaces a local file + reference (numeric id or relative path) with a portable placeholder string + `"__dd_file_ref::"` (`PLACEHOLDER_PREFIX`, `payloadRefs.js`), looking + the file up in `_dd_entity_ids` by `current_name` or `current_id` + (`lookupFileByValue`, `payloadRefs.js`). +- On apply, `fromPlaceholders` (`payloadRefs.js`) resolves the placeholder + back to the target's local relative path (`ent.current_name`). This runs in + `applyBatch` before the op handler executes (`apply.js`), + so handlers always see local paths. Writing the path back works because + Saltcorn's `/files/serve` accepts either a numeric id or a relative path + (`payloadRefs.js`, `payloadRefs.js`). + +Binary transfer (`apply.js` + `routes.js`): the placeholder only carries the +file's identity, not its bytes. The bytes move via a dedicated `create_file` +op and a pull endpoint: + +- `applyCreateFile` (`apply.js`) fetches the binary from the source peer + with a signed `GET /dev-deploy/api/file/` request + (`apply.js`), verifies the SHA-256 against `after.content_hash` + (`apply.js`; `sha256Buffer` from `files.js`), writes it to the local + absolute path, and creates the local `File` record (`apply.js`). +- The serving side is `apiFile` (`routes.js`, registered at + `routes.js`): it requires peer auth, looks the file up in + `_dd_entity_ids` by `uuid` and `kind = "file"` (`routes.js`), and + `sendFile`s the bytes. + +Tenant-scoped file paths: the absolute path is always reconstructed from the +file store root plus the tenant schema plus the relative serve path. On the +serving side `apiFile` builds it as +`path.join(db.connectObj.file_store, db.getTenantSchema(), mapping.current_name)` +(`routes.js`). On the applying side the same construction lives in +`toAbsolutePath(File, db, relPath)` (`files.js`), which joins +`db.connectObj.file_store`, `db.getTenantSchema()`, and the relative path. The +relative (serve) path is what is transported, because each instance has a +different file-store root (`files.js`); `toRelativePath` uses +`File.absPathToServePath` to produce it (`files.js`). `apiFile` passes +`{ dotfiles: "allow" }` to `sendFile` so paths containing dot-directories such +as `.dev-state` are not silently treated as not-found by Express +(`routes.js`). + +## Tables and columns + +| Table | Purpose | Key columns | +| --- | --- | --- | +| `_dd_table_modes` (`schema.js`) | One row per table that has a non-default mode. | `table_uuid` (PRIMARY KEY, the table's stable UUID), `data_mode`, `updated_at`, `starter_shipped_at` (`schema.js`). | +| `_dd_entity_ids` (`schema.js`) | Maps stable UUIDs to local integer ids for tables, files, etc. | `uuid`, `kind`, `current_name`, `current_id` (`schema.js`). Used to resolve `table_uuid` and file uuids on apply. | +| `` | The managed/starter table itself. | Hidden `_dd_row_uuid` TEXT column added by `ensureManagedSchema` (`rowIdentity.js`, `rowIdentity.js`). | + +The `_dd_row_uuid` column name is a single source of truth: it is exported as +`COLUMN_NAME` from `rowIdentity.js` and re-imported wherever needed +(for example as `ROW_UUID_COL` in `apply.js`, `wrap.js`, and `routes.js`). + +## Endpoints + +| Method | Path | Handler | Auth | Role in row sync | +| --- | --- | --- | --- | --- | +| GET | `/admin/dev-deploy/tables` | `tablesView` (`routes.js`) | admin (`routes.js`) | Lists tables and their data modes. | +| POST | `/admin/dev-deploy/tables/set` | `tablesSet` (`routes.js`) | admin (`routes.js`) | Sets a table's data mode; journals `set_table_mode` + the initial-ship `insert_row` ops. | +| GET | `/dev-deploy/api/file/:uuid` | `apiFile` (`routes.js`) | peer HMAC (`noCsrf`, `routes.js`) | Serves a file's bytes to a pulling peer during `create_file` apply. | + +The row ops themselves (`insert_row`, `update_row`, `drop_row`, +`set_table_mode`) are not standalone endpoints; they travel inside the normal +journal exchange (the `GET /dev-deploy/api/journal` and +`POST /dev-deploy/api/ingest` flow described in +[peering.md](./peering.md)) and are dispatched by the apply handler table +(`apply.js`). diff --git a/docs/multitenancy.md b/docs/multitenancy.md new file mode 100644 index 0000000..f5762db --- /dev/null +++ b/docs/multitenancy.md @@ -0,0 +1,222 @@ +# dev-deploy multi-tenancy + +dev-deploy runs unchanged inside a multi-tenant Saltcorn instance, where each +tenant lives in its own Postgres schema. The plugin treats every tenant as a +fully independent dev-deploy environment: each tenant has its own `_dd_*` +service tables (schema-qualified), its own `env_id` and journal, and its own set +of peers. Nothing is shared across tenants. + +This document covers how that isolation is achieved, the per-tenant installer, +and the Postgres-portability rules the code relies on. Code references are +`file:line` into the plugin source. + +See also: [architecture.md](architecture.md) for the core ops-journal / +entity-UUID / wrap / apply model that this builds on. + +## Contents + +- [Per-tenant schema qualification](#per-tenant-schema-qualification) +- [Tenant-keyed environment identity](#tenant-keyed-environment-identity) +- [The per-tenant installer](#the-per-tenant-installer) +- [Postgres-portability rules](#postgres-portability-rules) +- [Tenant introspection: getTenantSchema vs current_schema](#tenant-introspection-gettenantschema-vs-current_schema) + +## Per-tenant schema qualification + +On Postgres, Saltcorn places each tenant in its own schema and does NOT use +`SET search_path`; instead it qualifies queries with a schema prefix. dev-deploy +follows the same rule for every raw SQL statement it issues against its own +`_dd_*` tables and against managed user tables. + +The prefix comes from `db.getTenantSchemaPrefix()`. On SQLite it is empty; on +Postgres it is `"".`. Every `CREATE TABLE`, index, and raw query +interpolates it: + +- All six `_dd_*` tables are created with `${schema}_dd_...` in `schema.js` + (`schema.js`, `schema.js`, `schema.js`, `schema.js`, + `schema.js`, `schema.js`). +- Their indexes are likewise prefixed (`schema.js`, `schema.js`). +- Raw reads in the routes layer build SQL against `${schema}_dd_ops` etc. + (`routes.js`, `routes.js`, `routes.js`, `routes.js`, + `routes.js`, `routes.js`, `routes.js`). +- Conflict detection in apply queries `${schema}_dd_ops` (`apply.js`). +- The hidden-column infrastructure qualifies the user table it alters via + `tableSqlRef()`, which prepends the same prefix (`rowIdentity.js`). + +Note: `db.insert` / `db.select` / `db.selectMaybeOne` / `db.updateWhere` / +`db.deleteWhere` take an unqualified logical table name (e.g. `"_dd_ops"`) and +Saltcorn applies the tenant prefix internally. The explicit +`getTenantSchemaPrefix()` interpolation is only needed where the plugin drops to +raw `db.query` SQL. + +Because every statement is schema-scoped, a single Node process serving many +tenants reads and writes a different physical `_dd_*` table per tenant with no +code changes. + +## Tenant-keyed environment identity + +The environment identity (the singleton row in `_dd_env`) is schema-scoped, so a +process serving multiple tenants must not cache one identity row across them. +`env.js` keys its cache by tenant schema rather than using a module-level +singleton (`env.js`). + +| Symbol | Behavior | Location | +| --- | --- | --- | +| `cachedEnvByTenant` | `Map` keyed by tenant schema, not a single cached row | `env.js` | +| `tenantKey()` | Returns `db.getTenantSchema()`, or `"public"` if unavailable | `env.js` | +| `getEnv()` | Looks up / populates the cache entry for the current tenant key | `env.js` | +| `refreshEnvCache()` | Deletes only the current tenant's cache entry, then reloads | `env.js` | +| `initEnvIfMissing()` | Inserts a fresh `_dd_env` row with a new `env_id` if none exists, caches it under the current tenant key | `env.js` | +| `markBootstrapped()` | Stamps `bootstrapped_at`; only mutates the cached row if its `env_id` matches | `env.js` | + +Each tenant therefore gets: + +- its own `env_id` (a fresh v4 UUID generated in `initEnvIfMissing`, + `env.js`), +- its own `_dd_ops` journal (schema-qualified, above), +- its own `_dd_peers` rows and HMAC pairings. + +Two tenants pairing with the same remote base URL are independent: each has a +distinct `env_id`, a distinct shared secret, and a distinct journal, so promote +/ pull between tenants never cross-contaminates. + +## The per-tenant installer + +The plugin is installed into each tenant schema by +`scripts/installDevDeployTenant.js`, driven by the wrapper +`scripts/installDevDeployTenant.sh`. + +Prerequisites (`installDevDeployTenant.sh`): the tenants must already exist +(`saltcorn create-tenant `), and the plugin must already be installed into +the Postgres public schema once via the normal `install-plugin -d ./dev-deploy` +path so the shared `plugins_folder` copy exists. + +Usage (from project root, with the PG environment sourced from +`.dev-state-pg/env.sh`): + +``` +./dev-deploy/scripts/installDevDeployTenant.sh t1 t2 # named tenants +./dev-deploy/scripts/installDevDeployTenant.sh '*' # all tenants +``` + +What the script does: + +| Step | Detail | Location | +| --- | --- | --- | +| Re-root `@saltcorn/*` | Uses `createRequire` against the Saltcorn checkout's `node_modules` (up two dirs to project root, then `saltcorn/packages/...`) | `installDevDeployTenant.js` | +| Resolve tenants | Empty args or `*` means all tenants, read from `getAllTenants` run inside the default schema | `installDevDeployTenant.js` | +| Init per-tenant State | `init_multi_tenant(Plugin.loadAllPlugins, true, tenants)` so `getState()` resolves inside `runWithTenant`, without running migrations; this also re-runs each tenant's existing plugins' (idempotent) `onLoad` | `installDevDeployTenant.js` | +| Permit local plugin on tenants | Sets root config `tenants_unsafe_plugins = true` | `installDevDeployTenant.js` | +| Install per tenant | For each tenant, `installInto(tenant)` | `installDevDeployTenant.js` | + +`installInto` (`installDevDeployTenant.js`) runs inside +`db.runWithTenant(tenant, ...)` and a transaction: + +1. Deletes any prior `_sc_plugins` row for `dev-deploy` so it converges on + exactly one row (no duplicate source of truth) (`installDevDeployTenant.js`). +2. Constructs a `Plugin` with `source: "local"`, `location: DEV_DEPLOY_DIR`, and + calls `Plugin.loadAndSaveNewPlugin(plugin, true, false)` (`installDevDeployTenant.js`). +3. Verifies against dev-deploy's own table: confirms the `_sc_plugins` row + exists AND `_dd_env` exists in this tenant's schema (via + `information_schema.tables` filtered on `db.getTenantSchema()`), so a stale + plugin row cannot pass; throws if `onLoad` did not run + (`installDevDeployTenant.js`). + +Why this script exists instead of the CLI: the CLI +`install-plugin -t -d ` cannot install a local (`-d`) plugin on a +non-root tenant. In this Saltcorn build `loadAndSaveNewPlugin` skips any +non-`npm` plugin on a non-root tenant before its `allowUnsafe` argument is +consulted; the supported lever is the root-only `tenants_unsafe_plugins` config, +which the CLI never sets and this script does +(`installDevDeployTenant.js`, `installDevDeployTenant.js`). + +Running `onLoad` per tenant is what creates that tenant's `_dd_*` tables and +bootstraps its `env_id`; because both `createAllTables()` and +`initEnvIfMissing()` are idempotent, re-running the installer is safe. + +## Postgres-portability rules + +dev-deploy supports both SQLite and Postgres from one DDL/SQL codebase. The +following rules were learned to keep the `_dd_*` tables portable; deviating from +them breaks on Postgres specifically. + +### Auto-increment primary key: integer vs serial + +SQLite's `integer primary key` auto-assigns rowids; Postgres needs `serial` +(`AUTOINCREMENT` is SQLite-only syntax). The DDL picks the type at runtime: + +``` +const serial = db.isSQLite ? "integer" : "serial"; +``` + +Used for `_dd_peers.peer_id` (`schema.js`, `schema.js`). `_dd_anchors` +uses `peer_id INTEGER` as a foreign reference, not an auto-increment, so it stays +`INTEGER` (`schema.js`). + +### ADD COLUMN IF NOT EXISTS on Postgres to avoid transaction poisoning + +For idempotent column migrations on already-installed instances, a bare `ALTER +TABLE ... ADD COLUMN` caught in a JS `try/catch` is NOT enough on Postgres: a +failed statement poisons the surrounding transaction. So the code branches on +`db.isSQLite`: + +- SQLite lacks `ADD COLUMN IF NOT EXISTS`, so it runs the bare `ALTER` and + swallows the "column already exists" error. +- Postgres uses `ADD COLUMN IF NOT EXISTS` so the statement never errors and the + transaction is never poisoned. + +This pattern appears for `_dd_ops.conflict_with_op_id` (`schema.js`) and +`_dd_table_modes.starter_shipped_at` (`schema.js`). + +A related case: `createDdOps` creates a partial index with a `WHERE status = +'conflict'` predicate and `.catch(() => {})` (`schema.js`); that swallow is +fine because it is a standalone `db.query`, not part of a migration that must +keep a transaction alive. + +### Every `_dd_*` insert needs `{ noid: true }` + +The `_dd_*` tables have no column literally named `id` (they use named PKs: +`env_id`, `peer_id`, `uuid`, `op_id`, `table_uuid`, or a composite PK). Saltcorn's +`db.insert` assumes an `id` column unless told otherwise, so every insert into a +`_dd_*` table passes `{ noid: true }`: + +| Insert target | Location | +| --- | --- | +| `_dd_env` | `env.js` | +| `_dd_ops` | `apply.js` | +| `_dd_anchors` (inbound/outbound) | `routes.js`, `routes.js` | +| `_dd_table_modes` | `routes.js`, `apply.js` | + +(`_dd_peers` and `_dd_entity_ids` inserts live in `peers.js` / `entityIds.js`, +outside the files reviewed here, but follow the same `{ noid: true }` rule by the +same constraint.) + +## Tenant introspection: getTenantSchema vs current_schema + +NEVER use `current_schema()` to discover which tenant schema you are in. +Because Saltcorn qualifies queries with `getTenantSchemaPrefix()` rather than +issuing `SET search_path`, `current_schema()` returns `"public"` even while +serving inside a tenant. The correct source of truth is `db.getTenantSchema()`. + +This bit the hidden-column check in `rowIdentity.js`. The Postgres branch of +`columnExists()` queries `information_schema.columns` filtered by +`table_schema = db.getTenantSchema()` (`rowIdentity.js`, `rowIdentity.js`) +-- the same schema that `tableSqlRef()` / the `ALTER` target use +(`rowIdentity.js`). An earlier version used `current_schema()`, which made the +check falsely report the `_dd_row_uuid` column missing on every call after the +first; the explicitly-qualified `ALTER` then failed with `column +"_dd_row_uuid" already exists`, breaking apply (which calls +`ensureManagedSchema` once per `set_table_mode` and per `insert_row`) +(`rowIdentity.js`). + +Consistent introspection sources used throughout: + +| Need | Use | Examples | +| --- | --- | --- | +| Qualify raw SQL | `db.getTenantSchemaPrefix()` | `schema.js`, `routes.js`, `rowIdentity.js` | +| Name the current schema (for `information_schema`, file paths) | `db.getTenantSchema()` | `rowIdentity.js`, `installDevDeployTenant.js`, `routes.js` | +| Cache key per tenant | `db.getTenantSchema()` (via `tenantKey()`) | `env.js` | + +The same `db.getTenantSchema()` is used to locate per-tenant file storage in +the binary file endpoint: `path.join(file_store, db.getTenantSchema(), +relative_path)` (`routes.js`). diff --git a/docs/operations.md b/docs/operations.md new file mode 100644 index 0000000..90ebaaa --- /dev/null +++ b/docs/operations.md @@ -0,0 +1,204 @@ +# dev-deploy operations + +Running and maintaining the local development instances for the dev-deploy +plugin. Three Saltcorn instances back development and testing: a SQLite MAIN +instance on :3000 (which also hosts the saltcorn-idp LDAPS listener), a SQLite +TEST instance on :3001, and a Postgres multi-tenant instance on :3002. + +See also: [architecture.md](architecture.md), [peering.md](peering.md), +[multitenancy.md](multitenancy.md), [testing.md](testing.md). + +## Contents + +- [Layout](#layout) +- [The three dev instances](#the-three-dev-instances) +- [Environment files](#environment-files) +- [Starting the instances](#starting-the-instances) +- [Installing the plugin](#installing-the-plugin) +- [Per-tenant install on Postgres](#per-tenant-install-on-postgres) +- [Known issues](#known-issues) + +## Layout + +The project root is `~/claude/saltcorn`. The dev-deploy plugin source lives in +the `dev-deploy/` subfolder; the saltcorn-idp plugin source is a sibling in +`idp/`; upstream Saltcorn is checked out under `saltcorn/`. Each instance keeps +its database, file store, and session store under its own state directory in the +project root: + +| Instance | Port | State dir | Database | LDAPS port | +| --- | --- | --- | --- | --- | +| MAIN | 3000 | `.dev-state/` | SQLite (`saltcorn.sqlite`) | 1636 | +| TEST | 3001 | `.dev-state-test/` | SQLite (`saltcorn.sqlite`) | none | +| PG (multi-tenant) | 3002 | `.dev-state-pg/` | Postgres `saltcorn_idp` | 1637 | + +The start scripts `cd` into the state dir before running `saltcorn serve` so +each instance writes its own `sessions.sqlite` (Saltcorn's SQLite session store +writes `sessions.sqlite` at the process cwd, per +`packages/server/routes/utils.js`). + +## The three dev instances + +### MAIN (:3000, SQLite) + +`startServer.sh` sources `.dev-state/env.sh`, runs `saltcorn install-plugin -d +./dev-deploy` (non-fatal on failure -- the previously installed version still +loads), then `cd .dev-state` and `exec saltcorn serve "$@"`. MAIN is the only +instance that sets `SALTCORN_IDP_LDAP_PORT=1636`, so saltcorn-idp opens its +LDAPS listener there. The other instances leave that port unset (PG sets 1637; +see below), so the three do not collide. saltcorn-idp is installed via +`reinstallIdp.sh` rather than in the start script (the shared plugins_folder +makes a per-boot install race on concurrent starts); it loads at serve time from +that install. + +### TEST (:3001, SQLite) + +`startServerTest.sh` sources `.dev-state-test/env.sh` (which sets +`SALTCORN_PORT=3001`), installs the plugin the same way, then +`cd .dev-state-test` and `exec saltcorn serve -p "$SALTCORN_PORT" "$@"`. TEST is +the promote/pull peer used by the e2e suite. It does not set +`SALTCORN_IDP_LDAP_PORT`, so no LDAP listener runs there. + +### PG (:3002, Postgres multi-tenant) + +`startServerPg.sh` sources `.dev-state-pg/env.sh`, then `cd .dev-state-pg` and +`exec saltcorn serve -p 3002 "$@"`. The PG env deliberately does NOT set +`SQLITE_FILEPATH`, so `getConnectObject()` selects Postgres; it sets +`SALTCORN_MULTI_TENANT=true` to enable schema-per-tenant (Postgres only). Unlike +the SQLite start scripts, `startServerPg.sh` does NOT run `install-plugin` at +boot -- the plugin is installed into the public schema by `reinstallDevDeploy.sh` +and activated per tenant by `installDevDeployTenant.sh` (see below). After +per-tenant activation, each tenant's `onLoad` re-runs automatically on every +boot via `init_multi_tenant` -> `loadAllPlugins`. The PG env also sets +`SALTCORN_IDP_LDAP_PORT=1637` (distinct from MAIN's 1636) so the multi-tenant +LDAP gate can exercise tenant-in-DN binds against this instance. + +## Environment files + +Each state dir has an `env.sh` that is sourced before running `saltcorn`. They +all prepend the in-tree CLI (`saltcorn/packages/saltcorn-cli/bin`) to `PATH` so +`saltcorn ...` resolves to this checkout, and load nvm. + +| Variable | MAIN | TEST | PG | +| --- | --- | --- | --- | +| `SQLITE_FILEPATH` | `.dev-state/saltcorn.sqlite` | `.dev-state-test/saltcorn.sqlite` | unset (selects Postgres) | +| `SALTCORN_FILE_STORE` | `.dev-state/files` | `.dev-state-test/files` | `.dev-state-pg/files` | +| `SALTCORN_SESSION_SECRET` | set | set | set | +| `SALTCORN_PORT` | unset (defaults to 3000) | `3001` | unset (passed `-p 3002`) | +| `SALTCORN_IDP_LDAP_PORT` | `1636` | unset | `1637` | +| `SALTCORN_MULTI_TENANT` | unset | unset | `true` | +| `SALTCORN_JWT_SECRET` | unset | unset | set | +| `PGHOST` / `PGUSER` / `PGDATABASE` / `PGPASSWORD` | unset | unset | `/var/run/postgresql` / `scott` / `saltcorn_idp` / `peer` | + +On PG, Saltcorn only selects Postgres when user, password, and database are all +set (`connect.ts` `getConnectObject`). Authentication is via the Unix socket +with peer auth, which ignores the password, so `PGPASSWORD=peer` is a dummy that +just satisfies that check. + +## Starting the instances + +Run from the project root: + +``` +cd ~/claude/saltcorn +./startServer.sh & # MAIN :3000 (+ LDAPS :1636) +./startServerTest.sh & # TEST :3001 +./startServerPg.sh & # PG multi-tenant :3002 (+ LDAPS :1637) +``` + +The two SQLite start scripts each attempt `install-plugin -d ./dev-deploy` at +boot (failures are non-fatal). The PG instance does not; install it explicitly +as described below. + +## Installing the plugin + +After editing the plugin source, reinstall it into all three instances with the +servers stopped: + +``` +cd ~/claude/saltcorn +./reinstallDevDeploy.sh +``` + +`reinstallDevDeploy.sh` installs dev-deploy into MAIN (`.dev-state`), TEST +(`.dev-state-test`), and the PG public schema (`.dev-state-pg`). It is a separate +script (not folded into the start scripts) because the saltcorn plugins_folder +(`~/.local/share/saltcorn-plugins`) is shared by all instances, and `saltcorn +install-plugin`: + +- needs an ABSOLUTE `-d` path (it `path.join()`s then `require()`s, so a leading + `./` is collapsed and resolved as a node module), and +- aborts (EEXIST) if the per-plugin-dir `node_modules` symlinks already exist. + +Doing this in each start script would race when instances boot concurrently, so +reinstalls are centralized here. Before each install the script clears the +`node_modules` symlinks under the plugins root so `install-plugin` can recreate +them cleanly. + +For the PG instance, installing into the public schema is only step one; you must +then activate the plugin per tenant (next section). + +## Per-tenant install on Postgres + +The public-schema install does not create the `_dd_*` tables in each tenant +schema. To register + enable dev-deploy in a tenant schema and run its `onLoad` +(creating the `_dd_*` tables and bootstrapping the env row), use: + +``` +./dev-deploy/scripts/installDevDeployTenant.sh t1 t2 # named tenants +./dev-deploy/scripts/installDevDeployTenant.sh '*' # all tenants +``` + +The shell wrapper resolves the project root, sources `.dev-state-pg/env.sh`, and +runs `installDevDeployTenant.js`. The JS uses Saltcorn's supported +`Plugin.loadAndSaveNewPlugin` inside `runWithTenant` (replacing the old manual +`INSERT INTO ._sc_plugins` SQL hack). It sets the root-only config +`tenants_unsafe_plugins=true` so a LOCAL plugin can be installed into tenant +schemas, converges each tenant to exactly one `_sc_plugins` row, and verifies the +install by confirming both the `_sc_plugins` row and the tenant-schema `_dd_env` +table exist. With no args or a single `*`, it installs into every tenant from the +public `_sc_tenants` list. + +Prerequisites: + +- the tenants must already exist (`saltcorn create-tenant `), and +- the plugin must be installed into the PG public schema once (the + `reinstallDevDeploy.sh` path) so the shared plugins_folder copy exists. + +After per-tenant activation, each tenant's `onLoad` re-runs automatically on +every boot of `startServerPg.sh`. The SQLite MAIN/:3000 and TEST/:3001 instances +are unaffected (they set `SQLITE_FILEPATH`). See +[multitenancy.md](multitenancy.md) for the schema-qualification details. + +## Known issues + +### Shared node_modules symlinks couple the two plugins + +The `node_modules` symlinks under the plugins root +(`~/.local/share/saltcorn-plugins`) are SHARED across plugins. The +`clearSymlinks()` step in `reinstallDevDeploy.sh` therefore also clears +saltcorn-idp's symlinks. After running `reinstallDevDeploy.sh`, you MUST also +re-run `reinstallIdp.sh` (and vice versa) to keep both plugins' installs +consistent, then restart the servers. `reinstallIdp.sh` installs saltcorn-idp +into MAIN and TEST only (it does not touch the PG instance). + +Recommended sequence after editing plugin source: + +``` +# stop the servers first +./reinstallDevDeploy.sh +./dev-deploy/scripts/installDevDeployTenant.sh '*' # PG only, if using tenants +./reinstallIdp.sh +# restart the servers +``` + +### Themeless freshly-created tenant returns 500 from sendWrap + +A freshly created Postgres tenant has no theme/layout configured, so Saltcorn's +own admin pages that call `res.sendWrap` (for example `GET /table/new/`) return +HTTP 500. dev-deploy's admin pages self-render and are unaffected. When you need +a CSRF token for a Saltcorn admin POST against a themeless tenant, grab the token +from a dev-deploy page (for example `GET /admin/dev-deploy/peers`) instead -- the +mutate-then-redirect POST (such as `POST /table`) does not call `sendWrap` and so +succeeds. This is exploited throughout the PG gates; see +[testing.md](testing.md). diff --git a/docs/peering.md b/docs/peering.md new file mode 100644 index 0000000..0fdefae --- /dev/null +++ b/docs/peering.md @@ -0,0 +1,328 @@ +# dev-deploy peering + +How two dev-deploy instances find, authenticate, and exchange ops with each +other: the peer model, the pairing flow, the HMAC wire protocol, the sync +anchors that drive promote and pull, and how a standalone instance pairs with a +single tenant on a multi-tenant server. + +See also: [architecture.md](architecture.md) for the ops journal, stable UUIDs, +and the apply pipeline; the [README](../README.md) for the full table and +endpoint inventory. + +## Contents + +- [The peer model](#the-peer-model) +- [Pairing flow](#pairing-flow) +- [The HMAC wire protocol](#the-hmac-wire-protocol) +- [Promote, pull, and anchors](#promote-pull-and-anchors) +- [Mixed-topology peering](#mixed-topology-peering) +- [Endpoint reference](#endpoint-reference) +- [File reference](#file-reference) + +## The peer model + +A peer is one row in `_dd_peers` (defined in `lib/schema.js`). Each instance +stores a row per peer it talks to; the relationship is configured independently +on both sides (there is no central registry). + +| Column | Type | Meaning | +| --- | --- | --- | +| `peer_id` | `serial` / `integer` PK | Local surrogate id (auto-assigned). | +| `env_id` | `TEXT` UNIQUE | The peer's dev-deploy `env_id` (the other side's `_dd_env.env_id`). | +| `label` | `TEXT` | Optional human label (e.g. `test`, `prod`). | +| `base_url` | `TEXT` | Where to reach the peer (e.g. `http://localhost:3001` or `https://tenant.example.com`). | +| `peer_secret_ciphertext` | `TEXT` | Sealed shared secret (hex). | +| `peer_secret_iv` | `TEXT` | AES-GCM IV (hex). | +| `peer_secret_tag` | `TEXT` | AES-GCM auth tag (hex). | +| `require_tls` | `INTEGER` | TLS-required flag (stored as 0/1). | +| `created_at` | `TEXT` | ISO 8601 creation time. | +| `last_seen_at` | `TEXT` | ISO 8601 of the last verified inbound request from this peer; `null` until first contact. | + +### The sealed shared secret + +The shared secret is 32 random bytes (`randomSecret()`, `lib/crypto.js`). It +is never stored in plaintext. At rest it is sealed with AES-256-GCM +(`seal()`, `lib/crypto.js`) and split across the three `peer_secret_*` +columns as hex. The hex-text storage is deliberate: Saltcorn's SQLite insert +layer JSON-stringifies object values, which would mangle a raw `Buffer` column +(`lib/schema.js`). + +The 32-byte key-encryption key (KEK) used by `seal`/`open` is derived once per +process via HKDF-SHA256 from `SALTCORN_SESSION_SECRET` (`getKek()`, +`lib/crypto.js`; falls back to the Saltcorn `session_secret` config). Because +the KEK is tied to the session secret, rotating `SALTCORN_SESSION_SECRET` +invalidates every stored pairing -- existing ciphertexts no longer decrypt +(documented in `lib/crypto.js`). + +Plaintext only crosses the process boundary at two moments: + +- At pairing time, when the operator copies the secret into the other side's + pairing form. +- At HMAC sign/verify time, when `peerSecret()` (`lib/peers.js`) opens the + sealed bytes to compute or check a signature. + +`rowToPeer()` (`lib/peers.js`) deliberately omits the sealed columns from the +plain accessor; callers must go through `peerSecret()` / `peerSecretByEnvId()`. + +## Pairing flow + +Pairing is symmetric: each side ends up with a `_dd_peers` row pointing at the +other side's `env_id` and `base_url`, and both rows seal the *same* shared +secret. One side generates the secret; the operator pastes it into the other. + +Each instance's own `env_id` is shown on its Peers page (`peersView`, +`lib/routes.js`): "This instance's env_id is ... Paste this into the other +instance's peer form." The `env_id` itself is a random UUID minted once at +bootstrap (`lib/env.js`). + +Steps: + +1. On instance A, open `/admin/dev-deploy/peers` and submit the **Add peer** + form (`peersAdd`, `lib/routes.js`) with the peer's `env_id` (B's), an + optional `label`, B's `base_url`, and an optional `require_tls` checkbox. + Leave **Existing secret** blank. +2. `addPeer` (`lib/peers.js`) generates a fresh 32-byte secret, seals it, and + inserts the row. The plaintext secret is rendered once as 64 hex characters + on the confirmation page (`lib/routes.js`) -- "it will not be shown + again." +3. On instance B, open its own Peers page and submit **Add peer** with A's + `env_id`, A's `base_url`, and paste the 64-hex secret into the **Existing + secret** field. `peersAdd` validates it against `/^[0-9a-fA-F]{64}$/` + (`lib/routes.js`) and passes it to `addPeer` as `existingSecret`, so B + seals the identical secret rather than generating a new one. + +After both rows exist, A and B share one secret and each knows the other's +`env_id` and `base_url`. + +`env_id` is enforced UNIQUE, so re-adding the same peer fails with "peer with +env_id ... already exists" (`lib/peers.js`). + +### Rotation and deletion + +- **Rotate** (`peersRotate`, `lib/routes.js` -> `rotatePeerSecret`, + `lib/peers.js`) mints a new secret for an existing peer, re-seals it, and + shows the new value once. The operator must paste the new secret on the other + side (re-pair or rotate there) or the pairing breaks. +- **Delete** (`peersDelete`, `lib/routes.js` -> `deletePeer`, + `lib/peers.js`) removes the `_dd_peers` row *and* deletes that peer's + `_dd_anchors` rows, so a later re-pair starts syncing from the epoch again. + +## The HMAC wire protocol + +Every machine-API request is signed with the shared secret using HMAC-SHA256. +The outbound side is `lib/transport.js`; the inbound check is `requirePeerAuth` +(`lib/peerAuth.js`). + +### Headers + +| Header | Source | Meaning | +| --- | --- | --- | +| `X-DD-Env-Id` | sender's own `env_id` | Caller identity; the receiver looks it up in `_dd_peers` via `findPeerByEnvId` to find the matching secret. | +| `X-DD-Timestamp` | `String(Date.now())` | Milliseconds since epoch. | +| `X-DD-Nonce` | `randomNonce().toString("hex")` | 16 random bytes, hex (replay padding). | +| `X-DD-Signature` | `sign(secret, canonical)` | Hex HMAC-SHA256 over the canonical string. | + +All four headers are required; a missing one returns `400 missing header ...` +(`lib/peerAuth.js`, `lib/peerAuth.js`). + +When there is a request body, the sender sets +`Content-Type: application/vnd.dev-deploy+json` (`lib/transport.js`). This +custom type stops Saltcorn's `express.json()` middleware from consuming the +request stream, so the receiver can read the exact raw bytes and HMAC them +verbatim -- no re-serialization, no whitespace or key-order assumptions +(`lib/peerAuth.js`, `lib/peerAuth.js`). + +### The canonical string + +Both sides build the signed string with `buildCanonical` (`lib/crypto.js`). +It is six fields joined by newlines (`\n`): + +``` +timestamp +nonce +METHOD +path +targetHost +sha256hex(body) +``` + +- `METHOD` is uppercased. +- `path` is the request path including query string. Outbound it is the literal + `path` argument; inbound it is `req.originalUrl || req.url` + (`lib/peerAuth.js`). +- `body` is hashed with SHA-256 (`sha256Hex`, `lib/crypto.js`); an empty body + hashes the empty string. GET/HEAD never have a body + (`lib/peerAuth.js`). + +### Host binding (anti-cross-tenant replay) + +`targetHost` is the normalized host the request is aimed at, and binding it into +the signature is what stops a request signed for one tenant from being replayed +against another tenant on the same multi-tenant server. + +- Outbound, the host is derived from the peer's `base_url`: + `normalizeHost(new URL(baseUrl).host)` (`lib/transport.js`). +- Inbound, it is derived from the request: prefer `X-Forwarded-Host` (first + value, set by a trusted proxy), else the `Host` header, then normalized the + same way (`lib/peerAuth.js` to `lib/peerAuth.js`). + +`normalizeHost` (`lib/crypto.js`) lowercases, trims, and drops a trailing +`:80` or `:443` so both sides produce byte-identical strings (clients omit the +default port from the `Host` header). Because the canonical includes +`targetHost`, a signature computed for `t1.example.com` will not verify when the +same bytes are re-sent to `t2.example.com`: the receiver rebuilds the canonical +with its own host, the MAC differs, and verification fails with +`401 bad signature`. + +Note (`lib/peerAuth.js`): the receiver derives the host from the request, NOT +from `peerRow.base_url`. Inbound, `base_url` is the *sender's* address (used for +pull-back), not the receiver's own host. + +### Verification order + +`requirePeerAuth` (`lib/peerAuth.js`) checks, in order, and returns `null` +(after sending a 4xx) on the first failure: + +1. All four required headers present, else `400`. +2. Timestamp within the +/- 5 minute skew window + (`timestampWithinSkew`, `lib/crypto.js`; `SKEW_TOLERANCE_MS = 5 * 60 * + 1000`, `lib/crypto.js`), else `401 timestamp out of skew window`. +3. `X-DD-Env-Id` resolves to a `_dd_peers` row, else + `401 unknown peer env_id`. +4. The peer has a sealed secret that opens, else `401 peer not provisioned`. +5. Signature matches via constant-time compare (`verifySignature`, + `lib/crypto.js`, uses `crypto.timingSafeEqual`), else + `401 bad signature`. +6. If there was a body, it parses as JSON (after the signature already covered + the raw bytes), else `400 body is not valid JSON`. + +On success it parses the body into `req.body`, advances the peer's +`last_seen_at` (`touchPeerLastSeen`, `lib/peers.js`), sets `req.dd_peer` to +the peer row, and returns it. + +The nonce is sent and signed but the current code does not maintain a +server-side seen-nonce cache; replay protection rests on the skew window and the +host binding. (Stated to avoid over-claiming; no nonce store exists in the code +read.) + +## Promote, pull, and anchors + +Sync direction is per peer and per direction, tracked in `_dd_anchors` +(`lib/schema.js`): + +| Column | Meaning | +| --- | --- | +| `peer_id` | FK-by-convention to `_dd_peers.peer_id` (PK part). | +| `direction` | `outbound` or `inbound` (PK part). | +| `last_op_id` | The last op id synced in that direction for that peer. | +| `updated_at` | ISO 8601 of the last advance. | + +`PRIMARY KEY (peer_id, direction)` means at most one outbound and one inbound +watermark per peer. + +Both promote and pull select only ops authored by the *local* env +(`source_env_id = env.env_id`) and only those `created_at >` the anchor op's +`created_at`. If there is no anchor, sync starts from the epoch (the whole +journal). Helpers: `getOutboundAnchor` / `getInboundAnchor` / `upsertAnchor` +(`lib/routes.js` to `lib/routes.js`). + +### Promote (push ops to a peer) + +`promote` (`lib/routes.js`): + +1. Look up the peer and the local env; read the outbound anchor. +2. Select the local env's ops after the anchor, oldest first, `LIMIT 500` + (`lib/routes.js`). If none, redirect with "no ops to promote". +3. `signedFetch` `POST /dev-deploy/api/ingest` with `{ ops }` and the peer's + secret (`lib/routes.js`). +4. On success, advance the outbound anchor to the last op's `op_id` + (`upsertAnchor(peerId, "outbound", ...)`, `lib/routes.js`). +5. Summarize applied/error counts from the response and append any plugin- + version warnings from `diffPluginsWithPeer` (`lib/routes.js`, which calls + `/dev-deploy/api/health`). + +`planView` (`lib/routes.js`) is the dry run: same anchor-relative selection +(`LIMIT 500`) but rendered as a preview table instead of being sent. + +The receiving side, `apiIngest` (`lib/routes.js`), authenticates, applies +the batch with `applyBatch`, and advances *its* `inbound` anchor for the sender +to the last received `op_id` (`lib/routes.js`). + +### Pull (fetch a peer's ops) + +`pull` (`lib/routes.js`): + +1. Read the inbound anchor; build the path + `/dev-deploy/api/journal?since=` (or no `since` if no anchor) + (`lib/routes.js`). +2. `signedFetch` `GET` that path (`lib/routes.js`). +3. Apply the returned `ops` with `applyBatch` (`lib/routes.js`). +4. Advance the inbound anchor to the last pulled op's `op_id` + (`lib/routes.js`). +5. Summarize applied/error/conflict counts and plugin warnings. + +The serving side, `apiJournal` (`lib/routes.js`), returns the local env's +ops after `since` (resolved to the op's `created_at`), oldest first, `LIMIT +1000` (`lib/routes.js`), as `{ source_env_id, ops }`. + +## Mixed-topology peering + +A standalone instance and a specific tenant on a multi-tenant server peer the +same way as two standalone instances; the only difference is the `base_url`. + +- Address the tenant by its tenant hostname as `base_url`, e.g. + `https://tenant.example.com`. Saltcorn routes the request to that tenant by + host, and dev-deploy's tables are schema-qualified per tenant + (`db.getTenantSchemaPrefix()` is used throughout, e.g. `lib/routes.js`), + so the peer row, ops, and anchors all live in that tenant's schema. +- The host binding makes this safe: the signature is computed over the tenant + hostname (outbound from `base_url`; inbound from `X-Forwarded-Host` / `Host`). + A request signed for one tenant cannot be replayed against another tenant on + the same server, because each tenant's host produces a different canonical + string (see [Host binding](#host-binding-anti-cross-tenant-replay)). +- Each side still stores the other's `env_id` and `base_url` in its own + `_dd_peers`. A standalone instance points `base_url` at the tenant's hostname; + the tenant points `base_url` back at the standalone instance's hostname. + +If a reverse proxy fronts the tenants, it must set `X-Forwarded-Host` to the +tenant hostname so the inbound canonical matches the outbound one +(`lib/peerAuth.js`). + +## Endpoint reference + +All four machine-API routes are registered with `noCsrf: true` +(`lib/routes.js` to `lib/routes.js`) and require HMAC peer auth via +`requirePeerAuth`. The admin peer/sync routes require a session with admin role +(`role_id === 1`, `isAdmin`, `lib/routes.js`) and use CSRF fields. + +### Machine API (HMAC peer auth) + +| Method | Path | Handler | File:line | Purpose | +| --- | --- | --- | --- | --- | +| GET | `/dev-deploy/api/journal?since=op_id` | `apiJournal` | `lib/routes.js` | Return local env ops after `since`, oldest first, max 1000. Returns `{ source_env_id, ops }`. | +| POST | `/dev-deploy/api/ingest` | `apiIngest` | `lib/routes.js` | Apply `{ ops }` from a peer; advance that peer's inbound anchor. Returns `{ received, results }`. | +| GET | `/dev-deploy/api/file/:uuid` | `apiFile` | `lib/routes.js` | Stream a file entity's bytes by UUID (octet-stream). 404 if no `_dd_entity_ids` mapping for kind `file`. | +| GET | `/dev-deploy/api/health` | `apiHealth` | `lib/routes.js` | Return `{ env_id, label, plugins }` for plugin-drift checks. | + +### Admin peer and sync routes (session + admin role) + +| Method | Path | Handler | File:line | Purpose | +| --- | --- | --- | --- | --- | +| GET | `/admin/dev-deploy/peers` | `peersView` | `lib/routes.js` | List peers, show this env's `env_id`, add-peer form. | +| POST | `/admin/dev-deploy/peers/add` | `peersAdd` | `lib/routes.js` | Pair a peer; generate or accept a 64-hex secret. | +| POST | `/admin/dev-deploy/peers/rotate` | `peersRotate` | `lib/routes.js` | Rotate a peer's shared secret (shown once). | +| POST | `/admin/dev-deploy/peers/delete` | `peersDelete` | `lib/routes.js` | Delete a peer and its anchors. | +| GET | `/admin/dev-deploy/plan` | `planView` | `lib/routes.js` | Preview ops that would be promoted to a peer. | +| POST | `/admin/dev-deploy/promote` | `promote` | `lib/routes.js` | Push outbound ops to a peer via signed `ingest`. | +| POST | `/admin/dev-deploy/pull` | `pull` | `lib/routes.js` | Pull a peer's ops via signed `journal` and apply them. | + +## File reference + +| File | Responsibility | +| --- | --- | +| `lib/peers.js` | `_dd_peers` CRUD; seal/open the shared secret; `peerSecret`, `addPeer`, `rotatePeerSecret`, `deletePeer`, `touchPeerLastSeen`. | +| `lib/crypto.js` | AES-256-GCM seal/open, HKDF KEK, HMAC sign/verify, `buildCanonical`, `normalizeHost`, skew check, random secret/nonce. | +| `lib/transport.js` | Outbound signed requests: `signedFetch` (JSON) and `signedFetchBinary` (raw bytes). | +| `lib/peerAuth.js` | Inbound `requirePeerAuth`: header check, skew, peer lookup, raw-body HMAC verify, host binding. | +| `lib/routes.js` | Admin UI for pairing/plan/promote/pull and the four machine-API handlers. | +| `lib/schema.js` | `_dd_peers` (`:38`) and `_dd_anchors` (`:116`) table definitions. | diff --git a/docs/testing.md b/docs/testing.md new file mode 100644 index 0000000..5bcf609 --- /dev/null +++ b/docs/testing.md @@ -0,0 +1,114 @@ +# dev-deploy testing + +The plugin has five test gates under `test/`. The e2e suite covers the +single-server SQLite feature surface (MAIN :3000 paired with TEST :3001); the +four Postgres gates cover multi-tenant isolation, mixed-topology push peering, +mixed-topology + tenant<->tenant pull peering, and managed-row data sync against +the multi-tenant instance on :3002. + +See also: [operations.md](operations.md) for starting the instances, +[peering.md](peering.md), [multitenancy.md](multitenancy.md), +[architecture.md](architecture.md). + +## Contents + +- [The five gates](#the-five-gates) +- [Postgres gate specifics](#postgres-gate-specifics) +- [Running everything](#running-everything) + +## The five gates + +| Gate | File | Backend | What it covers | Prerequisites | Run command | Passes | +| --- | --- | --- | --- | --- | --- | --- | +| e2e | `test/e2e.js` | SQLite (MAIN :3000 + TEST :3001) | Schema + bootstrap identity, mutation capture, pairing, promote/pull, conflict detect/resolve/merge, table constraints, file propagation, file refs in page layout, page_group, workflow_step, config + menu, plugin-config propagation, managed/starter row data, revert, data_mode locks, machine-endpoint HMAC security, admin auth | Both SQLite servers up; plugin installed + bootstrapped; `admin@local` / `AdminP@ss1` on both | `node test/e2e.js` (from `dev-deploy/`; also `npm test`) | 93 | +| mtGate | `test/mtGate.js` | Postgres (:3002) | Phase 0 multi-tenant ISOLATION: each tenant is its own dev-deploy env (distinct env_id), per-tenant Ops/Entities counts, mutating t1 moves only t1's numbers | PG :3002 up; tenants `t1` and `t2` exist; dev-deploy installed per-tenant | `node test/mtGate.js` | 14 | +| mixedTopologyGate | `test/mixedTopologyGate.js` | Postgres (:3002) + SQLite MAIN (:3000) | Phase 1 mixed-topology peering: standalone dev (:3000) promotes to tenant t1 on :3002, ops land in t1 not t2; tenant-bound HMAC: a request signed for t1 is rejected (401) at t2, the same request signed for t2 passes auth | MAIN :3000 + PG :3002 up; dev-deploy on MAIN and per-tenant on t1/t2 | `node test/mixedTopologyGate.js` | 12 | +| managedRowsGate | `test/managedRowsGate.js` | Postgres (:3002) + `psql` | Phase 2 managed-rows on PG: mark managed adds the hidden `_dd_row_uuid` column and backfills rows; isolation (t2 gets no `_dd_table_modes` row); promote t1 -> t2 recreates the table with the SAME row UUIDs | PG :3002 up; `psql` reachable; tenants `t1`/`t2` with dev-deploy installed per-tenant | `node test/managedRowsGate.js` | 16 | +| pullGate | `test/pullGate.js` | Postgres (:3002) + SQLite MAIN (:3000) | Phase 1 PULL peering (reverse of mixedTopologyGate's push): t1 PULLS standalone dev's journal (host-bound HMAC on the GET `/dev-deploy/api/journal` path); isolation (t2 unchanged by t1's pull); idempotent re-pull ("nothing to pull"); tenant<->tenant pull (t2 pulls t1, dev untouched). `apiJournal` serves only first-party ops, so a pull relays a node's OWN ops only -- no transitive loops | MAIN :3000 + PG :3002 up; dev-deploy on MAIN and per-tenant on t1/t2 | `node test/pullGate.js` | 23 | + +Notes: + +- The e2e suite runs its tests in order and shares state across them; do not + reorder. Assertion failures are caught and counted (one failure does not stop + the suite); the runner exits non-zero if any test failed. +- Each Postgres gate self-skips with exit 0 (printing `SKIP`) if its required + port is not reachable; `managedRowsGate` also skips if `psql` is unavailable. +- e2e drives both instances over a mix of SQLite (`sqlite3`), `curl`, and a + `saltcorn run-js` shim (`test/sc-exec.js`) that gives the JS body full + `require()` access where `saltcorn run-js`'s vm sandbox falls short (for + `Field`, `TableConstraint`, `File`, etc.). It logs in as `admin@local` / + `AdminP@ss1`, scraping the CSRF token from each form's source GET page. +- e2e's "plugin mismatch" section assumes both instances carry identical plugin + lists (`base`, `sbadmin2`, `dev-deploy`) so a promote emits no WARNINGS + suffix. + +## Postgres gate specifics + +All four PG gates address tenants by Host header +(`.localhost.localdomain:3002` selects tenant `tNN`), keep a separate +cookie jar per tenant, and bootstrap each tenant admin over HTTP +(`admin@.local` / `AdminP@ss1`, creating the first user if needed). + +### psql introspection + +`mtGate`, `mixedTopologyGate`, and `pullGate` observe state entirely over HTTP by +scraping the dev-deploy dashboard (`/admin/dev-deploy/`) for the `Env ID`, +`Ops recorded`, and `Entities tracked` rows. `managedRowsGate` additionally uses +`psql` because the +hidden `_dd_row_uuid` column and per-row UUID values are not observable over +HTTP. Its psql connection is fixed in the file: + +| Setting | Value | +| --- | --- | +| `PGHOST` | `/var/run/postgresql` | +| `PGUSER` | `scott` | +| `PGDATABASE` | `saltcorn_idp` | +| `PGPASSWORD` | `peer` (dummy; socket peer auth ignores it) | + +It queries `information_schema.columns` for the `_dd_row_uuid` column and reads +schema-qualified tenant tables (for example `"t1".""`, `"t1"._dd_entity_ids`, +`"t2"._dd_table_modes`) to confirm the column add/backfill, per-tenant isolation, +and that promoted rows carry the SAME UUIDs across tenants. It cleans up its +probe table and `_dd_*` rows in both tenant schemas on exit. + +### The themeless-tenant sendWrap 500 gotcha + +A freshly created tenant has no theme/layout configured, so Saltcorn's own admin +pages that call `res.sendWrap` (for example `GET /table/new/`) return HTTP 500. +dev-deploy's admin pages self-render, so the PG gates grab the per-session +`_csrf` token from a dev-deploy page (`GET /admin/dev-deploy/peers`) rather than +from `/table/new`. The actual mutating POST (`POST /table`, +`POST /table/delete/`) only mutates and redirects (no `sendWrap`), so it +succeeds on a themeless tenant and journals the expected op (for example +`create_table`). This same trick is used to obtain CSRF for dev-deploy's own +admin POSTs (`/admin/dev-deploy/peers/add`, `/promote`, `/tables/set`, etc.). + +The gates also use idempotent cleanup so reruns start clean: peer rows are +cleared by env_id before pairing, and `managedRowsGate` uses a timestamped table +name (`mr`) and drops it afterward. `pullGate` likewise uses +timestamped probe tables (`pull_dev_`, `pull_t1_`) but drops each ONLY on +its SOURCE instance: a drop is a first-party op too, so dropping on the source +keeps the journal create->drop linear and the next run's pull replays it in order +to clean up the pulled copy. Dropping a PULLED copy instead would author a +competing local op and the next run would legitimately conflict with it. + +## Running everything + +``` +cd ~/claude/saltcorn +./startServer.sh & # MAIN :3000 +./startServerTest.sh & # TEST :3001 +./startServerPg.sh & # PG multi-tenant :3002 + +cd dev-deploy +node test/e2e.js # 93 +node test/mtGate.js # 14 (PG) +node test/mixedTopologyGate.js # 12 (MAIN + PG) +node test/managedRowsGate.js # 16 (PG + psql) +node test/pullGate.js # 23 (MAIN + PG) +``` + +The PG gates self-skip if :3002 (or, for `managedRowsGate`, `psql`) is not +available, so on a SQLite-only setup only the e2e suite runs its assertions. See +[operations.md](operations.md) for installing the plugin per tenant before the PG +gates can pass. diff --git a/index.js b/index.js new file mode 100644 index 0000000..ccd7679 --- /dev/null +++ b/index.js @@ -0,0 +1,62 @@ +// dev-deploy: Saltcorn plugin for migrating metadata changes across +// Dev/Test/Prod environments via an ops journal with stable UUIDs. + +const { PLUGIN_NAME, PLUGIN_VERSION } = require("./lib/constants"); +const { createAllTables } = require("./lib/schema"); +const { getEnv, initEnvIfMissing, markBootstrapped } = require("./lib/env"); +const { backfillAll } = require("./lib/entityIds"); +const { installAllWraps } = require("./lib/wrap"); +const { routes } = require("./lib/routes"); + + +const log = (msg) => { + // eslint-disable-next-line no-console + console.log(`[${PLUGIN_NAME}] ${msg}`); +}; + + +const ensureCsrfBypass = async () => { + try { + const { getState } = require("@saltcorn/data/db/state"); + const current = getState().getConfig("disable_csrf_routes", ""); + const want = "/dev-deploy/api/"; + const entries = current.split(",").map((s) => s.trim()).filter(Boolean); + if (!entries.includes(want)) { + entries.push(want); + await getState().setConfig("disable_csrf_routes", entries.join(",")); + } + } catch (e) { + // eslint-disable-next-line no-console + console.error(`[${PLUGIN_NAME}] failed to register csrf bypass:`, e); + } +}; + + +const onLoad = async (cfg) => { + try { + await createAllTables(); + const env = await initEnvIfMissing(); + if (!env.bootstrapped_at) { + const counts = await backfillAll(); + const total = Object.values(counts).reduce((a, b) => a + b, 0); + await markBootstrapped(env.env_id); + log(`v${PLUGIN_VERSION} bootstrapped env_id=${env.env_id} backfilled ${total} entities ${JSON.stringify(counts)}`); + } else { + log(`v${PLUGIN_VERSION} loaded env_id=${env.env_id}`); + } + installAllWraps(); + await ensureCsrfBypass(); + } catch (err) { + // eslint-disable-next-line no-console + console.error(`[${PLUGIN_NAME}] onLoad failed:`, err); + throw err; + } +}; + + +module.exports = { + sc_plugin_api_version: 1, + plugin_name: PLUGIN_NAME, + onLoad: onLoad, + routes: routes +}; diff --git a/lib/apply.js b/lib/apply.js new file mode 100644 index 0000000..11f0023 --- /dev/null +++ b/lib/apply.js @@ -0,0 +1,1146 @@ +// Apply handlers — replay an op from a peer onto this instance. +// +// Each handler: +// - Resolves the op's entity_uuid (and any parent_uuid) to this instance's +// local integer id via _dd_entity_ids. +// - Invokes the Saltcorn model method to reproduce the change. +// - Updates _dd_entity_ids (insert / rename / remove) accordingly. +// +// While a handler runs, the CRUD wraps are suppressed (no journal writes, no +// auto-UUID assignment) — we manage those side effects ourselves so the +// foreign op_id and entity_uuid are preserved across instances. + +const db = require("@saltcorn/data/db"); + +const Table = require("@saltcorn/data/models/table"); +const Field = require("@saltcorn/data/models/field"); +const View = require("@saltcorn/data/models/view"); +const Page = require("@saltcorn/data/models/page"); +const Trigger = require("@saltcorn/data/models/trigger"); +const Role = require("@saltcorn/data/models/role"); +const Library = require("@saltcorn/data/models/library"); +const Tag = require("@saltcorn/data/models/tag"); +const TableConstraint = require("@saltcorn/data/models/table_constraints"); +const File = require("@saltcorn/data/models/file"); +const PageGroup = require("@saltcorn/data/models/page_group"); +const PageGroupMember = require("@saltcorn/data/models/page_group_member"); +const WorkflowStep = require("@saltcorn/data/models/workflow_step"); + +const { + lookupByUuid, + adoptUuid, + updateName, + removeEntityRow, + lookupByCurrent, + constraintDisplayName +} = require("./entityIds"); +const { runSuppressed } = require("./context"); +const { ENTITY_KINDS, fileLocationToId } = require("./constants"); +const { refreshState } = require("./state"); +const { sha256Buffer, writeFileBytes, toAbsolutePath } = require("./files"); +const { signedFetchBinary } = require("./transport"); +const peers = require("./peers"); +const { getEnv } = require("./env"); +const { fromPlaceholders } = require("./payloadRefs"); +const { + ensureManagedSchema, + setRowUuid, + findIdByRowUuid, + COLUMN_NAME: ROW_UUID_COL +} = require("./rowIdentity"); +const { portableToRow } = require("./rowPayload"); + + +// Strip surrogate keys that don't translate across instances. Returns a new +// object with all non-id properties. +const stripSurrogateKeys = (obj, extraIdKeys) => { + const drop = new Set(["id", "table_id", "view_id", "page_id", "role_id_for_create", ...(extraIdKeys || [])]); + const out = {}; + for (const k of Object.keys(obj || {})) { + if (drop.has(k)) continue; + out[k] = obj[k]; + } + return out; +}; + + +const requireLocalEntity = async (uuid, kind) => { + const m = await lookupByUuid(uuid); + if (!m) { + throw new Error(`local entity for uuid=${uuid} (kind=${kind}) not found`); + } + return m; +}; + + +// ---------------- Tables ---------------- + +const applyCreateTable = async ({ op, payload }) => { + const after = payload.after || {}; + if (!after.name) { + throw new Error("create_table missing after.name"); + } + const existing = await lookupByUuid(op.entity_uuid); + if (existing) { + return { status: "noop", reason: "uuid already mapped" }; + } + const opts = stripSurrogateKeys(after); + const tableName = opts.name; + delete opts.name; + const t = await Table.create(tableName, opts); + await adoptUuid(op.entity_uuid, ENTITY_KINDS.TABLE, t.id, t.name, null); + return { status: "created", local_id: t.id }; +}; + + +const applyUpdateTable = async ({ op, payload }) => { + const mapping = await requireLocalEntity(op.entity_uuid, ENTITY_KINDS.TABLE); + const t = Table.findOne({ id: mapping.current_id }); + if (!t) { + throw new Error(`table id=${mapping.current_id} not found`); + } + const patch = stripSurrogateKeys(payload.patch || {}); + if (Object.keys(patch).length === 0) { + return { status: "noop", reason: "empty patch" }; + } + await t.update(patch); + if (patch.name !== undefined && patch.name !== mapping.current_name) { + await updateName(ENTITY_KINDS.TABLE, mapping.current_id, patch.name); + } + return { status: "updated" }; +}; + + +const applyDropTable = async ({ op, payload }) => { + const mapping = await lookupByUuid(op.entity_uuid); + if (!mapping) { + return { status: "noop", reason: "already dropped" }; + } + const t = Table.findOne({ id: mapping.current_id }); + if (!t) { + await removeEntityRow(ENTITY_KINDS.TABLE, mapping.current_id); + return { status: "noop", reason: "table not in saltcorn" }; + } + await t.delete(); + await removeEntityRow(ENTITY_KINDS.TABLE, mapping.current_id); + return { status: "dropped" }; +}; + + +// ---------------- Fields ---------------- + +const applyCreateField = async ({ op, payload }) => { + const after = payload.after || {}; + const parentUuid = payload.parent_uuid; + if (!parentUuid) { + throw new Error("create_field missing parent_uuid"); + } + const existing = await lookupByUuid(op.entity_uuid); + if (existing) { + return { status: "noop", reason: "uuid already mapped" }; + } + const parent = await requireLocalEntity(parentUuid, ENTITY_KINDS.TABLE); + const table = Table.findOne({ id: parent.current_id }); + if (!table) { + throw new Error(`parent table id=${parent.current_id} not found`); + } + const cfg = stripSurrogateKeys(after); + cfg.table_id = table.id; + const f = await Field.create(cfg); + await adoptUuid(op.entity_uuid, ENTITY_KINDS.FIELD, f.id, f.name, parentUuid); + return { status: "created", local_id: f.id }; +}; + + +const applyUpdateField = async ({ op, payload }) => { + const mapping = await requireLocalEntity(op.entity_uuid, ENTITY_KINDS.FIELD); + const f = await Field.findOne({ id: mapping.current_id }); + if (!f) { + throw new Error(`field id=${mapping.current_id} not found`); + } + const patch = stripSurrogateKeys(payload.patch || {}); + if (Object.keys(patch).length === 0) { + return { status: "noop", reason: "empty patch" }; + } + await f.update(patch); + if (patch.name !== undefined && patch.name !== mapping.current_name) { + await updateName(ENTITY_KINDS.FIELD, mapping.current_id, patch.name); + } + return { status: "updated" }; +}; + + +const applyDropField = async ({ op, payload }) => { + const mapping = await lookupByUuid(op.entity_uuid); + if (!mapping) { + return { status: "noop", reason: "already dropped" }; + } + const f = await Field.findOne({ id: mapping.current_id }); + if (!f) { + await removeEntityRow(ENTITY_KINDS.FIELD, mapping.current_id); + return { status: "noop" }; + } + await f.delete(); + await removeEntityRow(ENTITY_KINDS.FIELD, mapping.current_id); + return { status: "dropped" }; +}; + + +// ---------------- Views ---------------- + +const applyCreateView = async ({ op, payload }) => { + const after = payload.after || {}; + const parentUuid = payload.parent_uuid; + const existing = await lookupByUuid(op.entity_uuid); + if (existing) { + return { status: "noop", reason: "uuid already mapped" }; + } + const cfg = stripSurrogateKeys(after); + if (parentUuid) { + const parent = await requireLocalEntity(parentUuid, ENTITY_KINDS.TABLE); + cfg.table_id = parent.current_id; + } + const v = await View.create(cfg); + await adoptUuid(op.entity_uuid, ENTITY_KINDS.VIEW, v.id, v.name, parentUuid || null); + return { status: "created", local_id: v.id }; +}; + + +const applyUpdateView = async ({ op, payload }) => { + const mapping = await requireLocalEntity(op.entity_uuid, ENTITY_KINDS.VIEW); + const patch = stripSurrogateKeys(payload.patch || {}); + if (Object.keys(patch).length === 0) { + return { status: "noop", reason: "empty patch" }; + } + await View.update(patch, mapping.current_id); + if (patch.name !== undefined && patch.name !== mapping.current_name) { + await updateName(ENTITY_KINDS.VIEW, mapping.current_id, patch.name); + } + return { status: "updated" }; +}; + + +const applyDropView = async ({ op, payload }) => { + const mapping = await lookupByUuid(op.entity_uuid); + if (!mapping) { + return { status: "noop", reason: "already dropped" }; + } + const v = View.findOne({ id: mapping.current_id }); + if (!v) { + await removeEntityRow(ENTITY_KINDS.VIEW, mapping.current_id); + return { status: "noop" }; + } + await v.delete(); + await removeEntityRow(ENTITY_KINDS.VIEW, mapping.current_id); + return { status: "dropped" }; +}; + + +// ---------------- Pages ---------------- + +const applyCreatePage = async ({ op, payload }) => { + const after = payload.after || {}; + const existing = await lookupByUuid(op.entity_uuid); + if (existing) { + return { status: "noop", reason: "uuid already mapped" }; + } + const cfg = stripSurrogateKeys(after); + const p = await Page.create(cfg); + await adoptUuid(op.entity_uuid, ENTITY_KINDS.PAGE, p.id, p.name, null); + return { status: "created", local_id: p.id }; +}; + + +const applyUpdatePage = async ({ op, payload }) => { + const mapping = await requireLocalEntity(op.entity_uuid, ENTITY_KINDS.PAGE); + const patch = stripSurrogateKeys(payload.patch || {}); + if (Object.keys(patch).length === 0) { + return { status: "noop", reason: "empty patch" }; + } + await Page.update(mapping.current_id, patch); + if (patch.name !== undefined && patch.name !== mapping.current_name) { + await updateName(ENTITY_KINDS.PAGE, mapping.current_id, patch.name); + } + return { status: "updated" }; +}; + + +const applyDropPage = async ({ op, payload }) => { + const mapping = await lookupByUuid(op.entity_uuid); + if (!mapping) { + return { status: "noop", reason: "already dropped" }; + } + const p = Page.findOne({ id: mapping.current_id }); + if (!p) { + await removeEntityRow(ENTITY_KINDS.PAGE, mapping.current_id); + return { status: "noop" }; + } + await p.delete(); + await removeEntityRow(ENTITY_KINDS.PAGE, mapping.current_id); + return { status: "dropped" }; +}; + + +// ---------------- Triggers ---------------- + +const applyCreateTrigger = async ({ op, payload }) => { + const after = payload.after || {}; + const parentUuid = payload.parent_uuid; + const existing = await lookupByUuid(op.entity_uuid); + if (existing) { + return { status: "noop", reason: "uuid already mapped" }; + } + const cfg = stripSurrogateKeys(after); + if (parentUuid) { + const parent = await requireLocalEntity(parentUuid, ENTITY_KINDS.TABLE); + cfg.table_id = parent.current_id; + } + const tr = await Trigger.create(cfg); + await adoptUuid(op.entity_uuid, ENTITY_KINDS.TRIGGER, tr.id, tr.name || `trigger_${tr.id}`, parentUuid || null); + return { status: "created", local_id: tr.id }; +}; + + +const applyUpdateTrigger = async ({ op, payload }) => { + const mapping = await requireLocalEntity(op.entity_uuid, ENTITY_KINDS.TRIGGER); + const patch = stripSurrogateKeys(payload.patch || {}); + if (Object.keys(patch).length === 0) { + return { status: "noop", reason: "empty patch" }; + } + await Trigger.update(mapping.current_id, patch); + if (patch.name !== undefined && patch.name !== mapping.current_name) { + await updateName(ENTITY_KINDS.TRIGGER, mapping.current_id, patch.name); + } + return { status: "updated" }; +}; + + +const applyDropTrigger = async ({ op, payload }) => { + const mapping = await lookupByUuid(op.entity_uuid); + if (!mapping) { + return { status: "noop", reason: "already dropped" }; + } + const tr = Trigger.findOne({ id: mapping.current_id }); + if (!tr) { + await removeEntityRow(ENTITY_KINDS.TRIGGER, mapping.current_id); + return { status: "noop" }; + } + await tr.delete(); + await removeEntityRow(ENTITY_KINDS.TRIGGER, mapping.current_id); + return { status: "dropped" }; +}; + + +// ---------------- Roles ---------------- + +const applyCreateRole = async ({ op, payload }) => { + const after = payload.after || {}; + if (!after.id || !after.role) { + throw new Error("create_role missing after.id or after.role"); + } + const existing = await lookupByUuid(op.entity_uuid); + if (existing) { + return { status: "noop", reason: "uuid already mapped" }; + } + await Role.create({ id: after.id, role: after.role }); + await adoptUuid(op.entity_uuid, ENTITY_KINDS.ROLE, after.id, after.role, null); + return { status: "created", local_id: after.id }; +}; + + +const applyUpdateRole = async ({ op, payload }) => { + const mapping = await requireLocalEntity(op.entity_uuid, ENTITY_KINDS.ROLE); + const r = await Role.findOne({ id: mapping.current_id }); + const patch = payload.patch || {}; + if (Object.keys(patch).length === 0) { + return { status: "noop", reason: "empty patch" }; + } + await r.update(patch); + if (patch.role !== undefined && patch.role !== mapping.current_name) { + await updateName(ENTITY_KINDS.ROLE, mapping.current_id, patch.role); + } + return { status: "updated" }; +}; + + +const applyDropRole = async ({ op, payload }) => { + const mapping = await lookupByUuid(op.entity_uuid); + if (!mapping) { + return { status: "noop" }; + } + const r = await Role.findOne({ id: mapping.current_id }); + if (r) { + await r.delete(); + } + await removeEntityRow(ENTITY_KINDS.ROLE, mapping.current_id); + return { status: "dropped" }; +}; + + +// ---------------- Library ---------------- + +const applyCreateLibrary = async ({ op, payload }) => { + const after = payload.after || {}; + if (!after.name) { + throw new Error("create_library missing after.name"); + } + const existing = await lookupByUuid(op.entity_uuid); + if (existing) { + return { status: "noop", reason: "uuid already mapped" }; + } + const cfg = stripSurrogateKeys(after); + await Library.create(cfg); + const fresh = await Library.findOne({ name: after.name }); + if (!fresh) { + throw new Error("library not found after create"); + } + await adoptUuid(op.entity_uuid, ENTITY_KINDS.LIBRARY, fresh.id, fresh.name, null); + return { status: "created", local_id: fresh.id }; +}; + + +const applyUpdateLibrary = async ({ op, payload }) => { + const mapping = await requireLocalEntity(op.entity_uuid, ENTITY_KINDS.LIBRARY); + const li = await Library.findOne({ id: mapping.current_id }); + const patch = stripSurrogateKeys(payload.patch || {}); + if (Object.keys(patch).length === 0) { + return { status: "noop", reason: "empty patch" }; + } + await li.update(patch); + if (patch.name !== undefined && patch.name !== mapping.current_name) { + await updateName(ENTITY_KINDS.LIBRARY, mapping.current_id, patch.name); + } + return { status: "updated" }; +}; + + +const applyDropLibrary = async ({ op, payload }) => { + const mapping = await lookupByUuid(op.entity_uuid); + if (!mapping) { + return { status: "noop" }; + } + const li = await Library.findOne({ id: mapping.current_id }); + if (li) { + await li.delete(); + } + await removeEntityRow(ENTITY_KINDS.LIBRARY, mapping.current_id); + return { status: "dropped" }; +}; + + +// ---------------- Tags ---------------- + +const applyCreateTag = async ({ op, payload }) => { + const after = payload.after || {}; + const existing = await lookupByUuid(op.entity_uuid); + if (existing) { + return { status: "noop", reason: "uuid already mapped" }; + } + const cfg = stripSurrogateKeys(after); + const tg = await Tag.create(cfg); + await adoptUuid(op.entity_uuid, ENTITY_KINDS.TAG, tg.id, tg.name, null); + return { status: "created", local_id: tg.id }; +}; + + +const applyUpdateTag = async ({ op, payload }) => { + const mapping = await requireLocalEntity(op.entity_uuid, ENTITY_KINDS.TAG); + const tg = await Tag.findOne({ id: mapping.current_id }); + const patch = stripSurrogateKeys(payload.patch || {}); + if (Object.keys(patch).length === 0) { + return { status: "noop", reason: "empty patch" }; + } + await tg.update(patch); + if (patch.name !== undefined && patch.name !== mapping.current_name) { + await updateName(ENTITY_KINDS.TAG, mapping.current_id, patch.name); + } + return { status: "updated" }; +}; + + +const applyDropTag = async ({ op, payload }) => { + const mapping = await lookupByUuid(op.entity_uuid); + if (!mapping) { + return { status: "noop" }; + } + const tg = await Tag.findOne({ id: mapping.current_id }); + if (tg) { + await tg.delete(); + } + await removeEntityRow(ENTITY_KINDS.TAG, mapping.current_id); + return { status: "dropped" }; +}; + + +// ---------------- Files ---------------- + +const applyCreateFile = async ({ op, payload, opts }) => { + const after = payload.after || {}; + const relPath = after.relative_path; + if (!relPath) throw new Error("create_file missing relative_path"); + if (!after.content_hash) throw new Error("create_file missing content_hash"); + + const existing = await lookupByUuid(op.entity_uuid); + if (existing) { + return { status: "noop", reason: "uuid already mapped" }; + } + if (!opts || !opts.peerId) { + throw new Error("file apply requires opts.peerId to fetch binary from"); + } + + const peer = await peers.findPeer(opts.peerId); + if (!peer) throw new Error(`peer ${opts.peerId} not found`); + const secret = await peers.peerSecret(opts.peerId); + const env = await getEnv(); + + const r = await signedFetchBinary({ + baseUrl: peer.base_url, + method: "GET", + path: `/dev-deploy/api/file/${encodeURIComponent(op.entity_uuid)}`, + body: null, + sourceEnvId: env.env_id, + secret: secret, + requireTls: peer.require_tls + }); + if (!r.ok) { + throw new Error(`binary fetch returned ${r.status}`); + } + const bytes = r.bytes; + const actualHash = sha256Buffer(bytes); + if (actualHash !== after.content_hash) { + throw new Error(`content hash mismatch: expected ${after.content_hash}, got ${actualHash}`); + } + + const absPath = toAbsolutePath(File, db, relPath); + await writeFileBytes(absPath, bytes); + + const localFile = await File.create({ + filename: after.filename, + location: absPath, + uploaded_at: new Date(), + size_kb: after.size_kb, + mime_super: after.mime_super, + mime_sub: after.mime_sub, + min_role_read: after.min_role_read + }); + + const synthId = fileLocationToId(relPath); + await adoptUuid(op.entity_uuid, ENTITY_KINDS.FILE, synthId, relPath, null); + + return { status: "created", location: absPath, bytes: bytes.length }; +}; + + +const applyDropFile = async ({ op, payload }) => { + const mapping = await lookupByUuid(op.entity_uuid); + if (!mapping) { + return { status: "noop", reason: "already dropped" }; + } + const absPath = toAbsolutePath(File, db, mapping.current_name); + const file = await File.findOne({ location: absPath }); + if (file) { + await file.delete(); + } + await removeEntityRow(ENTITY_KINDS.FILE, mapping.current_id); + return { status: "dropped" }; +}; + + +// ---------------- TableConstraints ---------------- + +const applyCreateConstraint = async ({ op, payload }) => { + const after = payload.after || {}; + const parentUuid = payload.parent_uuid; + if (!parentUuid) throw new Error("create_constraint missing parent_uuid"); + if (!after.type) throw new Error("create_constraint missing type"); + const existing = await lookupByUuid(op.entity_uuid); + if (existing) { + return { status: "noop", reason: "uuid already mapped" }; + } + const parent = await requireLocalEntity(parentUuid, "table"); + const cfg = { + table_id: parent.current_id, + type: after.type, + configuration: after.configuration || {} + }; + const result = await TableConstraint.create(cfg); + await adoptUuid(op.entity_uuid, "constraint", result.id, constraintDisplayName(result), parentUuid); + return { status: "created", local_id: result.id }; +}; + + +const applyDropConstraint = async ({ op, payload }) => { + const mapping = await lookupByUuid(op.entity_uuid); + if (!mapping) { + return { status: "noop", reason: "already dropped" }; + } + const con = await TableConstraint.findOne({ id: mapping.current_id }); + if (con) { + await con.delete(); + } + await removeEntityRow("constraint", mapping.current_id); + return { status: "dropped" }; +}; + + +// ---------------- PageGroup ---------------- + +const applyCreatePageGroup = async ({ op, payload }) => { + const after = payload.after || {}; + if (!after.name) throw new Error("create_page_group missing after.name"); + const existing = await lookupByUuid(op.entity_uuid); + if (existing) { + return { status: "noop", reason: "uuid already mapped" }; + } + // Create the group first, empty members. PageGroupMember's constructor + // demands page_group_id which doesn't exist until after the insert. + const cfg = stripSurrogateKeys(after); + cfg.members = []; + const result = await PageGroup.create(cfg); + + // Now resolve each member's page_uuid to local page_id and addMember. + for (const m of payload.members || []) { + if (!m.page_uuid) { + throw new Error("page_group member missing page_uuid"); + } + const local = await lookupByUuid(m.page_uuid); + if (!local) { + throw new Error(`page_group member references unmapped page_uuid=${m.page_uuid}`); + } + await result.addMember({ + page_id: local.current_id, + eligible_formula: m.eligible_formula, + description: m.description, + sequence: m.sequence + }); + } + await adoptUuid(op.entity_uuid, "page_group", result.id, result.name, null); + return { status: "created", local_id: result.id }; +}; + + +const applyUpdatePageGroup = async ({ op, payload }) => { + const mapping = await requireLocalEntity(op.entity_uuid, "page_group"); + const patch = stripSurrogateKeys(payload.patch || {}); + if (Object.keys(patch).length === 0) { + return { status: "noop", reason: "empty patch" }; + } + await PageGroup.update(mapping.current_id, patch); + if (patch.name !== undefined && patch.name !== mapping.current_name) { + await updateName("page_group", mapping.current_id, patch.name); + } + return { status: "updated" }; +}; + + +const applyDropPageGroup = async ({ op, payload }) => { + const mapping = await lookupByUuid(op.entity_uuid); + if (!mapping) return { status: "noop" }; + const pg = PageGroup.findOne({ id: mapping.current_id }); + if (pg) await pg.delete(); + await removeEntityRow("page_group", mapping.current_id); + return { status: "dropped" }; +}; + + +// ---------------- Table rows (managed/starter) ---------------- + +const findLocalTableByUuid = async (tableUuid) => { + const ent = await lookupByUuid(tableUuid); + if (!ent || ent.kind !== "table") return null; + await refreshState(); + return Table.findOne({ id: ent.current_id }); +}; + + +const applyInsertRow = async ({ op, payload }) => { + if (!payload || !payload.table_uuid) throw new Error("insert_row missing table_uuid"); + if (!op.entity_uuid) throw new Error("insert_row missing row_uuid (entity_uuid)"); + const tbl = await findLocalTableByUuid(payload.table_uuid); + if (!tbl) throw new Error(`local table for uuid=${payload.table_uuid} not found`); + await ensureManagedSchema(tbl.name); + // Idempotency: if a row with this uuid already exists, skip + const existing = await findIdByRowUuid(tbl.name, op.entity_uuid); + if (existing) { + return { status: "noop", reason: "row uuid already present" }; + } + const rowData = await portableToRow(payload.after || {}, tbl); + const newId = await tbl.insertRow(rowData); + if (!newId) throw new Error("insertRow returned no id"); + await setRowUuid(tbl.name, newId, op.entity_uuid); + return { status: "inserted", local_id: newId }; +}; + + +const applyUpdateRow = async ({ op, payload }) => { + if (!payload || !payload.table_uuid) throw new Error("update_row missing table_uuid"); + if (!op.entity_uuid) throw new Error("update_row missing row_uuid"); + const tbl = await findLocalTableByUuid(payload.table_uuid); + if (!tbl) throw new Error(`local table for uuid=${payload.table_uuid} not found`); + await ensureManagedSchema(tbl.name); + const localId = await findIdByRowUuid(tbl.name, op.entity_uuid); + if (!localId) { + // The row doesn't exist on target yet — treat as insert + const rowData = await portableToRow(payload.patch || {}, tbl); + const newId = await tbl.insertRow(rowData); + await setRowUuid(tbl.name, newId, op.entity_uuid); + return { status: "inserted_for_update", local_id: newId }; + } + const patch = await portableToRow(payload.patch || {}, tbl); + if (Object.keys(patch).length === 0) { + return { status: "noop", reason: "empty patch" }; + } + await tbl.updateRow(patch, localId); + return { status: "updated", local_id: localId }; +}; + + +const applyDropRow = async ({ op, payload }) => { + if (!payload || !payload.table_uuid) throw new Error("drop_row missing table_uuid"); + if (!op.entity_uuid) throw new Error("drop_row missing row_uuid"); + const tbl = await findLocalTableByUuid(payload.table_uuid); + if (!tbl) return { status: "noop", reason: "table not present locally" }; + const localId = await findIdByRowUuid(tbl.name, op.entity_uuid); + if (!localId) return { status: "noop", reason: "row uuid not present locally" }; + await tbl.deleteRows({ id: localId }); + return { status: "dropped", local_id: localId }; +}; + + +const applySetTableMode = async ({ op, payload }) => { + if (!payload || !payload.table_uuid) throw new Error("set_table_mode missing table_uuid"); + const tbl = await findLocalTableByUuid(payload.table_uuid); + if (!tbl) throw new Error(`local table for uuid=${payload.table_uuid} not found`); + const mode = payload.data_mode || "user"; + if (mode === "managed" || mode === "starter") { + await ensureManagedSchema(tbl.name); + } + const now = new Date().toISOString(); + const existing = await db.selectMaybeOne("_dd_table_modes", { table_uuid: payload.table_uuid }); + if (existing) { + await db.updateWhere("_dd_table_modes", { data_mode: mode, updated_at: now }, { table_uuid: payload.table_uuid }); + } else { + await db.insert("_dd_table_modes", { table_uuid: payload.table_uuid, data_mode: mode, updated_at: now }, { noid: true }); + } + return { status: "set", mode: mode }; +}; + + +// ---------------- Plugin configuration ---------------- + +const applyUpdatePluginConfig = async ({ op, payload }) => { + const Plugin = require("@saltcorn/data/models/plugin"); + if (!payload || !payload.name) { + throw new Error("update_plugin_config missing payload.name"); + } + if (payload.name === "dev-deploy") { + return { status: "noop", reason: "skipping our own plugin" }; + } + const plugin = await Plugin.findOne({ name: payload.name }); + if (!plugin) { + throw new Error(`plugin "${payload.name}" is not installed on this instance`); + } + plugin.configuration = payload.configuration || {}; + await plugin.upsert(); + return { status: "updated", plugin: payload.name }; +}; + + +// ---------------- Config (menu_items etc.) ---------------- + +const applySetConfig = async ({ op, payload }) => { + const { key, value } = payload || {}; + if (!key) throw new Error("set_config missing key"); + const { getState } = require("@saltcorn/data/db/state"); + await getState().setConfig(key, value); + return { status: "set", key: key }; +}; + + +// ---------------- PageGroupMember ---------------- + +const applyCreatePageGroupMember = async ({ op, payload }) => { + const after = payload.after || {}; + const parentUuid = payload.parent_uuid; + if (!parentUuid) throw new Error("create_page_group_member missing parent_uuid"); + if (!after.page_uuid) throw new Error("create_page_group_member missing page_uuid"); + const existing = await lookupByUuid(op.entity_uuid); + if (existing) return { status: "noop", reason: "uuid already mapped" }; + const group = await requireLocalEntity(parentUuid, "page_group"); + const page = await requireLocalEntity(after.page_uuid, "page"); + const pg = PageGroup.findOne({ id: group.current_id }); + if (!pg) throw new Error(`local page_group id=${group.current_id} not found`); + const member = await pg.addMember({ + page_id: page.current_id, + eligible_formula: after.eligible_formula, + description: after.description + }); + await adoptUuid(op.entity_uuid, "page_group_member", member.id, `member_${member.id}`, parentUuid); + return { status: "created", local_id: member.id }; +}; + + +const applyDropPageGroupMember = async ({ op, payload }) => { + const mapping = await lookupByUuid(op.entity_uuid); + if (!mapping) return { status: "noop" }; + await PageGroupMember.delete(mapping.current_id); + await removeEntityRow("page_group_member", mapping.current_id); + return { status: "dropped" }; +}; + + +// ---------------- WorkflowStep ---------------- + +const applyCreateWorkflowStep = async ({ op, payload }) => { + const after = payload.after || {}; + const parentUuid = payload.parent_uuid; + const existing = await lookupByUuid(op.entity_uuid); + if (existing) return { status: "noop", reason: "uuid already mapped" }; + let parentTriggerId = null; + if (parentUuid) { + const parent = await requireLocalEntity(parentUuid, "trigger"); + parentTriggerId = parent.current_id; + } + const cfg = stripSurrogateKeys(after); + if (parentTriggerId) cfg.trigger_id = parentTriggerId; + const result = await WorkflowStep.create(cfg); + // WorkflowStep.create returns the inserted integer id, not an instance. + const id = typeof result === "number" ? result : (result && result.id); + if (!id) throw new Error("WorkflowStep.create returned no id"); + await adoptUuid(op.entity_uuid, "workflow_step", id, after.name || `step_${id}`, parentUuid || null); + return { status: "created", local_id: id }; +}; + + +const applyUpdateWorkflowStep = async ({ op, payload }) => { + const mapping = await requireLocalEntity(op.entity_uuid, "workflow_step"); + const ws = await WorkflowStep.findOne({ id: mapping.current_id }); + if (!ws) throw new Error(`workflow_step id=${mapping.current_id} not found`); + const patch = stripSurrogateKeys(payload.patch || {}); + if (Object.keys(patch).length === 0) { + return { status: "noop", reason: "empty patch" }; + } + await ws.update(patch); + if (patch.name !== undefined && patch.name !== mapping.current_name) { + await updateName("workflow_step", mapping.current_id, patch.name); + } + return { status: "updated" }; +}; + + +const applyDropWorkflowStep = async ({ op, payload }) => { + const mapping = await lookupByUuid(op.entity_uuid); + if (!mapping) return { status: "noop" }; + const ws = await WorkflowStep.findOne({ id: mapping.current_id }); + if (ws) await ws.delete(); + await removeEntityRow("workflow_step", mapping.current_id); + return { status: "dropped" }; +}; + + +// ---------------- Dispatch ---------------- + +const HANDLERS = { + create_table: applyCreateTable, + update_table: applyUpdateTable, + drop_table: applyDropTable, + create_field: applyCreateField, + update_field: applyUpdateField, + drop_field: applyDropField, + create_view: applyCreateView, + update_view: applyUpdateView, + drop_view: applyDropView, + create_page: applyCreatePage, + update_page: applyUpdatePage, + drop_page: applyDropPage, + create_trigger: applyCreateTrigger, + update_trigger: applyUpdateTrigger, + drop_trigger: applyDropTrigger, + create_role: applyCreateRole, + update_role: applyUpdateRole, + drop_role: applyDropRole, + create_library: applyCreateLibrary, + update_library: applyUpdateLibrary, + drop_library: applyDropLibrary, + create_tag: applyCreateTag, + update_tag: applyUpdateTag, + drop_tag: applyDropTag, + create_constraint: applyCreateConstraint, + drop_constraint: applyDropConstraint, + create_file: applyCreateFile, + drop_file: applyDropFile, + create_page_group: applyCreatePageGroup, + update_page_group: applyUpdatePageGroup, + drop_page_group: applyDropPageGroup, + create_page_group_member: applyCreatePageGroupMember, + drop_page_group_member: applyDropPageGroupMember, + create_workflow_step: applyCreateWorkflowStep, + update_workflow_step: applyUpdateWorkflowStep, + drop_workflow_step: applyDropWorkflowStep, + set_config: applySetConfig, + update_plugin_config: applyUpdatePluginConfig, + insert_row: applyInsertRow, + update_row: applyUpdateRow, + drop_row: applyDropRow, + set_table_mode: applySetTableMode +}; + + +// Record an op into our journal with its source-side identity preserved. +// applied_at is set; status may be 'committed' (apply succeeded), 'skipped_cascade' +// (parent in same batch handled it), 'error' (apply failed), or 'conflict' +// (a local op touched the same entity since the last sync with this peer). +const persistOp = async (op, status, applied, extra) => { + const payload = typeof op.payload === "string" ? op.payload : JSON.stringify(op.payload || {}); + const row = { + op_id: op.op_id, + source_env_id: op.source_env_id, + op_type: op.op_type, + entity_kind: op.entity_kind, + entity_uuid: op.entity_uuid, + payload: payload, + parent_op_id: op.parent_op_id, + correlation_id: op.correlation_id, + schema_version: op.schema_version || 1, + created_at: op.created_at, + applied_at: applied ? new Date().toISOString() : null, + status: status, + conflict_with_op_id: (extra && extra.conflict_with_op_id) || null + }; + await db.insert("_dd_ops", row, { noid: true }); +}; + + +// Find the most recent local op on the same entity_uuid that was applied +// after our last inbound sync from this peer. If one exists, this incoming op +// represents concurrent divergent changes -- a conflict. +// +// Returns the local op_id if conflicting, null otherwise. +const findConflictingLocalOp = async (op, opts) => { + if (!op.entity_uuid || !opts.peerId || !opts.myEnvId) return null; + const anchor = await db.selectMaybeOne("_dd_anchors", { peer_id: opts.peerId, direction: "inbound" }); + let cutoff = "1970-01-01T00:00:00.000Z"; + if (anchor) { + const anchorOp = await db.selectMaybeOne("_dd_ops", { op_id: anchor.last_op_id }); + if (anchorOp && anchorOp.applied_at) cutoff = anchorOp.applied_at; + } + const schema = db.getTenantSchemaPrefix(); + const rs = await db.query( + `SELECT op_id FROM ${schema}_dd_ops + WHERE entity_uuid = $1 + AND source_env_id = $2 + AND applied_at IS NOT NULL + AND applied_at > $3 + AND status NOT IN ('rejected', 'reverted') + ORDER BY applied_at DESC + LIMIT 1`, + [op.entity_uuid, opts.myEnvId, cutoff] + ); + return rs.rows.length > 0 ? rs.rows[0].op_id : null; +}; + + +// Apply a batch of ops in created_at order. Children whose parent_op_id is in +// the same batch are journaled but not re-applied — their parent's apply will +// reproduce the cascade locally. +// +// opts.peerId + opts.myEnvId enable conflict detection: if an incoming op's +// entity_uuid has a local op applied since the last sync with this peer, the +// incoming op is journaled with status='conflict' instead of being applied. +// The admin resolves via /admin/dev-deploy/conflicts. +const applyBatch = async (ops, opts) => { + opts = opts || {}; + await refreshState(); + const sorted = [...ops].sort((a, b) => String(a.created_at).localeCompare(String(b.created_at))); + const opIdSet = new Set(sorted.map((o) => o.op_id)); + const results = []; + + for (const op of sorted) { + const existing = await db.selectMaybeOne("_dd_ops", { op_id: op.op_id }); + if (existing) { + results.push({ op_id: op.op_id, status: "already_applied" }); + continue; + } + if (op.parent_op_id && opIdSet.has(op.parent_op_id)) { + await persistOp(op, "skipped_cascade", true); + results.push({ op_id: op.op_id, status: "skipped_cascade" }); + continue; + } + const conflictWith = await findConflictingLocalOp(op, opts); + if (conflictWith) { + await persistOp(op, "conflict", false, { conflict_with_op_id: conflictWith }); + results.push({ op_id: op.op_id, status: "conflict", conflict_with: conflictWith }); + continue; + } + const handler = HANDLERS[op.op_type]; + if (!handler) { + await persistOp(op, "error", true); + results.push({ op_id: op.op_id, status: "error", error: `no handler for ${op.op_type}` }); + continue; + } + try { + const payload = typeof op.payload === "string" ? JSON.parse(op.payload) : (op.payload || {}); + // Resolve __dd_file_ref:: placeholders to local file paths + // before handing the payload to the model handler. + try { await fromPlaceholders(payload); } catch (e) { /* best-effort */ } + const r = await runSuppressed(() => handler({ op: op, payload: payload, opts: opts })); + await persistOp(op, "committed", true); + results.push({ op_id: op.op_id, status: "applied", detail: r }); + } catch (err) { + // eslint-disable-next-line no-console + console.error(`[dev-deploy] apply ${op.op_type} (op=${op.op_id.slice(0,8)}) failed:`, err && err.stack ? err.stack : err); + await persistOp(op, "error", true); + results.push({ op_id: op.op_id, status: "error", error: err.message }); + } + } + return results; +}; + + +// Resolve a pending conflict. +// action='theirs': apply the conflicting incoming op now (suppressed), +// mark status='committed', clear conflict_with_op_id. +// action='mine' : mark the incoming op status='rejected', clear conflict_with. +// The local op stays as-is. The peer keeps sending this op_id +// on future pulls; we skip-by-id-already-present at the top of +// applyBatch (status check covers 'rejected' too via idempotency). +const resolveConflict = async (opId, action, opts) => { + opts = opts || {}; + const op = await db.selectMaybeOne("_dd_ops", { op_id: opId }); + if (!op) throw new Error(`op ${opId} not found`); + if (op.status !== "conflict") throw new Error(`op ${opId} is not in conflict status (status=${op.status})`); + + if (action === "mine") { + // "Use mine" on a create_file/drop_file conflict is a pure journal + // status change just like the metadata case -- the local state stands + // and the incoming op is rejected; no file bytes are touched. + await db.updateWhere("_dd_ops", { + status: "rejected", + conflict_with_op_id: null, + applied_at: new Date().toISOString() + }, { op_id: opId }); + return { status: "rejected" }; + } + if (action === "theirs") { + const handler = HANDLERS[op.op_type]; + if (!handler) throw new Error(`no handler for ${op.op_type}`); + await refreshState(); + const payload = typeof op.payload === "string" ? JSON.parse(op.payload) : (op.payload || {}); + // create_file's handler needs opts.peerId to fetch the file bytes from + // the originating peer. Pass opts through so file conflicts resolve too. + const r = await runSuppressed(() => handler({ op: op, payload: payload, opts: opts })); + await db.updateWhere("_dd_ops", { + status: "committed", + conflict_with_op_id: null, + applied_at: new Date().toISOString() + }, { op_id: opId }); + return { status: "applied", detail: r }; + } + throw new Error(`unknown action '${action}', expected 'theirs' or 'mine'`); +}; + + +// Find the entity instance for a conflict op's entity_uuid. +// Returns { instance, kind } or null if the entity isn't present locally. +const findEntityForConflict = async (op) => { + if (!op.op_type || !op.entity_uuid) return null; + const dash = op.op_type.indexOf("_"); + if (dash < 0) return null; + const kind = op.op_type.substring(dash + 1); + const m = await lookupByUuid(op.entity_uuid); + if (!m) return null; + await refreshState(); + const Cls = { + table: require("@saltcorn/data/models/table"), + field: require("@saltcorn/data/models/field"), + view: require("@saltcorn/data/models/view"), + page: require("@saltcorn/data/models/page"), + trigger: require("@saltcorn/data/models/trigger"), + role: require("@saltcorn/data/models/role"), + library: require("@saltcorn/data/models/library"), + tag: require("@saltcorn/data/models/tag") + }[kind]; + if (!Cls) return null; + const inst = await Cls.findOne({ id: m.current_id }); + if (!inst) return null; + return { instance: inst, kind: kind }; +}; + + +// Compute fields where the incoming op's patch diverges from the local entity's +// current state. Returns an array of { field, currentValue, incomingValue }. +// Only meaningful for update_X conflicts; returns [] for other op types. +const conflictFieldDiff = async (incomingOp) => { + if (!incomingOp.op_type || !incomingOp.op_type.startsWith("update_")) { + return { diffs: [], reason: "merge only meaningful for update ops" }; + } + const found = await findEntityForConflict(incomingOp); + if (!found) { + return { diffs: [], reason: "entity not present locally" }; + } + const payload = typeof incomingOp.payload === "string" ? JSON.parse(incomingOp.payload) : (incomingOp.payload || {}); + const patch = payload.patch || {}; + const diffs = []; + for (const [field, incomingValue] of Object.entries(patch)) { + const currentValue = found.instance[field]; + if (JSON.stringify(currentValue) === JSON.stringify(incomingValue)) { + continue; + } + diffs.push({ + field: field, + currentValue: currentValue, + incomingValue: incomingValue + }); + } + return { diffs: diffs, kind: found.kind, instance: found.instance }; +}; + + +// Apply a manual per-field merge. choices is { field: chosenValue } -- only +// fields that should be written are included; if empty, no model update runs. +// The original conflict op is marked status='merged' regardless. +const resolveConflictByMerge = async (opId, choices) => { + const op = await db.selectMaybeOne("_dd_ops", { op_id: opId }); + if (!op) throw new Error(`op ${opId} not found`); + if (op.status !== "conflict") throw new Error(`op ${opId} is not in conflict status (status=${op.status})`); + if (op.op_type === "create_file" || op.op_type === "drop_file") { + throw new Error(`merge is not supported for file ops (${op.op_type}); use "use theirs" or "use mine"`); + } + if (!op.op_type.startsWith("update_")) { + throw new Error(`merge only supported for update_X ops, got ${op.op_type}`); + } + const found = await findEntityForConflict(op); + if (!found) throw new Error(`entity ${op.entity_uuid} not present locally`); + + if (choices && Object.keys(choices).length > 0) { + const View = require("@saltcorn/data/models/view"); + const Page = require("@saltcorn/data/models/page"); + const Trigger = require("@saltcorn/data/models/trigger"); + if (found.kind === "view") { + await View.update(choices, found.instance.id); + } else if (found.kind === "page") { + await Page.update(found.instance.id, choices); + } else if (found.kind === "trigger") { + await Trigger.update(found.instance.id, choices); + } else { + await found.instance.update(choices); + } + } + + await db.updateWhere("_dd_ops", { + status: "merged", + conflict_with_op_id: null, + applied_at: new Date().toISOString() + }, { op_id: opId }); + + return { status: "merged", applied: choices || {} }; +}; + + +module.exports = { + applyBatch, + resolveConflict, + resolveConflictByMerge, + conflictFieldDiff, + HANDLERS +}; diff --git a/lib/constants.js b/lib/constants.js new file mode 100644 index 0000000..824c2c8 --- /dev/null +++ b/lib/constants.js @@ -0,0 +1,65 @@ +// Compile-time constants for the dev-deploy plugin. + +const PLUGIN_NAME = "dev-deploy"; +const PLUGIN_VERSION = "0.0.1"; + +// Namespace UUID for deterministic IDs derived from (kind, name). +// Generated once via crypto.randomUUID() and frozen here forever. +// Two environments that bootstrap from the same metadata population +// will assign identical UUIDs to entities with the same (kind, name). +const ID_NAMESPACE = "8b3a1e0d-4f6c-4d2a-9c5b-7e8f9a0b1c2d"; + +const OP_SCHEMA_VERSION = 1; + +const DATA_MODES = { + MANAGED: "managed", + STARTER: "starter", + USER: "user" +}; + +const DESTRUCTIVE_POLICY = { + AUTO: "auto", + CONFIRM: "confirm", + REFUSE: "refuse" +}; + +// Entity kinds the plugin tracks. Children point at parents via parent_uuid. +const ENTITY_KINDS = { + TABLE: "table", + FIELD: "field", + VIEW: "view", + PAGE: "page", + TRIGGER: "trigger", + ROLE: "role", + LIBRARY: "library", + TAG: "tag", + CONSTRAINT: "constraint", + FILE: "file", + PAGE_GROUP: "page_group", + PAGE_GROUP_MEMBER: "page_group_member", + WORKFLOW_STEP: "workflow_step" +}; + + +// Files don't have a meaningful integer id in current Saltcorn (File.create +// doesn't insert into _sc_files anymore — files are identified by their disk +// location and metadata via xattrs). We derive a stable 31-bit int from the +// location string so files fit the existing UNIQUE(kind, current_id) shape of +// _dd_entity_ids without a schema change. +const fileLocationToId = (location) => { + const crypto = require("crypto"); + const hash = crypto.createHash("sha256").update(String(location)).digest(); + return hash.readUInt32BE(0) & 0x7FFFFFFF; +}; + + +module.exports = { + PLUGIN_NAME, + PLUGIN_VERSION, + ID_NAMESPACE, + OP_SCHEMA_VERSION, + DATA_MODES, + DESTRUCTIVE_POLICY, + ENTITY_KINDS, + fileLocationToId +}; diff --git a/lib/context.js b/lib/context.js new file mode 100644 index 0000000..40da110 --- /dev/null +++ b/lib/context.js @@ -0,0 +1,65 @@ +// Per-async-flow context for the mutation wraps. +// +// AsyncLocalStorage gives every wrap a stack of in-flight op_ids so that ops +// triggered while another op is running (e.g. cascading field deletes inside +// a table delete) can attach themselves to the outer op as parent_op_id, and +// share its correlation_id. + +const { AsyncLocalStorage } = require("async_hooks"); + +const { randomUuid } = require("./ids"); + + +const storage = new AsyncLocalStorage(); + + +const enterOp = async (opId, fn) => { + const current = storage.getStore(); + const correlationId = current ? current.correlationId : randomUuid(); + const parentStack = current ? current.stack : []; + const newStore = { + stack: [...parentStack, opId], + correlationId: correlationId + }; + return await storage.run(newStore, fn); +}; + + +const currentParentOpId = () => { + const store = storage.getStore(); + if (!store || store.stack.length < 2) { + return null; + } + return store.stack[store.stack.length - 2]; +}; + + +const currentCorrelationId = () => { + const store = storage.getStore(); + return store ? store.correlationId : null; +}; + + +// Suppression mode: when applying ingested ops we run the underlying Saltcorn +// model methods but want the CRUD wraps to pass through silently — no journal +// writes, no entity_id assignment. The apply handler manages those side effects +// itself with the source-env's UUID. +const runSuppressed = async (fn) => { + const current = storage.getStore() || { stack: [], correlationId: null }; + return await storage.run({ ...current, suppressed: true }, fn); +}; + + +const isSuppressed = () => { + const s = storage.getStore(); + return !!(s && s.suppressed); +}; + + +module.exports = { + enterOp, + currentParentOpId, + currentCorrelationId, + runSuppressed, + isSuppressed +}; diff --git a/lib/crypto.js b/lib/crypto.js new file mode 100644 index 0000000..92bab1c --- /dev/null +++ b/lib/crypto.js @@ -0,0 +1,163 @@ +// Crypto primitives for dev-deploy peer auth. +// +// seal/open — AES-256-GCM for at-rest encryption of peer secrets. +// The 32-byte KEK is derived once per process via +// HKDF-SHA256 from SALTCORN_SESSION_SECRET. +// sign/verify — HMAC-SHA256 for signed peer-to-peer requests. +// buildCanonical — canonical string format every signed request agrees on. +// randomSecret — 32 random bytes (peer_secret) at pairing time. +// +// Rotating SALTCORN_SESSION_SECRET invalidates all peer pairings (the KEK +// changes, so existing ciphertexts no longer decrypt). Documented behavior. + +const crypto = require("crypto"); + + +const KEK_INFO = "dev-deploy:peer-secrets:aes-gcm-key:v1"; +const HMAC_ALGORITHM = "sha256"; +const GCM_ALGORITHM = "aes-256-gcm"; +const IV_BYTES = 12; +const TAG_BYTES = 16; +const SECRET_BYTES = 32; +const NONCE_BYTES = 16; +const SKEW_TOLERANCE_MS = 5 * 60 * 1000; + + +let cachedKek = null; + + +const getSessionSecret = () => { + const fromEnv = process.env.SALTCORN_SESSION_SECRET; + if (fromEnv && fromEnv.length > 0) { + return fromEnv; + } + // Fallback to Saltcorn state config if available + try { + const { getState } = require("@saltcorn/data/db/state"); + const v = getState().getConfig("session_secret"); + if (v) return v; + } catch (e) { + // ignore + } + throw new Error("dev-deploy: SALTCORN_SESSION_SECRET not available; cannot derive KEK"); +}; + + +const getKek = () => { + if (cachedKek) { + return cachedKek; + } + const sessionSecret = getSessionSecret(); + const salt = Buffer.from(KEK_INFO, "utf8"); + const ikm = Buffer.from(sessionSecret, "utf8"); + cachedKek = crypto.hkdfSync(HMAC_ALGORITHM, ikm, salt, Buffer.from(KEK_INFO, "utf8"), 32); + return cachedKek; +}; + + +const seal = (plaintext) => { + const iv = crypto.randomBytes(IV_BYTES); + const cipher = crypto.createCipheriv(GCM_ALGORITHM, getKek(), iv); + const buf = Buffer.isBuffer(plaintext) ? plaintext : Buffer.from(plaintext); + const ct = Buffer.concat([cipher.update(buf), cipher.final()]); + const tag = cipher.getAuthTag(); + return { ciphertext: ct, iv: iv, tag: tag }; +}; + + +const open = (sealed) => { + const decipher = crypto.createDecipheriv(GCM_ALGORITHM, getKek(), sealed.iv); + decipher.setAuthTag(sealed.tag); + return Buffer.concat([decipher.update(sealed.ciphertext), decipher.final()]); +}; + + +const randomSecret = () => { + return crypto.randomBytes(SECRET_BYTES); +}; + + +const randomNonce = () => { + return crypto.randomBytes(NONCE_BYTES); +}; + + +const sha256Hex = (data) => { + const buf = Buffer.isBuffer(data) ? data : Buffer.from(data || ""); + return crypto.createHash("sha256").update(buf).digest("hex"); +}; + + +const buildCanonical = ({ timestamp, nonce, method, path, targetHost, body }) => { + const bodyHash = sha256Hex(body || ""); + return [ + String(timestamp), + String(nonce), + String(method).toUpperCase(), + String(path), + String(targetHost || ""), + bodyHash + ].join("\n"); +}; + + +// Normalize a host[:port] so the signing side (outbound, derived from the peer +// base_url) and the verifying side (inbound, from the Host header) produce +// byte-identical canonicals: lowercase, trim, and drop the default port +// (clients omit :80/:443 from the Host header). Binding this into the canonical +// stops a request signed for one tenant being replayed against another tenant +// on the same multi-tenant server. +const normalizeHost = (h) => { + return String(h || "").trim().toLowerCase().replace(/:(?:80|443)$/, ""); +}; + + +const sign = (secret, canonical) => { + const mac = crypto.createHmac(HMAC_ALGORITHM, secret); + mac.update(canonical, "utf8"); + return mac.digest("hex"); +}; + + +const verifySignature = (secret, canonical, providedHex) => { + if (!providedHex || typeof providedHex !== "string") { + return false; + } + let expectedBuf; + let providedBuf; + try { + expectedBuf = Buffer.from(sign(secret, canonical), "hex"); + providedBuf = Buffer.from(providedHex, "hex"); + } catch (e) { + return false; + } + if (expectedBuf.length !== providedBuf.length) { + return false; + } + return crypto.timingSafeEqual(expectedBuf, providedBuf); +}; + + +const timestampWithinSkew = (tsString) => { + const ts = Number(tsString); + if (!Number.isFinite(ts)) { + return false; + } + const now = Date.now(); + return Math.abs(now - ts) <= SKEW_TOLERANCE_MS; +}; + + +module.exports = { + seal, + open, + randomSecret, + randomNonce, + buildCanonical, + normalizeHost, + sign, + verifySignature, + timestampWithinSkew, + sha256Hex, + SKEW_TOLERANCE_MS +}; diff --git a/lib/entityIds.js b/lib/entityIds.js new file mode 100644 index 0000000..4323919 --- /dev/null +++ b/lib/entityIds.js @@ -0,0 +1,349 @@ +// Stable UUIDs for Saltcorn metadata entities. +// +// Saltcorn core identifies entities by integer id and human name. dev-deploy +// needs a stable identity that survives across environments and renames, so we +// maintain a side-table _dd_entity_ids mapping (kind, current_id) -> uuid. +// +// First-run "backfill" assigns deterministic UUIDs (hash of namespace + kind +// + canonical name) so two environments installed from the same pack converge +// on the same UUIDs without a coordination step. + +const db = require("@saltcorn/data/db"); + +const Table = require("@saltcorn/data/models/table"); +const Field = require("@saltcorn/data/models/field"); +const View = require("@saltcorn/data/models/view"); +const Page = require("@saltcorn/data/models/page"); +const Trigger = require("@saltcorn/data/models/trigger"); +const Role = require("@saltcorn/data/models/role"); +const Library = require("@saltcorn/data/models/library"); +const Tag = require("@saltcorn/data/models/tag"); +const TableConstraint = require("@saltcorn/data/models/table_constraints"); +const PageGroup = require("@saltcorn/data/models/page_group"); +const WorkflowStep = require("@saltcorn/data/models/workflow_step"); + +const { deterministicUuid, randomUuid } = require("./ids"); +const { ENTITY_KINDS } = require("./constants"); + + +const assignNewUuid = async (kind, currentId, currentName, parentUuid) => { + const row = { + uuid: randomUuid(), + kind: kind, + current_name: currentName, + current_id: currentId, + parent_uuid: parentUuid || null, + created_at: new Date().toISOString() + }; + await db.insert("_dd_entity_ids", row, { noid: true }); + return row.uuid; +}; + + +const canonicalKey = (kind, entity, parentName) => { + if (kind === ENTITY_KINDS.FIELD) { + return `${parentName || "?"}.${entity.name}`; + } + return entity.name; +}; + + +// Stable fingerprint for a constraint, scoped to its parent table's UUID so +// it's globally unique across the journal. Two instances installed from the +// same base agree on this fingerprint and so derive identical UUIDs. +const constraintFingerprint = (con, parentUuid) => { + const cfg = con.configuration || {}; + let detail; + if (con.type === "Unique" && cfg.fields) { + detail = [...cfg.fields].sort().join(","); + } else if (con.type === "Index") { + detail = cfg.field || ""; + } else if (con.type === "Formula") { + detail = cfg.formula || ""; + } else { + detail = JSON.stringify(cfg); + } + return `${parentUuid || "?"}|${con.type}|${detail}`; +}; + + +// Friendly display name for _dd_entity_ids.current_name. Not used for identity. +const constraintDisplayName = (con) => { + const cfg = con.configuration || {}; + if (con.type === "Unique") return `unique(${(cfg.fields || []).join(",")})`; + if (con.type === "Index") return `index(${cfg.field || ""})`; + if (con.type === "Formula") return `formula`; + return `${con.type}_${con.id || "new"}`; +}; + + +const ensureUuid = async (kind, currentId, currentName, parentUuid, canonical) => { + const existing = await db.selectMaybeOne("_dd_entity_ids", { kind: kind, current_id: currentId }); + if (existing) { + return existing.uuid; + } + const uuid = deterministicUuid(kind, canonical || currentName); + const row = { + uuid: uuid, + kind: kind, + current_name: currentName, + current_id: currentId, + parent_uuid: parentUuid || null, + created_at: new Date().toISOString() + }; + await db.insert("_dd_entity_ids", row, { noid: true }); + return uuid; +}; + + +const lookupByCurrent = async (kind, currentId) => { + return await db.selectMaybeOne("_dd_entity_ids", { kind: kind, current_id: currentId }); +}; + + +const lookupByUuid = async (uuid) => { + return await db.selectMaybeOne("_dd_entity_ids", { uuid: uuid }); +}; + + +// Insert a row with a specific UUID (used by apply.js when ingesting an op +// that created an entity on a peer — we want to preserve the peer's UUID). +const adoptUuid = async (uuid, kind, currentId, currentName, parentUuid) => { + const row = { + uuid: uuid, + kind: kind, + current_name: currentName, + current_id: currentId, + parent_uuid: parentUuid || null, + created_at: new Date().toISOString() + }; + await db.insert("_dd_entity_ids", row, { noid: true }); + return uuid; +}; + + +const updateName = async (kind, currentId, newName) => { + await db.updateWhere("_dd_entity_ids", { current_name: newName }, { kind: kind, current_id: currentId }); +}; + + +const removeEntityRow = async (kind, currentId) => { + await db.deleteWhere("_dd_entity_ids", { kind: kind, current_id: currentId }); +}; + + +// Backfill helpers: each returns the count of new UUIDs inserted. + +const backfillTables = async () => { + const tables = await Table.find({}, { cached: false }); + let added = 0; + for (const t of tables) { + const before = await lookupByCurrent(ENTITY_KINDS.TABLE, t.id); + await ensureUuid(ENTITY_KINDS.TABLE, t.id, t.name, null, t.name); + if (!before) { + added += 1; + } + } + return added; +}; + + +const backfillFields = async () => { + const tables = await Table.find({}, { cached: false }); + let added = 0; + for (const t of tables) { + const tableUuidRow = await lookupByCurrent(ENTITY_KINDS.TABLE, t.id); + if (!tableUuidRow) { + continue; + } + const fields = t.getFields ? t.getFields() : (await Field.find({ table_id: t.id })); + for (const f of fields) { + const before = await lookupByCurrent(ENTITY_KINDS.FIELD, f.id); + await ensureUuid(ENTITY_KINDS.FIELD, f.id, f.name, tableUuidRow.uuid, `${t.name}.${f.name}`); + if (!before) { + added += 1; + } + } + } + return added; +}; + + +const backfillViews = async () => { + const views = await View.find({}); + let added = 0; + for (const v of views) { + const before = await lookupByCurrent(ENTITY_KINDS.VIEW, v.id); + let parentUuid = null; + if (v.table_id) { + const t = await lookupByCurrent(ENTITY_KINDS.TABLE, v.table_id); + parentUuid = t ? t.uuid : null; + } + await ensureUuid(ENTITY_KINDS.VIEW, v.id, v.name, parentUuid, v.name); + if (!before) { + added += 1; + } + } + return added; +}; + + +const backfillPages = async () => { + const pages = await Page.find({}); + let added = 0; + for (const p of pages) { + const before = await lookupByCurrent(ENTITY_KINDS.PAGE, p.id); + await ensureUuid(ENTITY_KINDS.PAGE, p.id, p.name, null, p.name); + if (!before) { + added += 1; + } + } + return added; +}; + + +const backfillTriggers = async () => { + const triggers = Trigger.find({}); + let added = 0; + for (const tr of triggers) { + const before = await lookupByCurrent(ENTITY_KINDS.TRIGGER, tr.id); + let parentUuid = null; + if (tr.table_id) { + const t = await lookupByCurrent(ENTITY_KINDS.TABLE, tr.table_id); + parentUuid = t ? t.uuid : null; + } + await ensureUuid(ENTITY_KINDS.TRIGGER, tr.id, tr.name || `trigger_${tr.id}`, parentUuid, tr.name || `trigger_${tr.id}`); + if (!before) { + added += 1; + } + } + return added; +}; + + +const backfillRoles = async () => { + const roles = await Role.find({}); + let added = 0; + for (const r of roles) { + const before = await lookupByCurrent(ENTITY_KINDS.ROLE, r.id); + await ensureUuid(ENTITY_KINDS.ROLE, r.id, r.role, null, r.role); + if (!before) { + added += 1; + } + } + return added; +}; + + +const backfillLibrary = async () => { + const items = await Library.find({}); + let added = 0; + for (const li of items) { + const before = await lookupByCurrent(ENTITY_KINDS.LIBRARY, li.id); + await ensureUuid(ENTITY_KINDS.LIBRARY, li.id, li.name, null, li.name); + if (!before) { + added += 1; + } + } + return added; +}; + + +const backfillTags = async () => { + const tags = await Tag.find({}); + let added = 0; + for (const tg of tags) { + const before = await lookupByCurrent(ENTITY_KINDS.TAG, tg.id); + await ensureUuid(ENTITY_KINDS.TAG, tg.id, tg.name, null, tg.name); + if (!before) { + added += 1; + } + } + return added; +}; + + +const backfillTableConstraints = async () => { + const cons = await TableConstraint.find({}); + let added = 0; + for (const c of cons) { + const before = await lookupByCurrent(ENTITY_KINDS.CONSTRAINT, c.id); + let parentUuid = null; + if (c.table_id) { + const t = await lookupByCurrent(ENTITY_KINDS.TABLE, c.table_id); + parentUuid = t ? t.uuid : null; + } + const canonical = constraintFingerprint(c, parentUuid); + await ensureUuid(ENTITY_KINDS.CONSTRAINT, c.id, constraintDisplayName(c), parentUuid, canonical); + if (!before) { + added += 1; + } + } + return added; +}; + + +const backfillPageGroups = async () => { + const groups = await PageGroup.find({}); + let added = 0; + for (const g of groups) { + const before = await lookupByCurrent(ENTITY_KINDS.PAGE_GROUP, g.id); + await ensureUuid(ENTITY_KINDS.PAGE_GROUP, g.id, g.name, null, g.name); + if (!before) { + added += 1; + } + } + return added; +}; + + +const backfillWorkflowSteps = async () => { + const steps = await WorkflowStep.find({}); + let added = 0; + for (const s of steps) { + const before = await lookupByCurrent(ENTITY_KINDS.WORKFLOW_STEP, s.id); + let parentUuid = null; + if (s.trigger_id) { + const tr = await lookupByCurrent(ENTITY_KINDS.TRIGGER, s.trigger_id); + parentUuid = tr ? tr.uuid : null; + } + const canonical = `${parentUuid || "?"}|${s.name || s.id}`; + await ensureUuid(ENTITY_KINDS.WORKFLOW_STEP, s.id, s.name || `step_${s.id}`, parentUuid, canonical); + if (!before) { + added += 1; + } + } + return added; +}; + + +const backfillAll = async () => { + const counts = {}; + counts.tables = await backfillTables(); + counts.fields = await backfillFields(); + counts.views = await backfillViews(); + counts.pages = await backfillPages(); + counts.triggers = await backfillTriggers(); + counts.roles = await backfillRoles(); + counts.library = await backfillLibrary(); + counts.tags = await backfillTags(); + counts.constraints = await backfillTableConstraints(); + counts.page_groups = await backfillPageGroups(); + counts.workflow_steps = await backfillWorkflowSteps(); + return counts; +}; + + +module.exports = { + backfillAll, + ensureUuid, + assignNewUuid, + lookupByCurrent, + lookupByUuid, + adoptUuid, + updateName, + removeEntityRow, + canonicalKey, + constraintFingerprint, + constraintDisplayName +}; diff --git a/lib/env.js b/lib/env.js new file mode 100644 index 0000000..4827454 --- /dev/null +++ b/lib/env.js @@ -0,0 +1,71 @@ +// This Saltcorn instance's dev-deploy identity (env_id, label, policies). +// Stored as a singleton row in _dd_env. + +const db = require("@saltcorn/data/db"); + +const { randomUuid } = require("./ids"); +const { DESTRUCTIVE_POLICY } = require("./constants"); + + +// The env identity is schema-scoped, so a single process serving multiple +// tenants must NOT share one cached row across them. Key the cache by tenant +// schema (db.getTenantSchema()), not a module-level singleton. +const cachedEnvByTenant = new Map(); + +const tenantKey = () => (db.getTenantSchema ? db.getTenantSchema() : "public"); + + +const getEnv = async () => { + const key = tenantKey(); + if (cachedEnvByTenant.has(key)) { + return cachedEnvByTenant.get(key); + } + const rows = await db.select("_dd_env", {}); + const env = rows.length > 0 ? rows[0] : null; + cachedEnvByTenant.set(key, env); + return env; +}; + + +const refreshEnvCache = async () => { + cachedEnvByTenant.delete(tenantKey()); + return await getEnv(); +}; + + +const initEnvIfMissing = async () => { + const existing = await getEnv(); + if (existing) { + return existing; + } + const now = new Date().toISOString(); + const row = { + env_id: randomUuid(), + env_label: null, + on_destructive_op: DESTRUCTIVE_POLICY.CONFIRM, + require_tls: 0, + created_at: now, + bootstrapped_at: null + }; + await db.insert("_dd_env", row, { noid: true }); + cachedEnvByTenant.set(tenantKey(), row); + return row; +}; + + +const markBootstrapped = async (envId) => { + const now = new Date().toISOString(); + await db.updateWhere("_dd_env", { bootstrapped_at: now }, { env_id: envId }); + const cached = cachedEnvByTenant.get(tenantKey()); + if (cached && cached.env_id === envId) { + cached.bootstrapped_at = now; + } +}; + + +module.exports = { + getEnv, + initEnvIfMissing, + markBootstrapped, + refreshEnvCache +}; diff --git a/lib/files.js b/lib/files.js new file mode 100644 index 0000000..e02a110 --- /dev/null +++ b/lib/files.js @@ -0,0 +1,60 @@ +// File helpers: content hashing, relative-path normalization, reading bytes. + +const crypto = require("crypto"); +const fs = require("fs"); + + +// SHA-256 of the file's contents, returned as lowercase hex. +const sha256File = async (absPath) => { + return new Promise((resolve, reject) => { + const hash = crypto.createHash("sha256"); + const stream = fs.createReadStream(absPath); + stream.on("data", (chunk) => hash.update(chunk)); + stream.on("end", () => resolve(hash.digest("hex"))); + stream.on("error", reject); + }); +}; + + +const sha256Buffer = (buf) => { + return crypto.createHash("sha256").update(buf).digest("hex"); +}; + + +// Convert Saltcorn's absolute file location to a tenant-relative serve path, +// using File.absPathToServePath. The relative path is what we transport +// between instances (each has a different file_store root). +const toRelativePath = (File, absPath) => { + if (!absPath) return ""; + return File.absPathToServePath(absPath); +}; + + +// Convert a relative serve path back to the absolute path on this instance. +const toAbsolutePath = (File, db, relPath) => { + const path = require("path"); + const tenant = db.getTenantSchema(); + return path.join(db.connectObj.file_store, tenant, relPath); +}; + + +const readFileBytes = async (absPath) => { + return await fs.promises.readFile(absPath); +}; + + +const writeFileBytes = async (absPath, buf) => { + const path = require("path"); + await fs.promises.mkdir(path.dirname(absPath), { recursive: true }); + await fs.promises.writeFile(absPath, buf); +}; + + +module.exports = { + sha256File, + sha256Buffer, + toRelativePath, + toAbsolutePath, + readFileBytes, + writeFileBytes +}; diff --git a/lib/ids.js b/lib/ids.js new file mode 100644 index 0000000..d6a6455 --- /dev/null +++ b/lib/ids.js @@ -0,0 +1,34 @@ +// UUID helpers for dev-deploy. +// +// Two flavors: +// - randomUuid(): a fresh v4 UUID, used for newly-created ops and entities +// - deterministicUuid(kind, name): a stable, repeatable UUID derived from +// (namespace, kind, name) using SHA-256. Used during first-run backfill so +// that two environments installed from the same base produce identical +// UUIDs for the same (kind, name) pair. RFC 4122 v5 shape with namespace. + +const crypto = require("crypto"); + +const { ID_NAMESPACE } = require("./constants"); + + +const deterministicUuid = (kind, name) => { + const input = `${ID_NAMESPACE}|${kind}|${name}`; + const hash = crypto.createHash("sha256").update(input).digest(); + const bytes = Buffer.from(hash.subarray(0, 16)); + bytes[6] = (bytes[6] & 0x0f) | 0x50; + bytes[8] = (bytes[8] & 0x3f) | 0x80; + const hex = bytes.toString("hex"); + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20, 32)}`; +}; + + +const randomUuid = () => { + return crypto.randomUUID(); +}; + + +module.exports = { + deterministicUuid, + randomUuid +}; diff --git a/lib/ops.js b/lib/ops.js new file mode 100644 index 0000000..cef1c05 --- /dev/null +++ b/lib/ops.js @@ -0,0 +1,47 @@ +// Journal append. + +const db = require("@saltcorn/data/db"); + +const { getEnv } = require("./env"); +const { randomUuid } = require("./ids"); +const { OP_SCHEMA_VERSION } = require("./constants"); +const { currentParentOpId, currentCorrelationId } = require("./context"); + + +const recordOp = async (rec) => { + const env = await getEnv(); + const now = new Date().toISOString(); + const row = { + op_id: rec.op_id || randomUuid(), + source_env_id: env.env_id, + op_type: rec.op_type, + entity_kind: rec.entity_kind || null, + entity_uuid: rec.entity_uuid || null, + payload: JSON.stringify(rec.payload || {}), + parent_op_id: rec.parent_op_id || currentParentOpId(), + correlation_id: rec.correlation_id || currentCorrelationId(), + schema_version: OP_SCHEMA_VERSION, + created_at: now, + applied_at: now, + status: "committed" + }; + await db.insert("_dd_ops", row, { noid: true }); + return row.op_id; +}; + + +const recordOpSafely = async (rec) => { + try { + return await recordOp(rec); + } catch (err) { + // eslint-disable-next-line no-console + console.error(`[dev-deploy] failed to record op ${rec.op_type}:`, err); + return null; + } +}; + + +module.exports = { + recordOp, + recordOpSafely +}; diff --git a/lib/payloadRefs.js b/lib/payloadRefs.js new file mode 100644 index 0000000..c981a35 --- /dev/null +++ b/lib/payloadRefs.js @@ -0,0 +1,98 @@ +// Recursive walker for op payloads: translate file references between this +// instance's local form (numeric id or relative path) and a portable +// placeholder ("__dd_file_ref::"). Promote-time wraps call +// toPlaceholders to journal portable payloads; apply-time handlers call +// fromPlaceholders to resolve back to the target's local file path. +// +// Saltcorn's /files/serve route accepts either a numeric file id or a +// relative path, so writing the path back on apply works regardless of which +// form the source used. + +const db = require("@saltcorn/data/db"); + +const { lookupByUuid } = require("./entityIds"); + + +// Common keys in page/view layout JSON that reference a file. +const FILE_REF_KEYS = new Set([ + "fileid", + "file_id", + "bgFileId", + "image_id" +]); + + +const PLACEHOLDER_PREFIX = "__dd_file_ref::"; + + +// Look up a file entity by value (string path or numeric id). Returns the +// entity_ids row or null. +const lookupFileByValue = async (v) => { + if (v === null || v === undefined || v === "") return null; + // Match by current_name (relative path string) + let row = await db.selectMaybeOne("_dd_entity_ids", { kind: "file", current_name: String(v) }); + if (row) return row; + // Also try numeric id, in case the source stored an integer id + const asNum = Number(v); + if (Number.isFinite(asNum)) { + row = await db.selectMaybeOne("_dd_entity_ids", { kind: "file", current_id: asNum }); + if (row) return row; + } + return null; +}; + + +// Walk obj, mutating in place. For each key in FILE_REF_KEYS with a non-empty +// value, replace with the result of `transform(key, value)`. Async-aware. +const transformFileRefs = async (obj, transform) => { + if (Array.isArray(obj)) { + for (let i = 0; i < obj.length; i++) { + if (obj[i] && typeof obj[i] === "object") { + await transformFileRefs(obj[i], transform); + } + } + return; + } + if (!obj || typeof obj !== "object") return; + for (const k of Object.keys(obj)) { + const v = obj[k]; + if (FILE_REF_KEYS.has(k) && v !== null && v !== undefined && v !== "" && typeof v !== "object") { + obj[k] = await transform(k, v); + } else if (v && typeof v === "object") { + await transformFileRefs(v, transform); + } + } +}; + + +// Convert local file refs (id or path) -> portable placeholders. +const toPlaceholders = async (payload) => { + await transformFileRefs(payload, async (k, v) => { + const ent = await lookupFileByValue(v); + if (ent) return PLACEHOLDER_PREFIX + ent.uuid; + return v; + }); + return payload; +}; + + +// Convert portable placeholders -> local file refs (write back as relative +// path, which Saltcorn's /files/serve accepts). +const fromPlaceholders = async (payload) => { + await transformFileRefs(payload, async (k, v) => { + if (typeof v !== "string") return v; + if (!v.startsWith(PLACEHOLDER_PREFIX)) return v; + const uuid = v.substring(PLACEHOLDER_PREFIX.length); + const ent = await lookupByUuid(uuid); + return ent ? ent.current_name : v; + }); + return payload; +}; + + +module.exports = { + toPlaceholders, + fromPlaceholders, + PLACEHOLDER_PREFIX, + FILE_REF_KEYS +}; diff --git a/lib/peerAuth.js b/lib/peerAuth.js new file mode 100644 index 0000000..9dbe9bb --- /dev/null +++ b/lib/peerAuth.js @@ -0,0 +1,141 @@ +// Verify incoming HMAC-signed peer requests. +// +// Required headers: +// X-DD-Env-Id caller's env UUID (looked up in _dd_peers) +// X-DD-Timestamp ms-since-epoch; rejected if skew > 5 min +// X-DD-Nonce random per-request opaque bytes (replay padding) +// X-DD-Signature hex HMAC-SHA256 over canonical string +// +// On success: req.dd_peer is set (peer row), req.body is the parsed JSON body, +// and the function returns the peer. On failure: a 4xx response is sent and +// null is returned. +// +// Peer requests use Content-Type: application/vnd.dev-deploy+json so Saltcorn's +// express.json() middleware doesn't consume the stream upstream. That lets us +// read the exact raw bytes here and use them in the HMAC -- no JSON-canonical +// fragility, no re-serialization assumptions about whitespace or key order. + +const { + buildCanonical, + normalizeHost, + verifySignature, + timestampWithinSkew +} = require("./crypto"); +const { + findPeerByEnvId, + peerSecret, + touchPeerLastSeen +} = require("./peers"); +const { getEnv } = require("./env"); + + +const REQUIRED_HEADERS = ["x-dd-env-id", "x-dd-timestamp", "x-dd-nonce", "x-dd-signature"]; + + +const readRawBody = async (req) => { + const chunks = []; + for await (const chunk of req) { + chunks.push(typeof chunk === "string" ? Buffer.from(chunk) : chunk); + } + return Buffer.concat(chunks).toString("utf8"); +}; + + +const requirePeerAuth = async (req, res) => { + for (const h of REQUIRED_HEADERS) { + if (!req.headers[h]) { + res.status(400).json({ error: `missing header ${h}` }); + return null; + } + } + const callerEnvId = req.headers["x-dd-env-id"]; + const timestamp = req.headers["x-dd-timestamp"]; + const nonce = req.headers["x-dd-nonce"]; + const signature = req.headers["x-dd-signature"]; + + if (!timestampWithinSkew(timestamp)) { + res.status(401).json({ error: "timestamp out of skew window" }); + return null; + } + + const peerRow = await findPeerByEnvId(callerEnvId); + if (!peerRow) { + res.status(401).json({ error: "unknown peer env_id" }); + return null; + } + + let secret; + try { + secret = await peerSecret(peerRow.peer_id); + } catch (e) { + res.status(401).json({ error: "peer not provisioned" }); + return null; + } + + // Read raw bytes (empty for GET/HEAD). HMAC covers exactly what arrived. + let bodyRaw = ""; + if (req.method !== "GET" && req.method !== "HEAD") { + try { + bodyRaw = await readRawBody(req); + } catch (e) { + res.status(400).json({ error: "failed to read request body" }); + return null; + } + } + + // Bind the target host (which tenant Saltcorn routed this request to) into the + // signed canonical: a request signed for tenant t1 must NOT verify when + // replayed against tenant t2 on the same server. Prefer X-Forwarded-Host (set + // by a trusted proxy) then the Host header, normalized identically to the + // outbound side (transport.js derives it from the peer base_url). NOTE: do NOT + // compare against peerRow.base_url -- inbound that is the SENDER's address + // (for pull-back), not this receiver's host. + const fullPath = req.originalUrl || req.url; + const fwdHost = req.headers["x-forwarded-host"]; + const rawHost = fwdHost ? String(fwdHost).split(",")[0] : req.headers.host; + const targetHost = normalizeHost(rawHost); + const canonical = buildCanonical({ + timestamp: timestamp, + nonce: nonce, + method: req.method, + path: fullPath, + targetHost: targetHost, + body: bodyRaw + }); + + if (!verifySignature(secret, canonical, signature)) { + res.status(401).json({ error: "bad signature" }); + return null; + } + + if (bodyRaw) { + try { + req.body = JSON.parse(bodyRaw); + } catch (e) { + res.status(400).json({ error: "body is not valid JSON" }); + return null; + } + } + + // If this env requires TLS for inbound peer traffic, reject any request + // that did not arrive over HTTPS. req.secure reflects a direct TLS socket; + // x-forwarded-proto covers the case where a trusted proxy terminates TLS. + const env = await getEnv(); + if (env && env.require_tls) { + const fwdProto = req.headers["x-forwarded-proto"]; + const overTls = req.secure || (fwdProto && String(fwdProto).split(",")[0].trim() === "https"); + if (!overTls) { + res.status(403).json({ error: "TLS required" }); + return null; + } + } + + await touchPeerLastSeen(peerRow.peer_id); + req.dd_peer = peerRow; + return peerRow; +}; + + +module.exports = { + requirePeerAuth +}; diff --git a/lib/peers.js b/lib/peers.js new file mode 100644 index 0000000..5d7cd46 --- /dev/null +++ b/lib/peers.js @@ -0,0 +1,145 @@ +// Peer model: CRUD on _dd_peers. +// +// Peer secrets are stored AES-256-GCM sealed; plaintext only crosses the +// process boundary at pairing time (when the operator copies the secret to +// the other instance's UI) and at HMAC sign/verify time. + +const db = require("@saltcorn/data/db"); + +const { + seal, + open, + randomSecret +} = require("./crypto"); + + +const rowToPeer = (row) => { + if (!row) return null; + return { + peer_id: row.peer_id, + env_id: row.env_id, + label: row.label, + base_url: row.base_url, + require_tls: !!row.require_tls, + created_at: row.created_at, + last_seen_at: row.last_seen_at, + // sealed components kept out of plain accessors -- use peerSecret() + }; +}; + + +const listPeers = async () => { + const rows = await db.select("_dd_peers", {}, { orderBy: "peer_id" }); + return rows.map(rowToPeer); +}; + + +const findPeer = async (peerId) => { + const row = await db.selectMaybeOne("_dd_peers", { peer_id: peerId }); + return rowToPeer(row); +}; + + +const findPeerByEnvId = async (envId) => { + return await db.selectMaybeOne("_dd_peers", { env_id: envId }); +}; + + +// Returns the plaintext 32-byte secret for an existing peer by id. +// Throws if the peer is missing or has no sealed secret. +const peerSecret = async (peerId) => { + const row = await db.selectMaybeOne("_dd_peers", { peer_id: peerId }); + if (!row) { + throw new Error(`peer ${peerId} not found`); + } + if (!row.peer_secret_ciphertext || !row.peer_secret_iv || !row.peer_secret_tag) { + throw new Error(`peer ${peerId} has no sealed secret`); + } + return open({ + ciphertext: Buffer.from(row.peer_secret_ciphertext, "hex"), + iv: Buffer.from(row.peer_secret_iv, "hex"), + tag: Buffer.from(row.peer_secret_tag, "hex") + }); +}; + + +const peerSecretByEnvId = async (envId) => { + const row = await findPeerByEnvId(envId); + if (!row) { + return null; + } + return await peerSecret(row.peer_id); +}; + + +// Create a new peer. If `existingSecret` is provided, use it; otherwise +// generate a fresh one. Returns { peer, secret } where secret is the plaintext +// Buffer -- only available at this single moment. +const addPeer = async ({ envId, label, baseUrl, requireTls, existingSecret }) => { + if (!envId || !baseUrl) { + throw new Error("addPeer requires envId and baseUrl"); + } + const dup = await findPeerByEnvId(envId); + if (dup) { + throw new Error(`peer with env_id ${envId} already exists`); + } + const secret = existingSecret || randomSecret(); + const sealed = seal(secret); + const row = { + env_id: envId, + label: label || null, + base_url: baseUrl, + peer_secret_ciphertext: sealed.ciphertext.toString("hex"), + peer_secret_iv: sealed.iv.toString("hex"), + peer_secret_tag: sealed.tag.toString("hex"), + require_tls: requireTls ? 1 : 0, + created_at: new Date().toISOString(), + last_seen_at: null + }; + // noid: _dd_peers' PK is peer_id (serial on PG), not "id"; without this + // Saltcorn's default RETURNING id makes the insert fail on Postgres with + // 'column "id" does not exist'. The peer_id is auto-assigned; re-select below. + await db.insert("_dd_peers", row, { noid: true }); + const fresh = await findPeerByEnvId(envId); + return { peer: rowToPeer(fresh), secret: secret }; +}; + + +const rotatePeerSecret = async (peerId) => { + const peer = await findPeer(peerId); + if (!peer) { + throw new Error(`peer ${peerId} not found`); + } + const secret = randomSecret(); + const sealed = seal(secret); + await db.updateWhere("_dd_peers", { + peer_secret_ciphertext: sealed.ciphertext.toString("hex"), + peer_secret_iv: sealed.iv.toString("hex"), + peer_secret_tag: sealed.tag.toString("hex") + }, { peer_id: peerId }); + return { peer: peer, secret: secret }; +}; + + +const deletePeer = async (peerId) => { + await db.deleteWhere("_dd_peers", { peer_id: peerId }); + await db.deleteWhere("_dd_anchors", { peer_id: peerId }); +}; + + +const touchPeerLastSeen = async (peerId) => { + await db.updateWhere("_dd_peers", { last_seen_at: new Date().toISOString() }, { peer_id: peerId }); +}; + + +module.exports = { + listPeers, + findPeer, + findPeerByEnvId, + peerSecret, + peerSecretByEnvId, + addPeer, + rotatePeerSecret, + deletePeer, + touchPeerLastSeen +}; diff --git a/lib/revert.js b/lib/revert.js new file mode 100644 index 0000000..f2fc5a8 --- /dev/null +++ b/lib/revert.js @@ -0,0 +1,393 @@ +// Revert: append a compensating op for any local op_id. +// +// create_X → drop the entity that was created +// drop_X → recreate the entity from payload.before +// update_X → re-apply payload.before as a new patch +// set_X → restore the prior value (payload.before / before_mode) +// insert_row → delete the row; drop_row → re-insert payload.before +// +// The revert calls the same Saltcorn model methods (or wrapped setters) the +// admin UI does, so the existing CRUD wraps fire and journal the inverse op +// naturally with this instance's env_id. The inverse op then promotes to peers +// like any other op. +// +// Dispatch keys off the op's stored `entity_kind` column (reliable) for the +// kind and the op_type prefix for the action -- NOT a naive op_type split, +// which mis-parses multi-word kinds (e.g. page_group_member -> "page"). +// +// Where the original op did not retain enough to rebuild the prior state +// (e.g. a dropped file's bytes, an old op journaled before before-capture was +// added), revert returns a clear { status: "unsupported", reason } rather than +// throwing a cryptic error. +// +// Caveats: +// - Reverting a drop produces a *new* entity with a new random UUID. The +// original UUID is gone for good. Content is restored; identity is fresh. +// - Reverting a field/view/trigger/constraint drop requires the parent table +// to still exist locally. parent_uuid was captured by the drop wrap; we +// resolve it to the current local id. + +const Table = require("@saltcorn/data/models/table"); +const Field = require("@saltcorn/data/models/field"); +const View = require("@saltcorn/data/models/view"); +const Page = require("@saltcorn/data/models/page"); +const Trigger = require("@saltcorn/data/models/trigger"); +const Role = require("@saltcorn/data/models/role"); +const Library = require("@saltcorn/data/models/library"); +const Tag = require("@saltcorn/data/models/tag"); +const TableConstraint = require("@saltcorn/data/models/table_constraints"); +const PageGroup = require("@saltcorn/data/models/page_group"); +const WorkflowStep = require("@saltcorn/data/models/workflow_step"); +const File = require("@saltcorn/data/models/file"); + +const db = require("@saltcorn/data/db"); + +const { lookupByUuid } = require("./entityIds"); +const { ENTITY_KINDS, DATA_MODES } = require("./constants"); +const { refreshState } = require("./state"); +const { randomUuid } = require("./ids"); +const { enterOp } = require("./context"); +const { recordOpSafely } = require("./ops"); +const { portableToRow } = require("./rowPayload"); +const { ensureManagedSchema, findIdByRowUuid } = require("./rowIdentity"); + + +const MODELS = { + [ENTITY_KINDS.TABLE]: Table, + [ENTITY_KINDS.FIELD]: Field, + [ENTITY_KINDS.VIEW]: View, + [ENTITY_KINDS.PAGE]: Page, + [ENTITY_KINDS.TRIGGER]: Trigger, + [ENTITY_KINDS.ROLE]: Role, + [ENTITY_KINDS.LIBRARY]: Library, + [ENTITY_KINDS.TAG]: Tag, + [ENTITY_KINDS.CONSTRAINT]: TableConstraint, + [ENTITY_KINDS.PAGE_GROUP]: PageGroup, + [ENTITY_KINDS.WORKFLOW_STEP]: WorkflowStep +}; + + +const stripIds = (obj) => { + const out = { ...(obj || {}) }; + for (const k of ["id", "table_id", "view_id", "page_id"]) delete out[k]; + return out; +}; + + +const unsupported = (reason) => { + return { status: "unsupported", reason: reason }; +}; + + +const findLocalInstance = async (kind, entityUuid) => { + const m = await lookupByUuid(entityUuid); + if (!m) return null; + const Cls = MODELS[kind]; + if (!Cls || typeof Cls.findOne !== "function") return null; + await refreshState(); + const inst = await Cls.findOne({ id: m.current_id }); + return inst || null; +}; + + +const resolveParentLocalId = async (parentUuid, kind) => { + if (!parentUuid) return null; + const parent = await lookupByUuid(parentUuid); + if (!parent) { + throw new Error(`parent uuid=${parentUuid} not present locally; cannot recreate ${kind}`); + } + return parent.current_id; +}; + + +// --- Drop the entity (revert of create_X) --- +const revertCreate = async (kind, op) => { + const inst = await findLocalInstance(kind, op.entity_uuid); + if (!inst) { + return { status: "noop", reason: "entity no longer present" }; + } + await inst.delete(); + return { status: "dropped" }; +}; + + +// --- Recreate from before-snapshot (revert of drop_X) --- +const recreateFromSnapshot = async (kind, payload) => { + const before = stripIds(payload.before || {}); + if (Object.keys(before).length === 0) { + return unsupported(`drop_${kind} did not retain a before-snapshot`); + } + const parentLocalId = await resolveParentLocalId(payload.parent_uuid, kind); + + switch (kind) { + case ENTITY_KINDS.TABLE: { + if (!before.name) throw new Error("missing before.name"); + const opts = { ...before }; + const name = opts.name; + delete opts.name; + await Table.create(name, opts); + return { status: "recreated" }; + } + case ENTITY_KINDS.FIELD: { + await Field.create({ ...before, table_id: parentLocalId }); + return { status: "recreated" }; + } + case ENTITY_KINDS.VIEW: { + const cfg = { ...before }; + if (parentLocalId) cfg.table_id = parentLocalId; + await View.create(cfg); + return { status: "recreated" }; + } + case ENTITY_KINDS.PAGE: { + await Page.create(before); + return { status: "recreated" }; + } + case ENTITY_KINDS.TRIGGER: { + const cfg = { ...before }; + if (parentLocalId) cfg.table_id = parentLocalId; + await Trigger.create(cfg); + return { status: "recreated" }; + } + case ENTITY_KINDS.ROLE: { + if (!before.id || !before.role) throw new Error("missing role identity"); + await Role.create({ id: before.id, role: before.role }); + return { status: "recreated" }; + } + case ENTITY_KINDS.LIBRARY: { + await Library.create(before); + return { status: "recreated" }; + } + case ENTITY_KINDS.TAG: { + await Tag.create(before); + return { status: "recreated" }; + } + case ENTITY_KINDS.CONSTRAINT: { + if (parentLocalId === null) throw new Error("constraint recreate needs parent table"); + if (!before.type) return unsupported("drop_constraint snapshot missing type"); + await TableConstraint.create({ + table_id: parentLocalId, + type: before.type, + configuration: before.configuration || {} + }); + return { status: "recreated" }; + } + case ENTITY_KINDS.PAGE_GROUP: { + if (!before.name) return unsupported("drop_page_group snapshot missing name"); + const cfg = { ...before }; + cfg.members = []; + await PageGroup.create(cfg); + return { status: "recreated" }; + } + case ENTITY_KINDS.WORKFLOW_STEP: { + const cfg = { ...before }; + if (parentLocalId) cfg.trigger_id = parentLocalId; + await WorkflowStep.create(cfg); + return { status: "recreated" }; + } + default: + return unsupported(`recreate not implemented for kind '${kind}'`); + } +}; + + +// --- Re-apply payload.before as a new patch (revert of update_X) --- +const reapplyBefore = async (kind, op, payload) => { + const inst = await findLocalInstance(kind, op.entity_uuid); + if (!inst) { + throw new Error(`entity_uuid=${op.entity_uuid} (${kind}) not present locally`); + } + const patch = stripIds(payload.before || {}); + if (Object.keys(patch).length === 0) { + return unsupported(`update_${kind} did not retain a before-snapshot`); + } + switch (kind) { + case ENTITY_KINDS.TABLE: + case ENTITY_KINDS.FIELD: + case ENTITY_KINDS.ROLE: + case ENTITY_KINDS.LIBRARY: + case ENTITY_KINDS.TAG: + await inst.update(patch); + return { status: "updated" }; + case ENTITY_KINDS.VIEW: + await View.update(patch, inst.id); + return { status: "updated" }; + case ENTITY_KINDS.PAGE: + await Page.update(inst.id, patch); + return { status: "updated" }; + case ENTITY_KINDS.TRIGGER: + await Trigger.update(inst.id, patch); + return { status: "updated" }; + case ENTITY_KINDS.PAGE_GROUP: + await PageGroup.update(inst.id, patch); + return { status: "updated" }; + case ENTITY_KINDS.WORKFLOW_STEP: + await inst.update(patch); + return { status: "updated" }; + default: + return unsupported(`reapply not implemented for kind '${kind}'`); + } +}; + + +// --- Table rows: insert<->delete, update<->before, drop<->reinsert --- +const findLocalTableByUuid = async (tableUuid) => { + const ent = await lookupByUuid(tableUuid); + if (!ent || ent.kind !== ENTITY_KINDS.TABLE) return null; + await refreshState(); + return Table.findOne({ id: ent.current_id }); +}; + + +const revertRow = async (action, op, payload) => { + const tableUuid = payload.table_uuid; + if (!tableUuid) return unsupported("row op missing table_uuid"); + const tbl = await findLocalTableByUuid(tableUuid); + if (!tbl) return { status: "noop", reason: "table not present locally" }; + await ensureManagedSchema(tbl.name); + const localId = await findIdByRowUuid(tbl.name, op.entity_uuid); + + if (action === "insert") { + // Revert of insert_row = delete the row (wrapped deleteRows journals drop_row). + if (!localId) return { status: "noop", reason: "row already gone" }; + await tbl.deleteRows({ id: localId }); + return { status: "dropped", local_id: localId }; + } + if (action === "drop") { + // Revert of drop_row = re-insert payload.before (fresh row uuid, per caveat). + const before = payload.before || {}; + if (Object.keys(before).length === 0) return unsupported("drop_row did not retain a before-snapshot"); + if (localId) return { status: "noop", reason: "row already present" }; + const rowData = await portableToRow(before, tbl); + await tbl.insertRow(rowData); + return { status: "reinserted" }; + } + if (action === "update") { + // Revert of update_row = re-apply payload.before (wrapped updateRow journals update_row). + const before = payload.before || {}; + if (Object.keys(before).length === 0) return unsupported("update_row did not retain a before-snapshot"); + if (!localId) return { status: "noop", reason: "row not present locally" }; + const patch = await portableToRow(before, tbl); + await tbl.updateRow(patch, localId); + return { status: "updated", local_id: localId }; + } + return unsupported(`unknown row action '${action}'`); +}; + + +// --- set_table_mode: restore before_mode + journal the compensating op --- +const revertTableMode = async (op, payload) => { + const tableUuid = payload.table_uuid; + if (!tableUuid) return unsupported("set_table_mode missing table_uuid"); + const beforeMode = payload.before_mode; + if (!beforeMode) return unsupported("set_table_mode did not retain the prior mode"); + if (beforeMode === DATA_MODES.MANAGED || beforeMode === DATA_MODES.STARTER) { + const tbl = await findLocalTableByUuid(tableUuid); + if (tbl) await ensureManagedSchema(tbl.name); + } + const now = new Date().toISOString(); + const existing = await db.selectMaybeOne("_dd_table_modes", { table_uuid: tableUuid }); + if (existing) { + await db.updateWhere("_dd_table_modes", { data_mode: beforeMode, updated_at: now }, { table_uuid: tableUuid }); + } else { + await db.insert("_dd_table_modes", { table_uuid: tableUuid, data_mode: beforeMode, updated_at: now }, { noid: true }); + } + const opId = randomUuid(); + await enterOp(opId, async () => { + await recordOpSafely({ + op_id: opId, + op_type: "set_table_mode", + entity_kind: "table_mode", + entity_uuid: tableUuid, + payload: { table_uuid: tableUuid, data_mode: beforeMode, before_mode: payload.data_mode } + }); + }); + return { status: "set", mode: beforeMode }; +}; + + +// --- set_config: restore the prior value via the wrapped setter (which journals) --- +const revertConfig = async (payload) => { + if (!payload.key) return unsupported("set_config missing key"); + if (!("before" in payload)) return unsupported("set_config did not retain the prior value"); + const { getState } = require("@saltcorn/data/db/state"); + await getState().setConfig(payload.key, payload.before); + return { status: "restored", key: payload.key }; +}; + + +// --- update_plugin_config: restore prior configuration via wrapped upsert --- +const revertPluginConfig = async (payload) => { + if (!payload.name) return unsupported("update_plugin_config missing name"); + if (!("before_configuration" in payload)) { + return unsupported("update_plugin_config did not retain the prior configuration"); + } + const Plugin = require("@saltcorn/data/models/plugin"); + const plugin = await Plugin.findOne({ name: payload.name }); + if (!plugin) return { status: "noop", reason: `plugin ${payload.name} not installed` }; + plugin.configuration = payload.before_configuration || {}; + await plugin.upsert(); + return { status: "restored", plugin: payload.name }; +}; + + +// --- file: create<->delete; drop content cannot be recovered --- +const revertFile = async (action, op) => { + if (action === "create") { + const inst = await findLocalInstance(ENTITY_KINDS.FILE, op.entity_uuid); + if (!inst) return { status: "noop", reason: "file no longer present" }; + await inst.delete(); + return { status: "dropped" }; + } + if (action === "drop") { + return unsupported("revert of drop_file is not supported: file content was not retained (restore the file manually)"); + } + return unsupported(`unknown file action '${action}'`); +}; + + +const ACTION_HANDLERS = { + create: revertCreate, + drop: recreateFromSnapshot, + update: reapplyBefore +}; + + +// Look up the original op, parse its payload, dispatch to the inverse action. +// The local wrap journals the inverse as a new op with a fresh op_id (except +// the bespoke table_mode path, which records its own compensating op). +const revertOp = async (opId) => { + const orig = await db.selectMaybeOne("_dd_ops", { op_id: opId }); + if (!orig) { + throw new Error(`op ${opId} not found`); + } + const opType = orig.op_type || ""; + const us = opType.indexOf("_"); + const action = us < 0 ? opType : opType.slice(0, us); + const kind = orig.entity_kind || (us < 0 ? "" : opType.slice(us + 1)); + const payload = typeof orig.payload === "string" ? JSON.parse(orig.payload) : (orig.payload || {}); + + // Bespoke (non entity-model) kinds. + if (kind === "table_row") return await revertRow(action, orig, payload); + if (kind === "table_mode") return await revertTableMode(orig, payload); + if (kind === "config") return await revertConfig(payload); + if (kind === "plugin_config") return await revertPluginConfig(payload); + if (kind === ENTITY_KINDS.FILE) return await revertFile(action, orig); + + // Entity-model kinds (create/update/drop via the model classes). + const handler = ACTION_HANDLERS[action]; + if (!handler) { + return unsupported(`no revert for action '${action}' (op_type=${opType})`); + } + if (!MODELS[kind]) { + return unsupported(`no revert for entity kind '${kind}' (op_type=${opType})`); + } + if (action === "drop") { + return await handler(kind, payload); + } + return await handler(kind, orig, payload); +}; + + +module.exports = { + revertOp +}; diff --git a/lib/routes.js b/lib/routes.js new file mode 100644 index 0000000..8322809 --- /dev/null +++ b/lib/routes.js @@ -0,0 +1,1128 @@ +// HTTP routes for dev-deploy. +// +// Admin UI (session + admin role): +// GET /admin/dev-deploy/ +// GET /admin/dev-deploy/ops +// GET /admin/dev-deploy/peers +// POST /admin/dev-deploy/peers/add +// POST /admin/dev-deploy/peers/rotate +// POST /admin/dev-deploy/peers/delete +// GET /admin/dev-deploy/plan +// POST /admin/dev-deploy/promote +// +// Machine API (HMAC peer auth): +// GET /dev-deploy/api/journal?since=op_id +// POST /dev-deploy/api/ingest + +const db = require("@saltcorn/data/db"); + +const { PLUGIN_NAME, PLUGIN_VERSION } = require("./constants"); +const { getEnv } = require("./env"); +const peers = require("./peers"); +const { requirePeerAuth } = require("./peerAuth"); +const { signedFetch } = require("./transport"); +const { applyBatch, resolveConflict, resolveConflictByMerge, conflictFieldDiff } = require("./apply"); +const { revertOp } = require("./revert"); +const { DATA_MODES } = require("./constants"); + + +const getInboundAnchor = async (peerId) => { + return await db.selectMaybeOne("_dd_anchors", { peer_id: peerId, direction: "inbound" }); +}; + + +const getOutboundAnchor = async (peerId) => { + return await db.selectMaybeOne("_dd_anchors", { peer_id: peerId, direction: "outbound" }); +}; + + +const upsertAnchor = async (peerId, direction, opId) => { + const now = new Date().toISOString(); + const existing = await db.selectMaybeOne("_dd_anchors", { peer_id: peerId, direction: direction }); + if (existing) { + await db.updateWhere("_dd_anchors", { last_op_id: opId, updated_at: now }, { peer_id: peerId, direction: direction }); + } else { + await db.insert("_dd_anchors", { peer_id: peerId, direction: direction, last_op_id: opId, updated_at: now }, { noid: true }); + } +}; + + +// Safety cap on the number of journal pages a single pull will fetch. The peer's +// apiJournal caps each response at 1000 ops; pull loops until drained or this cap. +const PULL_MAX_ITERS = 100; + + +const isAdmin = (req) => !!(req && req.user && req.user.role_id === 1); + + +const escape = (s) => { + if (s === null || s === undefined) return ""; + return String(s) + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """); +}; + + +const csrfField = (req) => { + const t = req.csrfToken ? req.csrfToken() : ""; + return ``; +}; + + +const layout = (title, body, flash) => ` + + +${escape(title)} + + + +${flash || ""} +${body} +`; + + +const flashMsg = (req) => { + const m = req.query.msg; + const e = req.query.err; + if (m) return `
${escape(m)}
`; + if (e) return `
${escape(e)}
`; + return ""; +}; + + +// ---------------- Admin dashboard ---------------- + +const dashboard = async (req, res) => { + if (!isAdmin(req)) { res.status(403).type("text/plain").send("admin only"); return; } + const env = await getEnv(); + const schema = db.getTenantSchemaPrefix(); + const opCount = (await db.query(`SELECT COUNT(*) AS c FROM ${schema}_dd_ops`)).rows[0].c; + const opsByKind = (await db.query(`SELECT op_type, COUNT(*) AS c FROM ${schema}_dd_ops GROUP BY op_type ORDER BY op_type`)).rows; + const entCounts = (await db.query(`SELECT kind, COUNT(*) AS c FROM ${schema}_dd_entity_ids GROUP BY kind ORDER BY kind`)).rows; + const peerList = await peers.listPeers(); + const conflictCount = (await db.query(`SELECT COUNT(*) AS c FROM ${schema}_dd_ops WHERE status='conflict'`)).rows[0].c; + + const opsByKindRows = opsByKind.length === 0 + ? `` + : opsByKind.map((r) => ``).join(""); + + const entRows = entCounts.length === 0 + ? `` + : entCounts.map((r) => ``).join(""); + + const body = ` +

dev-deploy dashboard

+
No ops recorded yet
${escape(r.op_type)}${escape(r.c)}
No entities tracked
${escape(r.kind)}${escape(r.c)}
+ + + + + + + + +
Env ID${escape(env ? env.env_id : "?")}
Label${env && env.env_label ? escape(env.env_label) : '(unset)'}
Destructive-op policy${escape(env ? env.on_destructive_op : "?")}
Require TLS (default)${env && env.require_tls ? "yes" : "no"}
Bootstrapped at${escape(env ? env.bootstrapped_at : "")}
Ops recorded${escape(opCount)}
Peers configured${escape(peerList.length)}
Pending conflicts${conflictCount > 0 ? `${escape(conflictCount)}` : "0"}
+

Ops by type

+ ${opsByKindRows}
op_typecount
+

Entities tracked

+ ${entRows}
kindcount
+ `; + res.type("text/html").send(layout("dev-deploy dashboard", body, flashMsg(req))); +}; + + +// ---------------- Ops viewer ---------------- + +const fetchOps = async (limit, since, offset) => { + const schema = db.getTenantSchemaPrefix(); + let sql = `SELECT op_id, source_env_id, op_type, entity_kind, entity_uuid, payload, parent_op_id, correlation_id, schema_version, created_at, applied_at, status FROM ${schema}_dd_ops`; + const params = []; + if (since) { + const anchor = (await db.query(`SELECT created_at FROM ${schema}_dd_ops WHERE op_id = $1`, [since])).rows[0]; + if (anchor) { + sql += ` WHERE created_at > $${params.length + 1}`; + params.push(anchor.created_at); + } + } + sql += ` ORDER BY created_at DESC LIMIT $${params.length + 1}`; + params.push(limit || 100); + sql += ` OFFSET $${params.length + 1}`; + params.push(offset || 0); + return (await db.query(sql, params)).rows; +}; + + +const opsView = async (req, res) => { + if (!isAdmin(req)) { res.status(403).type("text/plain").send("admin only"); return; } + const wantJson = (req.headers.accept || "").includes("application/json"); + const limit = Math.min(parseInt(req.query.limit || "100", 10) || 100, 500); + const offset = Math.max(parseInt(req.query.offset || "0", 10) || 0, 0); + const since = req.query.since; + const ops = await fetchOps(limit, since, offset); + if (wantJson) { res.json({ ops: ops }); return; } + const rows = ops.length === 0 + ? `Journal is empty` + : ops.map((o) => ` + ${escape(o.op_id.slice(0, 8))} + ${escape(o.op_type)} + ${escape((o.entity_uuid || "").slice(0, 8))} + ${escape((o.parent_op_id || "").slice(0, 8))} + ${escape(o.status)} + ${escape(o.created_at)} + +
+ ${csrfField(req)} + + +
+ + 
${escape(o.payload)}
+ `).join(""); + const qs = (off) => { + const parts = [`offset=${off}`, `limit=${limit}`]; + if (since) { parts.push(`since=${encodeURIComponent(since)}`); } + return "/admin/dev-deploy/ops?" + parts.join("&"); + }; + const prevLink = offset > 0 + ? `« Prev` + : `« Prev`; + const nextLink = ops.length === limit + ? `Next »` + : `Next »`; + const body = ` +

Journal

+

Showing up to ${escape(limit)} ops${since ? `, since op ${escape(since.slice(0, 8))}` : ""}, offset ${escape(offset)}. Newest first. Revert appends a compensating op rather than rewriting history.

+ + + ${rows} +
opop_typeentityparentstatuscreatedactionspayload
+

${prevLink}   ${nextLink}

+ `; + res.type("text/html").send(layout("dev-deploy journal", body, flashMsg(req))); +}; + + +// ---------------- Peers ---------------- + +const peersView = async (req, res) => { + if (!isAdmin(req)) { res.status(403).type("text/plain").send("admin only"); return; } + const env = await getEnv(); + const list = await peers.listPeers(); + const rows = list.length === 0 + ? `No peers yet` + : list.map((p) => ` + ${escape(p.peer_id)} + ${escape(p.label || "(unset)")} + ${escape(p.env_id)} + ${escape(p.base_url)} + ${escape(p.last_seen_at || "never")} + +
+ ${csrfField(req)} + + +
+
+ ${csrfField(req)} + + +
+
+ ${csrfField(req)} + + +
+
+ ${csrfField(req)} + + +
+ + `).join(""); + + const body = ` +

Peers

+

This instance's env_id is ${escape(env ? env.env_id : "?")}. Paste this into the other instance's peer form.

+ + + ${rows} +
idlabelenv_idbase_urllast seenactions
+ +

Add peer

+
+ ${csrfField(req)} +
+ Peer info +

+

+

+

+
+
+ Shared secret +

Leave blank to generate one (shown once after submit). Paste the same secret in the peer's own pairing form.

+

+
+

+
+ `; + res.type("text/html").send(layout("dev-deploy peers", body, flashMsg(req))); +}; + + +const peersAdd = async (req, res) => { + if (!isAdmin(req)) { res.status(403).type("text/plain").send("admin only"); return; } + try { + const envId = (req.body.env_id || "").trim(); + const label = (req.body.label || "").trim() || null; + const baseUrl = (req.body.base_url || "").trim(); + const requireTls = !!req.body.require_tls; + const provided = (req.body.existing_secret || "").trim(); + let existingSecret = null; + if (provided) { + if (!/^[0-9a-fA-F]{64}$/.test(provided)) { + throw new Error("existing_secret must be 64 hex characters"); + } + existingSecret = Buffer.from(provided, "hex"); + } + const { peer, secret } = await peers.addPeer({ envId: envId, label: label, baseUrl: baseUrl, requireTls: requireTls, existingSecret: existingSecret }); + const secretHex = secret.toString("hex"); + const body = ` +

Peer ${escape(peer.label || peer.env_id)} paired

+

Copy this secret into the peer's pairing form (it will not be shown again):

+

${escape(secretHex)}

+

If this is a brand-new pairing, give the peer this side's env_id too.

+

Back to peers

+ `; + res.type("text/html").send(layout("Peer paired", body)); + } catch (err) { + res.redirect("/admin/dev-deploy/peers?err=" + encodeURIComponent(err.message)); + } +}; + + +const peersRotate = async (req, res) => { + if (!isAdmin(req)) { res.status(403).type("text/plain").send("admin only"); return; } + try { + const peerId = parseInt(req.body.peer_id, 10); + const { peer, secret } = await peers.rotatePeerSecret(peerId); + const body = ` +

Peer ${escape(peer.label || peer.env_id)} secret rotated

+

New secret (shown once):

+

${escape(secret.toString("hex"))}

+

Paste this on the other side via Rotate or by re-pairing.

+

Back to peers

+ `; + res.type("text/html").send(layout("Secret rotated", body)); + } catch (err) { + res.redirect("/admin/dev-deploy/peers?err=" + encodeURIComponent(err.message)); + } +}; + + +const peersDelete = async (req, res) => { + if (!isAdmin(req)) { res.status(403).type("text/plain").send("admin only"); return; } + try { + const peerId = parseInt(req.body.peer_id, 10); + await peers.deletePeer(peerId); + res.redirect("/admin/dev-deploy/peers?msg=" + encodeURIComponent("peer deleted")); + } catch (err) { + res.redirect("/admin/dev-deploy/peers?err=" + encodeURIComponent(err.message)); + } +}; + + +// ---------------- Plan + promote ---------------- + +const planView = async (req, res) => { + if (!isAdmin(req)) { res.status(403).type("text/plain").send("admin only"); return; } + const peerIdRaw = req.query.peer; + const peerList = await peers.listPeers(); + if (!peerIdRaw) { + const opts = peerList.length === 0 + ? `` + : peerList.map((p) => ``).join(""); + const body = ` +

Plan

+
+ + +
+ `; + res.type("text/html").send(layout("dev-deploy plan", body, flashMsg(req))); + return; + } + const peerId = parseInt(peerIdRaw, 10); + const peer = await peers.findPeer(peerId); + if (!peer) { res.status(404).send("peer not found"); return; } + const limit = Math.min(parseInt(req.query.limit || "100", 10) || 100, 500); + const offset = Math.max(parseInt(req.query.offset || "0", 10) || 0, 0); + const env = await getEnv(); + // Anchor: the last_op_id we sent outbound to this peer (or epoch) + const anchor = await db.selectMaybeOne("_dd_anchors", { peer_id: peerId, direction: "outbound" }); + const schema = db.getTenantSchemaPrefix(); + let sql = `SELECT op_id, op_type, entity_kind, entity_uuid, payload, parent_op_id, status, created_at FROM ${schema}_dd_ops WHERE source_env_id = $1`; + const params = [env.env_id]; + if (anchor) { + const anchorRow = await db.selectMaybeOne("_dd_ops", { op_id: anchor.last_op_id }); + if (anchorRow) { + sql += ` AND created_at > $${params.length + 1}`; + params.push(anchorRow.created_at); + } + } + sql += ` ORDER BY created_at ASC LIMIT $${params.length + 1}`; + params.push(limit); + sql += ` OFFSET $${params.length + 1}`; + params.push(offset); + const planRows = (await db.query(sql, params)).rows; + const rowsHtml = planRows.length === 0 + ? `No new ops to send` + : planRows.map((o) => ` + ${escape(o.op_id.slice(0, 8))} + ${escape(o.op_type)} + ${escape((o.entity_uuid || "").slice(0, 8))} + ${escape(o.status)} + ${escape(o.created_at)} + `).join(""); + const body = ` +

Plan: promote to ${escape(peer.label || peer.env_id)}

+

Anchor: ${anchor ? `${escape(anchor.last_op_id.slice(0, 8))}` : '(none — will send from epoch)'}

+

Ops that would be sent: ${escape(planRows.length)}

+ + + ${rowsHtml} +
opop_typeentitystatuscreated
+
+ ${csrfField(req)} + +

+
+ `; + res.type("text/html").send(layout("dev-deploy plan", body, flashMsg(req))); +}; + + +const promote = async (req, res) => { + if (!isAdmin(req)) { res.status(403).type("text/plain").send("admin only"); return; } + try { + const peerId = parseInt(req.body.peer_id, 10); + const peer = await peers.findPeer(peerId); + if (!peer) throw new Error(`peer ${peerId} not found`); + const env = await getEnv(); + const anchor = await getOutboundAnchor(peerId); + const schema = db.getTenantSchemaPrefix(); + let sql = `SELECT op_id, source_env_id, op_type, entity_kind, entity_uuid, payload, parent_op_id, correlation_id, schema_version, created_at, status FROM ${schema}_dd_ops WHERE source_env_id = $1`; + const params = [env.env_id]; + if (anchor) { + const anchorRow = await db.selectMaybeOne("_dd_ops", { op_id: anchor.last_op_id }); + if (anchorRow) { + sql += ` AND created_at > $${params.length + 1}`; + params.push(anchorRow.created_at); + } + } + sql += ` ORDER BY created_at ASC LIMIT 500`; + const ops = (await db.query(sql, params)).rows; + if (ops.length === 0) { + res.redirect("/admin/dev-deploy/peers?msg=" + encodeURIComponent("no ops to promote")); + return; + } + const secret = await peers.peerSecret(peerId); + const r = await signedFetch({ + baseUrl: peer.base_url, + method: "POST", + path: "/dev-deploy/api/ingest", + body: { ops: ops }, + sourceEnvId: env.env_id, + secret: secret, + requireTls: peer.require_tls + }); + if (!r.ok) { + throw new Error(`peer responded ${r.status}: ${JSON.stringify(r.body)}`); + } + await upsertAnchor(peerId, "outbound", ops[ops.length - 1].op_id); + const applied = (r.body && r.body.results || []).filter((x) => x.status === "applied").length; + const errors = (r.body && r.body.results || []).filter((x) => x.status === "error").length; + let msg = `promoted ${ops.length} ops (${applied} applied, ${errors} errors)`; + const localPlugins = (await db.query(`SELECT name, source, version FROM _sc_plugins ORDER BY name`)).rows; + const warnings = await diffPluginsWithPeer(peer, env, localPlugins); + if (warnings.length > 0) { + msg += " | WARNINGS: " + warnings.join("; "); + } + res.redirect("/admin/dev-deploy/peers?msg=" + encodeURIComponent(msg)); + } catch (err) { + res.redirect("/admin/dev-deploy/peers?err=" + encodeURIComponent(err.message)); + } +}; + + +const pull = async (req, res) => { + if (!isAdmin(req)) { res.status(403).type("text/plain").send("admin only"); return; } + try { + const peerId = parseInt(req.body.peer_id, 10); + const peer = await peers.findPeer(peerId); + if (!peer) throw new Error(`peer ${peerId} not found`); + const env = await getEnv(); + const secret = await peers.peerSecret(peerId); + + // apiJournal caps each response at 1000 ops, so loop: fetch from the + // advancing inbound anchor, apply, advance the anchor, repeat until a + // fetch returns 0 ops (drained) or we hit the safety cap. + let total = 0; + let applied = 0; + let errors = 0; + let conflicts = 0; + let iters = 0; + let cappedOut = false; + for (;;) { + if (iters >= PULL_MAX_ITERS) { + cappedOut = true; + break; + } + iters += 1; + const anchor = await getInboundAnchor(peerId); + const since = anchor ? anchor.last_op_id : null; + const path = since ? `/dev-deploy/api/journal?since=${encodeURIComponent(since)}` : "/dev-deploy/api/journal"; + const r = await signedFetch({ + baseUrl: peer.base_url, + method: "GET", + path: path, + body: null, + sourceEnvId: env.env_id, + secret: secret, + requireTls: peer.require_tls + }); + if (!r.ok) { + throw new Error(`peer responded ${r.status}: ${JSON.stringify(r.body)}`); + } + const ops = (r.body && r.body.ops) || []; + if (ops.length === 0) { + break; + } + const results = await applyBatch(ops, { peerId: peerId, myEnvId: env.env_id }); + applied += results.filter((x) => x.status === "applied").length; + errors += results.filter((x) => x.status === "error").length; + conflicts += results.filter((x) => x.status === "conflict").length; + total += ops.length; + await upsertAnchor(peerId, "inbound", ops[ops.length - 1].op_id); + } + if (total === 0) { + res.redirect("/admin/dev-deploy/peers?msg=" + encodeURIComponent("nothing to pull")); + return; + } + let sum = `pulled ${total} ops (${applied} applied, ${errors} errors, ${conflicts} conflicts)`; + if (cappedOut) { + sum += " (stopped at safety cap; pull again)"; + } + const localPlugins = (await db.query(`SELECT name, source, version FROM _sc_plugins ORDER BY name`)).rows; + const warnings = await diffPluginsWithPeer(peer, env, localPlugins); + if (warnings.length > 0) { + sum += " | WARNINGS: " + warnings.join("; "); + } + const dest = conflicts > 0 ? "/admin/dev-deploy/conflicts?msg=" : "/admin/dev-deploy/peers?msg="; + res.redirect(dest + encodeURIComponent(sum)); + } catch (err) { + res.redirect("/admin/dev-deploy/peers?err=" + encodeURIComponent(err.message)); + } +}; + + +// ---------------- Conflicts ---------------- + +const conflictsView = async (req, res) => { + if (!isAdmin(req)) { res.status(403).type("text/plain").send("admin only"); return; } + + const schema = db.getTenantSchemaPrefix(); + const conflicts = (await db.query(` + SELECT i.op_id AS i_op_id, + i.source_env_id AS i_source, + i.op_type AS i_op_type, + i.entity_kind AS i_kind, + i.entity_uuid AS i_uuid, + i.payload AS i_payload, + i.created_at AS i_created, + l.op_id AS l_op_id, + l.op_type AS l_op_type, + l.payload AS l_payload, + l.applied_at AS l_applied + FROM ${schema}_dd_ops i + LEFT JOIN ${schema}_dd_ops l ON l.op_id = i.conflict_with_op_id + WHERE i.status = 'conflict' + ORDER BY i.created_at ASC + `)).rows; + + const isMergeable = (c) => + c.i_op_type && c.l_op_type && + c.i_op_type.startsWith("update_") && + c.l_op_type.startsWith("update_") && + c.i_op_type === c.l_op_type; + + const rowsHtml = conflicts.length === 0 + ? `No pending conflicts` + : conflicts.map((c) => ` + + incoming ${escape(c.i_op_id.slice(0, 8))} from ${escape((c.i_source || "").slice(0, 8))}
+ ${escape(c.i_op_type)} on entity ${escape((c.i_uuid || "").slice(0, 8))}
+ created ${escape(c.i_created)} + 
${escape(c.i_payload)}
+ + + ${c.l_op_id ? `local ${escape(c.l_op_id.slice(0, 8))}
+ ${escape(c.l_op_type)}
+ applied ${escape(c.l_applied || "")} + 
${escape(c.l_payload)}
` : '(no local op recorded)'} + + + ${isMergeable(c) ? `

` : ""} +
+ ${csrfField(req)} + + + +
+
+ ${csrfField(req)} + + + +
+ + `).join(""); + + const body = ` +

Pending conflicts

+

A conflict means an incoming op and a local op both touched the same entity since the last sync. The incoming op was NOT applied; pick which version wins.

+
    +
  • Use theirs: applies the incoming op now (overwrites local change). The local op stays in the journal but its effect is overwritten.
    • +
  • Use mine: marks the incoming op as rejected. The local state stands. The peer may re-send the op on future syncs; subsequent pulls will skip it via idempotency.
    • +
+ + + ${rowsHtml} +
TheirsMineAction
+ `; + res.type("text/html").send(layout("dev-deploy conflicts", body, flashMsg(req))); +}; + + +const conflictsResolve = async (req, res) => { + if (!isAdmin(req)) { res.status(403).type("text/plain").send("admin only"); return; } + try { + const opId = (req.body.op_id || "").trim(); + const action = (req.body.action || "").trim(); + if (!opId) throw new Error("op_id required"); + if (!["theirs", "mine"].includes(action)) throw new Error("action must be 'theirs' or 'mine'"); + // For file ops, "use theirs" re-applies create_file which fetches bytes + // from the originating peer -- resolve the peer from the op's source env. + const op = await db.selectMaybeOne("_dd_ops", { op_id: opId }); + const peer = op ? await peers.findPeerByEnvId(op.source_env_id) : null; + const env = await getEnv(); + const opts = { + peerId: peer ? peer.peer_id : null, + myEnvId: env ? env.env_id : null + }; + const r = await resolveConflict(opId, action, opts); + res.redirect("/admin/dev-deploy/conflicts?msg=" + encodeURIComponent(`resolved ${opId.slice(0, 8)} with action=${action}: ${JSON.stringify(r)}`)); + } catch (err) { + res.redirect("/admin/dev-deploy/conflicts?err=" + encodeURIComponent(err.message)); + } +}; + + +const renderValue = (v) => { + if (v === null || v === undefined) { + return '(unset)'; + } + if (typeof v === "object") { + return `
${escape(JSON.stringify(v, null, 2))}
`; + } + return escape(String(v)); +}; + + +const conflictsMergeView = async (req, res) => { + if (!isAdmin(req)) { res.status(403).type("text/plain").send("admin only"); return; } + const opId = (req.query.op_id || "").trim(); + if (!opId) { res.redirect("/admin/dev-deploy/conflicts?err=op_id+required"); return; } + try { + const op = await db.selectMaybeOne("_dd_ops", { op_id: opId }); + if (!op) throw new Error(`op ${opId} not found`); + if (op.status !== "conflict") throw new Error(`op ${opId} is not in conflict status`); + + const diff = await conflictFieldDiff(op); + const diffs = diff.diffs || []; + + const ent = diff.instance + ? `

Entity: ${escape(diff.kind)} ${escape(diff.instance.name || diff.instance.role || diff.instance.id)} (local id ${escape(diff.instance.id)})

` + : `

${escape(diff.reason || "no entity diff available")}

`; + + let formBody; + if (diffs.length === 0) { + formBody = `

No field-level differences detected (current state already matches the incoming op's patch on every field). Applying the merge will just mark this conflict as resolved.

`; + } else { + const rows = diffs.map((d) => ` + ${escape(d.field)} + ${renderValue(d.currentValue)} + ${renderValue(d.incomingValue)} + +
+
+ + + `).join(""); + formBody = ` + + + ${rows} +
fieldcurrent (mine)incoming (theirs)resolution
+

Defaults to keep current for every field — submit as-is for a no-op resolution that just clears the conflict marker.

+ `; + } + + const body = ` +

Merge conflict per field

+ ${ent} +

Incoming op ${escape(op.op_id.slice(0, 8))} from ${escape((op.source_env_id || "").slice(0, 8))}; ${escape(op.op_type)}.

+
+ ${csrfField(req)} + + ${formBody} +

+ + Cancel +

+
+ `; + res.type("text/html").send(layout("dev-deploy merge", body, flashMsg(req))); + } catch (err) { + res.redirect("/admin/dev-deploy/conflicts?err=" + encodeURIComponent(err.message)); + } +}; + + +// Parse number/boolean/null literal strings from a text input. JSON.parse first +// (handles "true", "42", "null"); fall back to the raw string. +const coerce = (s) => { + if (s === undefined || s === null) return s; + try { + const parsed = JSON.parse(s); + if (parsed === null || ["boolean", "number"].includes(typeof parsed)) return parsed; + return s; + } catch (e) { + return s; + } +}; + + +const conflictsMergeApply = async (req, res) => { + if (!isAdmin(req)) { res.status(403).type("text/plain").send("admin only"); return; } + try { + const opId = (req.body.op_id || "").trim(); + if (!opId) throw new Error("op_id required"); + + const op = await db.selectMaybeOne("_dd_ops", { op_id: opId }); + if (!op) throw new Error(`op ${opId} not found`); + if (op.status !== "conflict") throw new Error(`op ${opId} is not in conflict status`); + const payload = typeof op.payload === "string" ? JSON.parse(op.payload) : (op.payload || {}); + const incomingPatch = payload.patch || {}; + + // For each "choice_" entry in the form body, decide what value + // to write -- if any. "current" means don't touch the field. + const choices = {}; + for (const [k, v] of Object.entries(req.body || {})) { + if (!k.startsWith("choice_")) continue; + const field = k.substring("choice_".length); + if (v === "incoming") { + choices[field] = incomingPatch[field]; + } else if (v === "custom") { + const customVal = req.body[`custom_${field}`]; + choices[field] = coerce(customVal); + } + } + + const r = await resolveConflictByMerge(opId, choices); + res.redirect("/admin/dev-deploy/conflicts?msg=" + encodeURIComponent(`merged ${opId.slice(0, 8)}: ${JSON.stringify(r)}`)); + } catch (err) { + res.redirect("/admin/dev-deploy/conflicts?err=" + encodeURIComponent(err.message)); + } +}; + + +// ---------------- Tables (data_mode) ---------------- + +const tablesView = async (req, res) => { + if (!isAdmin(req)) { res.status(403).type("text/plain").send("admin only"); return; } + const schema = db.getTenantSchemaPrefix(); + const rows = (await db.query(` + SELECT e.uuid, e.current_name, e.current_id, + COALESCE(m.data_mode, 'user') AS data_mode, + m.updated_at, + m.starter_shipped_at + FROM ${schema}_dd_entity_ids e + LEFT JOIN ${schema}_dd_table_modes m ON m.table_uuid = e.uuid + WHERE e.kind = 'table' + ORDER BY e.current_name + `)).rows; + + const lockedNames = new Set(["users"]); + const modeOpts = [DATA_MODES.USER, DATA_MODES.STARTER, DATA_MODES.MANAGED]; + + const rowsHtml = rows.length === 0 + ? `No tables tracked yet` + : rows.map((r) => { + const locked = lockedNames.has(r.current_name); + const select = locked + ? `user (locked)` + : `
+ ${csrfField(req)} + + + +
`; + const shipped = r.starter_shipped_at ? `
shipped ${escape(r.starter_shipped_at)}` : ""; + return ` + ${escape(r.current_name)} + ${escape(r.uuid.slice(0, 8))} + ${escape(r.current_id)} + ${select}${shipped} + ${escape(r.updated_at || "—")} + `; + }).join(""); + + const body = ` +

Tables — data mode

+

Controls how each table's row content propagates between environments. Choose carefully — switching from user to managed or starter rewrites the table's schema (adds a hidden _dd_row_uuid column) and ships existing rows.

+
    +
  • user (default) — rows belong to the local environment; deploys never touch them. The only safe choice for end-user-entered data.
    • +
  • starter — rows ship to target on first install, then the target owns them; future changes on this side don't propagate. Good for default user roles, sample categories, template data the user expects to customize.
    • +
  • managed — rows always sync from source. Source is canonical; target's edits get overwritten or surface as conflicts. Good for catalogs, lookup tables, anything dev-curated.
    • +
+

The Saltcorn users table is locked to user and cannot be changed.

+ + + ${rowsHtml} +
tableuuidlocal iddata_modeupdated_at
+ `; + res.type("text/html").send(layout("dev-deploy tables", body, flashMsg(req))); +}; + + +const tablesSet = async (req, res) => { + if (!isAdmin(req)) { res.status(403).type("text/plain").send("admin only"); return; } + try { + const tableUuid = (req.body.table_uuid || "").trim(); + const dataMode = (req.body.data_mode || "").trim(); + if (!tableUuid) throw new Error("table_uuid required"); + const allowed = new Set(Object.values(DATA_MODES)); + if (!allowed.has(dataMode)) throw new Error(`data_mode must be one of ${[...allowed].join(", ")}`); + const ent = await db.selectMaybeOne("_dd_entity_ids", { uuid: tableUuid }); + if (!ent || ent.kind !== "table") throw new Error("table not found"); + if (ent.current_name === "users") throw new Error("the users table is locked to data_mode=user"); + + const { ensureManagedSchema, dropManagedSchema, allRowsWithUuid, setRowUuid, newRowUuid, COLUMN_NAME: ROW_UUID_COL } = require("./rowIdentity"); + const { rowToPortable, markStarterShipped } = require("./rowPayload"); + const Table = require("@saltcorn/data/models/table"); + const { randomUuid } = require("./ids"); + const { enterOp } = require("./context"); + const { recordOpSafely } = require("./ops"); + const { refreshState } = require("./state"); + + await refreshState(); + + const prior = await db.selectMaybeOne("_dd_table_modes", { table_uuid: tableUuid }); + const now = new Date().toISOString(); + + // Upsert mode row first. + if (prior) { + await db.updateWhere("_dd_table_modes", { data_mode: dataMode, updated_at: now, starter_shipped_at: null }, { table_uuid: tableUuid }); + } else { + await db.insert("_dd_table_modes", { table_uuid: tableUuid, data_mode: dataMode, updated_at: now, starter_shipped_at: null }, { noid: true }); + } + + // Journal set_table_mode FIRST so target's apply sees mode change before any row ops. + { + const opId = randomUuid(); + await enterOp(opId, async () => { + await recordOpSafely({ + op_id: opId, + op_type: "set_table_mode", + entity_kind: "table_mode", + entity_uuid: tableUuid, + payload: { table_uuid: tableUuid, data_mode: dataMode, before_mode: (prior && prior.data_mode) || DATA_MODES.USER } + }); + }); + } + + let initialShipped = 0; + if (dataMode === DATA_MODES.MANAGED || dataMode === DATA_MODES.STARTER) { + // Make sure THIS instance has the hidden column + UUIDs on existing rows. + await ensureManagedSchema(ent.current_name); + + // Initial ship: journal an insert_row op for every existing row. + const table = Table.findOne({ id: ent.current_id }); + if (table) { + const rows = await allRowsWithUuid(ent.current_name); + for (const row of rows) { + let rowUuid = row[ROW_UUID_COL]; + if (!rowUuid) { + rowUuid = newRowUuid(); + await setRowUuid(ent.current_name, row.id, rowUuid); + } + const { portable } = await rowToPortable(row, table); + const opId = randomUuid(); + await enterOp(opId, async () => { + await recordOpSafely({ + op_id: opId, + op_type: "insert_row", + entity_kind: "table_row", + entity_uuid: rowUuid, + payload: { table_uuid: tableUuid, after: portable } + }); + }); + initialShipped++; + } + } + // For starter: lock out further row ops. + if (dataMode === DATA_MODES.STARTER) { + await markStarterShipped(tableUuid); + } + } else if (prior && (prior.data_mode === DATA_MODES.MANAGED || prior.data_mode === DATA_MODES.STARTER)) { + // Reverting to user — drop the hidden column for cleanliness. Best-effort. + try { + await dropManagedSchema(ent.current_name); + } catch (e) { + // ignore on older SQLite that doesn't support DROP COLUMN + } + } + + const summary = initialShipped > 0 + ? `set ${ent.current_name} to ${dataMode}; shipped ${initialShipped} rows` + : `set ${ent.current_name} to ${dataMode}`; + res.redirect("/admin/dev-deploy/tables?msg=" + encodeURIComponent(summary)); + } catch (err) { + res.redirect("/admin/dev-deploy/tables?err=" + encodeURIComponent(err.message)); + } +}; + + +// ---------------- Revert ---------------- + +const revertView = async (req, res) => { + if (!isAdmin(req)) { res.status(403).type("text/plain").send("admin only"); return; } + try { + const opId = (req.body.op_id || "").trim(); + if (!opId) throw new Error("op_id required"); + const result = await revertOp(opId); + res.redirect("/admin/dev-deploy/ops?msg=" + encodeURIComponent(`reverted op ${opId.slice(0, 8)}: ${JSON.stringify(result)}`)); + } catch (err) { + res.redirect("/admin/dev-deploy/ops?err=" + encodeURIComponent(err.message)); + } +}; + + +// ---------------- Machine endpoints ---------------- + +const apiJournal = async (req, res) => { + const peer = await requirePeerAuth(req, res); + if (!peer) return; + const since = req.query.since; + const env = await getEnv(); + const schema = db.getTenantSchemaPrefix(); + let sql = `SELECT op_id, source_env_id, op_type, entity_kind, entity_uuid, payload, parent_op_id, correlation_id, schema_version, created_at, status FROM ${schema}_dd_ops WHERE source_env_id = $1`; + const params = [env.env_id]; + if (since) { + const anchorRow = await db.selectMaybeOne("_dd_ops", { op_id: since }); + if (anchorRow) { + sql += ` AND created_at > $${params.length + 1}`; + params.push(anchorRow.created_at); + } + } + sql += ` ORDER BY created_at ASC LIMIT 1000`; + const ops = (await db.query(sql, params)).rows; + res.json({ source_env_id: env.env_id, ops: ops }); +}; + + +const apiHealth = async (req, res) => { + const peer = await requirePeerAuth(req, res); + if (!peer) return; + const env = await getEnv(); + const plugins = (await db.query(`SELECT name, source, version FROM _sc_plugins ORDER BY name`)).rows; + res.json({ + env_id: env.env_id, + label: env.env_label, + plugins: plugins + }); +}; + + +// Compare local plugin list with peer's. Returns array of human-readable +// warning strings (empty if all match). Best-effort: if the peer's health +// endpoint is unreachable or returns non-200, returns a single "couldn't +// reach peer's health endpoint" warning and lets the caller proceed. +const diffPluginsWithPeer = async (peerRow, env, localPlugins) => { + let r; + try { + const secret = await peers.peerSecret(peerRow.peer_id); + r = await signedFetch({ + baseUrl: peerRow.base_url, + method: "GET", + path: "/dev-deploy/api/health", + body: null, + sourceEnvId: env.env_id, + secret: secret, + requireTls: peerRow.require_tls + }); + } catch (e) { + return [`could not check peer plugin list: ${e.message}`]; + } + if (!r.ok || !r.body || !Array.isArray(r.body.plugins)) { + return [`peer's health endpoint returned ${r.status}`]; + } + const localByName = new Map(localPlugins.map((p) => [p.name, p])); + const peerByName = new Map(r.body.plugins.map((p) => [p.name, p])); + const warnings = []; + for (const [name, mine] of localByName) { + const theirs = peerByName.get(name); + if (!theirs) { + warnings.push(`peer missing plugin "${name}"`); + } else if ((mine.version || "") !== (theirs.version || "")) { + warnings.push(`plugin version mismatch on "${name}": local ${mine.version || "?"}, peer ${theirs.version || "?"}`); + } + } + for (const [name, theirs] of peerByName) { + if (!localByName.has(name)) { + warnings.push(`peer has plugin not installed here: "${name}"`); + } + } + return warnings; +}; + + +const apiFile = async (req, res) => { + try { + const peer = await requirePeerAuth(req, res); + if (!peer) return; + const uuid = req.params.uuid; + if (!uuid) { + res.status(400).json({ error: "uuid required" }); + return; + } + const mapping = await db.selectMaybeOne("_dd_entity_ids", { uuid: uuid, kind: "file" }); + if (!mapping) { + res.status(404).json({ error: "file not found", uuid: uuid }); + return; + } + const path = require("path"); + const dbMod = require("@saltcorn/data/db"); + const absPath = path.join(dbMod.connectObj.file_store, dbMod.getTenantSchema(), mapping.current_name); + res.type("application/octet-stream"); + // dotfiles: 'allow' so paths containing .dev-state (etc.) aren't + // silently treated as not-found by Express's default dotfile policy. + res.sendFile(absPath, { dotfiles: "allow" }, (err) => { + if (err && !res.headersSent) { + // eslint-disable-next-line no-console + console.error(`[dev-deploy] sendFile failed for ${absPath}:`, err.message); + res.status(500).json({ error: "failed to read file: " + err.message, path: absPath }); + } + }); + } catch (err) { + // eslint-disable-next-line no-console + console.error(`[dev-deploy] apiFile crashed:`, err && err.stack ? err.stack : err); + if (!res.headersSent) { + res.status(500).json({ error: err.message }); + } + } +}; + + +const apiIngest = async (req, res) => { + const peer = await requirePeerAuth(req, res); + if (!peer) return; + const ops = (req.body && req.body.ops) || []; + if (!Array.isArray(ops)) { + res.status(400).json({ error: "ops must be an array" }); + return; + } + const env = await getEnv(); + const results = await applyBatch(ops, { peerId: peer.peer_id, myEnvId: env.env_id }); + // Advance inbound anchor to the last op_id from the source side + if (ops.length > 0) { + const lastOp = ops[ops.length - 1]; + const now = new Date().toISOString(); + const existing = await db.selectMaybeOne("_dd_anchors", { peer_id: peer.peer_id, direction: "inbound" }); + if (existing) { + await db.updateWhere("_dd_anchors", { last_op_id: lastOp.op_id, updated_at: now }, { peer_id: peer.peer_id, direction: "inbound" }); + } else { + await db.insert("_dd_anchors", { peer_id: peer.peer_id, direction: "inbound", last_op_id: lastOp.op_id, updated_at: now }, { noid: true }); + } + } + res.json({ received: ops.length, results: results }); +}; + + +// ---------------- Route registration ---------------- + +const routes = [ + { url: "/admin/dev-deploy/", method: "get", callback: dashboard }, + { url: "/admin/dev-deploy/ops", method: "get", callback: opsView }, + { url: "/admin/dev-deploy/peers", method: "get", callback: peersView }, + { url: "/admin/dev-deploy/peers/add", method: "post", callback: peersAdd }, + { url: "/admin/dev-deploy/peers/rotate", method: "post", callback: peersRotate }, + { url: "/admin/dev-deploy/peers/delete", method: "post", callback: peersDelete }, + { url: "/admin/dev-deploy/plan", method: "get", callback: planView }, + { url: "/admin/dev-deploy/promote", method: "post", callback: promote }, + { url: "/admin/dev-deploy/pull", method: "post", callback: pull }, + { url: "/admin/dev-deploy/revert", method: "post", callback: revertView }, + { url: "/admin/dev-deploy/tables", method: "get", callback: tablesView }, + { url: "/admin/dev-deploy/tables/set", method: "post", callback: tablesSet }, + { url: "/admin/dev-deploy/conflicts", method: "get", callback: conflictsView }, + { url: "/admin/dev-deploy/conflicts/resolve", method: "post", callback: conflictsResolve }, + { url: "/admin/dev-deploy/conflicts/merge", method: "get", callback: conflictsMergeView }, + { url: "/admin/dev-deploy/conflicts/merge/apply", method: "post", callback: conflictsMergeApply }, + { url: "/dev-deploy/api/journal", method: "get", callback: apiJournal, noCsrf: true }, + { url: "/dev-deploy/api/ingest", method: "post", callback: apiIngest, noCsrf: true }, + { url: "/dev-deploy/api/file/:uuid", method: "get", callback: apiFile, noCsrf: true }, + { url: "/dev-deploy/api/health", method: "get", callback: apiHealth, noCsrf: true } +]; + + +module.exports = { + routes +}; diff --git a/lib/rowIdentity.js b/lib/rowIdentity.js new file mode 100644 index 0000000..eb0656b --- /dev/null +++ b/lib/rowIdentity.js @@ -0,0 +1,131 @@ +// Hidden _dd_row_uuid column infrastructure for managed/starter tables. +// +// When an admin marks a user table as managed or starter, this module +// adds a TEXT column _dd_row_uuid to the underlying SQL table via raw ALTER +// (NOT registered in _sc_fields, so Saltcorn's table builder doesn't show +// it). Existing rows are backfilled with random UUIDs. From that point, +// the wrap layer reads/writes _dd_row_uuid as the cross-environment identity +// for each row. + +const crypto = require("crypto"); + +const db = require("@saltcorn/data/db"); + + +const COLUMN_NAME = "_dd_row_uuid"; + + +const tableSqlRef = (tableName) => { + const schema = db.getTenantSchemaPrefix(); + return `${schema}"${db.sqlsanitize(tableName)}"`; +}; + + +const columnExists = async (tableName) => { + if (db.isSQLite) { + const rs = await db.query(`PRAGMA table_info("${db.sqlsanitize(tableName)}")`); + return rs.rows.some((r) => r.name === COLUMN_NAME); + } + // Check the tenant's OWN schema -- the same one tableSqlRef()/ALTER target. + // current_schema() is NOT reliable: Saltcorn qualifies queries with + // getTenantSchemaPrefix() rather than SET search_path, so current_schema() is + // "public" even inside a tenant. Using it made this falsely report the column + // missing on every call after the first, and the explicitly-qualified ALTER + // then failed with 'column "_dd_row_uuid" already exists' (breaking apply, + // which calls ensureManagedSchema once per set_table_mode + per insert_row). + const rs = await db.query( + `SELECT 1 FROM information_schema.columns + WHERE table_schema = $1 + AND table_name = $2 + AND column_name = $3`, + [db.getTenantSchema(), tableName, COLUMN_NAME] + ); + return rs.rows.length > 0; +}; + + +const ensureManagedSchema = async (tableName) => { + if (await columnExists(tableName)) { + return { added: false }; + } + await db.query(`ALTER TABLE ${tableSqlRef(tableName)} ADD COLUMN ${COLUMN_NAME} TEXT`); + // Backfill existing rows with random UUIDs. + const rs = await db.query(`SELECT id FROM ${tableSqlRef(tableName)} WHERE ${COLUMN_NAME} IS NULL`); + let backfilled = 0; + for (const r of rs.rows) { + await db.query( + `UPDATE ${tableSqlRef(tableName)} SET ${COLUMN_NAME} = $1 WHERE id = $2`, + [crypto.randomUUID(), r.id] + ); + backfilled++; + } + // Index for fast lookups by uuid. + await db.query( + `CREATE INDEX IF NOT EXISTS "${db.sqlsanitize(tableName)}_dd_row_uuid_idx" + ON ${tableSqlRef(tableName)} (${COLUMN_NAME})` + ).catch(() => {}); + return { added: true, backfilled: backfilled }; +}; + + +const dropManagedSchema = async (tableName) => { + if (!(await columnExists(tableName))) { + return { dropped: false }; + } + // SQLite 3.35+ and Postgres both support DROP COLUMN. If the SQLite is + // older it will throw — surface the error. + await db.query(`ALTER TABLE ${tableSqlRef(tableName)} DROP COLUMN ${COLUMN_NAME}`); + return { dropped: true }; +}; + + +const getRowUuid = async (tableName, id) => { + const rs = await db.query( + `SELECT ${COLUMN_NAME} AS uuid FROM ${tableSqlRef(tableName)} WHERE id = $1`, + [id] + ); + return rs.rows.length > 0 ? rs.rows[0].uuid : null; +}; + + +const findIdByRowUuid = async (tableName, uuid) => { + const rs = await db.query( + `SELECT id FROM ${tableSqlRef(tableName)} WHERE ${COLUMN_NAME} = $1`, + [uuid] + ); + return rs.rows.length > 0 ? rs.rows[0].id : null; +}; + + +const setRowUuid = async (tableName, id, uuid) => { + await db.query( + `UPDATE ${tableSqlRef(tableName)} SET ${COLUMN_NAME} = $1 WHERE id = $2`, + [uuid, id] + ); +}; + + +const newRowUuid = () => crypto.randomUUID(); + + +// All rows currently in the table, including their _dd_row_uuid. Used during +// the initial managed/starter ship to journal an insert_row op for each. +const allRowsWithUuid = async (tableName) => { + const rs = await db.query( + `SELECT * FROM ${tableSqlRef(tableName)} ORDER BY id ASC` + ); + return rs.rows; +}; + + +module.exports = { + COLUMN_NAME, + columnExists, + ensureManagedSchema, + dropManagedSchema, + getRowUuid, + findIdByRowUuid, + setRowUuid, + newRowUuid, + allRowsWithUuid +}; diff --git a/lib/rowPayload.js b/lib/rowPayload.js new file mode 100644 index 0000000..d0aa880 --- /dev/null +++ b/lib/rowPayload.js @@ -0,0 +1,144 @@ +// Row payload translation: FK ids -> row uuids (on journal) and back (on apply). +// +// For each FK field on the row's table (field.is_fkey), look up the referenced +// row's _dd_row_uuid. Store under "__uuid" to avoid colliding with +// the original field name. On apply, resolve back to the local row id. +// +// FKs to user-mode tables are NULLed on the target, with a warning attached +// to the payload so it surfaces in the admin UI. + +const db = require("@saltcorn/data/db"); + +const { getRowUuid, findIdByRowUuid, COLUMN_NAME } = require("./rowIdentity"); +const { lookupByCurrent } = require("./entityIds"); + + +const UUID_SUFFIX = "__uuid"; + + +// Returns data_mode for the table with given current_id (Saltcorn table id). +// 'user' (default), 'starter', or 'managed'. +const tableModeByCurrentId = async (tableId) => { + const ent = await lookupByCurrent("table", tableId); + if (!ent) return "user"; + const row = await db.selectMaybeOne("_dd_table_modes", { table_uuid: ent.uuid }); + return row ? row.data_mode : "user"; +}; + + +const tableModeByUuid = async (tableUuid) => { + if (!tableUuid) return "user"; + const row = await db.selectMaybeOne("_dd_table_modes", { table_uuid: tableUuid }); + return row ? row.data_mode : "user"; +}; + + +// rowData: a plain row from the table. +// table: the Saltcorn Table instance whose fields define the FK shape. +// Returns { portable: {...with __uuid keys for FKs}, warnings: [...] }. +const rowToPortable = async (rowData, table) => { + const portable = {}; + const warnings = []; + for (const field of table.fields || []) { + if (field.name === COLUMN_NAME || field.name === "id") continue; + const v = rowData[field.name]; + if (field.is_fkey && field.reftable_name && v !== null && v !== undefined) { + const refMode = await tableModeByCurrentId_byName(field.reftable_name); + if (refMode === "managed" || refMode === "starter") { + const refUuid = await getRowUuid(field.reftable_name, v); + if (refUuid) { + portable[field.name + UUID_SUFFIX] = refUuid; + } else { + portable[field.name + UUID_SUFFIX] = null; + warnings.push(`${field.name}: source row references ${field.reftable_name}.id=${v} but it has no _dd_row_uuid yet`); + } + } else { + // FK into a user-mode table — can't translate. Drop and warn. + portable[field.name + UUID_SUFFIX] = null; + warnings.push(`${field.name} → user-mode table ${field.reftable_name}; FK will be null on target`); + } + } else { + portable[field.name] = v; + } + } + return { portable: portable, warnings: warnings }; +}; + + +// portable: payload from a journaled row op. +// table: the Saltcorn Table instance on the local (target) side. +const portableToRow = async (portable, table) => { + const row = {}; + for (const field of table.fields || []) { + if (field.name === COLUMN_NAME || field.name === "id") continue; + const uuidKey = field.name + UUID_SUFFIX; + if (uuidKey in portable) { + const refUuid = portable[uuidKey]; + if (!refUuid) { + row[field.name] = null; + } else if (field.is_fkey && field.reftable_name) { + const localId = await findIdByRowUuid(field.reftable_name, refUuid); + row[field.name] = localId; // may be null if target doesn't have that row yet + } else { + row[field.name] = null; + } + } else if (field.name in portable) { + row[field.name] = portable[field.name]; + } + } + return row; +}; + + +// Helper: lookup mode by table name (via entity_ids). +const tableModeByCurrentId_byName = async (tableName) => { + const Table = require("@saltcorn/data/models/table"); + const t = Table.findOne({ name: tableName }); + if (!t) return "user"; + return await tableModeByCurrentId(t.id); +}; + + +// True if a starter table has already had its initial-ship completed; managed +// tables ignore this (they always journal). +const isStarterShipped = async (tableUuid) => { + const row = await db.selectMaybeOne("_dd_table_modes", { table_uuid: tableUuid }); + return !!(row && row.starter_shipped_at); +}; + + +const markStarterShipped = async (tableUuid) => { + await db.updateWhere("_dd_table_modes", { starter_shipped_at: new Date().toISOString() }, { table_uuid: tableUuid }); +}; + + +// For a given Saltcorn Table id, returns { mode, tableUuid, shouldJournal }. +// shouldJournal is the wrap-level decision: true → record the row op; false → +// pass through silently. +const journalDecision = async (tableId) => { + const { lookupByCurrent } = require("./entityIds"); + const ent = await lookupByCurrent("table", tableId); + if (!ent) return { mode: "user", tableUuid: null, shouldJournal: false }; + const mode = await tableModeByUuid(ent.uuid); + if (mode === "managed") { + return { mode: mode, tableUuid: ent.uuid, shouldJournal: true }; + } + if (mode === "starter") { + const shipped = await isStarterShipped(ent.uuid); + return { mode: mode, tableUuid: ent.uuid, shouldJournal: !shipped }; + } + return { mode: "user", tableUuid: ent.uuid, shouldJournal: false }; +}; + + +module.exports = { + rowToPortable, + portableToRow, + tableModeByCurrentId, + tableModeByUuid, + tableModeByCurrentId_byName, + isStarterShipped, + markStarterShipped, + journalDecision, + UUID_SUFFIX +}; diff --git a/lib/schema.js b/lib/schema.js new file mode 100644 index 0000000..906ad1e --- /dev/null +++ b/lib/schema.js @@ -0,0 +1,161 @@ +// DDL for dev-deploy's six plugin tables. +// +// Saltcorn supports SQLite and PostgreSQL. We use a portable subset: +// - TEXT for all strings, JSON payloads, ISO 8601 timestamps +// - INTEGER for booleans (0/1) and surrogate ids +// - No JSONB; payloads are TEXT containing JSON, parsed/stringified at the +// application layer +// +// Tables are created idempotently in onLoad. Re-running is safe. + +const db = require("@saltcorn/data/db"); + + +const createDdEnv = async () => { + const schema = db.getTenantSchemaPrefix(); + await db.query(` + CREATE TABLE IF NOT EXISTS ${schema}_dd_env ( + env_id TEXT PRIMARY KEY, + env_label TEXT, + on_destructive_op TEXT NOT NULL DEFAULT 'confirm', + require_tls INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL, + bootstrapped_at TEXT + ) + `); +}; + + +const createDdPeers = async () => { + // Secret components stored as hex TEXT rather than BLOB: Saltcorn's + // SQLite insert layer JSON-stringifies any object value, which would + // mangle Buffer columns. Hex is portable and survives that path intact. + const schema = db.getTenantSchemaPrefix(); + // Portable auto-increment PK: sqlite "integer primary key" auto-assigns + // rowids; postgres uses "serial" (AUTOINCREMENT is sqlite-only syntax). + const serial = db.isSQLite ? "integer" : "serial"; + await db.query(` + CREATE TABLE IF NOT EXISTS ${schema}_dd_peers ( + peer_id ${serial} PRIMARY KEY, + env_id TEXT NOT NULL UNIQUE, + label TEXT, + base_url TEXT NOT NULL, + peer_secret_ciphertext TEXT, + peer_secret_iv TEXT, + peer_secret_tag TEXT, + require_tls INTEGER, + created_at TEXT NOT NULL, + last_seen_at TEXT + ) + `); +}; + + +const createDdEntityIds = async () => { + const schema = db.getTenantSchemaPrefix(); + await db.query(` + CREATE TABLE IF NOT EXISTS ${schema}_dd_entity_ids ( + uuid TEXT PRIMARY KEY, + kind TEXT NOT NULL, + current_name TEXT NOT NULL, + current_id INTEGER NOT NULL, + parent_uuid TEXT, + created_at TEXT NOT NULL, + UNIQUE (kind, current_id) + ) + `); + await db.query(`CREATE INDEX IF NOT EXISTS _dd_entity_ids_kind_name ON ${schema}_dd_entity_ids (kind, current_name)`); + await db.query(`CREATE INDEX IF NOT EXISTS _dd_entity_ids_parent ON ${schema}_dd_entity_ids (parent_uuid)`); +}; + + +const createDdOps = async () => { + const schema = db.getTenantSchemaPrefix(); + await db.query(` + CREATE TABLE IF NOT EXISTS ${schema}_dd_ops ( + op_id TEXT PRIMARY KEY, + source_env_id TEXT NOT NULL, + op_type TEXT NOT NULL, + entity_kind TEXT, + entity_uuid TEXT, + payload TEXT NOT NULL, + parent_op_id TEXT, + correlation_id TEXT, + schema_version INTEGER NOT NULL DEFAULT 1, + created_at TEXT NOT NULL, + applied_at TEXT, + status TEXT NOT NULL DEFAULT 'committed', + conflict_with_op_id TEXT + ) + `); + await db.query(`CREATE INDEX IF NOT EXISTS _dd_ops_created ON ${schema}_dd_ops (created_at)`); + await db.query(`CREATE INDEX IF NOT EXISTS _dd_ops_source ON ${schema}_dd_ops (source_env_id, created_at)`); + await db.query(`CREATE INDEX IF NOT EXISTS _dd_ops_entity ON ${schema}_dd_ops (entity_uuid)`); + await db.query(`CREATE INDEX IF NOT EXISTS _dd_ops_correlation ON ${schema}_dd_ops (correlation_id)`); + await db.query(`CREATE INDEX IF NOT EXISTS _dd_ops_status ON ${schema}_dd_ops (status) WHERE status = 'conflict'`).catch(() => {}); + + // Idempotent migration for instances installed before conflict_with_op_id + // existed. On PG a failed statement poisons the surrounding transaction, so a + // bare ALTER caught in JS is not enough -- use ADD COLUMN IF NOT EXISTS there; + // sqlite lacks that clause, so run the bare ALTER and swallow the error. + if (db.isSQLite) { + try { + await db.query(`ALTER TABLE ${schema}_dd_ops ADD COLUMN conflict_with_op_id TEXT`); + } catch (e) { + // column already exists; ignore + } + } else { + await db.query(`ALTER TABLE ${schema}_dd_ops ADD COLUMN IF NOT EXISTS conflict_with_op_id TEXT`); + } +}; + + +const createDdAnchors = async () => { + const schema = db.getTenantSchemaPrefix(); + await db.query(` + CREATE TABLE IF NOT EXISTS ${schema}_dd_anchors ( + peer_id INTEGER NOT NULL, + direction TEXT NOT NULL, + last_op_id TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (peer_id, direction) + ) + `); +}; + + +const createDdTableModes = async () => { + const schema = db.getTenantSchemaPrefix(); + await db.query(` + CREATE TABLE IF NOT EXISTS ${schema}_dd_table_modes ( + table_uuid TEXT PRIMARY KEY, + data_mode TEXT NOT NULL, + updated_at TEXT NOT NULL, + starter_shipped_at TEXT + ) + `); + // Idempotent migration for older installs that lack starter_shipped_at. + // (See createDdOps: PG needs IF NOT EXISTS or a failed ALTER poisons the txn.) + if (db.isSQLite) { + try { + await db.query(`ALTER TABLE ${schema}_dd_table_modes ADD COLUMN starter_shipped_at TEXT`); + } catch (e) { /* column exists */ } + } else { + await db.query(`ALTER TABLE ${schema}_dd_table_modes ADD COLUMN IF NOT EXISTS starter_shipped_at TEXT`); + } +}; + + +const createAllTables = async () => { + await createDdEnv(); + await createDdPeers(); + await createDdEntityIds(); + await createDdOps(); + await createDdAnchors(); + await createDdTableModes(); +}; + + +module.exports = { + createAllTables +}; diff --git a/lib/state.js b/lib/state.js new file mode 100644 index 0000000..d2ce190 --- /dev/null +++ b/lib/state.js @@ -0,0 +1,27 @@ +// Shared helpers for Saltcorn state-cache interactions. + +// Refresh Saltcorn's in-memory model caches. Mutations done in another process +// (e.g. saltcorn run-js, prior ingest, or a peer's push) are not visible to +// Model.findOne / Model.find until we refresh, because those methods read from +// state.tables/views/pages/triggers arrays populated at startup. +// +// Best-effort: if Saltcorn changes its API and a refresh function disappears, +// callers fall back to whatever's in the cache. +const refreshState = async () => { + try { + const { getState } = require("@saltcorn/data/db/state"); + const s = getState(); + if (s.refresh_tables) await s.refresh_tables(true); + if (s.refresh_views) await s.refresh_views(true); + if (s.refresh_pages) await s.refresh_pages(true); + if (s.refresh_triggers) await s.refresh_triggers(true); + if (s.refresh_page_groups) await s.refresh_page_groups(true); + } catch (e) { + // ignore — best-effort + } +}; + + +module.exports = { + refreshState +}; diff --git a/lib/transport.js b/lib/transport.js new file mode 100644 index 0000000..d868206 --- /dev/null +++ b/lib/transport.js @@ -0,0 +1,109 @@ +// Outbound signed peer requests. +// +// Build a canonical string, sign it with the peer's shared secret, and send +// it via fetch. Body is JSON-serialized once and used for both the HTTP body +// and the signature payload to keep client/server hash agreement trivial. + +const { + buildCanonical, + normalizeHost, + sign, + randomNonce +} = require("./crypto"); + + +const signedFetch = async ({ baseUrl, method, path, body, sourceEnvId, secret, requireTls }) => { + if (requireTls && new URL(baseUrl).protocol !== "https:") { + throw new Error(`peer requires TLS but base_url is not https: ${baseUrl}`); + } + const timestamp = String(Date.now()); + const nonce = randomNonce().toString("hex"); + const bodyStr = body ? JSON.stringify(body) : ""; + const targetHost = normalizeHost(new URL(baseUrl).host); + const canonical = buildCanonical({ + timestamp: timestamp, + nonce: nonce, + method: method, + path: path, + targetHost: targetHost, + body: bodyStr + }); + const signature = sign(secret, canonical); + + const url = baseUrl.replace(/\/+$/, "") + path; + const headers = { + "X-DD-Env-Id": sourceEnvId, + "X-DD-Timestamp": timestamp, + "X-DD-Nonce": nonce, + "X-DD-Signature": signature + }; + if (bodyStr) { + // Custom Content-Type so Saltcorn's express.json() middleware leaves the + // request stream untouched; the server reads exact bytes for HMAC. + headers["Content-Type"] = "application/vnd.dev-deploy+json"; + } + + const init = { method: method, headers: headers }; + if (bodyStr) { + init.body = bodyStr; + } + + const res = await fetch(url, init); + const text = await res.text(); + let parsed = null; + if (text) { + try { + parsed = JSON.parse(text); + } catch (e) { + parsed = { raw: text }; + } + } + return { status: res.status, ok: res.ok, body: parsed }; +}; + + +// Like signedFetch but returns the response body as a Buffer (raw bytes). +// For binary endpoints like GET /dev-deploy/api/file/:uuid. +const signedFetchBinary = async ({ baseUrl, method, path, body, sourceEnvId, secret, requireTls }) => { + if (requireTls && new URL(baseUrl).protocol !== "https:") { + throw new Error(`peer requires TLS but base_url is not https: ${baseUrl}`); + } + const timestamp = String(Date.now()); + const nonce = randomNonce().toString("hex"); + const bodyStr = body ? JSON.stringify(body) : ""; + const targetHost = normalizeHost(new URL(baseUrl).host); + const canonical = buildCanonical({ + timestamp: timestamp, + nonce: nonce, + method: method, + path: path, + targetHost: targetHost, + body: bodyStr + }); + const signature = sign(secret, canonical); + + const url = baseUrl.replace(/\/+$/, "") + path; + const headers = { + "X-DD-Env-Id": sourceEnvId, + "X-DD-Timestamp": timestamp, + "X-DD-Nonce": nonce, + "X-DD-Signature": signature + }; + if (bodyStr) { + headers["Content-Type"] = "application/vnd.dev-deploy+json"; + } + + const init = { method: method, headers: headers }; + if (bodyStr) { + init.body = bodyStr; + } + const res = await fetch(url, init); + const bytes = Buffer.from(await res.arrayBuffer()); + return { status: res.status, ok: res.ok, bytes: bytes }; +}; + + +module.exports = { + signedFetch, + signedFetchBinary +}; diff --git a/lib/wrap.js b/lib/wrap.js new file mode 100644 index 0000000..5fed1ee --- /dev/null +++ b/lib/wrap.js @@ -0,0 +1,1012 @@ +// Monkey-patches Saltcorn metadata model classes to journal every CRUD action. +// +// Each wrap: +// 1. Pre-generates an op_id and enters an AsyncLocalStorage scope so any +// child mutations triggered by the original method (e.g. cascading field +// deletes from a table delete) see this op as their parent. +// 2. Optionally captures pre-state via a "before" hook. +// 3. Invokes the original method. +// 4. On success, runs an "after" hook to compute entity uuid + payload and +// appends the op to _dd_ops. +// +// Failures inside the after-hook are logged but do not throw -- we never want +// the journal to corrupt user-facing operations. Failures inside the original +// method propagate normally (no op is recorded for an aborted mutation). + +const Table = require("@saltcorn/data/models/table"); +const Field = require("@saltcorn/data/models/field"); +const View = require("@saltcorn/data/models/view"); +const Page = require("@saltcorn/data/models/page"); +const Trigger = require("@saltcorn/data/models/trigger"); +const Role = require("@saltcorn/data/models/role"); +const Library = require("@saltcorn/data/models/library"); +const Tag = require("@saltcorn/data/models/tag"); +const TableConstraint = require("@saltcorn/data/models/table_constraints"); +const File = require("@saltcorn/data/models/file"); +const PageGroup = require("@saltcorn/data/models/page_group"); +const PageGroupMember = require("@saltcorn/data/models/page_group_member"); +const WorkflowStep = require("@saltcorn/data/models/workflow_step"); +const Plugin = require("@saltcorn/data/models/plugin"); +const db = require("@saltcorn/data/db"); + +const { randomUuid } = require("./ids"); +const { enterOp, isSuppressed } = require("./context"); +const { recordOpSafely } = require("./ops"); +const { + assignNewUuid, + lookupByCurrent, + updateName, + removeEntityRow, + constraintDisplayName +} = require("./entityIds"); +const { ENTITY_KINDS, fileLocationToId } = require("./constants"); +const { sha256File, toRelativePath } = require("./files"); +const { toPlaceholders } = require("./payloadRefs"); +const { + setRowUuid, + getRowUuid, + newRowUuid, + COLUMN_NAME: ROW_UUID_COL +} = require("./rowIdentity"); +const { + rowToPortable, + journalDecision +} = require("./rowPayload"); + + +const snapshotInstance = (inst, keys) => { + const out = {}; + for (const k of keys) { + if (inst[k] !== undefined) { + out[k] = inst[k]; + } + } + return out; +}; + + +// Shared before/after hooks for instance `delete()` methods. +// Captures the entity's UUID, parent UUID (if any), and snapshot, then removes +// the entity_ids row after the original delete completes so reused integer ids +// don't collide. parent_uuid is preserved so revert can find the parent later. +const standardDropHooks = (kind, keys) => ({ + before: async function () { + const existing = await lookupByCurrent(kind, this.id); + return { + uuid: existing ? existing.uuid : null, + currentId: this.id, + parentUuid: existing ? existing.parent_uuid : null, + snapshot: snapshotInstance(this, keys) + }; + }, + after: async function ({ before }) { + if (!before || !before.uuid) return null; + await removeEntityRow(kind, before.currentId); + return { + entityUuid: before.uuid, + payload: { before: before.snapshot, parent_uuid: before.parentUuid } + }; + } +}); + + +const wrap = (target, method, kind, action, hooks) => { + const original = target[method]; + if (typeof original !== "function") { + // eslint-disable-next-line no-console + console.warn(`[dev-deploy] no method ${kind}.${method} to wrap`); + return; + } + if (original.__ddWrapped) { + return; + } + const wrapped = async function (...args) { + if (isSuppressed()) { + return await original.apply(this, args); + } + const opId = randomUuid(); + const beforeCtx = hooks.before ? await safeCall(hooks.before, this, [args]) : null; + return await enterOp(opId, async () => { + const result = await original.apply(this, args); + const details = await safeCall(hooks.after, this, [{ args: args, result: result, before: beforeCtx }]); + if (details) { + // Translate any local file refs (numeric ids or paths) in the + // payload to portable __dd_file_ref:: placeholders so + // the journal stores cross-environment-stable references. + if (details.payload) { + try { + await toPlaceholders(details.payload); + } catch (e) { + // best-effort; payload still journals as-is + } + } + await recordOpSafely({ + op_id: opId, + op_type: `${action}_${kind}`, + entity_kind: kind, + entity_uuid: details.entityUuid, + payload: details.payload + }); + } + return result; + }); + }; + wrapped.__ddWrapped = true; + wrapped.__ddOriginal = original; + target[method] = wrapped; +}; + + +const safeCall = async (fn, ctx, args) => { + if (!fn) { + return null; + } + try { + return await fn.apply(ctx, args); + } catch (err) { + // eslint-disable-next-line no-console + console.error(`[dev-deploy] hook error:`, err); + return null; + } +}; + + +// Common keys to snapshot for each entity. Skips heavy/derived fields. +const TABLE_KEYS = ["id", "name", "min_role_read", "min_role_write", "versioned", "description", "ownership_field_id", "ownership_formula", "external", "provider_name", "provider_cfg"]; +const FIELD_KEYS = ["id", "name", "label", "type", "table_id", "required", "is_unique", "calculated", "stored", "expression", "reftable_name", "reftype", "refname", "primary_key", "attributes"]; +const VIEW_KEYS = ["id", "name", "viewtemplate", "table_id", "configuration", "min_role", "default_render_page", "exttable_name", "description", "slug"]; +const PAGE_KEYS = ["id", "name", "title", "description", "min_role", "layout", "fixed_states", "menu_label"]; +const TRIGGER_KEYS = ["id", "name", "action", "when_trigger", "table_id", "configuration", "min_role", "description", "channel"]; +const ROLE_KEYS = ["id", "role"]; +const LIBRARY_KEYS = ["id", "name", "icon", "layout"]; +const TAG_KEYS = ["id", "name"]; +const CONSTRAINT_KEYS = ["id", "type", "table_id", "configuration"]; +const PAGE_GROUP_KEYS = ["id", "name", "description", "min_role", "random_allocation"]; +const WORKFLOW_STEP_KEYS = ["id", "name", "trigger_id", "action_name", "next_step", "initial_step", "configuration", "only_if"]; + + +// ----- Table ----- + +const wrapTable = () => { + wrap(Table, "create", ENTITY_KINDS.TABLE, "create", { + after: async ({ result }) => { + if (!result || !result.id) return null; + const uuid = await assignNewUuid(ENTITY_KINDS.TABLE, result.id, result.name, null); + return { + entityUuid: uuid, + payload: { after: snapshotInstance(result, TABLE_KEYS) } + }; + } + }); + + wrap(Table.prototype, "update", ENTITY_KINDS.TABLE, "update", { + before: async function (args) { + const existing = await lookupByCurrent(ENTITY_KINDS.TABLE, this.id); + return { + uuid: existing ? existing.uuid : null, + oldName: existing ? existing.current_name : this.name, + snapshot: snapshotInstance(this, TABLE_KEYS) + }; + }, + after: async function ({ args, before }) { + if (!before || !before.uuid) return null; + const patch = args[0] || {}; + const newName = patch.name !== undefined ? patch.name : before.oldName; + if (newName !== before.oldName) { + await updateName(ENTITY_KINDS.TABLE, this.id, newName); + } + return { + entityUuid: before.uuid, + payload: { + before: before.snapshot, + patch: patch, + after: snapshotInstance(this, TABLE_KEYS) + } + }; + } + }); + + wrap(Table.prototype, "delete", ENTITY_KINDS.TABLE, "drop", { + before: async function (args) { + const existing = await lookupByCurrent(ENTITY_KINDS.TABLE, this.id); + return { + uuid: existing ? existing.uuid : null, + currentId: this.id, + snapshot: snapshotInstance(this, TABLE_KEYS), + onlyForget: !!args[0] + }; + }, + after: async function ({ before }) { + if (!before || !before.uuid) return null; + await removeEntityRow(ENTITY_KINDS.TABLE, before.currentId); + return { + entityUuid: before.uuid, + payload: { before: before.snapshot, only_forget: before.onlyForget } + }; + } + }); +}; + + +// ----- Field ----- + +const wrapField = () => { + wrap(Field, "create", ENTITY_KINDS.FIELD, "create", { + after: async ({ args, result }) => { + if (!result || !result.id) return null; + let parentUuid = null; + if (result.table_id) { + const t = await lookupByCurrent(ENTITY_KINDS.TABLE, result.table_id); + parentUuid = t ? t.uuid : null; + } + const uuid = await assignNewUuid(ENTITY_KINDS.FIELD, result.id, result.name, parentUuid); + return { + entityUuid: uuid, + payload: { + after: snapshotInstance(result, FIELD_KEYS), + parent_uuid: parentUuid + } + }; + } + }); + + wrap(Field.prototype, "update", ENTITY_KINDS.FIELD, "update", { + before: async function (args) { + const existing = await lookupByCurrent(ENTITY_KINDS.FIELD, this.id); + return { + uuid: existing ? existing.uuid : null, + oldName: existing ? existing.current_name : this.name, + snapshot: snapshotInstance(this, FIELD_KEYS) + }; + }, + after: async function ({ args, before }) { + if (!before || !before.uuid) return null; + const patch = args[0] || {}; + const newName = patch.name !== undefined ? patch.name : before.oldName; + if (newName !== before.oldName) { + await updateName(ENTITY_KINDS.FIELD, this.id, newName); + } + return { + entityUuid: before.uuid, + payload: { + before: before.snapshot, + patch: patch + } + }; + } + }); + + wrap(Field.prototype, "delete", ENTITY_KINDS.FIELD, "drop", standardDropHooks(ENTITY_KINDS.FIELD, FIELD_KEYS)); +}; + + +// ----- View ----- + +const wrapView = () => { + wrap(View, "create", ENTITY_KINDS.VIEW, "create", { + after: async ({ result }) => { + if (!result || !result.id) return null; + let parentUuid = null; + if (result.table_id) { + const t = await lookupByCurrent(ENTITY_KINDS.TABLE, result.table_id); + parentUuid = t ? t.uuid : null; + } + const uuid = await assignNewUuid(ENTITY_KINDS.VIEW, result.id, result.name, parentUuid); + return { + entityUuid: uuid, + payload: { after: snapshotInstance(result, VIEW_KEYS), parent_uuid: parentUuid } + }; + } + }); + + // View.update is static (v, id) + wrap(View, "update", ENTITY_KINDS.VIEW, "update", { + before: async ({ args }) => { + const patch = args[0] || {}; + const id = args[1]; + const existing = await lookupByCurrent(ENTITY_KINDS.VIEW, id); + return { + id: id, + uuid: existing ? existing.uuid : null, + oldName: existing ? existing.current_name : null, + patch: patch + }; + }, + after: async ({ before }) => { + if (!before || !before.uuid) return null; + const newName = before.patch.name !== undefined ? before.patch.name : before.oldName; + if (newName !== before.oldName) { + await updateName(ENTITY_KINDS.VIEW, before.id, newName); + } + return { + entityUuid: before.uuid, + payload: { patch: before.patch } + }; + } + }); + + wrap(View.prototype, "delete", ENTITY_KINDS.VIEW, "drop", standardDropHooks(ENTITY_KINDS.VIEW, VIEW_KEYS)); +}; + + +// ----- Page ----- + +const wrapPage = () => { + wrap(Page, "create", ENTITY_KINDS.PAGE, "create", { + after: async ({ result }) => { + if (!result || !result.id) return null; + const uuid = await assignNewUuid(ENTITY_KINDS.PAGE, result.id, result.name, null); + return { entityUuid: uuid, payload: { after: snapshotInstance(result, PAGE_KEYS) } }; + } + }); + + // Page.update is static (id, row) + wrap(Page, "update", ENTITY_KINDS.PAGE, "update", { + before: async ({ args }) => { + const id = args[0]; + const row = args[1] || {}; + const existing = await lookupByCurrent(ENTITY_KINDS.PAGE, id); + return { + id: id, + uuid: existing ? existing.uuid : null, + oldName: existing ? existing.current_name : null, + patch: row + }; + }, + after: async ({ before }) => { + if (!before || !before.uuid) return null; + const newName = before.patch.name !== undefined ? before.patch.name : before.oldName; + if (newName !== before.oldName) { + await updateName(ENTITY_KINDS.PAGE, before.id, newName); + } + return { entityUuid: before.uuid, payload: { patch: before.patch } }; + } + }); + + wrap(Page.prototype, "delete", ENTITY_KINDS.PAGE, "drop", standardDropHooks(ENTITY_KINDS.PAGE, PAGE_KEYS)); +}; + + +// ----- Trigger ----- + +const wrapTrigger = () => { + wrap(Trigger, "create", ENTITY_KINDS.TRIGGER, "create", { + after: async ({ result }) => { + if (!result || !result.id) return null; + let parentUuid = null; + if (result.table_id) { + const t = await lookupByCurrent(ENTITY_KINDS.TABLE, result.table_id); + parentUuid = t ? t.uuid : null; + } + const name = result.name || `trigger_${result.id}`; + const uuid = await assignNewUuid(ENTITY_KINDS.TRIGGER, result.id, name, parentUuid); + return { entityUuid: uuid, payload: { after: snapshotInstance(result, TRIGGER_KEYS), parent_uuid: parentUuid } }; + } + }); + + // Trigger.update is static (id, row) + wrap(Trigger, "update", ENTITY_KINDS.TRIGGER, "update", { + before: async ({ args }) => { + const id = args[0]; + const row = args[1] || {}; + const existing = await lookupByCurrent(ENTITY_KINDS.TRIGGER, id); + return { + id: id, + uuid: existing ? existing.uuid : null, + oldName: existing ? existing.current_name : null, + patch: row + }; + }, + after: async ({ before }) => { + if (!before || !before.uuid) return null; + const newName = before.patch.name !== undefined ? before.patch.name : before.oldName; + if (newName && newName !== before.oldName) { + await updateName(ENTITY_KINDS.TRIGGER, before.id, newName); + } + return { entityUuid: before.uuid, payload: { patch: before.patch } }; + } + }); + + wrap(Trigger.prototype, "delete", ENTITY_KINDS.TRIGGER, "drop", standardDropHooks(ENTITY_KINDS.TRIGGER, TRIGGER_KEYS)); +}; + + +// ----- Role ----- + +const wrapRole = () => { + wrap(Role, "create", ENTITY_KINDS.ROLE, "create", { + after: async ({ args, result }) => { + // Role.create returns inserted row; id is in result. Some return shapes vary. + const id = result && result.id ? result.id : (args[0] && args[0].id); + const role = result && result.role ? result.role : (args[0] && args[0].role); + if (!id || !role) return null; + const uuid = await assignNewUuid(ENTITY_KINDS.ROLE, id, role, null); + return { entityUuid: uuid, payload: { after: { id: id, role: role } } }; + } + }); + + wrap(Role.prototype, "update", ENTITY_KINDS.ROLE, "update", { + before: async function () { + const existing = await lookupByCurrent(ENTITY_KINDS.ROLE, this.id); + return { uuid: existing ? existing.uuid : null, oldName: existing ? existing.current_name : this.role, snapshot: snapshotInstance(this, ROLE_KEYS) }; + }, + after: async function ({ args, before }) { + if (!before || !before.uuid) return null; + const patch = args[0] || {}; + const newName = patch.role !== undefined ? patch.role : before.oldName; + if (newName !== before.oldName) { + await updateName(ENTITY_KINDS.ROLE, this.id, newName); + } + return { entityUuid: before.uuid, payload: { before: before.snapshot, patch: patch } }; + } + }); + + wrap(Role.prototype, "delete", ENTITY_KINDS.ROLE, "drop", standardDropHooks(ENTITY_KINDS.ROLE, ROLE_KEYS)); +}; + + +// ----- Library ----- + +const wrapLibrary = () => { + wrap(Library, "create", ENTITY_KINDS.LIBRARY, "create", { + after: async ({ args, result }) => { + // Library.create returns void; we look up by name from args + const cfg = args[0] || {}; + if (!cfg.name) return null; + const Library = require("@saltcorn/data/models/library"); + const lib = await Library.findOne({ name: cfg.name }); + if (!lib) return null; + const uuid = await assignNewUuid(ENTITY_KINDS.LIBRARY, lib.id, lib.name, null); + return { entityUuid: uuid, payload: { after: snapshotInstance(lib, LIBRARY_KEYS) } }; + } + }); + + wrap(Library.prototype, "update", ENTITY_KINDS.LIBRARY, "update", { + before: async function () { + const existing = await lookupByCurrent(ENTITY_KINDS.LIBRARY, this.id); + return { uuid: existing ? existing.uuid : null, oldName: existing ? existing.current_name : this.name, snapshot: snapshotInstance(this, LIBRARY_KEYS) }; + }, + after: async function ({ args, before }) { + if (!before || !before.uuid) return null; + const patch = args[0] || {}; + const newName = patch.name !== undefined ? patch.name : before.oldName; + if (newName !== before.oldName) { + await updateName(ENTITY_KINDS.LIBRARY, this.id, newName); + } + return { entityUuid: before.uuid, payload: { before: before.snapshot, patch: patch } }; + } + }); + + wrap(Library.prototype, "delete", ENTITY_KINDS.LIBRARY, "drop", standardDropHooks(ENTITY_KINDS.LIBRARY, LIBRARY_KEYS)); +}; + + +// ----- Tag ----- + +const wrapTag = () => { + wrap(Tag, "create", ENTITY_KINDS.TAG, "create", { + after: async ({ result }) => { + if (!result || !result.id) return null; + const uuid = await assignNewUuid(ENTITY_KINDS.TAG, result.id, result.name, null); + return { entityUuid: uuid, payload: { after: snapshotInstance(result, TAG_KEYS) } }; + } + }); + + wrap(Tag.prototype, "update", ENTITY_KINDS.TAG, "update", { + before: async function () { + const existing = await lookupByCurrent(ENTITY_KINDS.TAG, this.id); + return { uuid: existing ? existing.uuid : null, oldName: existing ? existing.current_name : this.name, snapshot: snapshotInstance(this, TAG_KEYS) }; + }, + after: async function ({ args, before }) { + if (!before || !before.uuid) return null; + const patch = args[0] || {}; + const newName = patch.name !== undefined ? patch.name : before.oldName; + if (newName !== before.oldName) { + await updateName(ENTITY_KINDS.TAG, this.id, newName); + } + return { entityUuid: before.uuid, payload: { before: before.snapshot, patch: patch } }; + } + }); + + wrap(Tag.prototype, "delete", ENTITY_KINDS.TAG, "drop", standardDropHooks(ENTITY_KINDS.TAG, TAG_KEYS)); +}; + + +// ----- File ----- + +const FILE_KEYS = ["filename", "location", "mime_super", "mime_sub", "size_kb", "min_role_read"]; + +const wrapFile = () => { + wrap(File, "create", ENTITY_KINDS.FILE, "create", { + after: async ({ result }) => { + if (!result || !result.location) return null; + const relPath = toRelativePath(File, result.location); + const synthId = fileLocationToId(relPath); + let contentHash = null; + try { + contentHash = await sha256File(result.location); + } catch (e) { + // best-effort; some create paths may set up metadata only + } + const uuid = await assignNewUuid(ENTITY_KINDS.FILE, synthId, relPath, null); + return { + entityUuid: uuid, + payload: { + after: { + filename: result.filename, + relative_path: relPath, + mime_super: result.mime_super, + mime_sub: result.mime_sub, + size_kb: result.size_kb, + min_role_read: result.min_role_read, + content_hash: contentHash + } + } + }; + } + }); + + wrap(File.prototype, "delete", ENTITY_KINDS.FILE, "drop", { + before: async function () { + const relPath = toRelativePath(File, this.location); + const synthId = fileLocationToId(relPath); + const existing = await lookupByCurrent(ENTITY_KINDS.FILE, synthId); + return { + uuid: existing ? existing.uuid : null, + currentId: synthId, + relPath: relPath, + parentUuid: null, + snapshot: snapshotInstance(this, FILE_KEYS) + }; + }, + after: async function ({ before }) { + if (!before || !before.uuid) return null; + await removeEntityRow(ENTITY_KINDS.FILE, before.currentId); + return { + entityUuid: before.uuid, + payload: { before: { ...before.snapshot, relative_path: before.relPath } } + }; + } + }); +}; + + +// ----- TableConstraint ----- + +const wrapTableConstraint = () => { + wrap(TableConstraint, "create", ENTITY_KINDS.CONSTRAINT, "create", { + after: async ({ result }) => { + if (!result || !result.id) return null; + // Guard against double-firing (e.g. the wrap may be installed in + // multiple module-resolution contexts when sc-exec uses createRequire). + const already = await lookupByCurrent(ENTITY_KINDS.CONSTRAINT, result.id); + if (already) return null; + let parentUuid = null; + if (result.table_id) { + const t = await lookupByCurrent(ENTITY_KINDS.TABLE, result.table_id); + parentUuid = t ? t.uuid : null; + } + const name = constraintDisplayName(result); + const uuid = await assignNewUuid(ENTITY_KINDS.CONSTRAINT, result.id, name, parentUuid); + return { + entityUuid: uuid, + payload: { after: snapshotInstance(result, CONSTRAINT_KEYS), parent_uuid: parentUuid } + }; + } + }); + + wrap(TableConstraint.prototype, "delete", ENTITY_KINDS.CONSTRAINT, "drop", standardDropHooks(ENTITY_KINDS.CONSTRAINT, CONSTRAINT_KEYS)); +}; + + +// ----- Table row CRUD (managed/starter data_mode) ----- +// +// Wraps Table.prototype.insertRow / updateRow / deleteRows. Each checks the +// table's data_mode (looked up via _dd_table_modes). 'user' tables pass +// through silently. 'managed' tables always journal. 'starter' tables journal +// only during their initial ship (when starter_shipped_at is NULL); subsequent +// row changes pass through. + +const safeJournal = async (opType, entityUuid, payload) => { + try { + const opId = randomUuid(); + await enterOp(opId, async () => { + await recordOpSafely({ + op_id: opId, + op_type: opType, + entity_kind: "table_row", + entity_uuid: entityUuid, + payload: payload + }); + }); + } catch (e) { + // eslint-disable-next-line no-console + console.error(`[dev-deploy] ${opType} journal failed:`, e); + } +}; + + +const wrapTableRows = () => { + // --- insertRow --- + const origInsert = Table.prototype.insertRow; + if (origInsert && !origInsert.__ddWrapped) { + const wrapped = async function (...args) { + if (isSuppressed()) { + return await origInsert.apply(this, args); + } + const decision = await journalDecision(this.id); + if (!decision.shouldJournal) { + return await origInsert.apply(this, args); + } + const v_in = args[0] || {}; + const newId = await origInsert.apply(this, args); + if (!newId) return newId; + + const rowUuid = newRowUuid(); + await setRowUuid(this.name, newId, rowUuid); + + const { portable, warnings } = await rowToPortable({ ...v_in, id: newId }, this); + await safeJournal("insert_row", rowUuid, { + table_uuid: decision.tableUuid, + after: portable, + warnings: warnings.length > 0 ? warnings : undefined + }); + return newId; + }; + wrapped.__ddWrapped = true; + wrapped.__ddOriginal = origInsert; + Table.prototype.insertRow = wrapped; + } + + // --- updateRow --- + const origUpdate = Table.prototype.updateRow; + if (origUpdate && !origUpdate.__ddWrapped) { + const wrapped = async function (v_in, id_in, ...rest) { + if (isSuppressed()) { + return await origUpdate.apply(this, [v_in, id_in, ...rest]); + } + const decision = await journalDecision(this.id); + if (!decision.shouldJournal) { + return await origUpdate.apply(this, [v_in, id_in, ...rest]); + } + const id = typeof id_in === "object" && id_in !== null ? id_in.id : id_in; + const rowUuid = await getRowUuid(this.name, id); + + // Capture the row's prior values (portable) BEFORE the update so the + // op is revertible (revert re-applies this as the inverse patch). + let beforePortable = null; + if (rowUuid) { + const priorRows = await db.select(this.name, { id: id }); + if (priorRows && priorRows[0]) { + beforePortable = (await rowToPortable(priorRows[0], this)).portable; + } + } + + const result = await origUpdate.apply(this, [v_in, id_in, ...rest]); + + if (rowUuid) { + const { portable } = await rowToPortable({ ...v_in, id: id }, this); + await safeJournal("update_row", rowUuid, { + table_uuid: decision.tableUuid, + patch: portable, + before: beforePortable || undefined + }); + } + return result; + }; + wrapped.__ddWrapped = true; + wrapped.__ddOriginal = origUpdate; + Table.prototype.updateRow = wrapped; + } + + // --- deleteRows --- + const origDelete = Table.prototype.deleteRows; + if (origDelete && !origDelete.__ddWrapped) { + const wrapped = async function (where, ...rest) { + if (isSuppressed()) { + return await origDelete.apply(this, [where, ...rest]); + } + const decision = await journalDecision(this.id); + if (!decision.shouldJournal) { + return await origDelete.apply(this, [where, ...rest]); + } + // Capture rows BEFORE deletion so we can journal their uuids. + const rows = await db.select(this.name, where || {}); + const result = await origDelete.apply(this, [where, ...rest]); + for (const row of rows) { + const rowUuid = row[ROW_UUID_COL]; + if (!rowUuid) continue; + const { portable } = await rowToPortable(row, this); + await safeJournal("drop_row", rowUuid, { + table_uuid: decision.tableUuid, + before: portable + }); + } + return result; + }; + wrapped.__ddWrapped = true; + wrapped.__ddOriginal = origDelete; + Table.prototype.deleteRows = wrapped; + } +}; + + +// ----- Plugin configuration ----- + +// Wrap Plugin.prototype.upsert to journal configuration changes. Skip the +// dev-deploy plugin itself (no self-reference). Skip cases where the upsert +// doesn't change the configuration — Saltcorn calls upsert on every plugin +// load, which would otherwise flood the journal. +const wrapPlugin = () => { + wrap(Plugin.prototype, "upsert", "plugin_config", "update", { + before: async function () { + if (!this.name || this.name === "dev-deploy") return null; + let priorConfig = null; + if (this.id) { + const existing = await db.selectMaybeOne("_sc_plugins", { id: this.id }); + if (existing) { + try { + priorConfig = typeof existing.configuration === "string" + ? JSON.parse(existing.configuration) + : (existing.configuration || null); + } catch (e) { + priorConfig = null; + } + } + } + return { name: this.name, priorConfig: priorConfig }; + }, + after: async function ({ before }) { + if (!before) return null; + const newConfig = this.configuration || {}; + if (JSON.stringify(before.priorConfig || {}) === JSON.stringify(newConfig)) { + return null; + } + return { + entityUuid: null, + payload: { name: before.name, configuration: newConfig, before_configuration: before.priorConfig || {} } + }; + } + }); +}; + + +// ----- Config keys (menu, etc.) ----- + +// Config keys we journal when state.setConfig is called. The op carries the +// full key+value pair; apply replays it on the target via setConfig. +// menu_items references pages/views by NAME (not id), which is naturally +// stable across instances, so no UUID translation is needed. +const TRACKED_CONFIG_KEYS = new Set([ + "menu_items", + "site_name", + "site_logo_id", + "base_url" +]); + + +const wrapSetConfig = () => { + const { getState } = require("@saltcorn/data/db/state"); + const state = getState(); + if (!state || typeof state.setConfig !== "function") return; + if (state.setConfig.__ddWrapped) return; + + const original = state.setConfig.bind(state); + const wrapped = async function (key, value) { + if (isSuppressed()) { + return await original(key, value); + } + // Capture the prior value BEFORE setting so the op is revertible. + const beforeValue = (typeof state.getConfig === "function") ? state.getConfig(key) : undefined; + const result = await original(key, value); + if (!TRACKED_CONFIG_KEYS.has(key)) { + return result; + } + const opId = randomUuid(); + await enterOp(opId, async () => { + await recordOpSafely({ + op_id: opId, + op_type: "set_config", + entity_kind: "config", + entity_uuid: null, // config keys don't have UUIDs + payload: { key: key, value: value, before: beforeValue } + }); + }); + return result; + }; + wrapped.__ddWrapped = true; + wrapped.__ddOriginal = original; + state.setConfig = wrapped; +}; + + +// ----- PageGroup ----- + +// Members of a PageGroup reference page_id (integer). For cross-env identity +// we translate page_id <-> page_uuid in payloads. +const translateMembersToPortable = async (members) => { + const out = []; + for (const m of members || []) { + const pageEnt = m.page_id ? await lookupByCurrent(ENTITY_KINDS.PAGE, m.page_id) : null; + out.push({ + page_uuid: pageEnt ? pageEnt.uuid : null, + eligible_formula: m.eligible_formula, + description: m.description, + sequence: m.sequence + }); + } + return out; +}; + + +const wrapPageGroup = () => { + wrap(PageGroup, "create", ENTITY_KINDS.PAGE_GROUP, "create", { + after: async ({ result }) => { + if (!result || !result.id) return null; + const already = await lookupByCurrent(ENTITY_KINDS.PAGE_GROUP, result.id); + if (already) return null; + const uuid = await assignNewUuid(ENTITY_KINDS.PAGE_GROUP, result.id, result.name, null); + const members = await translateMembersToPortable(result.members || []); + return { + entityUuid: uuid, + payload: { after: snapshotInstance(result, PAGE_GROUP_KEYS), members: members } + }; + } + }); + + wrap(PageGroup, "update", ENTITY_KINDS.PAGE_GROUP, "update", { + before: async ({ args }) => { + const id = args[0]; + const row = args[1] || {}; + const existing = await lookupByCurrent(ENTITY_KINDS.PAGE_GROUP, id); + return { + id: id, + uuid: existing ? existing.uuid : null, + oldName: existing ? existing.current_name : null, + patch: row + }; + }, + after: async ({ before }) => { + if (!before || !before.uuid) return null; + const newName = before.patch.name !== undefined ? before.patch.name : before.oldName; + if (newName && newName !== before.oldName) { + await updateName(ENTITY_KINDS.PAGE_GROUP, before.id, newName); + } + return { entityUuid: before.uuid, payload: { patch: before.patch } }; + } + }); + + wrap(PageGroup.prototype, "delete", ENTITY_KINDS.PAGE_GROUP, "drop", standardDropHooks(ENTITY_KINDS.PAGE_GROUP, PAGE_GROUP_KEYS)); +}; + + +// ----- PageGroupMember ----- + +const wrapPageGroupMember = () => { + wrap(PageGroupMember, "create", ENTITY_KINDS.PAGE_GROUP_MEMBER, "create", { + after: async ({ result }) => { + if (!result || !result.id) return null; + const already = await lookupByCurrent(ENTITY_KINDS.PAGE_GROUP_MEMBER, result.id); + if (already) return null; + let groupUuid = null; + if (result.page_group_id) { + const g = await lookupByCurrent(ENTITY_KINDS.PAGE_GROUP, result.page_group_id); + groupUuid = g ? g.uuid : null; + } + let pageUuid = null; + if (result.page_id) { + const p = await lookupByCurrent(ENTITY_KINDS.PAGE, result.page_id); + pageUuid = p ? p.uuid : null; + } + const uuid = await assignNewUuid(ENTITY_KINDS.PAGE_GROUP_MEMBER, result.id, `member_${result.id}`, groupUuid); + return { + entityUuid: uuid, + payload: { + after: { + page_uuid: pageUuid, + eligible_formula: result.eligible_formula, + description: result.description, + sequence: result.sequence + }, + parent_uuid: groupUuid + } + }; + } + }); + + wrap(PageGroupMember, "delete", ENTITY_KINDS.PAGE_GROUP_MEMBER, "drop", { + before: async ({ args }) => { + const id = args[0]; + const existing = await lookupByCurrent(ENTITY_KINDS.PAGE_GROUP_MEMBER, id); + return { + uuid: existing ? existing.uuid : null, + currentId: id, + parentUuid: existing ? existing.parent_uuid : null + }; + }, + after: async ({ before }) => { + if (!before || !before.uuid) return null; + await removeEntityRow(ENTITY_KINDS.PAGE_GROUP_MEMBER, before.currentId); + return { + entityUuid: before.uuid, + payload: { before: { id: before.currentId }, parent_uuid: before.parentUuid } + }; + } + }); +}; + + +// ----- WorkflowStep ----- + +const wrapWorkflowStep = () => { + wrap(WorkflowStep, "create", ENTITY_KINDS.WORKFLOW_STEP, "create", { + after: async ({ args, result }) => { + // WorkflowStep.create returns the inserted integer id (db.insert), + // not an instance. Use the args + id to reconstruct the snapshot. + const id = typeof result === "number" ? result : (result && result.id); + if (!id) return null; + const already = await lookupByCurrent(ENTITY_KINDS.WORKFLOW_STEP, id); + if (already) return null; + const step = await WorkflowStep.findOne({ id: id }); + if (!step) return null; + let parentUuid = null; + if (step.trigger_id) { + const tr = await lookupByCurrent(ENTITY_KINDS.TRIGGER, step.trigger_id); + parentUuid = tr ? tr.uuid : null; + } + const name = step.name || `step_${id}`; + const uuid = await assignNewUuid(ENTITY_KINDS.WORKFLOW_STEP, id, name, parentUuid); + return { + entityUuid: uuid, + payload: { after: snapshotInstance(step, WORKFLOW_STEP_KEYS), parent_uuid: parentUuid } + }; + } + }); + + wrap(WorkflowStep.prototype, "update", ENTITY_KINDS.WORKFLOW_STEP, "update", { + before: async function () { + const existing = await lookupByCurrent(ENTITY_KINDS.WORKFLOW_STEP, this.id); + return { + uuid: existing ? existing.uuid : null, + oldName: existing ? existing.current_name : this.name, + snapshot: snapshotInstance(this, WORKFLOW_STEP_KEYS) + }; + }, + after: async function ({ args, before }) { + if (!before || !before.uuid) return null; + const patch = args[0] || {}; + const newName = patch.name !== undefined ? patch.name : before.oldName; + if (newName && newName !== before.oldName) { + await updateName(ENTITY_KINDS.WORKFLOW_STEP, this.id, newName); + } + return { entityUuid: before.uuid, payload: { before: before.snapshot, patch: patch } }; + } + }); + + wrap(WorkflowStep.prototype, "delete", ENTITY_KINDS.WORKFLOW_STEP, "drop", standardDropHooks(ENTITY_KINDS.WORKFLOW_STEP, WORKFLOW_STEP_KEYS)); +}; + + +const installAllWraps = () => { + wrapTable(); + wrapField(); + wrapView(); + wrapPage(); + wrapTrigger(); + wrapRole(); + wrapLibrary(); + wrapTag(); + wrapTableConstraint(); + wrapFile(); + wrapPageGroup(); + wrapPageGroupMember(); + wrapWorkflowStep(); + wrapSetConfig(); + wrapPlugin(); + wrapTableRows(); +}; + + +module.exports = { + installAllWraps +}; diff --git a/package.json b/package.json new file mode 100644 index 0000000..5dc334f --- /dev/null +++ b/package.json @@ -0,0 +1,21 @@ +{ + "name": "dev-deploy", + "version": "0.0.1", + "description": "Saltcorn plugin: migrate metadata changes (tables, fields, views, pages, triggers, roles, library, tags) across Dev/Test/Prod environments via an ops journal with stable UUIDs and HMAC-authenticated peer endpoints. Detects concurrent-edit conflicts and offers theirs/mine resolution. User-table row data is never touched.", + "main": "index.js", + "scripts": { + "test": "node test/e2e.js" + }, + "keywords": [ + "deploy", + "migration", + "ops-journal", + "hmac", + "metadata-sync" + ], + "engines": { + "node": ">=20" + }, + "author": "Scott Duensing", + "license": "MIT" +} diff --git a/scripts/installDevDeployTenant.js b/scripts/installDevDeployTenant.js new file mode 100644 index 0000000..f3337c4 --- /dev/null +++ b/scripts/installDevDeployTenant.js @@ -0,0 +1,99 @@ +// Clean per-tenant installer for dev-deploy on the Postgres multi-tenant +// instance. Registers + enables the plugin in each named tenant schema and runs +// its onLoad (creating the _dd_* tables + bootstrapping the env), using +// Saltcorn's supported Plugin.loadAndSaveNewPlugin API inside runWithTenant -- +// replacing the old manual "INSERT INTO ._sc_plugins" SQL hack. +// +// The CLI `install-plugin -t -d ` cannot do this: a local (-d) +// plugin is "unsafe" on a non-root tenant, so loadAndSaveNewPlugin returns +// before the upsert unless allowUnsafeOnTenantsWithoutConfigSetting (its 5th arg) +// is set, and the CLI never passes it. This script passes it. +// +// Usage (from project root, PG env sourced -- see installDevDeployTenant.sh): +// node dev-deploy/scripts/installDevDeployTenant.js t1 t2 (or '*' for all tenants) + +const { createRequire } = require("node:module"); +const path = require("node:path"); + +// Re-root @saltcorn/* against the Saltcorn checkout's node_modules. This file +// lives at dev-deploy/scripts/, so up 2 = project root, then saltcorn/packages/... +const scRequire = createRequire(path.join(__dirname, "..", "..", "saltcorn", "packages", "saltcorn-data", "package.json")); + +const Plugin = scRequire("@saltcorn/data/models/plugin"); +const db = scRequire("@saltcorn/data/db"); +const { init_multi_tenant, getRootState } = scRequire("@saltcorn/data/db/state"); +const { getAllTenants } = scRequire("@saltcorn/admin-models/models/tenant"); + +const PLUGIN_NAME = "dev-deploy"; +const DEV_DEPLOY_DIR = path.resolve(__dirname, ".."); + + +const installInto = async (tenant) => { + await db.runWithTenant(tenant, async () => { + await db.withTransaction(async () => { + // Remove any prior rows (including the old manual-hack row and earlier + // installs) so we converge on exactly one _sc_plugins row -- no + // duplicate source of truth. + await db.deleteWhere("_sc_plugins", { name: PLUGIN_NAME }); + const plugin = new Plugin({ name: PLUGIN_NAME, source: "local", location: DEV_DEPLOY_DIR, configuration: {} }); + await Plugin.loadAndSaveNewPlugin(plugin, true, false); + }); + // Verify against dev-deploy's own table so a stale row can't pass. + const row = await db.selectMaybeOne("_sc_plugins", { name: PLUGIN_NAME }); + const svc = await db.query( + "SELECT 1 FROM information_schema.tables WHERE table_schema = $1 AND table_name = '_dd_env'", + [db.getTenantSchema()] + ); + const hasSvc = svc && svc.rows && svc.rows.length > 0; + // eslint-disable-next-line no-console + console.log(`[installDevDeployTenant] ${tenant}: _sc_plugins=${row ? "yes" : "NO"} _dd_env=${hasSvc ? "yes" : "NO"}`); + if (!row || !hasSvc) { + throw new Error("install verification failed for tenant " + tenant + " (onLoad did not run)"); + } + }); +}; + + +const main = async () => { + await Plugin.loadAllPlugins(); + + // Resolve the target tenants (from the public _sc_tenants list). + let tenants = process.argv.slice(2); + if (tenants.length === 0 || (tenants.length === 1 && tenants[0] === "*")) { + const all = await db.runWithTenant(db.connectObj.default_schema, getAllTenants); + tenants = (all || []).map((t) => (typeof t === "string" ? t : t.subdomain)).filter(Boolean); + } + if (!tenants || tenants.length === 0) { + // eslint-disable-next-line no-console + console.error("[installDevDeployTenant] no tenants to install into"); + process.exit(1); + } + + // Initialize per-tenant State (so getState() resolves inside runWithTenant) + // without running migrations. This also runs each tenant's existing plugins' + // onLoad, which is itself idempotent. + await init_multi_tenant(Plugin.loadAllPlugins, true, tenants); + + // Permit installing this LOCAL plugin into tenant schemas. In this Saltcorn + // build loadAndSaveNewPlugin skips any non-"npm" plugin on a non-root tenant + // BEFORE the allowUnsafe arg is consulted; the supported lever is this + // root-only config -- the intended setting for a multi-tenant deployment that + // offers the dev-deploy plugin to its tenants. + await getRootState().setConfig("tenants_unsafe_plugins", true); + + // eslint-disable-next-line no-console + console.log("[installDevDeployTenant] installing " + PLUGIN_NAME + " into: " + tenants.join(", ")); + for (const t of tenants) { + await installInto(t); + } + // eslint-disable-next-line no-console + console.log("[installDevDeployTenant] done"); + process.exit(0); +}; + + +main().catch((e) => { + // eslint-disable-next-line no-console + console.error("[installDevDeployTenant] ERROR:", e && (e.stack || e.message || e)); + process.exit(1); +}); diff --git a/scripts/installDevDeployTenant.sh b/scripts/installDevDeployTenant.sh new file mode 100755 index 0000000..22899a5 --- /dev/null +++ b/scripts/installDevDeployTenant.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Clean per-tenant installer for dev-deploy on the Postgres MULTI-TENANT +# instance (.dev-state-pg). Replaces the manual "INSERT INTO ._sc_plugins" +# SQL hack: registers + enables the plugin in each tenant schema and runs its +# onLoad (creating the _dd_* tables + bootstrapping the env). After this, per-tenant +# onLoad re-runs automatically on every boot via init_multi_tenant->loadAllPlugins. +# +# Prerequisites: the tenants must already exist (saltcorn create-tenant ), +# and the plugin must be installed into the PG public schema once (the normal +# install-plugin -d ./dev-deploy path) so the shared plugins_folder copy exists. +# +# Usage: ./dev-deploy/scripts/installDevDeployTenant.sh t1 t2 (or '*' for all tenants) +set -e +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +cd "$ROOT" +# shellcheck disable=SC1091 +source .dev-state-pg/env.sh +node "$ROOT/dev-deploy/scripts/installDevDeployTenant.js" "$@" diff --git a/test/e2e.js b/test/e2e.js new file mode 100644 index 0000000..adf7606 --- /dev/null +++ b/test/e2e.js @@ -0,0 +1,1040 @@ +// dev-deploy integration test suite. +// +// Prereqs: both saltcorn instances running and reachable, plugin installed + +// bootstrapped, admin@local / AdminP@ss1 present on both. Run from the +// dev-deploy/ directory: +// +// cd ~/claude/saltcorn && ./startServer.sh & ./startServerTest.sh & +// node test/e2e.js +// +// Each test resets only what it needs to. Tests run in order; assertion +// failures are caught and counted -- one failure does not stop the suite. + +const { execFileSync, spawn } = require("node:child_process"); +const assert = require("node:assert/strict"); +const path = require("node:path"); + +const MAIN = { url: "http://localhost:3000", db: "/home/scott/claude/saltcorn/.dev-state/saltcorn.sqlite", env: "/home/scott/claude/saltcorn/.dev-state/env.sh" }; +const TEST = { url: "http://localhost:3001", db: "/home/scott/claude/saltcorn/.dev-state-test/saltcorn.sqlite", env: "/home/scott/claude/saltcorn/.dev-state-test/env.sh" }; + +const MAIN_COOKIES = "/tmp/dd-test-jar-main.txt"; +const TEST_COOKIES = "/tmp/dd-test-jar-test.txt"; + + +// --- helpers --- + +const sql = (dbPath, query) => { + const out = execFileSync("sqlite3", [dbPath, query], { encoding: "utf8" }); + return out.trim(); +}; + + +const sqlRows = (dbPath, query) => { + const out = sql(dbPath, query); + if (!out) return []; + return out.split("\n"); +}; + + +const resetInstanceDb = (dbPath) => { + sql(dbPath, ` + DELETE FROM _dd_ops; + DELETE FROM _dd_peers; + DELETE FROM _dd_anchors; + DELETE FROM _dd_table_modes; + DELETE FROM _sc_table_constraints; + DELETE FROM _sc_workflow_steps; + DELETE FROM _sc_page_group_members; + DELETE FROM _sc_page_groups; + DELETE FROM _sc_pages; + DELETE FROM _sc_triggers; + DELETE FROM _sc_views; + DELETE FROM _sc_config WHERE key IN ('menu_items', 'unrolled_menu_items'); + UPDATE _sc_plugins SET configuration = NULL WHERE name != 'dev-deploy'; + `); + // Strip _dd_entity_ids back to Saltcorn's bootstrap baseline. + sql(dbPath, ` + DELETE FROM _dd_entity_ids WHERE kind NOT IN ('table','field','role'); + DELETE FROM _dd_entity_ids WHERE kind='table' AND current_name != 'users'; + DELETE FROM _dd_entity_ids WHERE kind='field' AND current_name NOT IN ('id','email','role_id'); + DELETE FROM _dd_entity_ids WHERE kind='role' AND current_name NOT IN ('admin','staff','user','public'); + `); + const userTables = sqlRows(dbPath, "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE '\\_%' ESCAPE '\\' AND name != 'users' AND name NOT LIKE 'sqlite_%';"); + for (const t of userTables) { + if (t) sql(dbPath, `DROP TABLE IF EXISTS "${t}"`); + } + sql(dbPath, "DELETE FROM _sc_tables WHERE name != 'users'; DELETE FROM _sc_fields WHERE table_id NOT IN (SELECT id FROM _sc_tables);"); +}; + + +const runJs = (envPath, code) => { + return execFileSync( + "bash", + ["-c", `source ${envPath} && saltcorn run-js --code ${JSON.stringify(code)}`], + { encoding: "utf8" } + ); +}; + + +// scExec is like runJs but uses our test/sc-exec.js shim, giving the JS body +// full require() access -- needed for Field, TableConstraint, File, etc. that +// saltcorn run-js's vm sandbox doesn't expose. Code is piped via stdin to +// dodge shell-escape pitfalls with multi-line strings. +const scExec = (envPath, code) => { + return execFileSync( + "bash", + ["-c", `source ${envPath} && node ${path.join(__dirname, "sc-exec.js")}`], + { encoding: "utf8", input: code } + ); +}; + + +// --- HTTP helpers (browser-style admin auth via cookie jar) --- + +const curl = (args) => { + const out = execFileSync("curl", args, { encoding: "utf8" }); + return out; +}; + + +const extractCsrfFromHtml = (html) => { + const m = html.match(/name="_csrf"[^>]*value="([^"]+)"/); + return m ? m[1] : null; +}; + + +const login = (instance, jar) => { + execFileSync("rm", ["-f", jar]); + const page = curl(["-sS", "-c", jar, `${instance.url}/auth/login`]); + const csrf = extractCsrfFromHtml(page); + if (!csrf) throw new Error(`no csrf on login page for ${instance.url}`); + curl([ + "-sS", "-c", jar, "-b", jar, "-o", "/dev/null", + "-X", "POST", `${instance.url}/auth/login`, + "-d", `email=admin@local&password=AdminP@ss1&_csrf=${csrf}` + ]); +}; + + +const envIdOf = (instance, jar) => { + const html = curl(["-sS", "-b", jar, `${instance.url}/admin/dev-deploy/`]); + const m = html.match(/([0-9a-f-]{36})<\/code>/); + if (!m) throw new Error(`no env_id in dashboard html for ${instance.url}`); + return m[1]; +}; + + +const freshCsrf = (instance, jar, urlPath) => { + const html = curl(["-sS", "-b", jar, `${instance.url}${urlPath}`]); + return extractCsrfFromHtml(html); +}; + + +// Each admin POST handler embeds CSRF in a form on a specific GET page. +// Match the POST path to the page that renders its form. +const CSRF_PAGE_FOR = [ + ["/admin/dev-deploy/peers/add", "/admin/dev-deploy/peers"], + ["/admin/dev-deploy/peers/rotate", "/admin/dev-deploy/peers"], + ["/admin/dev-deploy/peers/delete", "/admin/dev-deploy/peers"], + ["/admin/dev-deploy/promote", "/admin/dev-deploy/peers"], + ["/admin/dev-deploy/pull", "/admin/dev-deploy/peers"], + ["/admin/dev-deploy/conflicts/resolve", "/admin/dev-deploy/conflicts"], + ["/admin/dev-deploy/tables/set", "/admin/dev-deploy/tables"], + ["/admin/dev-deploy/revert", "/admin/dev-deploy/ops"] +]; + + +const adminPost = (instance, jar, urlPath, fields) => { + const mapping = CSRF_PAGE_FOR.find(([p]) => urlPath === p); + if (!mapping) throw new Error(`no CSRF source page mapped for POST ${urlPath}`); + const csrf = freshCsrf(instance, jar, mapping[1]); + if (!csrf) throw new Error(`no CSRF token on ${mapping[1]} for POST ${urlPath}`); + const args = ["-sS", "-b", jar, "-o", "/tmp/dd-test-postbody.html", + "-w", "%{http_code}|%{redirect_url}", + "-X", "POST", `${instance.url}${urlPath}`, + "--data-urlencode", `_csrf=${csrf}`]; + for (const [k, v] of Object.entries(fields)) { + args.push("--data-urlencode", `${k}=${v}`); + } + const out = curl(args); + const [code, redirect] = out.split("|"); + return { status: parseInt(code, 10), redirect: redirect, body: require("node:fs").readFileSync("/tmp/dd-test-postbody.html", "utf8") }; +}; + + +// --- test runner --- + +let passed = 0, failed = 0; +const failures = []; + +const test = async (name, fn) => { + try { + await fn(); + console.log(` PASS ${name}`); + passed++; + } catch (err) { + console.log(` FAIL ${name}`); + console.log(` ${err.message}`); + failures.push({ name, err }); + failed++; + } +}; + + +const section = (name) => { + console.log(`\n[${name}]`); +}; + + +// --- scenarios --- + +const main = async () => { + section("preconditions"); + await test("both servers respond to /auth/login", async () => { + for (const inst of [MAIN, TEST]) { + const out = curl(["-sS", "-o", "/dev/null", "-w", "%{http_code}", `${inst.url}/auth/login`]); + assert.equal(out, "200", `${inst.url} returned ${out}`); + } + }); + + section("reset both instances"); + resetInstanceDb(MAIN.db); + resetInstanceDb(TEST.db); + login(MAIN, MAIN_COOKIES); + login(TEST, TEST_COOKIES); + const mainEnv = envIdOf(MAIN, MAIN_COOKIES); + const testEnv = envIdOf(TEST, TEST_COOKIES); + console.log(` main env_id: ${mainEnv}`); + console.log(` test env_id: ${testEnv}`); + + section("schema"); + await test("six _dd_* tables exist on main", async () => { + const tables = sqlRows(MAIN.db, "SELECT name FROM sqlite_master WHERE type='table' AND name LIKE '_dd_%' ORDER BY name").join(","); + assert.equal(tables, "_dd_anchors,_dd_entity_ids,_dd_env,_dd_ops,_dd_peers,_dd_table_modes"); + }); + await test("_dd_ops has conflict_with_op_id column", async () => { + const cols = sqlRows(MAIN.db, "PRAGMA table_info(_dd_ops)").map((r) => r.split("|")[1]); + assert.ok(cols.includes("conflict_with_op_id"), `cols: ${cols.join(",")}`); + }); + + section("bootstrap identity"); + await test("env_ids are distinct UUIDs", async () => { + assert.notEqual(mainEnv, testEnv); + assert.match(mainEnv, /^[0-9a-f-]{36}$/); + }); + await test("'users' table has identical UUID on both instances (deterministic backfill)", async () => { + const m = sql(MAIN.db, "SELECT uuid FROM _dd_entity_ids WHERE kind='table' AND current_name='users'"); + const t = sql(TEST.db, "SELECT uuid FROM _dd_entity_ids WHERE kind='table' AND current_name='users'"); + assert.equal(m, t, "deterministic UUIDs diverged"); + assert.notEqual(m, ""); + }); + await test("all baseline role UUIDs match across instances", async () => { + const m = sqlRows(MAIN.db, "SELECT current_name||'|'||uuid FROM _dd_entity_ids WHERE kind='role' ORDER BY current_name").join(","); + const t = sqlRows(TEST.db, "SELECT current_name||'|'||uuid FROM _dd_entity_ids WHERE kind='role' ORDER BY current_name").join(","); + assert.equal(m, t); + }); + + section("mutation capture"); + runJs(MAIN.env, 'const t = await Table.create("widgets"); await t.update({ description: "initial" });'); + await test("create_table op is journaled", async () => { + const c = sql(MAIN.db, "SELECT COUNT(*) FROM _dd_ops WHERE op_type='create_table' AND source_env_id='" + mainEnv + "'"); + assert.equal(c, "1"); + }); + await test("update_table op is journaled", async () => { + const c = sql(MAIN.db, "SELECT COUNT(*) FROM _dd_ops WHERE op_type='update_table' AND source_env_id='" + mainEnv + "'"); + assert.ok(parseInt(c, 10) >= 1); + }); + + section("pairing"); + const addRes = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/peers/add", { + env_id: testEnv, + label: "test", + base_url: TEST.url + }); + await test("main /peers/add returns 200 with shared secret", async () => { + assert.equal(addRes.status, 200); + }); + const secretMatch = addRes.body.match(/class="secret">([0-9a-f]{64}) { + assert.equal(pairTest.status, 200); + }); + + section("promote main -> test"); + const peerIdOnMain = sql(MAIN.db, `SELECT peer_id FROM _dd_peers WHERE env_id='${testEnv}'`); + const promoteRes = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/promote", { peer_id: peerIdOnMain }); + await test("promote returns 302 with success message", async () => { + assert.equal(promoteRes.status, 302); + assert.match(promoteRes.redirect, /msg=promoted/); + }); + await test("test instance now has widgets table", async () => { + const r = sql(TEST.db, "SELECT name FROM _sc_tables WHERE name='widgets'"); + assert.equal(r, "widgets"); + }); + await test("widgets UUID matches between main and test", async () => { + const m = sql(MAIN.db, "SELECT uuid FROM _dd_entity_ids WHERE current_name='widgets'"); + const t = sql(TEST.db, "SELECT uuid FROM _dd_entity_ids WHERE current_name='widgets'"); + assert.equal(m, t); + }); + + section("pull test -> main (no conflicts)"); + runJs(TEST.env, 'await Table.create("test_added");'); + const peerIdOnMain2 = sql(MAIN.db, `SELECT peer_id FROM _dd_peers WHERE env_id='${testEnv}'`); + const pullRes = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/pull", { peer_id: peerIdOnMain2 }); + await test("pull returns 302 with success message", async () => { + assert.equal(pullRes.status, 302); + assert.match(pullRes.redirect, /msg=pulled/); + }); + await test("main now has test_added", async () => { + const r = sql(MAIN.db, "SELECT name FROM _sc_tables WHERE name='test_added'"); + assert.equal(r, "test_added"); + }); + await test("test_added UUID matches across instances", async () => { + const m = sql(MAIN.db, "SELECT uuid FROM _dd_entity_ids WHERE current_name='test_added'"); + const t = sql(TEST.db, "SELECT uuid FROM _dd_entity_ids WHERE current_name='test_added'"); + assert.equal(m, t); + }); + + section("conflict detection + resolution"); + runJs(MAIN.env, 'const t = Table.findOne({name: "widgets"}); await t.update({ description: "from MAIN" });'); + runJs(TEST.env, 'const t = Table.findOne({name: "widgets"}); await t.update({ description: "from TEST" });'); + const peerIdForConflict = sql(MAIN.db, `SELECT peer_id FROM _dd_peers WHERE env_id='${testEnv}'`); + const pullConflictRes = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/pull", { peer_id: peerIdForConflict }); + await test("pull with divergent edits returns 1 conflict", async () => { + assert.match(pullConflictRes.redirect, /1%20conflicts/); + }); + await test("conflict op recorded with status=conflict", async () => { + const c = sql(MAIN.db, "SELECT COUNT(*) FROM _dd_ops WHERE status='conflict'"); + assert.equal(c, "1"); + }); + await test("conflict op has conflict_with_op_id set", async () => { + const r = sql(MAIN.db, "SELECT conflict_with_op_id FROM _dd_ops WHERE status='conflict'"); + assert.match(r, /^[0-9a-f-]{36}$/); + }); + await test("widgets description on main is still 'from MAIN' (held op not applied)", async () => { + const r = sql(MAIN.db, "SELECT description FROM _sc_tables WHERE name='widgets'"); + assert.equal(r, "from MAIN"); + }); + + const conflictOp = sql(MAIN.db, "SELECT op_id FROM _dd_ops WHERE status='conflict' LIMIT 1"); + const resolveRes = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/conflicts/resolve", { + op_id: conflictOp, + action: "theirs" + }); + await test("resolve theirs returns 302 with success", async () => { + assert.equal(resolveRes.status, 302); + assert.match(resolveRes.redirect, /msg=resolved/); + }); + await test("widgets description on main is now 'from TEST'", async () => { + const r = sql(MAIN.db, "SELECT description FROM _sc_tables WHERE name='widgets'"); + assert.equal(r, "from TEST"); + }); + await test("resolved op status is committed", async () => { + const r = sql(MAIN.db, `SELECT status FROM _dd_ops WHERE op_id='${conflictOp}'`); + assert.equal(r, "committed"); + }); + await test("pending conflicts is 0", async () => { + const c = sql(MAIN.db, "SELECT COUNT(*) FROM _dd_ops WHERE status='conflict'"); + assert.equal(c, "0"); + }); + + section("conflict merge per-field"); + runJs(MAIN.env, 'const t = Table.findOne({name: "widgets"}); await t.update({ description: "M2" });'); + runJs(TEST.env, 'const t = Table.findOne({name: "widgets"}); await t.update({ description: "T2" });'); + const peerIdForMerge = sql(MAIN.db, `SELECT peer_id FROM _dd_peers WHERE env_id='${testEnv}'`); + adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/pull", { peer_id: peerIdForMerge }); + + await test("a fresh conflict appears for the second divergence", async () => { + const c = sql(MAIN.db, "SELECT COUNT(*) FROM _dd_ops WHERE status='conflict'"); + assert.equal(c, "1"); + }); + + const mergeOp = sql(MAIN.db, "SELECT op_id FROM _dd_ops WHERE status='conflict' LIMIT 1"); + + await test("merge view returns 200 for an update-vs-update conflict", async () => { + const out = curl(["-sS", "-b", MAIN_COOKIES, "-o", "/dev/null", "-w", "%{http_code}", + `${MAIN.url}/admin/dev-deploy/conflicts/merge?op_id=${mergeOp}`]); + assert.equal(out, "200"); + }); + + await test("merge view shows the diverging description field", async () => { + const html = curl(["-sS", "-b", MAIN_COOKIES, + `${MAIN.url}/admin/dev-deploy/conflicts/merge?op_id=${mergeOp}`]); + assert.match(html, /choice_description/); + assert.match(html, /T2/); + }); + + // Fetch the merge page for CSRF then POST a custom value + const mergeHtml = curl(["-sS", "-b", MAIN_COOKIES, + `${MAIN.url}/admin/dev-deploy/conflicts/merge?op_id=${mergeOp}`]); + const mergeCsrf = extractCsrfFromHtml(mergeHtml); + const mergeResp = execFileSync("curl", [ + "-sS", "-b", MAIN_COOKIES, "-o", "/dev/null", + "-w", "%{http_code}|%{redirect_url}", + "-X", "POST", `${MAIN.url}/admin/dev-deploy/conflicts/merge/apply`, + "--data-urlencode", `_csrf=${mergeCsrf}`, + "--data-urlencode", `op_id=${mergeOp}`, + "--data-urlencode", "choice_description=custom", + "--data-urlencode", "custom_description=hybrid value" + ], { encoding: "utf8" }); + const [mergeStatus, mergeRedirect] = mergeResp.split("|"); + + await test("merge apply returns 302 with success message", async () => { + assert.equal(mergeStatus, "302"); + assert.match(mergeRedirect, /msg=merged/); + }); + + await test("widgets description on main is now 'hybrid value'", async () => { + const r = sql(MAIN.db, "SELECT description FROM _sc_tables WHERE name='widgets'"); + assert.equal(r, "hybrid value"); + }); + + await test("merged op has status=merged, conflict_with cleared", async () => { + const r = sql(MAIN.db, `SELECT status||'|'||COALESCE(conflict_with_op_id,'') FROM _dd_ops WHERE op_id='${mergeOp}'`); + assert.equal(r, "merged|"); + }); + + await test("pending conflicts is 0 after merge", async () => { + const c = sql(MAIN.db, "SELECT COUNT(*) FROM _dd_ops WHERE status='conflict'"); + assert.equal(c, "0"); + }); + + section("table constraints"); + // Add a field to widgets and a unique constraint on it. Promote to test. + scExec(MAIN.env, ` + const t = Table.findOne({name: "widgets"}); + await Field.create({ table_id: t.id, name: "sku", label: "SKU", type: "String" }); + const fresh = Table.findOne({name: "widgets"}); + await TableConstraint.create({ table_id: fresh.id, type: "Unique", configuration: { fields: ["sku"] } }); + `); + + await test("create_field op journaled for sku", async () => { + const c = sql(MAIN.db, "SELECT COUNT(*) FROM _dd_ops WHERE op_type='create_field' AND payload LIKE '%sku%'"); + assert.equal(c, "1"); + }); + await test("create_constraint op journaled with type=Unique", async () => { + const c = sql(MAIN.db, "SELECT COUNT(*) FROM _dd_ops WHERE op_type='create_constraint' AND payload LIKE '%Unique%'"); + assert.equal(c, "1"); + }); + + const peerIdForConstraint = sql(MAIN.db, `SELECT peer_id FROM _dd_peers WHERE env_id='${testEnv}'`); + const promoteConstraintRes = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/promote", { peer_id: peerIdForConstraint }); + await test("promote with constraint returns success", async () => { + assert.equal(promoteConstraintRes.status, 302); + assert.match(promoteConstraintRes.redirect, /msg=promoted/); + }); + await test("test now has the sku field", async () => { + const r = sql(TEST.db, "SELECT COUNT(*) FROM _sc_fields WHERE name='sku' AND table_id=(SELECT id FROM _sc_tables WHERE name='widgets')"); + assert.equal(r, "1"); + }); + await test("test now has the unique constraint on sku", async () => { + // _sc_table_constraints uses JSON for configuration; SQLite stores TEXT + const r = sql(TEST.db, "SELECT COUNT(*) FROM _sc_table_constraints WHERE type='Unique' AND configuration LIKE '%sku%'"); + assert.equal(r, "1"); + }); + await test("constraint UUID matches across instances", async () => { + const m = sql(MAIN.db, "SELECT uuid FROM _dd_entity_ids WHERE kind='constraint' AND current_name='unique(sku)'"); + const t = sql(TEST.db, "SELECT uuid FROM _dd_entity_ids WHERE kind='constraint' AND current_name='unique(sku)'"); + assert.equal(m, t); + assert.notEqual(m, ""); + }); + + section("file propagation"); + const FILE_BYTES = "Hello, dev-deploy! This is a test document."; + scExec(MAIN.env, ` + const fs = require("fs"); + const path = require("path"); + const tenant = db.getTenantSchema(); + const absPath = path.join(db.connectObj.file_store, tenant, "test_doc.txt"); + fs.mkdirSync(path.dirname(absPath), { recursive: true }); + fs.writeFileSync(absPath, ${JSON.stringify(FILE_BYTES)}); + await File.create({ + filename: "test_doc.txt", + location: absPath, + uploaded_at: new Date(), + size_kb: 1, + mime_super: "text", + mime_sub: "plain", + min_role_read: 1 + }); + `); + + await test("create_file op journaled with content_hash + relative_path", async () => { + const r = sql(MAIN.db, "SELECT payload FROM _dd_ops WHERE op_type='create_file' LIMIT 1"); + assert.match(r, /content_hash/); + assert.match(r, /test_doc\.txt/); + }); + + await test("file entry in _dd_entity_ids on main", async () => { + const r = sql(MAIN.db, "SELECT current_name FROM _dd_entity_ids WHERE kind='file' LIMIT 1"); + assert.equal(r, "test_doc.txt"); + }); + + const peerIdForFile = sql(MAIN.db, `SELECT peer_id FROM _dd_peers WHERE env_id='${testEnv}'`); + const fileRes = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/promote", { peer_id: peerIdForFile }); + + await test("file promote returns success", async () => { + assert.equal(fileRes.status, 302); + assert.match(fileRes.redirect, /msg=promoted/); + }); + + await test("file exists on test instance with same bytes", async () => { + const fs = require("node:fs"); + const out = fs.readFileSync("/home/scott/claude/saltcorn/.dev-state-test/files/public/test_doc.txt", "utf8"); + assert.equal(out, FILE_BYTES); + }); + + await test("file UUID matches across instances", async () => { + const m = sql(MAIN.db, "SELECT uuid FROM _dd_entity_ids WHERE kind='file' AND current_name='test_doc.txt'"); + const t = sql(TEST.db, "SELECT uuid FROM _dd_entity_ids WHERE kind='file' AND current_name='test_doc.txt'"); + assert.equal(m, t); + assert.notEqual(m, ""); + }); + + section("plugin mismatch warning"); + await test("health endpoint is reachable and HMAC-required", async () => { + const out = curl(["-sS", "-o", "/dev/null", "-w", "%{http_code}", `${TEST.url}/dev-deploy/api/health`]); + assert.equal(out, "400"); // missing HMAC headers + }); + await test("promote includes no plugin warnings when peers match", async () => { + // Both instances have identical plugin lists (base, sbadmin2, dev-deploy) + // so the warning suffix should be absent. + runJs(MAIN.env, 'await Table.create("warning_demo_table");'); + const peerId = sql(MAIN.db, `SELECT peer_id FROM _dd_peers WHERE env_id='${testEnv}'`); + const r = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/promote", { peer_id: peerId }); + assert.equal(r.status, 302); + // Both sides have the same plugins so no WARNINGS suffix should appear. + assert.ok(!/WARNINGS/.test(r.redirect || ""), `unexpected warnings: ${r.redirect}`); + }); + + section("file refs in page layout"); + scExec(MAIN.env, ` + const Page = require("@saltcorn/data/models/page"); + // Pages with a file ref in their layout — fileid as a relative-path + // string (the form Saltcorn produces on a fresh upload). + await Page.create({ + name: "page_with_image", + title: "Page With Image", + description: "Has an image", + min_role: 100, + layout: { type: "image", srctype: "File", fileid: "test_doc.txt", alt: "test" }, + fixed_states: {} + }); + `); + + await test("create_page op replaces fileid with placeholder in journal", async () => { + const r = sql(MAIN.db, "SELECT payload FROM _dd_ops WHERE op_type='create_page' AND payload LIKE '%page_with_image%' LIMIT 1"); + assert.match(r, /__dd_file_ref::/); + // The original string must NOT appear as the raw fileid value + assert.ok(!/"fileid":"test_doc\.txt"/.test(r), `fileid was not translated: ${r}`); + }); + + const peerIdForRefs = sql(MAIN.db, `SELECT peer_id FROM _dd_peers WHERE env_id='${testEnv}'`); + const refsPromote = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/promote", { peer_id: peerIdForRefs }); + await test("page-with-image promote returns success", async () => { + assert.equal(refsPromote.status, 302); + assert.match(refsPromote.redirect, /msg=promoted/); + }); + + await test("page exists on test with fileid resolved back to local path", async () => { + const layout = sql(TEST.db, "SELECT layout FROM _sc_pages WHERE name='page_with_image'"); + // On test, fileid should be the relative path that resolves locally + assert.match(layout, /"fileid":"test_doc\.txt"/); + // Placeholder must NOT leak into the live entity + assert.ok(!/__dd_file_ref::/.test(layout), `placeholder leaked into live page: ${layout}`); + }); + + section("page_group propagation"); + scExec(MAIN.env, ` + const Page = require("@saltcorn/data/models/page"); + const PageGroup = require("@saltcorn/data/models/page_group"); + const page = await Page.create({ + name: "intro_page", + title: "Intro", + description: "Intro page", + min_role: 100, + layout: { type: "blank", contents: "Hello" }, + fixed_states: {} + }); + const pg = await PageGroup.create({ + name: "home_group", + description: "home routing", + min_role: 100, + random_allocation: false, + members: [] + }); + await pg.addMember({ + page_id: page.id, + eligible_formula: "true", + description: "default" + }); + `); + + await test("create_page_group op journaled", async () => { + const r = sql(MAIN.db, "SELECT payload FROM _dd_ops WHERE op_type='create_page_group' LIMIT 1"); + assert.match(r, /home_group/); + }); + await test("create_page_group_member op journaled with page_uuid", async () => { + const r = sql(MAIN.db, "SELECT payload FROM _dd_ops WHERE op_type='create_page_group_member' LIMIT 1"); + assert.match(r, /page_uuid/); + }); + + const peerIdForPG = sql(MAIN.db, `SELECT peer_id FROM _dd_peers WHERE env_id='${testEnv}'`); + const pgPromote = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/promote", { peer_id: peerIdForPG }); + await test("page_group promote returns success", async () => { + assert.equal(pgPromote.status, 302); + assert.match(pgPromote.redirect, /msg=promoted/); + }); + await test("test instance has home_group page_group", async () => { + const r = sql(TEST.db, "SELECT name FROM _sc_page_groups WHERE name='home_group'"); + assert.equal(r, "home_group"); + }); + await test("page_group on test has its member row", async () => { + const r = sql(TEST.db, "SELECT COUNT(*) FROM _sc_page_group_members m JOIN _sc_page_groups g ON m.page_group_id=g.id WHERE g.name='home_group'"); + assert.equal(r, "1"); + }); + await test("page_group UUID matches across instances", async () => { + const m = sql(MAIN.db, "SELECT uuid FROM _dd_entity_ids WHERE kind='page_group' AND current_name='home_group'"); + const t = sql(TEST.db, "SELECT uuid FROM _dd_entity_ids WHERE kind='page_group' AND current_name='home_group'"); + assert.equal(m, t); + assert.notEqual(m, ""); + }); + + section("workflow_step propagation"); + scExec(MAIN.env, ` + const Trigger = require("@saltcorn/data/models/trigger"); + const WorkflowStep = require("@saltcorn/data/models/workflow_step"); + // clean any prior + const oldT = Trigger.findOne({name: "demo_workflow"}); + if (oldT) await oldT.delete(); + const tr = await Trigger.create({ + name: "demo_workflow", + action: "Workflow", + when_trigger: "Never", + min_role: 1, + configuration: {} + }); + await WorkflowStep.create({ + name: "step1", + trigger_id: tr.id, + action_name: "set_context", + next_step: "", + initial_step: true, + configuration: { ctx_values: '{ "hello": "world" }' }, + only_if: "" + }); + `); + + await test("create_workflow_step op journaled", async () => { + const r = sql(MAIN.db, "SELECT COUNT(*) FROM _dd_ops WHERE op_type='create_workflow_step'"); + assert.equal(r, "1"); + }); + + const peerIdForWf = sql(MAIN.db, `SELECT peer_id FROM _dd_peers WHERE env_id='${testEnv}'`); + const wfPromote = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/promote", { peer_id: peerIdForWf }); + await test("workflow_step promote returns success", async () => { + assert.equal(wfPromote.status, 302); + assert.match(wfPromote.redirect, /msg=promoted/); + }); + await test("test has the demo_workflow trigger", async () => { + const r = sql(TEST.db, "SELECT name FROM _sc_triggers WHERE name='demo_workflow'"); + assert.equal(r, "demo_workflow"); + }); + await test("test has the workflow step", async () => { + const r = sql(TEST.db, "SELECT COUNT(*) FROM _sc_workflow_steps s JOIN _sc_triggers t ON s.trigger_id=t.id WHERE t.name='demo_workflow' AND s.name='step1'"); + assert.equal(r, "1"); + }); + await test("workflow_step UUID matches across instances", async () => { + const m = sql(MAIN.db, "SELECT uuid FROM _dd_entity_ids WHERE kind='workflow_step' AND current_name='step1'"); + const t = sql(TEST.db, "SELECT uuid FROM _dd_entity_ids WHERE kind='workflow_step' AND current_name='step1'"); + assert.equal(m, t); + assert.notEqual(m, ""); + }); + + section("config + menu propagation"); + scExec(MAIN.env, ` + const { getState } = require("@saltcorn/data/db/state"); + await getState().setConfig("menu_items", [ + { label: "Widgets", type: "View", viewname: "widget_list" }, + { label: "About", type: "Page", pagename: "about" } + ]); + `); + await test("set_config op journaled for menu_items", async () => { + const r = sql(MAIN.db, "SELECT COUNT(*) FROM _dd_ops WHERE op_type='set_config' AND payload LIKE '%menu_items%'"); + assert.ok(parseInt(r, 10) >= 1, `got ${r} set_config ops`); + }); + const peerIdForMenu = sql(MAIN.db, `SELECT peer_id FROM _dd_peers WHERE env_id='${testEnv}'`); + const menuPromote = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/promote", { peer_id: peerIdForMenu }); + await test("menu promote returns success", async () => { + assert.equal(menuPromote.status, 302); + assert.match(menuPromote.redirect, /msg=promoted/); + }); + await test("test instance has matching menu_items", async () => { + const v = sql(TEST.db, "SELECT value FROM _sc_config WHERE key='menu_items'"); + assert.match(v, /widget_list/); + assert.match(v, /about/); + }); + + section("plugin configuration propagation"); + scExec(MAIN.env, ` + const Plugin = require("@saltcorn/data/models/plugin"); + const p = await Plugin.findOne({ name: "sbadmin2" }); + if (!p) throw new Error("sbadmin2 plugin not found"); + p.configuration = { test_key: "from MAIN", color_scheme: "dark" }; + await p.upsert(); + `); + await test("update_plugin_config op journaled for sbadmin2", async () => { + const r = sql(MAIN.db, "SELECT COUNT(*) FROM _dd_ops WHERE op_type='update_plugin_config' AND payload LIKE '%sbadmin2%'"); + assert.equal(r, "1"); + }); + const peerIdForPC = sql(MAIN.db, `SELECT peer_id FROM _dd_peers WHERE env_id='${testEnv}'`); + const pcPromote = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/promote", { peer_id: peerIdForPC }); + await test("plugin-config promote returns success", async () => { + assert.equal(pcPromote.status, 302); + assert.match(pcPromote.redirect, /msg=promoted/); + }); + await test("test sbadmin2 has updated configuration", async () => { + const cfg = sql(TEST.db, "SELECT configuration FROM _sc_plugins WHERE name='sbadmin2'"); + assert.match(cfg, /from MAIN/); + assert.match(cfg, /color_scheme/); + }); + + section("managed row data propagation"); + + // Create two tables on main: categories + products with FK products.category_id -> categories.id. + // Add some rows. Then mark both managed. Promote. Verify rows on test with same UUIDs. + scExec(MAIN.env, ` + const Table = require("@saltcorn/data/models/table"); + const Field = require("@saltcorn/data/models/field"); + + const cats = await Table.create("dd_categories"); + await Field.create({ table_id: cats.id, name: "name", label: "Name", type: "String" }); + + const prods = await Table.create("dd_products"); + await Field.create({ table_id: prods.id, name: "name", label: "Name", type: "String" }); + await Field.create({ table_id: prods.id, name: "price", label: "Price", type: "Float" }); + await Field.create({ table_id: prods.id, name: "category", label: "Category", type: "Key to dd_categories", reftable_name: "dd_categories", reftype: "Integer" }); + + const cats2 = Table.findOne({name: "dd_categories"}); + const electId = await cats2.insertRow({ name: "Electronics" }); + const apparelId = await cats2.insertRow({ name: "Apparel" }); + + const prods2 = Table.findOne({name: "dd_products"}); + await prods2.insertRow({ name: "Widget", price: 9.99, category: electId }); + await prods2.insertRow({ name: "Gadget", price: 19.95, category: electId }); + await prods2.insertRow({ name: "T-shirt", price: 14.99, category: apparelId }); + `); + + // Mark both tables managed via the admin UI (causes initial ship) + const catsUuid = sql(MAIN.db, "SELECT uuid FROM _dd_entity_ids WHERE kind='table' AND current_name='dd_categories'"); + const prodsUuid = sql(MAIN.db, "SELECT uuid FROM _dd_entity_ids WHERE kind='table' AND current_name='dd_products'"); + + const setCatsManaged = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/tables/set", { table_uuid: catsUuid, data_mode: "managed" }); + const setProdsManaged = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/tables/set", { table_uuid: prodsUuid, data_mode: "managed" }); + + await test("marking dd_categories managed ships 2 rows", async () => { + assert.match(setCatsManaged.redirect, /shipped.{1,3}2.{1,3}rows/); + }); + await test("marking dd_products managed ships 3 rows", async () => { + assert.match(setProdsManaged.redirect, /shipped.{1,3}3.{1,3}rows/); + }); + await test("_dd_row_uuid column exists on dd_categories on main", async () => { + const info = sqlRows(MAIN.db, "PRAGMA table_info(dd_categories);").join(","); + assert.match(info, /_dd_row_uuid/); + }); + await test("insert_row ops journaled for the seed rows", async () => { + const c = sql(MAIN.db, "SELECT COUNT(*) FROM _dd_ops WHERE op_type='insert_row'"); + assert.ok(parseInt(c, 10) >= 5, `expected >=5 insert_row ops, got ${c}`); + }); + + // Promote + const peerIdForRows = sql(MAIN.db, `SELECT peer_id FROM _dd_peers WHERE env_id='${testEnv}'`); + const rowPromote = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/promote", { peer_id: peerIdForRows }); + await test("managed row promote returns success", async () => { + assert.equal(rowPromote.status, 302); + assert.match(rowPromote.redirect, /msg=promoted/); + }); + await test("test has dd_categories rows", async () => { + const r = sql(TEST.db, "SELECT COUNT(*) FROM dd_categories"); + assert.equal(r, "2"); + }); + await test("test has dd_products rows", async () => { + const r = sql(TEST.db, "SELECT COUNT(*) FROM dd_products"); + assert.equal(r, "3"); + }); + await test("a product on test has its FK resolved to a local category id", async () => { + const r = sql(TEST.db, "SELECT p.name || '|' || c.name FROM dd_products p JOIN dd_categories c ON p.category=c.id WHERE p.name='Widget'"); + assert.equal(r, "Widget|Electronics"); + }); + await test("row UUIDs match across instances for a product", async () => { + const m = sql(MAIN.db, "SELECT _dd_row_uuid FROM dd_products WHERE name='Widget'"); + const t = sql(TEST.db, "SELECT _dd_row_uuid FROM dd_products WHERE name='Widget'"); + assert.equal(m, t); + assert.notEqual(m, ""); + }); + + // Update a row on main, promote again, verify on test. + scExec(MAIN.env, ` + const Table = require("@saltcorn/data/models/table"); + const prods = Table.findOne({name: "dd_products"}); + const widget = (await prods.getJoinedRows({ where: { name: "Widget" }}))[0]; + await prods.updateRow({ price: 12.50 }, widget.id); + `); + const rowPromote2 = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/promote", { peer_id: peerIdForRows }); + await test("update_row op promoted", async () => { + assert.match(rowPromote2.redirect, /msg=promoted/); + }); + await test("test sees Widget at new price 12.5", async () => { + const r = sql(TEST.db, "SELECT price FROM dd_products WHERE name='Widget'"); + assert.equal(r, "12.5"); + }); + + // Delete a row on main, promote, verify removed on test. + scExec(MAIN.env, ` + const Table = require("@saltcorn/data/models/table"); + const prods = Table.findOne({name: "dd_products"}); + await prods.deleteRows({ name: "T-shirt" }); + `); + const rowPromote3 = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/promote", { peer_id: peerIdForRows }); + await test("drop_row op promoted", async () => { + assert.match(rowPromote3.redirect, /msg=promoted/); + }); + await test("test no longer has T-shirt row", async () => { + const r = sql(TEST.db, "SELECT COUNT(*) FROM dd_products WHERE name='T-shirt'"); + assert.equal(r, "0"); + }); + + section("starter rows: ship once then detach"); + scExec(MAIN.env, ` + const Table = require("@saltcorn/data/models/table"); + const Field = require("@saltcorn/data/models/field"); + const seed = await Table.create("dd_templates"); + await Field.create({ table_id: seed.id, name: "name", label: "Name", type: "String" }); + const t = Table.findOne({name: "dd_templates"}); + await t.insertRow({ name: "Welcome" }); + await t.insertRow({ name: "Goodbye" }); + `); + const seedUuid = sql(MAIN.db, "SELECT uuid FROM _dd_entity_ids WHERE kind='table' AND current_name='dd_templates'"); + const setSeedStarter = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/tables/set", { table_uuid: seedUuid, data_mode: "starter" }); + await test("marking dd_templates starter ships 2 rows", async () => { + assert.match(setSeedStarter.redirect, /shipped.{1,3}2.{1,3}rows/); + }); + await test("dd_templates marked starter_shipped_at", async () => { + const r = sql(MAIN.db, `SELECT starter_shipped_at IS NOT NULL FROM _dd_table_modes WHERE table_uuid='${seedUuid}'`); + assert.equal(r, "1"); + }); + + // After starter ship, further row ops on main should NOT journal + const beforeCount = sql(MAIN.db, "SELECT COUNT(*) FROM _dd_ops WHERE op_type='insert_row' OR op_type='update_row'"); + scExec(MAIN.env, ` + const Table = require("@saltcorn/data/models/table"); + const t = Table.findOne({name: "dd_templates"}); + await t.insertRow({ name: "Post-ship row" }); + `); + const afterCount = sql(MAIN.db, "SELECT COUNT(*) FROM _dd_ops WHERE op_type='insert_row' OR op_type='update_row'"); + await test("subsequent inserts on starter table do NOT journal", async () => { + assert.equal(afterCount, beforeCount, `journal grew from ${beforeCount} to ${afterCount} on a starter table`); + }); + + const starterPromote = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/promote", { peer_id: peerIdForRows }); + await test("starter promote ships initial rows", async () => { + assert.match(starterPromote.redirect, /msg=promoted/); + }); + await test("test has the 2 initial templates (but not the post-ship row)", async () => { + const c = sql(TEST.db, "SELECT COUNT(*) FROM dd_templates"); + assert.equal(c, "2"); + const has = sql(TEST.db, "SELECT COUNT(*) FROM dd_templates WHERE name='Welcome'"); + assert.equal(has, "1"); + const postship = sql(TEST.db, "SELECT COUNT(*) FROM dd_templates WHERE name='Post-ship row'"); + assert.equal(postship, "0"); + }); + + section("revert"); + runJs(MAIN.env, 'await Table.create("ephemeral");'); + const createOp = sql(MAIN.db, "SELECT op_id FROM _dd_ops WHERE op_type='create_table' AND payload LIKE '%ephemeral%' LIMIT 1"); + const revertRes = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/revert", { op_id: createOp }); + await test("revert returns 302", async () => { + assert.equal(revertRes.status, 302); + }); + await test("ephemeral table is gone after revert", async () => { + const r = sql(MAIN.db, "SELECT COUNT(*) FROM _sc_tables WHERE name='ephemeral'"); + assert.equal(r, "0"); + }); + await test("drop_table op recorded as the revert", async () => { + const r = sql(MAIN.db, "SELECT COUNT(*) FROM _dd_ops WHERE op_type='drop_table' AND payload LIKE '%ephemeral%'"); + assert.ok(parseInt(r, 10) >= 1); + }); + + section("data_mode"); + const widgetsUuid = sql(MAIN.db, "SELECT uuid FROM _dd_entity_ids WHERE current_name='widgets'"); + const setRes = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/tables/set", { + table_uuid: widgetsUuid, + data_mode: "starter" + }); + await test("tables/set redirects with success", async () => { + assert.equal(setRes.status, 302); + assert.match(setRes.redirect, /msg=set/); + }); + await test("_dd_table_modes row inserted with starter", async () => { + const r = sql(MAIN.db, `SELECT data_mode FROM _dd_table_modes WHERE table_uuid='${widgetsUuid}'`); + assert.equal(r, "starter"); + }); + await test("users table is locked: cannot change data_mode", async () => { + const usersUuid = sql(MAIN.db, "SELECT uuid FROM _dd_entity_ids WHERE current_name='users' AND kind='table'"); + const r = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/tables/set", { + table_uuid: usersUuid, + data_mode: "managed" + }); + assert.match(r.redirect || "", /err=/); + const mode = sql(MAIN.db, `SELECT COUNT(*) FROM _dd_table_modes WHERE table_uuid='${usersUuid}'`); + assert.equal(mode, "0", "users table should have no row in _dd_table_modes"); + }); + + section("revert (extended op types)"); + // Validates the broadened revert handlers (rows / config / table-mode / + // constraint), which previously threw "no revert handler". Runs after the + // data_mode section so a set_table_mode op (with before_mode) exists. + const wUuidR = sql(MAIN.db, "SELECT uuid FROM _dd_entity_ids WHERE current_name='widgets' AND kind='table'"); + const modeBeforeRevert = sql(MAIN.db, `SELECT data_mode FROM _dd_table_modes WHERE table_uuid='${wUuidR}'`); + const setModeOp = sql(MAIN.db, `SELECT op_id FROM _dd_ops WHERE op_type='set_table_mode' AND entity_uuid='${wUuidR}' ORDER BY created_at DESC LIMIT 1`); + const rmode = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/revert", { op_id: setModeOp }); + await test("revert set_table_mode returns 302 (no crash)", async () => { + assert.equal(rmode.status, 302); + }); + await test("set_table_mode revert changed widgets' mode away from 'starter'", async () => { + assert.equal(modeBeforeRevert, "starter"); + const after = sql(MAIN.db, `SELECT data_mode FROM _dd_table_modes WHERE table_uuid='${wUuidR}'`); + assert.notEqual(after, "starter", "mode should have been restored to before_mode"); + }); + + const conOp = sql(MAIN.db, "SELECT op_id FROM _dd_ops WHERE op_type='create_constraint' AND payload LIKE '%Unique%' ORDER BY created_at DESC LIMIT 1"); + const rcon = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/revert", { op_id: conOp }); + await test("revert create_constraint returns 302", async () => { + assert.equal(rcon.status, 302); + }); + await test("unique(sku) constraint dropped on MAIN after revert", async () => { + const c = sql(MAIN.db, "SELECT COUNT(*) FROM _sc_table_constraints WHERE type='Unique' AND configuration LIKE '%sku%'"); + assert.equal(c, "0"); + }); + await test("drop_constraint op recorded as the constraint revert", async () => { + const c = sql(MAIN.db, "SELECT COUNT(*) FROM _dd_ops WHERE op_type='drop_constraint'"); + assert.ok(parseInt(c, 10) >= 1); + }); + + scExec(MAIN.env, `const {getState}=require("@saltcorn/data/db/state"); await getState().setConfig("site_name","REV_BEFORE");`); + scExec(MAIN.env, `const {getState}=require("@saltcorn/data/db/state"); await getState().setConfig("site_name","REV_AFTER");`); + const cfgOp = sql(MAIN.db, "SELECT op_id FROM _dd_ops WHERE op_type='set_config' AND payload LIKE '%REV_AFTER%' ORDER BY created_at DESC LIMIT 1"); + const rcfg = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/revert", { op_id: cfgOp }); + await test("revert set_config returns 302", async () => { + assert.equal(rcfg.status, 302); + }); + await test("set_config revert restored site_name to REV_BEFORE", async () => { + const v = sql(MAIN.db, "SELECT value FROM _sc_config WHERE key='site_name'"); + assert.match(v, /REV_BEFORE/); + }); + + scExec(MAIN.env, `const t = await Table.create("revrows"); await Field.create({table_id:t.id,name:"label",label:"Label",type:"String"});`); + const rrUuid = sql(MAIN.db, "SELECT uuid FROM _dd_entity_ids WHERE current_name='revrows' AND kind='table'"); + adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/tables/set", { table_uuid: rrUuid, data_mode: "managed" }); + scExec(MAIN.env, `const t = Table.findOne({name:"revrows"}); await t.insertRow({label:"to-revert"});`); + const rowOp = sql(MAIN.db, "SELECT op_id FROM _dd_ops WHERE op_type='insert_row' AND payload LIKE '%to-revert%' ORDER BY created_at DESC LIMIT 1"); + await test("insert_row op journaled for the managed table", async () => { + assert.notEqual(rowOp, ""); + }); + const rrow = adminPost(MAIN, MAIN_COOKIES, "/admin/dev-deploy/revert", { op_id: rowOp }); + await test("revert insert_row returns 302", async () => { + assert.equal(rrow.status, 302); + }); + await test("row deleted from revrows after insert_row revert", async () => { + const c = sql(MAIN.db, "SELECT COUNT(*) FROM revrows"); + assert.equal(c, "0"); + }); + + section("machine-endpoint security"); + // Test that requests without HMAC are rejected + await test("unsigned POST to ingest returns 400 (missing header)", async () => { + const out = curl([ + "-sS", "-o", "/dev/null", "-w", "%{http_code}", + "-X", "POST", + "-H", "Content-Type: application/vnd.dev-deploy+json", + "--data", '{"ops":[]}', + `${TEST.url}/dev-deploy/api/ingest` + ]); + assert.equal(out, "400"); + }); + await test("ingest with bogus signature returns 401", async () => { + const out = curl([ + "-sS", "-o", "/dev/null", "-w", "%{http_code}", + "-X", "POST", + "-H", "Content-Type: application/vnd.dev-deploy+json", + "-H", `X-DD-Env-Id: ${mainEnv}`, + "-H", `X-DD-Timestamp: ${Date.now()}`, + "-H", "X-DD-Nonce: deadbeefdeadbeefdeadbeefdeadbeef", + "-H", "X-DD-Signature: 0000000000000000000000000000000000000000000000000000000000000000", + "--data", '{"ops":[]}', + `${TEST.url}/dev-deploy/api/ingest` + ]); + assert.equal(out, "401"); + }); + await test("ingest from unknown env_id returns 401", async () => { + const out = curl([ + "-sS", "-o", "/dev/null", "-w", "%{http_code}", + "-X", "POST", + "-H", "Content-Type: application/vnd.dev-deploy+json", + "-H", "X-DD-Env-Id: 00000000-0000-4000-8000-000000000000", + "-H", `X-DD-Timestamp: ${Date.now()}`, + "-H", "X-DD-Nonce: deadbeefdeadbeefdeadbeefdeadbeef", + "-H", "X-DD-Signature: 0000000000000000000000000000000000000000000000000000000000000000", + "--data", '{"ops":[]}', + `${TEST.url}/dev-deploy/api/ingest` + ]); + assert.equal(out, "401"); + }); + await test("ingest with stale timestamp returns 401", async () => { + const stale = String(Date.now() - 10 * 60 * 1000); + const out = curl([ + "-sS", "-o", "/dev/null", "-w", "%{http_code}", + "-X", "POST", + "-H", "Content-Type: application/vnd.dev-deploy+json", + "-H", `X-DD-Env-Id: ${mainEnv}`, + "-H", `X-DD-Timestamp: ${stale}`, + "-H", "X-DD-Nonce: deadbeefdeadbeefdeadbeefdeadbeef", + "-H", "X-DD-Signature: 0000000000000000000000000000000000000000000000000000000000000000", + "--data", '{"ops":[]}', + `${TEST.url}/dev-deploy/api/ingest` + ]); + assert.equal(out, "401"); + }); + + section("admin auth"); + await test("unauthenticated GET /admin/dev-deploy/ returns 403", async () => { + const out = curl(["-sS", "-o", "/dev/null", "-w", "%{http_code}", `${MAIN.url}/admin/dev-deploy/`]); + assert.equal(out, "403"); + }); + + // --- summary --- + console.log(`\n${passed} passed, ${failed} failed`); + if (failed > 0) { + console.log("\nFailures:"); + for (const f of failures) { + console.log(` - ${f.name}`); + console.log(` ${f.err.stack.split("\n").slice(0, 4).join("\n ")}`); + } + process.exit(1); + } + process.exit(0); +}; + + +main().catch((err) => { + console.error("test runner crashed:", err); + process.exit(2); +}); diff --git a/test/managedRowsGate.js b/test/managedRowsGate.js new file mode 100644 index 0000000..4f91f70 --- /dev/null +++ b/test/managedRowsGate.js @@ -0,0 +1,255 @@ +// Phase 2 gate: the MANAGED-ROWS (row-data sync) feature on multi-tenant +// Postgres. Proves the rowIdentity PG path + per-tenant isolation + cross-tenant +// row sync end to end: +// 1. mark a table managed on tenant t1 -> rowIdentity.ensureManagedSchema adds +// the hidden _dd_row_uuid column to t1's tenant-schema table and backfills +// existing rows with UUIDs (the SQLite-vs-PG portable column path), and the +// initial ship journals a set_table_mode + one insert_row op per row; +// 2. ISOLATION: sibling tenant t2 gets no _dd_table_modes row for that table; +// 3. SYNC: promote t1 -> t2 (tenant-to-tenant peering, Phase 1) -> t2 applies +// the ops, creating the table with the SAME _dd_row_uuid values (stable +// cross-environment row identity) -- the apply-on-PG path. +// +// HTTP drives the feature; psql introspects each tenant's schema (the _dd_row_uuid +// column + row UUIDs aren't observable over HTTP). Self-skips if PG :3002 or psql +// is unavailable. Run: node test/managedRowsGate.js + +const http = require("http"); +const net = require("net"); +const { execFileSync } = require("child_process"); + +const PG_PORT = 3002; +const ADMIN_PW = "AdminP@ss1"; +const PGENV = Object.assign({}, process.env, { + PGHOST: "/var/run/postgresql", + PGUSER: "scott", + PGDATABASE: "saltcorn_idp", + PGPASSWORD: "peer" +}); + +const thost = (t) => t + ".localhost.localdomain:" + PG_PORT; +const jars = { t1: {}, t2: {} }; +let pass = 0; +let fail = 0; + + +const ok = (cond, msg) => { + if (cond) { + pass++; + console.log(" PASS " + msg); + } else { + fail++; + console.log(" FAIL " + msg); + } +}; + + +const portOpen = (port) => { + return new Promise((resolve) => { + const s = net.connect(port, "127.0.0.1"); + const done = (up) => { s.destroy(); resolve(up); }; + s.setTimeout(1000); + s.on("connect", () => done(true)); + s.on("timeout", () => done(false)); + s.on("error", () => done(false)); + }); +}; + + +const psql = (sql) => execFileSync("psql", ["-tAqc", sql], { env: PGENV }).toString().trim(); +const psqlLines = (sql) => psql(sql).split("\n").map((s) => s.trim()).filter(Boolean); + + +const storeCookies = (t, headers) => { + const sc = headers["set-cookie"]; + if (!sc) { + return; + } + for (const line of sc) { + const pair = line.split(";")[0]; + const eq = pair.indexOf("="); + if (eq > 0) { + jars[t][pair.slice(0, eq).trim()] = pair.slice(eq + 1).trim(); + } + } +}; + + +const request = (t, method, path, opts) => { + const options = opts || {}; + return new Promise((resolve, reject) => { + const headers = Object.assign({ Host: thost(t) }, options.headers || {}); + const jar = jars[t]; + if (Object.keys(jar).length > 0) { + headers["Cookie"] = Object.keys(jar).map((k) => k + "=" + jar[k]).join("; "); + } + let data = null; + if (options.body) { + data = new URLSearchParams(options.body).toString(); + headers["Content-Type"] = "application/x-www-form-urlencoded"; + headers["Content-Length"] = Buffer.byteLength(data); + } + const r = http.request({ host: "127.0.0.1", port: PG_PORT, method: method, path: path, headers: headers }, (resp) => { + storeCookies(t, resp.headers); + let body = ""; + resp.on("data", (c) => { body += c; }); + resp.on("end", () => resolve({ status: resp.statusCode, headers: resp.headers, loc: resp.headers.location, body: body })); + }); + r.on("error", reject); + if (data !== null) { + r.write(data); + } + r.end(); + }); +}; + + +const csrfOf = (h) => { const m = h.match(/name="_csrf" value="([^"]+)"/); return m ? m[1] : ""; }; +const opCountOf = (h) => { const m = h.match(/Ops recorded<\/th>(\d+)<\/td>/); return m ? parseInt(m[1], 10) : null; }; +const envIdOf = (h) => { const m = h.match(/env_id<\/strong> is ([0-9a-f-]{36})<\/code>/); return m ? m[1] : ""; }; +const secretOf = (h) => { const m = h.match(/

([0-9a-f]+)<\/p>/); return m ? m[1] : ""; }; +const ddCsrf = async (t) => csrfOf((await request(t, "GET", "/admin/dev-deploy/peers")).body); + + +const authed = async (t) => /true/.test((await request(t, "GET", "/auth/authenticated")).body); + + +const bootstrap = async (t) => { + const lp = await request(t, "GET", "/auth/login"); + await request(t, "POST", "/auth/login", { body: { email: "admin@" + t + ".local", password: ADMIN_PW, _csrf: csrfOf(lp.body) } }); + if (await authed(t)) { + return true; + } + const cp = await request(t, "GET", "/auth/create_first_user"); + const cc = csrfOf(cp.body); + if (cc) { + await request(t, "POST", "/auth/create_first_user", { body: { email: "admin@" + t + ".local", password: ADMIN_PW, default_language: "en", _csrf: cc } }); + } + return await authed(t); +}; + + +const peerIdFor = (html, envId) => { + for (const row of html.split("")) { + if (row.indexOf("" + envId + "") >= 0) { + const m = row.match(/name="peer_id" value="(\d+)"/); + if (m) { + return m[1]; + } + } + } + return null; +}; + + +const clearPeer = async (t, envId) => { + const pid = peerIdFor((await request(t, "GET", "/admin/dev-deploy/peers")).body, envId); + if (pid) { + await request(t, "POST", "/admin/dev-deploy/peers/delete", { body: { peer_id: pid, _csrf: await ddCsrf(t) } }); + } +}; + + +const cleanup = (tbl) => { + for (const t of ["t1", "t2"]) { + try { psql(`DROP TABLE IF EXISTS "${t}"."${tbl}"`); } catch (e) { /* ignore */ } + try { psql(`DELETE FROM "${t}"._dd_entity_ids WHERE current_name='${tbl}'`); } catch (e) { /* ignore */ } + try { psql(`DELETE FROM "${t}"._dd_table_modes WHERE table_uuid IN (SELECT uuid FROM "${t}"._dd_entity_ids WHERE current_name='${tbl}')`); } catch (e) { /* ignore */ } + } +}; + + +const run = async (tbl) => { + ok(await bootstrap("t1"), "t1 admin session"); + ok(await bootstrap("t2"), "t2 admin session"); + + // Create the table on t1 only; seed two rows directly (the wrap journals the + // initial ship from these existing rows when we mark the table managed). + const c1 = await ddCsrf("t1"); + const cr = await request("t1", "POST", "/table", { body: { name: tbl, _csrf: c1 } }); + ok(cr.status >= 300 && cr.status < 400 && /\/table\/\d+/.test(cr.loc || ""), "t1 created table " + tbl + " (HTTP " + cr.status + ")"); + psql(`INSERT INTO "t1"."${tbl}" DEFAULT VALUES`); + psql(`INSERT INTO "t1"."${tbl}" DEFAULT VALUES`); + ok(psql(`SELECT count(*) FROM "t1"."${tbl}"`) === "2", "t1 seeded 2 rows"); + const tblUuid = psql(`SELECT uuid FROM "t1"._dd_entity_ids WHERE kind='table' AND current_name='${tbl}'`); + ok(/^[0-9a-f-]{36}$/.test(tblUuid), "t1 tracked the table (uuid " + tblUuid + ")"); + + const t1Ops0 = opCountOf((await request("t1", "GET", "/admin/dev-deploy/")).body); + const t2Ops0 = opCountOf((await request("t2", "GET", "/admin/dev-deploy/")).body); + + // Mark managed on t1. + const setr = await request("t1", "POST", "/admin/dev-deploy/tables/set", { body: { table_uuid: tblUuid, data_mode: "managed", _csrf: await ddCsrf("t1") } }); + const setMsg = decodeURIComponent(setr.loc || ""); + ok(setr.status >= 300 && setr.status < 400 && /managed/.test(setMsg) && !/err=/.test(setMsg), "t1 marked table managed (" + setMsg.replace(/^.*\?/, "") + ")"); + + // rowIdentity PG path: column added + every existing row backfilled. + const colT1 = psql(`SELECT count(*) FROM information_schema.columns WHERE table_schema='t1' AND table_name='${tbl}' AND column_name='_dd_row_uuid'`); + ok(colT1 === "1", "t1 table gained the _dd_row_uuid column (PG ensureManagedSchema)"); + ok(psql(`SELECT count(*) FROM "t1"."${tbl}" WHERE _dd_row_uuid IS NOT NULL`) === "2", "t1 backfilled both rows with UUIDs"); + + // Initial ship journaled: set_table_mode + 2 insert_row = 3 ops. + const t1Ops1 = opCountOf((await request("t1", "GET", "/admin/dev-deploy/")).body); + ok(t1Ops1 !== null && t1Ops1 >= t1Ops0 + 3, "t1 journaled the managed ship (" + t1Ops0 + " -> " + t1Ops1 + ", +>=3)"); + + // ISOLATION: marking managed on t1 did not touch t2's _dd_table_modes. + ok(psql(`SELECT count(*) FROM "t2"._dd_table_modes WHERE table_uuid='${tblUuid}'`) === "0", "t2 has NO _dd_table_modes row for the table (isolation)"); + ok(t2Ops0 === opCountOf((await request("t2", "GET", "/admin/dev-deploy/")).body), "t2 journal unchanged by t1's managed ship"); + + // SYNC: pair t1 -> t2 and promote. t2 applies create_table + set_table_mode + + // insert_row, recreating the table with the SAME row UUIDs. + const t1Env = envIdOf((await request("t1", "GET", "/admin/dev-deploy/peers")).body); + const t2Env = envIdOf((await request("t2", "GET", "/admin/dev-deploy/peers")).body); + await clearPeer("t1", t2Env); + await clearPeer("t2", t1Env); + const addRes = await request("t1", "POST", "/admin/dev-deploy/peers/add", { body: { env_id: t2Env, label: "t2", base_url: "http://" + thost("t2"), _csrf: await ddCsrf("t1") } }); + const secret = secretOf(addRes.body); + ok(/^[0-9a-f]{64}$/.test(secret), "t1 paired with t2"); + await request("t2", "POST", "/admin/dev-deploy/peers/add", { body: { env_id: t1Env, label: "t1", base_url: "http://" + thost("t1"), existing_secret: secret, _csrf: await ddCsrf("t2") } }); + + const t2PeerId = peerIdFor((await request("t1", "GET", "/admin/dev-deploy/peers")).body, t2Env); + const prom = await request("t1", "POST", "/admin/dev-deploy/promote", { body: { peer_id: t2PeerId, _csrf: await ddCsrf("t1") } }); + ok(prom.status >= 200 && prom.status < 400, "t1 promote -> t2 (HTTP " + prom.status + ")"); + + // t2 now has the table, managed, with the SAME _dd_row_uuid values as t1. + const colT2 = psql(`SELECT count(*) FROM information_schema.columns WHERE table_schema='t2' AND table_name='${tbl}' AND column_name='_dd_row_uuid'`); + ok(colT2 === "1", "t2 received the table WITH the _dd_row_uuid column (apply on PG)"); + ok(psql(`SELECT count(*) FROM "t2"._dd_table_modes m JOIN "t2"._dd_entity_ids e ON e.uuid=m.table_uuid WHERE e.current_name='${tbl}' AND m.data_mode='managed'`) === "1", "t2's table is now data_mode=managed"); + const u1 = psqlLines(`SELECT _dd_row_uuid FROM "t1"."${tbl}" ORDER BY _dd_row_uuid`); + const u2 = psqlLines(`SELECT _dd_row_uuid FROM "t2"."${tbl}" ORDER BY _dd_row_uuid`); + ok(u2.length === 2 && JSON.stringify(u1) === JSON.stringify(u2), "t2 rows carry the SAME UUIDs as t1 (stable cross-tenant row identity)"); + + // Cleanup peers + tables. + await clearPeer("t1", t2Env); + await clearPeer("t2", t1Env); + cleanup(tbl); + + console.log("\nRESULT: " + pass + " passed, " + fail + " failed"); + process.exit(fail ? 1 : 0); +}; + + +const main = async () => { + if (!(await portOpen(PG_PORT))) { + console.log("SKIP: Postgres multi-tenant instance not reachable on 127.0.0.1:" + PG_PORT); + process.exit(0); + } + try { + execFileSync("psql", ["-tAqc", "SELECT 1"], { env: PGENV }); + } catch (e) { + console.log("SKIP: psql/Postgres not available for schema introspection"); + process.exit(0); + } + const tbl = "mr" + Date.now(); + try { + await run(tbl); + } catch (e) { + cleanup(tbl); + throw e; + } +}; + + +main().catch((e) => { + console.error("MANAGED-ROWS GATE ERROR:", e && (e.stack || e.message || e)); + process.exit(2); +}); diff --git a/test/mixedTopologyGate.js b/test/mixedTopologyGate.js new file mode 100644 index 0000000..88efad0 --- /dev/null +++ b/test/mixedTopologyGate.js @@ -0,0 +1,267 @@ +// Phase 1 mixed-topology peering gate. A STANDALONE dev instance (SQLite MAIN on +// :3000) pairs with a specific TENANT (t1) on the multi-tenant Postgres "prod" +// (:3002), promotes its metadata ops to t1, and we verify: +// 1. the ops land in t1's journal and NOT in sibling tenant t2 (mixed-topology +// isolation -- a standalone dev deploying to one tenant on a shared server); +// 2. the tenant-bound HMAC: a peer request signed for t1 is REJECTED (401) when +// replayed against t2 on the same server, while the SAME request signed for +// t2 passes auth -- proving the rejection is host-specific, not the secret. +// +// dev and t2 both hold a dev peer row with the SAME shared secret, so the only +// thing standing between a t1-signed request and t2 accepting it is the host +// binding in the canonical string (lib/crypto.js buildCanonical -> targetHost). +// +// Run: node test/mixedTopologyGate.js (MAIN :3000 + PG :3002 up, dev-deploy +// installed on MAIN and per-tenant on t1/t2). Self-skips if either is down. + +const http = require("http"); +const net = require("net"); + +const { buildCanonical, sign, normalizeHost } = require("../lib/crypto"); + +const DEV = { port: 3000, host: "localhost:3000", base: "http://localhost:3000" }; +const PROD_PORT = 3002; +const ADMIN_PW = "AdminP@ss1"; +const INGEST = "/dev-deploy/api/ingest"; + +const thost = (t) => t + ".localhost.localdomain:" + PROD_PORT; +const jars = { dev: {}, t1: {}, t2: {} }; +let pass = 0; +let fail = 0; + + +const ok = (cond, msg) => { + if (cond) { + pass++; + console.log(" PASS " + msg); + } else { + fail++; + console.log(" FAIL " + msg); + } +}; + + +const portOpen = (port) => { + return new Promise((resolve) => { + const s = net.connect(port, "127.0.0.1"); + const done = (up) => { s.destroy(); resolve(up); }; + s.setTimeout(1000); + s.on("connect", () => done(true)); + s.on("timeout", () => done(false)); + s.on("error", () => done(false)); + }); +}; + + +const HOSTOF = (inst) => (inst === "dev" ? DEV.host : thost(inst)); +const PORTOF = (inst) => (inst === "dev" ? DEV.port : PROD_PORT); +const ADMIN = (inst) => (inst === "dev" ? "admin@local" : "admin@" + inst + ".local"); + + +const storeCookies = (inst, headers) => { + const sc = headers["set-cookie"]; + if (!sc) { + return; + } + for (const line of sc) { + const pair = line.split(";")[0]; + const eq = pair.indexOf("="); + if (eq > 0) { + jars[inst][pair.slice(0, eq).trim()] = pair.slice(eq + 1).trim(); + } + } +}; + + +// opts.host overrides the Host header; opts.headers sets raw headers (for the +// signed replay); opts.rawBody sends an exact body string; opts.body is a form. +const request = (inst, method, path, opts) => { + const options = opts || {}; + return new Promise((resolve, reject) => { + const headers = Object.assign({ Host: options.host || HOSTOF(inst) }, options.headers || {}); + const jar = jars[inst]; + if (jar && Object.keys(jar).length > 0) { + headers["Cookie"] = Object.keys(jar).map((k) => k + "=" + jar[k]).join("; "); + } + let data = null; + if (options.rawBody !== undefined) { + data = options.rawBody; + headers["Content-Length"] = Buffer.byteLength(data); + } else if (options.body) { + data = new URLSearchParams(options.body).toString(); + headers["Content-Type"] = "application/x-www-form-urlencoded"; + headers["Content-Length"] = Buffer.byteLength(data); + } + const r = http.request({ host: "127.0.0.1", port: PORTOF(inst), method: method, path: path, headers: headers }, (resp) => { + storeCookies(inst, resp.headers); + let body = ""; + resp.on("data", (c) => { body += c; }); + resp.on("end", () => resolve({ status: resp.statusCode, headers: resp.headers, body: body })); + }); + r.on("error", reject); + if (data !== null && data !== undefined) { + r.write(data); + } + r.end(); + }); +}; + + +const csrfOf = (h) => { const m = h.match(/name="_csrf" value="([^"]+)"/); return m ? m[1] : ""; }; +const envIdOf = (h) => { const m = h.match(/env_id<\/strong> is ([0-9a-f-]{36})<\/code>/); return m ? m[1] : ""; }; +const opCountOf = (h) => { const m = h.match(/Ops recorded<\/th>(\d+)<\/td>/); return m ? parseInt(m[1], 10) : null; }; +const secretOf = (h) => { const m = h.match(/

([0-9a-f]+)<\/p>/); return m ? m[1] : ""; }; + + +// Find the peers-table row carrying envId and pull its peer_id out of the row's +// promote/delete form input. +const peerIdFor = (html, envId) => { + for (const row of html.split("")) { + if (row.indexOf("" + envId + "") >= 0) { + const m = row.match(/name="peer_id" value="(\d+)"/); + if (m) { + return m[1]; + } + } + } + return null; +}; + + +const authed = async (inst) => /true/.test((await request(inst, "GET", "/auth/authenticated")).body); + + +const bootstrap = async (inst) => { + const lp = await request(inst, "GET", "/auth/login"); + await request(inst, "POST", "/auth/login", { body: { email: ADMIN(inst), password: ADMIN_PW, _csrf: csrfOf(lp.body) } }); + if (await authed(inst)) { + return true; + } + const cp = await request(inst, "GET", "/auth/create_first_user"); + const cc = csrfOf(cp.body); + if (cc) { + await request(inst, "POST", "/auth/create_first_user", { body: { email: ADMIN(inst), password: ADMIN_PW, default_language: "en", _csrf: cc } }); + } + return await authed(inst); +}; + + +const ddCsrf = async (inst) => csrfOf((await request(inst, "GET", "/admin/dev-deploy/peers")).body); + + +// Remove any existing peer row for envId (idempotent reruns). +const clearPeer = async (inst, envId) => { + const pid = peerIdFor((await request(inst, "GET", "/admin/dev-deploy/peers")).body, envId); + if (pid) { + await request(inst, "POST", "/admin/dev-deploy/peers/delete", { body: { peer_id: pid, _csrf: await ddCsrf(inst) } }); + } +}; + + +const addPeer = async (inst, envId, label, baseUrl, secret) => { + const body = { env_id: envId, label: label, base_url: baseUrl, _csrf: await ddCsrf(inst) }; + if (secret) { + body.existing_secret = secret; + } + return await request(inst, "POST", "/admin/dev-deploy/peers/add", { body: body }); +}; + + +const run = async () => { + ok(await bootstrap("dev"), "dev (standalone MAIN :3000) admin session"); + ok(await bootstrap("t1"), "t1 admin session"); + ok(await bootstrap("t2"), "t2 admin session"); + + const devEnv = envIdOf((await request("dev", "GET", "/admin/dev-deploy/peers")).body); + const t1Env = envIdOf((await request("t1", "GET", "/admin/dev-deploy/peers")).body); + const t2Env = envIdOf((await request("t2", "GET", "/admin/dev-deploy/peers")).body); + ok(/^[0-9a-f-]{36}$/.test(devEnv), "dev env_id (" + devEnv + ")"); + ok(t1Env && t2Env && t1Env !== t2Env && t1Env !== devEnv, "dev/t1/t2 have distinct env_ids"); + + // Idempotent: drop any peer rows left by a prior run. + await clearPeer("dev", t1Env); + await clearPeer("t1", devEnv); + await clearPeer("t2", devEnv); + + // Pair dev -> t1 (dev generates the shared secret); t1 stores dev with it. + const addRes = await addPeer("dev", t1Env, "prod-t1", "http://" + thost("t1")); + const secret = secretOf(addRes.body); + ok(/^[0-9a-f]{64}$/.test(secret), "dev paired with t1 (shared secret issued)"); + await addPeer("t1", devEnv, "dev", DEV.base, secret); + // t2 also holds the dev peer with the SAME secret -> the cross-tenant replay + // can ONLY be stopped by the host binding, not by a differing secret. + await addPeer("t2", devEnv, "dev", DEV.base, secret); + + const t1Ops0 = opCountOf((await request("t1", "GET", "/admin/dev-deploy/")).body); + const t2Ops0 = opCountOf((await request("t2", "GET", "/admin/dev-deploy/")).body); + + // dev promotes its journal to t1. + const t1PeerId = peerIdFor((await request("dev", "GET", "/admin/dev-deploy/peers")).body, t1Env); + ok(!!t1PeerId, "dev has a peer_id for t1 (" + t1PeerId + ")"); + const prom = await request("dev", "POST", "/admin/dev-deploy/promote", { body: { peer_id: t1PeerId, _csrf: await ddCsrf("dev") } }); + ok(prom.status >= 200 && prom.status < 400, "dev promote -> t1 returned HTTP " + prom.status); + + const t1Ops1 = opCountOf((await request("t1", "GET", "/admin/dev-deploy/")).body); + const t2Ops1 = opCountOf((await request("t2", "GET", "/admin/dev-deploy/")).body); + ok(t1Ops1 !== null && t1Ops1 > t1Ops0, "t1 received dev's ops (" + t1Ops0 + " -> " + t1Ops1 + ")"); + ok(t2Ops1 === t2Ops0, "t2 journal UNCHANGED by dev->t1 promote (" + t2Ops0 + " -> " + t2Ops1 + ")"); + + // --- Tenant-bound HMAC --- + // Build a peer-signed ingest exactly as transport.js would, then aim it at the + // wrong tenant. Signature material is identical except the target host. + const secretBuf = Buffer.from(secret, "hex"); + const mkHeaders = (targetHost, body) => { + const ts = String(Date.now()); + const nonce = "00112233445566778899aabbccddeeff"; + const canonical = buildCanonical({ + timestamp: ts, + nonce: nonce, + method: "POST", + path: INGEST, + targetHost: normalizeHost(targetHost), + body: body + }); + return { + "X-DD-Env-Id": devEnv, + "X-DD-Timestamp": ts, + "X-DD-Nonce": nonce, + "X-DD-Signature": sign(secretBuf, canonical), + "Content-Type": "application/vnd.dev-deploy+json" + }; + }; + const body = "{}"; + // Signed for t1, replayed at t2 -> must be rejected by the host binding. + const replay = await request("t2", "POST", INGEST, { host: thost("t2"), headers: mkHeaders(thost("t1"), body), rawBody: body }); + ok(replay.status === 401, "ingest signed for t1 REJECTED at t2 (tenant-bound HMAC; HTTP " + replay.status + ")"); + // Same request, but signed for t2 -> auth must pass (proves the rejection was + // host-specific, not a secret/timestamp problem). Non-401 = auth accepted. + const control = await request("t2", "POST", INGEST, { host: thost("t2"), headers: mkHeaders(thost("t2"), body), rawBody: body }); + ok(control.status !== 401, "same ingest signed for t2 passes auth at t2 (HTTP " + control.status + ", host-specific rejection confirmed)"); + + // Cleanup (best-effort): drop the peer rows so reruns start clean. + await clearPeer("dev", t1Env); + await clearPeer("t1", devEnv); + await clearPeer("t2", devEnv); + + console.log("\nRESULT: " + pass + " passed, " + fail + " failed"); + process.exit(fail ? 1 : 0); +}; + + +const main = async () => { + if (!(await portOpen(DEV.port))) { + console.log("SKIP: standalone dev not reachable on 127.0.0.1:" + DEV.port); + process.exit(0); + } + if (!(await portOpen(PROD_PORT))) { + console.log("SKIP: Postgres multi-tenant prod not reachable on 127.0.0.1:" + PROD_PORT); + process.exit(0); + } + await run(); +}; + + +main().catch((e) => { + console.error("MIXED-TOPOLOGY GATE ERROR:", e); + process.exit(2); +}); diff --git a/test/mtGate.js b/test/mtGate.js new file mode 100644 index 0000000..741a2d3 --- /dev/null +++ b/test/mtGate.js @@ -0,0 +1,270 @@ +// Multi-tenant ISOLATION gate for dev-deploy against the Postgres instance +// (:3002). Phase 0 proof: each tenant is its OWN dev-deploy environment on the +// shared Postgres server (schema-qualified _dd_* tables + per-tenant env row). +// The gate bootstraps the t1 and t2 admins over HTTP, reads each tenant's +// dev-deploy admin dashboard, and asserts that: +// * each dashboard is reachable (200) and exposes a dev-deploy env_id, +// * t1's env_id and t2's env_id are DISTINCT (separate per-tenant env rows), +// * the dashboards' "Ops recorded" and "Entities tracked" counts are reported +// per tenant -- mutating t1 moves only t1's numbers, never t2's. +// Tenants are addressed by Host header (tNN.localhost.localdomain:3002 -> tenant +// tNN); each tenant uses a separate cookie jar. Run: node test/mtGate.js +// +// Prerequisites: +// * PG multi-tenant instance up on :3002 (./startServerPg.sh). +// * Tenants t1 and t2 exist (saltcorn create-tenant t1 / t2). +// * dev-deploy installed per-tenant: +// ./dev-deploy/scripts/installDevDeployTenant.sh t1 t2 +// (this creates the _dd_* tables + bootstraps each tenant's env row). +// If :3002 is not reachable the gate self-skips (exit 0, prints SKIP). + +const http = require("http"); +const net = require("net"); + +const PG_PORT = 3002; +const TENANTS = ["t1", "t2"]; +const ADMIN_PW = "AdminP@ss1"; + +const jars = { t1: {}, t2: {} }; +let pass = 0; +let fail = 0; + + +const ok = (cond, msg) => { + if (cond) { + pass++; + console.log(" PASS " + msg); + } else { + fail++; + console.log(" FAIL " + msg); + } +}; + + +const HOST = (t) => t + ".localhost.localdomain:" + PG_PORT; +const ADMIN = (t) => "admin@" + t + ".local"; + + +// TCP connect probe: resolve true if :3002 accepts a connection within 1s. +const portOpen = (port) => { + return new Promise((resolve) => { + const sock = net.connect(port, "127.0.0.1"); + const done = (up) => { + sock.destroy(); + resolve(up); + }; + sock.setTimeout(1000); + sock.on("connect", () => done(true)); + sock.on("timeout", () => done(false)); + sock.on("error", () => done(false)); + }); +}; + + +const storeCookies = (t, headers) => { + const sc = headers["set-cookie"]; + if (!sc) { + return; + } + for (const line of sc) { + const pair = line.split(";")[0]; + const eq = pair.indexOf("="); + if (eq > 0) { + jars[t][pair.slice(0, eq).trim()] = pair.slice(eq + 1).trim(); + } + } +}; + + +const request = (t, method, path, opts) => { + const options = opts || {}; + return new Promise((resolve, reject) => { + const headers = Object.assign({ Host: HOST(t) }, options.headers || {}); + const jar = jars[t]; + if (Object.keys(jar).length > 0) { + headers["Cookie"] = Object.keys(jar).map((k) => k + "=" + jar[k]).join("; "); + } + let data = null; + if (options.body) { + data = new URLSearchParams(options.body).toString(); + headers["Content-Type"] = "application/x-www-form-urlencoded"; + headers["Content-Length"] = Buffer.byteLength(data); + } + const r = http.request({ host: "127.0.0.1", port: PG_PORT, method: method, path: path, headers: headers }, (resp) => { + storeCookies(t, resp.headers); + let body = ""; + resp.on("data", (c) => { body += c; }); + resp.on("end", () => resolve({ status: resp.statusCode, headers: resp.headers, body: body })); + }); + r.on("error", reject); + if (data !== null) { + r.write(data); + } + r.end(); + }); +}; + + +const csrfOf = (html) => { + const m = html.match(/name="_csrf" value="([^"]+)"/); + return m ? m[1] : ""; +}; + + +const authed = async (t) => { + return /true/.test((await request(t, "GET", "/auth/authenticated")).body); +}; + + +// Ensure a tenant admin exists and the jar holds an authenticated session. +const bootstrapTenant = async (t) => { + const lp = await request(t, "GET", "/auth/login"); + await request(t, "POST", "/auth/login", { body: { email: ADMIN(t), password: ADMIN_PW, _csrf: csrfOf(lp.body) } }); + if (await authed(t)) { + return true; + } + const cp = await request(t, "GET", "/auth/create_first_user"); + const cc = csrfOf(cp.body); + if (cc) { + await request(t, "POST", "/auth/create_first_user", { body: { email: ADMIN(t), password: ADMIN_PW, default_language: "en", _csrf: cc } }); + } + if (await authed(t)) { + return true; + } + const lp2 = await request(t, "GET", "/auth/login"); + await request(t, "POST", "/auth/login", { body: { email: ADMIN(t), password: ADMIN_PW, _csrf: csrfOf(lp2.body) } }); + return await authed(t); +}; + + +// The dashboard renders the env_id in the "Env ID" row as: +// Env IDUUID +// (see lib/routes.js dashboard()). Anchor on that row so a later UUID +// cannot be mistaken for it. +const envIdOf = (html) => { + const m = html.match(/Env ID<\/th>([0-9a-f-]{36})<\/code>/); + return m ? m[1] : ""; +}; + + +// The "Ops recorded" row is: Ops recordedN +const opCountOf = (html) => { + const m = html.match(/Ops recorded<\/th>(\d+)<\/td>/); + return m ? parseInt(m[1], 10) : null; +}; + + +// The "Entities tracked" table renders one kindcount +// per kind; sum the counts for a single per-tenant entity total. +const entityTotalOf = (html) => { + const idx = html.indexOf("Entities tracked"); + if (idx < 0) { + return null; + } + const section = html.slice(idx); + let total = 0; + let matched = false; + const re = /[^<]+<\/td>(\d+)<\/td><\/tr>/g; + let m; + while ((m = re.exec(section)) !== null) { + total += parseInt(m[1], 10); + matched = true; + } + return matched ? total : null; +}; + + +const dashboardOf = async (t) => { + return await request(t, "GET", "/admin/dev-deploy/"); +}; + + +const run = async () => { + // Bootstrap both tenant admins. + for (const t of TENANTS) { + ok(await bootstrapTenant(t), t + " admin session established"); + } + + // Both dashboards reachable (200) and each exposes a dev-deploy env_id. + const d1 = await dashboardOf("t1"); + const d2 = await dashboardOf("t2"); + ok(d1.status === 200, "t1 dev-deploy dashboard reachable (HTTP " + d1.status + ")"); + ok(d2.status === 200, "t2 dev-deploy dashboard reachable (HTTP " + d2.status + ")"); + + const env1 = envIdOf(d1.body); + const env2 = envIdOf(d2.body); + ok(/^[0-9a-f-]{36}$/.test(env1), "t1 dashboard exposes an env_id (" + (env1 || "none") + ")"); + ok(/^[0-9a-f-]{36}$/.test(env2), "t2 dashboard exposes an env_id (" + (env2 || "none") + ")"); + + // Core Phase 0 assertion: the two tenants are DISTINCT dev-deploy envs. + ok(env1 && env2 && env1 !== env2, "t1 and t2 have DISTINCT env_ids (" + env1 + " / " + env2 + ")"); + + // Each tenant reports its OWN ops/entities counts (they are not one shared + // number). Capture the baselines first. + const ops1 = opCountOf(d1.body); + const ops2 = opCountOf(d2.body); + const ent1 = entityTotalOf(d1.body); + const ent2 = entityTotalOf(d2.body); + ok(ops1 !== null && ops2 !== null, "both dashboards report an 'Ops recorded' count (t1=" + ops1 + ", t2=" + ops2 + ")"); + ok(ent1 !== null && ent2 !== null, "both dashboards report an 'Entities tracked' total (t1=" + ent1 + ", t2=" + ent2 + ")"); + + // Mutate ONLY t1 (creating a table journals create_table + tracks a new + // 'table' entity), then re-read both dashboards. t1's counts must move and + // t2's must NOT -- proving the _dd_* tables are schema-qualified per tenant + // rather than a single shared journal. POST /table (Saltcorn's own admin + // create handler) calls Table.create, which dev-deploy's hooks journal as a + // create_table op. Its CSRF lives on GET /table/new; success redirects 302 + // to /table/. + const tableName = "mt_iso_probe_" + Date.now(); + // Saltcorn's own table admin pages (e.g. GET /table/new/) call res.sendWrap, + // which 500s on a freshly created tenant that has no theme/layout configured. + // dev-deploy's admin pages self-render, so grab the (per-session) _csrf token + // from one of those; POST /table itself only mutates + redirects (no sendWrap), + // so it succeeds on a themeless tenant and journals a create_table op. + const tcsrf = csrfOf((await request("t1", "GET", "/admin/dev-deploy/peers")).body); + const createRes = await request("t1", "POST", "/table", { body: { name: tableName, _csrf: tcsrf } }); + const created = createRes.status >= 300 && createRes.status < 400 && /\/table\/\d+/.test(createRes.headers.location || ""); + ok(created, "t1 created a table to move its journal (HTTP " + createRes.status + " -> " + (createRes.headers.location || "") + ")"); + + const d1b = await dashboardOf("t1"); + const d2b = await dashboardOf("t2"); + const ops1b = opCountOf(d1b.body); + const ops2b = opCountOf(d2b.body); + const ent1b = entityTotalOf(d1b.body); + const ent2b = entityTotalOf(d2b.body); + + ok(ops1b !== null && ops1b > ops1, "t1 'Ops recorded' increased after t1 mutation (" + ops1 + " -> " + ops1b + ")"); + ok(ops2b === ops2, "t2 'Ops recorded' UNCHANGED by t1 mutation (" + ops2 + " -> " + ops2b + ")"); + ok(ent1b !== null && ent1b > ent1, "t1 'Entities tracked' increased after t1 mutation (" + ent1 + " -> " + ent1b + ")"); + ok(ent2b === ent2, "t2 'Entities tracked' UNCHANGED by t1 mutation (" + ent2 + " -> " + ent2b + ")"); + + // Cleanup: drop the probe table so reruns stay deterministic. Best-effort. + try { + const tid = (createRes.headers.location || "").match(/\/table\/(\d+)/); + if (tid) { + // Same theme caveat: take the _csrf from dev-deploy's page, not /table/. + const dcsrf = csrfOf((await request("t1", "GET", "/admin/dev-deploy/peers")).body); + await request("t1", "POST", "/table/delete/" + tid[1], { body: { _csrf: dcsrf } }); + } + } catch (e) { + // ignore cleanup failures + } + + console.log("\nRESULT: " + pass + " passed, " + fail + " failed"); + process.exit(fail ? 1 : 0); +}; + + +const main = async () => { + if (!(await portOpen(PG_PORT))) { + console.log("SKIP: Postgres multi-tenant instance not reachable on 127.0.0.1:" + PG_PORT); + process.exit(0); + } + await run(); +}; + + +main().catch((e) => { + console.error("MT GATE ERROR:", e); + process.exit(2); +}); diff --git a/test/pullGate.js b/test/pullGate.js new file mode 100644 index 0000000..b8b0f48 --- /dev/null +++ b/test/pullGate.js @@ -0,0 +1,347 @@ +// Phase 1 PULL peering gate (the reverse of mixedTopologyGate's push/promote). +// mixedTopologyGate proves dev -> t1 PROMOTE (push to /api/ingest). This gate +// proves the PULL path (/admin/dev-deploy/pull -> the peer's signed +// /api/journal) across the same multi-tenant boundary, which nothing else +// covers: e2e.js exercises pull only single-instance (SQLite MAIN <-> MAIN), +// never across the Postgres tenant boundary or tenant<->tenant. +// +// Topology: STANDALONE dev (SQLite MAIN :3000) + tenants t1, t2 on the +// multi-tenant Postgres "prod" (:3002). We verify: +// 1. REVERSE PULL (mixed topology): t1 PULLS dev's journal -> dev's ops land +// in t1 (applied >= 1), proving the host-bound HMAC works on the GET pull +// path (signedFetch targetHost = dev's host), not just the push path. +// 2. ISOLATION ON PULL: t2's journal is UNCHANGED by t1 pulling from dev -- +// a pull into one tenant never leaks into a sibling tenant. +// 3. IDEMPOTENT RE-PULL: pulling again with the advanced inbound anchor +// returns "nothing to pull" (no duplicate apply). +// 4. TENANT<->TENANT PULL: t2 PULLS t1's journal (same Postgres server, two +// tenants) -> t1's OWN ops land in t2 while dev is untouched. apiJournal +// serves only first-party ops (source_env_id == self), so t2 receives +// t1's ops, NOT the dev ops t1 itself pulled in step 1 (no transitive +// relay / no loops). +// +// Run: node test/pullGate.js (MAIN :3000 + PG :3002 up, dev-deploy installed +// on MAIN and per-tenant on t1/t2). Self-skips if either is down. + +const http = require("http"); +const net = require("net"); + +const DEV = { port: 3000, host: "localhost:3000", base: "http://localhost:3000" }; +const PROD_PORT = 3002; +const ADMIN_PW = "AdminP@ss1"; +const RUN = Date.now(); +const DEV_PROBE = "pull_dev_" + RUN; +const T1_PROBE = "pull_t1_" + RUN; + +const thost = (t) => t + ".localhost.localdomain:" + PROD_PORT; +const jars = { dev: {}, t1: {}, t2: {} }; +let pass = 0; +let fail = 0; + + +const ok = (cond, msg) => { + if (cond) { + pass++; + console.log(" PASS " + msg); + } else { + fail++; + console.log(" FAIL " + msg); + } +}; + + +const portOpen = (port) => { + return new Promise((resolve) => { + const s = net.connect(port, "127.0.0.1"); + const done = (up) => { s.destroy(); resolve(up); }; + s.setTimeout(1000); + s.on("connect", () => done(true)); + s.on("timeout", () => done(false)); + s.on("error", () => done(false)); + }); +}; + + +const HOSTOF = (inst) => (inst === "dev" ? DEV.host : thost(inst)); +const PORTOF = (inst) => (inst === "dev" ? DEV.port : PROD_PORT); +const ADMIN = (inst) => (inst === "dev" ? "admin@local" : "admin@" + inst + ".local"); +const BASEOF = (inst) => (inst === "dev" ? DEV.base : "http://" + thost(inst)); + + +const storeCookies = (inst, headers) => { + const sc = headers["set-cookie"]; + if (!sc) { + return; + } + for (const line of sc) { + const pair = line.split(";")[0]; + const eq = pair.indexOf("="); + if (eq > 0) { + jars[inst][pair.slice(0, eq).trim()] = pair.slice(eq + 1).trim(); + } + } +}; + + +const request = (inst, method, path, opts) => { + const options = opts || {}; + return new Promise((resolve, reject) => { + const headers = Object.assign({ Host: options.host || HOSTOF(inst) }, options.headers || {}); + const jar = jars[inst]; + if (jar && Object.keys(jar).length > 0) { + headers["Cookie"] = Object.keys(jar).map((k) => k + "=" + jar[k]).join("; "); + } + let data = null; + if (options.body) { + data = new URLSearchParams(options.body).toString(); + headers["Content-Type"] = "application/x-www-form-urlencoded"; + headers["Content-Length"] = Buffer.byteLength(data); + } + const r = http.request({ host: "127.0.0.1", port: PORTOF(inst), method: method, path: path, headers: headers }, (resp) => { + storeCookies(inst, resp.headers); + let body = ""; + resp.on("data", (c) => { body += c; }); + resp.on("end", () => resolve({ status: resp.statusCode, headers: resp.headers, body: body })); + }); + r.on("error", reject); + if (data !== null) { + r.write(data); + } + r.end(); + }); +}; + + +const csrfOf = (h) => { const m = h.match(/name="_csrf" value="([^"]+)"/); return m ? m[1] : ""; }; +const envIdOf = (h) => { const m = h.match(/env_id<\/strong> is ([0-9a-f-]{36})<\/code>/); return m ? m[1] : ""; }; +const opCountOf = (h) => { const m = h.match(/Ops recorded<\/th>(\d+)<\/td>/); return m ? parseInt(m[1], 10) : null; }; +const secretOf = (h) => { const m = h.match(/

([0-9a-f]+)<\/p>/); return m ? m[1] : ""; }; + + +// pull redirects 302 with ?msg=... (success) or ?err=... (failure); the message +// reads "pulled N ops (A applied, E errors, C conflicts)" or "nothing to pull". +const redirectMsg = (res) => { + const loc = res.headers.location || ""; + const m = loc.match(/[?&](?:msg|err)=([^&]+)/); + return m ? decodeURIComponent(m[1]) : ""; +}; + + +const pulledCounts = (msg) => { + const m = msg.match(/pulled (\d+) ops \((\d+) applied, (\d+) errors, (\d+) conflicts\)/); + return m ? { n: +m[1], applied: +m[2], errors: +m[3], conflicts: +m[4] } : null; +}; + + +const peerIdFor = (html, envId) => { + for (const row of html.split("")) { + if (row.indexOf("" + envId + "") >= 0) { + const m = row.match(/name="peer_id" value="(\d+)"/); + if (m) { + return m[1]; + } + } + } + return null; +}; + + +// Tracked-tables admin page renders one row per tracked table: +// NAMEUUID8LOCAL_ID... +// Map a probe table's NAME to its LOCAL_ID so cleanup can drop it (the source +// copy AND any copy materialized by a pull) without tracking create locations. +const localIdByName = async (inst, name) => { + const html = (await request(inst, "GET", "/admin/dev-deploy/tables")).body; + const re = new RegExp("" + name + "\\s*[0-9a-f]{8}\\s*(\\d+)"); + const m = html.match(re); + return m ? m[1] : null; +}; + + +const authed = async (inst) => /true/.test((await request(inst, "GET", "/auth/authenticated")).body); + + +const bootstrap = async (inst) => { + const lp = await request(inst, "GET", "/auth/login"); + await request(inst, "POST", "/auth/login", { body: { email: ADMIN(inst), password: ADMIN_PW, _csrf: csrfOf(lp.body) } }); + if (await authed(inst)) { + return true; + } + const cp = await request(inst, "GET", "/auth/create_first_user"); + const cc = csrfOf(cp.body); + if (cc) { + await request(inst, "POST", "/auth/create_first_user", { body: { email: ADMIN(inst), password: ADMIN_PW, default_language: "en", _csrf: cc } }); + } + return await authed(inst); +}; + + +const ddCsrf = async (inst) => csrfOf((await request(inst, "GET", "/admin/dev-deploy/peers")).body); + + +const clearPeer = async (inst, envId) => { + const pid = peerIdFor((await request(inst, "GET", "/admin/dev-deploy/peers")).body, envId); + if (pid) { + await request(inst, "POST", "/admin/dev-deploy/peers/delete", { body: { peer_id: pid, _csrf: await ddCsrf(inst) } }); + } +}; + + +// Returns the issued shared secret when the peer is added WITHOUT one (the +// adding side generates it); returns "" when an existing secret is supplied. +const addPeer = async (inst, envId, label, baseUrl, secret) => { + const body = { env_id: envId, label: label, base_url: baseUrl, _csrf: await ddCsrf(inst) }; + if (secret) { + body.existing_secret = secret; + } + const res = await request(inst, "POST", "/admin/dev-deploy/peers/add", { body: body }); + return secretOf(res.body); +}; + + +// Create a table via Saltcorn's own POST /table (journaled by dev-deploy as a +// create_table op authored by THIS instance). The _csrf comes from dev-deploy's +// self-rendered page: a freshly created themeless tenant 500s on /table/new +// (sendWrap with no layout), but POST /table only mutates + redirects, so it +// succeeds. Returns true on the 302 -> /table/ success. +const createTable = async (inst, name) => { + const csrf = await ddCsrf(inst); + const res = await request(inst, "POST", "/table", { body: { name: name, _csrf: csrf } }); + return res.status >= 300 && res.status < 400 && /\/table\/\d+/.test(res.headers.location || ""); +}; + + +const dropTableByName = async (inst, name) => { + const id = await localIdByName(inst, name); + if (id) { + await request(inst, "POST", "/table/delete/" + id, { body: { _csrf: await ddCsrf(inst) } }); + } +}; + + +const opCount = async (inst) => opCountOf((await request(inst, "GET", "/admin/dev-deploy/")).body); + + +const doPull = async (inst, peerId) => { + return await request(inst, "POST", "/admin/dev-deploy/pull", { body: { peer_id: peerId, _csrf: await ddCsrf(inst) } }); +}; + + +const run = async () => { + ok(await bootstrap("dev"), "dev (standalone MAIN :3000) admin session"); + ok(await bootstrap("t1"), "t1 admin session"); + ok(await bootstrap("t2"), "t2 admin session"); + + const devEnv = envIdOf((await request("dev", "GET", "/admin/dev-deploy/peers")).body); + const t1Env = envIdOf((await request("t1", "GET", "/admin/dev-deploy/peers")).body); + const t2Env = envIdOf((await request("t2", "GET", "/admin/dev-deploy/peers")).body); + ok(/^[0-9a-f-]{36}$/.test(devEnv), "dev env_id (" + devEnv + ")"); + ok(t1Env && t2Env && t1Env !== t2Env && t1Env !== devEnv, "dev/t1/t2 have distinct env_ids"); + + // Fresh pairings (idempotent reruns): a new peer_id => a fresh inbound anchor + // => the pull starts from the beginning of the source journal. + await clearPeer("dev", t1Env); + await clearPeer("t1", devEnv); + await clearPeer("t1", t2Env); + await clearPeer("t2", t1Env); + + // Pair dev <-> t1 so t1 can pull from dev: t1 needs dev as a peer (base_url + + // secret to initiate); dev needs t1 as a peer (to AUTH t1's signed pull, which + // peerAuth looks up by t1's env_id). dev issues the secret. + const secretDevT1 = await addPeer("dev", t1Env, "prod-t1", BASEOF("t1")); + ok(/^[0-9a-f]{64}$/.test(secretDevT1), "dev<->t1 paired (shared secret issued)"); + await addPeer("t1", devEnv, "dev", BASEOF("dev"), secretDevT1); + + // Give dev a brand-new op to serve: a uniquely named table guarantees an op + // t1 has never seen (status "applied", not "already_applied"), so the assert + // holds on every rerun regardless of accumulated history. + ok(await createTable("dev", DEV_PROBE), "dev created a probe table to seed its journal"); + + const t1Before = await opCount("t1"); + const t2Before = await opCount("t2"); + + // (1) REVERSE PULL: t1 pulls dev's journal. + const t1DevPeer = peerIdFor((await request("t1", "GET", "/admin/dev-deploy/peers")).body, devEnv); + ok(!!t1DevPeer, "t1 has a peer_id for dev (" + t1DevPeer + ")"); + const pull1 = await doPull("t1", t1DevPeer); + const pull1Msg = redirectMsg(pull1); + const c1 = pulledCounts(pull1Msg); + ok(pull1.status === 302, "t1 pull from dev returned 302 (" + pull1.status + ")"); + ok(c1 !== null, "t1 pull reported a pulled-ops summary (\"" + pull1Msg + "\")"); + ok(c1 && c1.applied >= 1, "t1 APPLIED >= 1 op pulled from dev (applied=" + (c1 ? c1.applied : "?") + ")"); + ok(c1 && c1.errors === 0 && c1.conflicts === 0, "t1 pull had no errors/conflicts"); + + const t1After1 = await opCount("t1"); + const t2After1 = await opCount("t2"); + // (1) lands in t1. + ok(t1After1 > t1Before, "t1 journal grew from the pull (" + t1Before + " -> " + t1After1 + ")"); + // (2) ISOLATION ON PULL: t2 untouched. + ok(t2After1 === t2Before, "t2 journal UNCHANGED by t1 pulling from dev (" + t2Before + " -> " + t2After1 + ")"); + + // (3) IDEMPOTENT RE-PULL: anchor advanced, nothing new to fetch. + const pull2 = await doPull("t1", t1DevPeer); + const pull2Msg = redirectMsg(pull2); + ok(/nothing to pull/.test(pull2Msg), "re-pull from dev is idempotent (\"" + pull2Msg + "\")"); + const t1After2 = await opCount("t1"); + ok(t1After2 === t1After1, "t1 journal UNCHANGED by the idempotent re-pull (" + t1After1 + " -> " + t1After2 + ")"); + + // (4) TENANT<->TENANT PULL: pair t1<->t2 (t1 issues), seed t1 with its own op, + // then t2 pulls t1. t1's apiJournal serves only t1-authored ops, so t2 gets + // t1's probe -- NOT the dev ops t1 pulled in step 1 (no transitive relay). + const secretT1T2 = await addPeer("t1", t2Env, "prod-t2", BASEOF("t2")); + ok(/^[0-9a-f]{64}$/.test(secretT1T2), "t1<->t2 paired (shared secret issued)"); + await addPeer("t2", t1Env, "prod-t1", BASEOF("t1"), secretT1T2); + + ok(await createTable("t1", T1_PROBE), "t1 created a probe table to seed its own journal"); + + const devBefore4 = await opCount("dev"); + const t2Before4 = await opCount("t2"); + const t2T1Peer = peerIdFor((await request("t2", "GET", "/admin/dev-deploy/peers")).body, t1Env); + ok(!!t2T1Peer, "t2 has a peer_id for t1 (" + t2T1Peer + ")"); + const pull3 = await doPull("t2", t2T1Peer); + const pull3Msg = redirectMsg(pull3); + const c3 = pulledCounts(pull3Msg); + ok(c3 && c3.applied >= 1, "t2 APPLIED >= 1 op pulled from t1 (tenant<->tenant; applied=" + (c3 ? c3.applied : "?") + ")"); + ok(c3 && c3.errors === 0 && c3.conflicts === 0, "t2 pull from t1 had no errors/conflicts"); + + const t2After4 = await opCount("t2"); + const devAfter4 = await opCount("dev"); + ok(t2After4 > t2Before4, "t2 journal grew from pulling t1 (" + t2Before4 + " -> " + t2After4 + ")"); + ok(devAfter4 === devBefore4, "dev journal UNCHANGED by the t2<->t1 pull (" + devBefore4 + " -> " + devAfter4 + ")"); + + // Cleanup (best-effort): drop each probe ONLY on its SOURCE instance (dev + // authored DEV_PROBE, t1 authored T1_PROBE), never on the puller. A drop is a + // first-party op too: dropping on the source makes the journal create->drop + // LINEAR, so the next run's pull replays it in order and cleans up the pulled + // copy with no conflict. Dropping the PULLED copy instead would author a + // competing local op, and the next run (replaying the source's create) would + // legitimately conflict with it -- a self-inflicted divergence, not a bug. + await dropTableByName("dev", DEV_PROBE); + await dropTableByName("t1", T1_PROBE); + await clearPeer("dev", t1Env); + await clearPeer("t1", devEnv); + await clearPeer("t1", t2Env); + await clearPeer("t2", t1Env); + + console.log("\nRESULT: " + pass + " passed, " + fail + " failed"); + process.exit(fail ? 1 : 0); +}; + + +const main = async () => { + if (!(await portOpen(DEV.port))) { + console.log("SKIP: standalone dev not reachable on 127.0.0.1:" + DEV.port); + process.exit(0); + } + if (!(await portOpen(PROD_PORT))) { + console.log("SKIP: Postgres multi-tenant prod not reachable on 127.0.0.1:" + PROD_PORT); + process.exit(0); + } + await run(); +}; + + +main().catch((e) => { + console.error("PULL GATE ERROR:", e); + process.exit(2); +}); diff --git a/test/sc-exec.js b/test/sc-exec.js new file mode 100644 index 0000000..3750a9b --- /dev/null +++ b/test/sc-exec.js @@ -0,0 +1,57 @@ +// Like `saltcorn run-js` but without the vm sandbox -- the supplied JS body +// runs with full `require()` access so the e2e suite can drive Field / +// TableConstraint / File creation. Invoked from the test runner via: +// +// source && node test/sc-exec.js "" + +const { createRequire } = require("node:module"); +const path = require("node:path"); + +// Resolve @saltcorn/* against the Saltcorn checkout's node_modules. Node +// resolves require from the script's path by default, which doesn't see +// Saltcorn's deps; createRequire reroots the resolution. Layout: +// / +// dev-deploy/test/sc-exec.js (this file) +// saltcorn/packages/saltcorn-data/ +// So: up 2 to project root, then into saltcorn/packages/saltcorn-data/. +const scRequire = createRequire(path.join(__dirname, "..", "..", "saltcorn", "packages", "saltcorn-data", "package.json")); + + +const main = async () => { + // Read code from stdin -- avoids shell-escape gymnastics for multi-line + // bodies (notably \n which bash double-quotes pass through literally). + const code = require("node:fs").readFileSync(0, "utf8"); + if (!code) { + console.error("usage: pipe JS code into sc-exec.js's stdin"); + process.exit(2); + } + + const Plugin = scRequire("@saltcorn/data/models/plugin"); + const { init_multi_tenant } = scRequire("@saltcorn/data/db/state"); + await Plugin.loadAllPlugins(); + await init_multi_tenant(Plugin.loadAllPlugins, undefined, []); + + const Table = scRequire("@saltcorn/data/models/table"); + const Field = scRequire("@saltcorn/data/models/field"); + const View = scRequire("@saltcorn/data/models/view"); + const Page = scRequire("@saltcorn/data/models/page"); + const Trigger = scRequire("@saltcorn/data/models/trigger"); + const TableConstraint = scRequire("@saltcorn/data/models/table_constraints"); + const File = scRequire("@saltcorn/data/models/file"); + const db = scRequire("@saltcorn/data/db"); + + // Build an async closure with the models + a re-rooted require in scope. + // (`new Function` bodies don't inherit require from the script scope.) + const fn = new Function( + "Table", "Field", "View", "Page", "Trigger", "TableConstraint", "File", "db", "require", + `return (async () => { ${code} })();` + ); + await fn(Table, Field, View, Page, Trigger, TableConstraint, File, db, scRequire); + process.exit(0); +}; + + +main().catch((err) => { + console.error(err && err.stack ? err.stack : err); + process.exit(1); +});