// Live pull over REAL HTTP. Modes (each run against its own instance's DB, set by // runLivePull.sh): // init-target register the plugin (real onLoad) on a fresh TARGET DB; print env_id // init-source register + seed distinctive design state on the SOURCE DB, and pair // the target into it; print env_id (needs TARGET_ENV, SHARED) // pair-target pair the source into the TARGET DB (needs SOURCE_ENV, SOURCE_URL, SHARED) // drive (TARGET, in-process) pull the SOURCE over HTTP, plan, apply, verify // (needs SOURCE_ENV, SOURCE_URL) // // The source runs as a real `saltcorn serve` process; the target is driven // in-process. So the pull crosses a real network boundary: the target's outbound // signed request -> HTTP -> the source's real inbound route + HMAC verify -> // /api/state -> a signed response the target verifies before using it. const Table = require("@saltcorn/data/models/table"); const Field = require("@saltcorn/data/models/field"); const File = require("@saltcorn/data/models/file"); const Page = require("@saltcorn/data/models/page"); const { getState } = require("@saltcorn/data/db/state"); const { peerBytes, API_PATHS } = require("../lib/transport"); const FILE_BYTES = Buffer.from("SOURCE FILE BYTES over HTTP"); // layout/logo images must be real images (content-sniffed); vary trailing bytes. const PNG = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==", "base64"); const LOGO_BYTES = Buffer.concat([PNG, Buffer.from("logo")]); const HERO_BYTES = Buffer.concat([PNG, Buffer.from("hero")]); const { bootstrapPlugin, ok, summary } = require("./harness"); const peers = require("../lib/peers"); const env = require("../lib/env"); const capture = require("../lib/capture"); const rows = require("../lib/rows"); const ops = require("../lib/operations"); const planLib = require("../lib/plan"); const applyLib = require("../lib/apply"); const SHARED = process.env.SHARED ? Buffer.from(process.env.SHARED, "hex") : null; const initTarget = async () => { await bootstrapPlugin(); const e = await env.getEnv(); console.log("ENV_ID=" + e.env_id); }; const initSource = async () => { await bootstrapPlugin(); // distinctive design state the target does NOT have const planets = await Table.create("planets", { min_role_read: 1, min_role_write: 1 }); await Field.create({ table: planets, name: "pname", label: "pname", type: "String" }); await Field.create({ table: planets, name: "diameter", label: "diameter", type: "Integer" }); await getState().setConfig("site_name", "SourceSite"); // a managed reference table with rows (its manifest rides /api/state) const status = await Table.create("ref_status", { min_role_read: 1, min_role_write: 1 }); await Field.create({ table: status, name: "label", label: "label", type: "String" }); const fresh = Table.findOne({ name: "ref_status" }); await fresh.insertRow({ label: "active" }); await fresh.insertRow({ label: "archived" }); await rows.setDataMode("ref_status", "managed"); // a managed table with a FILE field (its bytes ride GET /api/file/:hash) const docs = await Table.create("docs", { min_role_read: 1, min_role_write: 1 }); await Field.create({ table: docs, name: "title", label: "title", type: "String" }); await Field.create({ table: docs, name: "attachment", label: "attachment", type: "File" }); const f = await File.from_contents("readme.txt", "text/plain", FILE_BYTES, 1, 1, "/"); await Table.findOne({ name: "docs" }).insertRow({ title: "doc1", attachment: f.field_value }); await rows.setDataMode("docs", "managed"); // a site logo + a page with a layout image (always-on file sync, not row data) const logo = await File.from_contents("logo.png", "image/png", LOGO_BYTES, 1, 100, "/"); await getState().setConfig("site_logo_id", logo.field_value); const hero = await File.from_contents("hero.png", "image/png", HERO_BYTES, 1, 100, "/"); await Page.create({ name: "landing", title: "Landing", description: "", min_role: 1, layout: { above: [{ type: "image", fileid: hero.field_value, srcType: "File" }] }, fixed_states: {} }); // pair the target INTO the source so the source can verify the target's pulls await peers.addPeer({ envId: process.env.TARGET_ENV, label: "target", baseUrl: "http://127.0.0.1:1/", existingSecret: SHARED }); const e = await env.getEnv(); console.log("ENV_ID=" + e.env_id); }; const pairTarget = async () => { await bootstrapPlugin(); await peers.addPeer({ envId: process.env.SOURCE_ENV, label: "source", baseUrl: process.env.SOURCE_URL, existingSecret: SHARED }); }; const drive = async () => { await bootstrapPlugin(); const e = await env.getEnv(); const peerRow = await peers.findPeerByEnvId(process.env.SOURCE_ENV); ok("source peer is paired", !!peerRow); console.log("\n# real HTTP: plugin-parity health check"); const warnings = await ops.diffPluginsWithPeer(peerRow, e, await require("../lib/common/pluginParity").localPluginList()); ok("signed GET /api/health round-tripped over HTTP", Array.isArray(warnings) && !warnings.some((w) => /could not|returned/.test(w)), JSON.stringify(warnings)); console.log("\n# real HTTP: pull desired state"); const state = await ops.fetchPeerState(peerRow, e); ok("signed GET /api/state returned a pack over HTTP", !!(state && state.pack && Array.isArray(state.pack.tables))); ok("source's planets table present in pulled pack", !!state.pack.tables.find((t) => t.name === "planets")); ok("source's managed ref rows rode /api/state", !!(state.rows && state.rows.ref_status && state.rows.ref_status.rows.length === 2)); console.log("\n# plan + apply (in-process) the pulled state"); const { plan_id, plan } = await planLib.buildPlan({ peerRow, env: e, desired: state.pack, desiredHash: state.hash, sourceEnvId: state.env_id, direction: "pull", sourceRows: state.rows }); ok("plan creates planets", plan.entities.tables.create.some((t) => t.name === "planets")); ok("plan sets site_name config", plan.config.sets.site_name === "SourceSite"); const res = await applyLib.applyPlan(plan_id, { applyDeletes: true }); ok("apply succeeded", res.applied === true); console.log("\n# verify the TARGET converged"); const p = Table.findOne({ name: "planets" }); ok("planets table now exists on target", !!p); ok("planets has diameter field", !!(p && p.getField("diameter"))); ok("ref_status table created on target", !!Table.findOne({ name: "ref_status" })); ok("site_name config converged", getState().getConfig("site_name") === "SourceSite", getState().getConfig("site_name")); console.log("\n# always-on layout/logo file sync over HTTP (materialized during apply)"); const landing = Page.findOne({ name: "landing" }); ok("landing page synced", !!landing); const heroFid = landing && landing.layout.above[0].fileid; ok("hero layout image resolved to a local file", !!heroFid && !heroFid.startsWith("__ss_file_ref::")); const heroBytes = heroFid && (await (await File.findOne(heroFid)).get_contents()); ok("hero image content matches over HTTP", !!heroBytes && Buffer.compare(heroBytes, HERO_BYTES) === 0); const logoVal = getState().getConfig("site_logo_id"); ok("site logo resolved to a local file", !!logoVal && !logoVal.startsWith("__ss_file_ref::")); const logoBytes = logoVal && (await (await File.findOne(logoVal)).get_contents()); ok("logo content matches over HTTP", !!logoBytes && Buffer.compare(logoBytes, LOGO_BYTES) === 0); console.log("\n# real HTTP: fetch a file by content hash"); const docRow = state.rows.docs && state.rows.docs.rows[0]; const fileHash = docRow && docRow.attachment__file && docRow.attachment__file.hash; ok("docs manifest carried a file placeholder", !!fileHash); const fileBytes = await peerBytes(peerRow, e.env_id, API_PATHS.file(fileHash)); ok("signed GET /api/file/:hash returned the verified bytes over HTTP", Buffer.compare(fileBytes, FILE_BYTES) === 0); console.log("\n# full file materialization over HTTP (target opts in, second pull)"); await rows.setDataMode("docs", "managed"); const state2 = await ops.fetchPeerState(peerRow, e); const built = await planLib.buildPlan({ peerRow, env: e, desired: state2.pack, desiredHash: state2.hash, sourceEnvId: state2.env_id, direction: "pull", sourceRows: state2.rows }); await applyLib.applyPlan(built.plan_id, { applyDeletes: true }); const doc = (await Table.findOne({ name: "docs" }).getRows({}, {}))[0]; ok("doc row synced to target", !!doc); const localFile = doc && (await File.findOne(doc.attachment)); const bytes = localFile && (await localFile.get_contents()); ok("attachment materialized over HTTP with correct bytes", !!bytes && Buffer.compare(bytes, FILE_BYTES) === 0); }; const MODES = { "init-target": initTarget, "init-source": initSource, "pair-target": pairTarget, "drive": drive }; const EXIT_USAGE = 2; const main = async () => { const run = MODES[process.argv[2]]; if (!run) { console.error("unknown mode: " + process.argv[2]); process.exit(EXIT_USAGE); } await run(); process.exit(summary()); }; main().catch((err) => { console.error(err && err.stack ? err.stack : err); process.exit(1); });