314 lines
12 KiB
JavaScript
314 lines
12 KiB
JavaScript
// Content-addressed transfer of files referenced INSIDE opaque metadata blobs:
|
|
// page layouts, view configurations, and the site_logo_id config value. This is
|
|
// distinct from rowPayload.js (structured managed-row File fields). ALWAYS ON.
|
|
//
|
|
// * CAPTURE: placeholderize() replaces every resolvable file reference with a
|
|
// content-addressed placeholder "__ss_file_ref::<hash>" -- HASH ONLY, so the
|
|
// placeholder is fully content-stable (filename, mime and role are NOT encoded;
|
|
// they either churn or make byte-identical files diff forever). Both sides do
|
|
// this symmetrically, so the pack hash and diff are content-based.
|
|
// * SERVE: indexLocalFiles() adds live layout/logo files to the source's
|
|
// content-hash index so GET /api/file/:hash can serve them.
|
|
// * APPLY: resolvePlaceholders() replaces placeholders with a LOCAL file value:
|
|
// dedup against the target's OWN layout/logo files (never row files, so a file
|
|
// landed via the unrestricted row path can't be laundered into a layout); else
|
|
// fetch the bytes, verify the hash, and SNIFF the mime FROM THE BYTES (an image
|
|
// magic-number allow-list -- the peer's claimed mime is never trusted, and SVG
|
|
// is excluded to avoid script-in-SVG). Unresolvable placeholders are REMOVED
|
|
// (never persisted verbatim, which would break rendering / clobber the logo).
|
|
|
|
const File = require("@saltcorn/data/models/file");
|
|
const fsp = require("fs").promises;
|
|
const { getState } = require("@saltcorn/data/db/state");
|
|
const { hashFileContents } = require("./common/fileStore");
|
|
const { sha256Hex } = require("./common/hash");
|
|
const { ADMIN_USER_ID, PUBLIC_ROLE_ID } = require("./constants");
|
|
|
|
|
|
const PREFIX = "__ss_file_ref::";
|
|
const FILE_REF_KEYS = ["fileid", "file_id", "bgFileId", "image_id"];
|
|
const LOGO_CONFIG_KEY = "site_logo_id";
|
|
const MAX_DEPTH = 100; // guard against a hostile peer's deeply-nested layout JSON
|
|
const HASH_CACHE_MAX = 10000; // bound on the stat-hash cache (entries)
|
|
const FILENAME_HASH_CHARS = 16; // hash prefix used to name a materialized file
|
|
const SHORT_HASH_CHARS = 12; // hash prefix shown in warnings
|
|
const SNIFF_HEADER_BYTES = 12; // bytes needed to recognize every allowed format
|
|
|
|
const MIME_EXT = {
|
|
"image/png": "png", "image/jpeg": "jpg", "image/gif": "gif",
|
|
"image/webp": "webp", "image/bmp": "bmp", "image/x-icon": "ico", "image/avif": "avif"
|
|
};
|
|
|
|
|
|
const isPlaceholder = (v) => typeof v === "string" && v.startsWith(PREFIX);
|
|
const makePlaceholder = (hash) => `${PREFIX}${hash}`;
|
|
const parsePlaceholder = (v) => (isPlaceholder(v) ? { hash: v.slice(PREFIX.length) } : null);
|
|
const shortHash = (hash) => hash.slice(0, SHORT_HASH_CHARS);
|
|
|
|
|
|
// Determine an image mime from the CONTENT (magic numbers), or null if not a
|
|
// recognized raster image. Never trusts a caller-supplied mime. SVG is excluded on
|
|
// purpose (SVG can carry script and is served inline).
|
|
const startsWithBytes = (b, sig, offset = 0) => sig.every((v, i) => b[offset + i] === v);
|
|
const IMAGE_SIGNATURES = [
|
|
{ mime: "image/png", test: (b) => startsWithBytes(b, [0x89, 0x50, 0x4e, 0x47]) },
|
|
{ mime: "image/jpeg", test: (b) => startsWithBytes(b, [0xff, 0xd8, 0xff]) },
|
|
{ mime: "image/gif", test: (b) => startsWithBytes(b, [0x47, 0x49, 0x46, 0x38]) },
|
|
{ mime: "image/bmp", test: (b) => startsWithBytes(b, [0x42, 0x4d]) },
|
|
{ mime: "image/x-icon", test: (b) => startsWithBytes(b, [0x00, 0x00, 0x01, 0x00]) },
|
|
{ mime: "image/webp", test: (b) => startsWithBytes(b, [0x52, 0x49, 0x46, 0x46]) && startsWithBytes(b, [0x57, 0x45, 0x42, 0x50], 8) },
|
|
{ mime: "image/avif", test: (b) => b.slice(4, 8).toString("ascii") === "ftyp" && b.slice(8, 12).toString("ascii").startsWith("avif") }
|
|
];
|
|
const sniffImageMime = (b) => {
|
|
if (!b || b.length < SNIFF_HEADER_BYTES) {
|
|
return null;
|
|
}
|
|
const hit = IMAGE_SIGNATURES.find((s) => s.test(b));
|
|
return hit ? hit.mime : null;
|
|
};
|
|
|
|
const deriveFilename = (hash, mime) => `${hash.slice(0, FILENAME_HASH_CHARS)}.${MIME_EXT[(mime || "").toLowerCase()] || "img"}`;
|
|
|
|
|
|
// stat-based hash cache: re-hash a file only when its (mtime,size) changes, so
|
|
// repeated buildDesiredState/index passes don't re-read every file's bytes.
|
|
// Bounded: cleared once it reaches HASH_CACHE_MAX entries.
|
|
const hashCache = new Map();
|
|
const statHash = async (file) => {
|
|
if (file.s3_store) {
|
|
return await hashFileContents(file);
|
|
}
|
|
try {
|
|
const st = await fsp.stat(file.location);
|
|
const c = hashCache.get(file.location);
|
|
if (c && c.mtimeMs === st.mtimeMs && c.size === st.size) {
|
|
return c.hash;
|
|
}
|
|
const h = await hashFileContents(file);
|
|
if (hashCache.size >= HASH_CACHE_MAX) {
|
|
hashCache.clear();
|
|
}
|
|
hashCache.set(file.location, { mtimeMs: st.mtimeMs, size: st.size, hash: h });
|
|
return h;
|
|
} catch (e) {
|
|
return await hashFileContents(file);
|
|
}
|
|
};
|
|
|
|
// Content hash of the file a File-field value points at, or null (no value,
|
|
// missing file, unreadable).
|
|
const localFileHash = async (value) => {
|
|
if (!value) {
|
|
return null;
|
|
}
|
|
try {
|
|
const file = await File.findOne(value);
|
|
return file ? await statHash(file) : null;
|
|
} catch (e) {
|
|
return null;
|
|
}
|
|
};
|
|
|
|
|
|
// Recursively collect { obj, key, value } for every FILE_REF_KEYS string value,
|
|
// bounded by MAX_DEPTH so a hostile deeply-nested blob can't overflow the stack.
|
|
const collectRefs = (node, out, depth) => {
|
|
if (!node || typeof node !== "object" || depth > MAX_DEPTH) {
|
|
return out;
|
|
}
|
|
if (Array.isArray(node)) {
|
|
for (const el of node) {
|
|
collectRefs(el, out, depth + 1);
|
|
}
|
|
return out;
|
|
}
|
|
for (const [k, v] of Object.entries(node)) {
|
|
if (FILE_REF_KEYS.includes(k) && typeof v === "string" && v) {
|
|
out.push({ obj: node, key: k, value: v });
|
|
} else if (v && typeof v === "object") {
|
|
collectRefs(v, out, depth + 1);
|
|
}
|
|
}
|
|
return out;
|
|
};
|
|
|
|
// Every file ref inside page layouts and view configurations (of a pack, or of the
|
|
// live State), each tagged with its owning entity's min_role -- who can see it --
|
|
// which becomes the materialized file's min_role_read.
|
|
const layoutRefs = (pages, views) => {
|
|
const refs = [];
|
|
const add = (blob, role) => {
|
|
for (const r of collectRefs(blob, [], 0)) {
|
|
refs.push(Object.assign(r, { role }));
|
|
}
|
|
};
|
|
for (const p of (pages || [])) {
|
|
if (p && p.layout) {
|
|
add(p.layout, p.min_role);
|
|
}
|
|
}
|
|
for (const vw of (views || [])) {
|
|
if (vw && vw.configuration) {
|
|
add(vw.configuration, vw.min_role);
|
|
}
|
|
}
|
|
return refs;
|
|
};
|
|
|
|
|
|
// CAPTURE: replace resolvable file refs (layouts/configs + the logo) with hash-only
|
|
// placeholders. Dedup-within-pass via the stat hash cache (same value -> same file).
|
|
const placeholderize = async (pack) => {
|
|
const refs = layoutRefs(pack.pages, pack.views);
|
|
let count = 0;
|
|
for (const ref of refs) {
|
|
if (isPlaceholder(ref.value)) {
|
|
continue;
|
|
}
|
|
const h = await localFileHash(ref.value);
|
|
if (h) {
|
|
ref.obj[ref.key] = makePlaceholder(h);
|
|
count += 1;
|
|
}
|
|
}
|
|
if (pack.config && pack.config[LOGO_CONFIG_KEY] && !isPlaceholder(pack.config[LOGO_CONFIG_KEY])) {
|
|
const h = await localFileHash(pack.config[LOGO_CONFIG_KEY]);
|
|
if (h) {
|
|
pack.config[LOGO_CONFIG_KEY] = makePlaceholder(h);
|
|
count += 1;
|
|
}
|
|
}
|
|
return { count };
|
|
};
|
|
|
|
|
|
// SERVE / dedup: add every LIVE layout/config/logo file (hash -> field value) to
|
|
// `map`. Uses the State cache (synchronous). This is layout/logo files ONLY.
|
|
const indexLocalFiles = async (map) => {
|
|
const state = getState();
|
|
for (const ref of layoutRefs(state.pages, state.views)) {
|
|
if (isPlaceholder(ref.value)) {
|
|
continue;
|
|
}
|
|
const h = await localFileHash(ref.value);
|
|
if (h) {
|
|
map.set(h, ref.value);
|
|
}
|
|
}
|
|
const logo = state.getConfig(LOGO_CONFIG_KEY, null);
|
|
if (logo && !isPlaceholder(logo)) {
|
|
const h = await localFileHash(logo);
|
|
if (h) {
|
|
map.set(h, logo);
|
|
}
|
|
}
|
|
return map;
|
|
};
|
|
|
|
|
|
// Strip a placeholder that could not be resolved so it is never persisted verbatim:
|
|
// for the logo, DELETE the key (so install_pack's setConfig leaves the target's real
|
|
// logo untouched); for a layout ref, blank it (the Image element renders nothing).
|
|
const removeRef = (pack, ref) => {
|
|
if (ref.obj === pack.config && ref.key === LOGO_CONFIG_KEY) {
|
|
delete pack.config[LOGO_CONFIG_KEY];
|
|
} else {
|
|
ref.obj[ref.key] = "";
|
|
}
|
|
};
|
|
|
|
|
|
// APPLY: replace placeholders in layouts/configs/logo with LOCAL file values.
|
|
// opts: { fileFetcher, fileIndexSeed (obj hash->value, e.g. rollback pre-apply), warnings }
|
|
const resolvePlaceholders = async (pack, opts = {}) => {
|
|
const warnings = opts.warnings || [];
|
|
const result = { materialized: 0, deduped: 0, skipped: 0 };
|
|
|
|
const refs = layoutRefs(pack.pages, pack.views);
|
|
if (pack.config && isPlaceholder(pack.config[LOGO_CONFIG_KEY])) {
|
|
refs.push({ obj: pack.config, key: LOGO_CONFIG_KEY, value: pack.config[LOGO_CONFIG_KEY], role: PUBLIC_ROLE_ID });
|
|
}
|
|
const placeholders = refs.filter((r) => isPlaceholder(r.value));
|
|
if (placeholders.length === 0) {
|
|
return result; // nothing to do; skip building the index
|
|
}
|
|
// A file shared by several entities must be readable by the most permissive of
|
|
// them (the highest min_role; an unknown role is treated as public).
|
|
const readRole = new Map();
|
|
for (const ref of placeholders) {
|
|
const h = parsePlaceholder(ref.value).hash;
|
|
const role = Number.isInteger(ref.role) ? ref.role : PUBLIC_ROLE_ID;
|
|
readRole.set(h, Math.max(readRole.get(h) || 0, role));
|
|
}
|
|
|
|
// Dedup index = the target's OWN layout/logo files only (+ optional seed for
|
|
// rollback). Never row files -- so a file landed via the unrestricted row path
|
|
// cannot be laundered into a layout.
|
|
const local = new Map();
|
|
await indexLocalFiles(local);
|
|
if (opts.fileIndexSeed) {
|
|
for (const [h, v] of Object.entries(opts.fileIndexSeed)) {
|
|
if (!local.has(h)) {
|
|
local.set(h, v);
|
|
}
|
|
}
|
|
}
|
|
|
|
for (const ref of placeholders) {
|
|
const spec = parsePlaceholder(ref.value);
|
|
if (!spec || !spec.hash) {
|
|
removeRef(pack, ref);
|
|
continue;
|
|
}
|
|
if (local.has(spec.hash)) {
|
|
ref.obj[ref.key] = local.get(spec.hash);
|
|
result.deduped += 1;
|
|
continue;
|
|
}
|
|
if (typeof opts.fileFetcher !== "function") {
|
|
warnings.push(`no file transport; a layout/logo image was left unset`);
|
|
removeRef(pack, ref);
|
|
result.skipped += 1;
|
|
continue;
|
|
}
|
|
let bytes;
|
|
try {
|
|
bytes = await opts.fileFetcher(spec.hash);
|
|
} catch (e) {
|
|
warnings.push(`fetch of a layout/logo image (hash ${shortHash(spec.hash)}) failed: ${e.message}`);
|
|
removeRef(pack, ref);
|
|
result.skipped += 1;
|
|
continue;
|
|
}
|
|
if (sha256Hex(bytes) !== spec.hash) {
|
|
warnings.push(`content hash mismatch for a layout/logo image; refusing`);
|
|
removeRef(pack, ref);
|
|
result.skipped += 1;
|
|
continue;
|
|
}
|
|
const mime = sniffImageMime(bytes);
|
|
if (!mime) {
|
|
warnings.push(`a layout/logo reference resolved to non-image content; refusing`);
|
|
removeRef(pack, ref);
|
|
result.skipped += 1;
|
|
continue;
|
|
}
|
|
const file = await File.from_contents(deriveFilename(spec.hash, mime), mime, bytes, ADMIN_USER_ID, readRole.get(spec.hash), "/");
|
|
local.set(spec.hash, file.field_value);
|
|
ref.obj[ref.key] = file.field_value;
|
|
result.materialized += 1;
|
|
}
|
|
return result;
|
|
};
|
|
|
|
|
|
module.exports = {
|
|
placeholderize,
|
|
resolvePlaceholders,
|
|
indexLocalFiles,
|
|
localFileHash,
|
|
shortHash,
|
|
isPlaceholder,
|
|
parsePlaceholder,
|
|
makePlaceholder
|
|
};
|