DVX_GUI/server/dvxserver.lua

-- The MIT License (MIT)
--
-- Copyright (C) 2026 Scott Duensing
--
-- Permission is hereby granted, free of charge, to any person obtaining a copy
-- of this software and associated documentation files (the "Software"), to
-- deal in the Software without restriction, including without limitation the
-- rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
-- sell copies of the Software, and to permit persons to whom the Software is
-- furnished to do so, subject to the following conditions:
--
-- The above copyright notice and this permission notice shall be included in
-- all copies or substantial portions of the Software.
--
-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
-- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-- FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
-- IN THE SOFTWARE.

-- dvxserver.lua -- the DVX package server:  the catalogue of add-on packages that DVX's Software
-- program browses, installs, and updates from, and where publishers upload them.
--
--     calog dvxserver.lua -- dvx.conf                     (in this folder, beside ../../singemaster)
--     calog ../../singemaster/master.lua dvxserver.lua -- singe=../../singemaster master=master.conf dvx=dvx.conf
--
-- Plain HTTP on loopback, behind a reverse proxy (as singemaster:  see its README).  Built on
-- singemaster's shared modules (accounts, the log, mail, keys, the blob store, and service.lua's
-- route wrapper, account routes, and operator checks), so the two can run in one calog process.
--
-- Nothing a package says is taken on the uploader's word:  the DVX package tool (dvxpkg check, the
-- code DVX installs with) checks every upload.  A version is published only when the operator
-- approves it with the repository's signature, made on the operator's own computer
-- (tools/review.lua):  the service holds no secret key, only the repository's public key, with which
-- it checks the signature it is given.  DOS machines download slowly, so several contexts serve,
-- and one download does not hold up everyone else.

-- calogScript() is this file, so this folder, singemaster (../../singemaster), and calog (../../calog)
-- are found from wherever calog was started;  singe=DIR and calog=DIR in calogArgs() say where the
-- other two are when they are elsewhere.  calogArgs() is one list for every script in the run, so
-- with another service in the same process dvx=PATH says which configuration is whose;  run alone,
-- the first plain argument is the configuration.  These come before anything is loaded, so they
-- cannot come from singemaster's modules (master.lua has its own).  A serving context is a task,
-- which no file started:  it is given this folder as DVX_HERE (see the end of this file).
local function named(name)
    for _, value in ipairs(calogArgs()) do
        local found = value:match("^" .. name .. "=(.+)$")
        if found then
            return found
        end
    end
    return nil
end

local function ownArg(name)
    local found = named(name)
    if found then
        return found
    end
    for _, value in ipairs(calogArgs()) do
        if not value:find("=", 1, true) then
            return value
        end
    end
    return nil
end

local HERE  = DVX_HERE or (calogScript():match("^(.*)[/\\]") or ".")
local SINGE = named("singe") or (HERE .. "/../../singemaster")
local CALOG = named("calog") or (HERE .. "/../../calog")

local function need(path)
    return load(fsRead(path), "@" .. path)()
end

local config   = need(SINGE .. "/lib/config.lua")
local db       = need(SINGE .. "/lib/db.lua")
local audit    = need(SINGE .. "/lib/audit.lua")
local web      = need(SINGE .. "/lib/web.lua")
local accounts = need(SINGE .. "/lib/accounts.lua")
local mail     = need(SINGE .. "/lib/mail.lua")
local keys     = need(SINGE .. "/lib/keys.lua")
local blobs    = need(SINGE .. "/lib/blobs.lua")
local service  = need(SINGE .. "/lib/service.lua")
local packages = need(HERE .. "/lib/packages.lua")
local httpd    = need(CALOG .. "/examples/httpd.lua")
local smtp     = need(CALOG .. "/examples/smtp.lua")

-- The keys dvx.conf must have, and the ones that are numbers
local REQUIRED = { "bindHost", "bindPort", "publicOrigin", "dataDir", "database", "pbkdf2Iterations", "mailFrom",
                   "dvxpkg", "repoKey", "serveContexts", "uploadMax" }
local NUMERIC  = { bindPort = true, pbkdf2Iterations = true, smtpPort = true, serveContexts = true, uploadMax = true }

-- Every event the service writes (audit.lua refuses a route naming one that is not here)
local EVENTS = {
    "account.register", "account.verify", "account.login", "account.logout", "account.read",
    "account.recover", "account.reset", "account.sweepPending",
    "catalogue.list", "package.get", "package.download", "package.signature", "updates.check",
    "package.upload", "catalogue.mine",
    "admin.pending", "admin.download", "admin.versionState", "version.approve", "version.reject",
    "admin.packageState", "admin.log", "key.grant", "key.revoke",
    "health", "service.start", "service.stop", "mail.sent", "mail.failed",
}

-- The service's own tables, after the shared ones (accounts, sessions, the log, rate limits, mail,
-- keys).  A package is vendor.name;  each version's bytes are a blob, stored once by SHA-256.
local SCHEMA = {
[[
CREATE TABLE blob (
    id      INTEGER PRIMARY KEY,
    size    INTEGER NOT NULL,
    sha256  TEXT    NOT NULL UNIQUE
)
]],
[[
CREATE TABLE package (
    id              TEXT    PRIMARY KEY,                  -- vendor.name
    ownerAccountId  INTEGER REFERENCES account(id),       -- NULL:  the operator's (DVX's own)
    state           TEXT    NOT NULL,                     -- published | withdrawn
    createdAt       INTEGER NOT NULL
)
]],
[[CREATE INDEX packageOwner ON package(ownerAccountId)]],
[[
CREATE TABLE packageVersion (
    packageId   TEXT    NOT NULL REFERENCES package(id) ON DELETE CASCADE,
    version     TEXT    NOT NULL,          -- major.minor.patch
    rank        INTEGER NOT NULL,          -- the version as one number, for ordering
    name        TEXT    NOT NULL,
    kind        TEXT    NOT NULL,
    summary     TEXT,
    publisher   TEXT,
    author      TEXT,
    licence     TEXT,
    homepage    TEXT,
    dvx         TEXT,                      -- the oldest DVX it runs on, or NULL
    abi         INTEGER,                   -- code:  the ABI it was made for
    manifest    TEXT    NOT NULL,          -- PACKAGE.INI exactly as signed
    signerKey   TEXT,                      -- the publisher's key id (PACKAGE.SIG), or NULL
    repoSig     TEXT,                      -- the repository's detached signature, set on approval
    blobId      INTEGER NOT NULL REFERENCES blob(id),
    state       TEXT    NOT NULL,          -- pending | approved | rejected
    note        TEXT,
    uploadedBy  INTEGER REFERENCES account(id),
    uploadedAt  INTEGER NOT NULL,
    reviewedAt  INTEGER,
    PRIMARY KEY (packageId, version)
)
]],
[[CREATE INDEX packageVersionState ON packageVersion(state, uploadedAt)]],
[[
CREATE TABLE packageDepend (
    packageId   TEXT    NOT NULL,
    version     TEXT    NOT NULL,
    dependId    TEXT    NOT NULL,
    minVersion  TEXT    NOT NULL,
    PRIMARY KEY (packageId, version, dependId),
    FOREIGN KEY (packageId, version) REFERENCES packageVersion(packageId, version) ON DELETE CASCADE
)
]],
}

local MIGRATIONS = { db.join(db.SHARED_SCHEMA, db.KEY_SCHEMA, SCHEMA) }

local CONFIG_KEY    = "dvxserver.config"        -- kv:  the configuration's path, for the serving contexts
local LISTENER_KEY  = "dvxserver.listener"      -- kv:  the listener the serving contexts accept on
local STOP_MS       = 500                       -- the longest a context waits (an accept, a pause) to see a stop
local RETRY_MS      = 20                        -- the pause after an accept that failed
local CHORE_SECONDS = 30
local PATH_ID_VERSION = "^([%w_.]+)/([%d.]+)$"   -- <id>/<version> in a path


-- A serving context started by the main one (see the end of this file) is numbered, and finds the
-- configuration and the listener in kv;  the main context reads its arguments and puts them there.
local worker     = DVX_WORKER
local configPath = worker and kvGet(CONFIG_KEY) or (ownArg("dvx") or (HERE .. "/dvx.conf"))
local settings, configError = config.load(configPath, REQUIRED, NUMERIC)
if not settings then
    calogPrint("dvxserver configuration:", configError)
    calogExit(1)
end

fsMkdir(settings.dataDir)
local handle, schemaVersion = db.open(settings.database, MIGRATIONS)
if not handle then
    calogPrint("dvxserver database:", schemaVersion)
    calogExit(1)
end
blobs.TABLE = "blob"

local log    = audit.new(handle, EVENTS)
local server = httpd.new()
local svc    = service.new({ handle = handle, log = log, settings = settings, server = server, product = "DVX",
                             audit = audit, web = web, accounts = accounts, mail = mail, keys = keys, smtp = smtp })


-- The operator:  the console token, or an account holding the admin key.  Unlike svc:operator,
-- asking does not mark the request refused.
local function isOperator(req, account)
    return svc:console(req) or (account ~= nil and keys.holds(handle, account.id, keys.ADMIN))
end


-- <id>/<version> at the end of a request's path, the version as the catalogue writes it ("2.0" is
-- 2.0.0), or nil.
local function idVersion(req)
    local id, text = (req.rest or ""):match(PATH_ID_VERSION)
    local version  = text and packages.versionOf(text)

    if not version then
        return nil
    end
    return id, version
end


-- A version's bytes, resumable, as id-version.dpk:  the reply, the bytes sent, and whether resumed.
local function sendVersion(req, id, version, row)
    return web.fileReply(req, { path = blobs.path(settings.dataDir, row.sha256), size = row.size, sha256 = row.sha256 },
                         id .. "-" .. version .. ".dpk")
end


-- Run the package tool:  its exit status and what it printed.
local function dvxpkg(args)
    local argv = { settings.dvxpkg }

    for _, a in ipairs(args) do
        argv[#argv + 1] = a
    end
    local r = procRun(argv)
    return r.exit, r.stdout or "", r.stderr or ""
end


-- What the package tool says about a refused file, without its "dvxpkg: " prefix.
local function toolReason(text)
    return (tostring(text):gsub("^dvxpkg:%s*", ""):gsub("%s+$", ""))
end


svc:route("GET", "/v1/health", "health", function(req, context)
    return web.json(200, { ok = true, service = "dvx-packages", schema = schemaVersion })
end)

svc:accountRoutes()
svc:keyRoutes()
svc:logRoute()


-- Every published package at its newest approved version.  Anonymous:  the catalogue is public.
svc:route("GET", "/v1/catalogue", "catalogue.list", function(req, context)
    local list = packages.catalogue(handle)

    context.detail = { count = #list }
    return web.json(200, { packages = list, count = #list })
end)


-- /v1/package/<id> -- one package and its approved versions, newest first.
svc:route("GET", "/v1/package/", "package.get", function(req, context)
    local id       = (req.rest or ""):match("^([%w_.]+)$")
    local versions = id and packages.versions(handle, id)

    if not versions then
        return web.fail(404, "noPackage", "no such package")
    end
    context.targetType = "package"
    context.targetId   = id
    return web.json(200, { id = id, versions = versions })
end)


-- /v1/download/<id>/<version> -- the bytes of an approved version, resumable.
svc:route("GET", "/v1/download/", "package.download", function(req, context)
    local id, version = idVersion(req)
    local row = id and packages.servable(handle, id, version)

    if not row then
        return web.fail(404, "noVersion", "no such package or version")
    end
    context.targetType    = "package"
    context.targetId      = id
    context.targetVersion = version
    local reply, sent, resumed = sendVersion(req, id, version, row)
    context.detail = { size = row.size, sent = sent, resumed = resumed }
    return reply
end)


-- /v1/signature/<id>/<version> -- the repository's signature of an approved version (a NAME.SIG
-- file), which DVX checks with the repository key it was installed with.
svc:route("GET", "/v1/signature/", "package.signature", function(req, context)
    local id, version = idVersion(req)
    local row = id and packages.servable(handle, id, version)

    if not row or not row.repoSig then
        return web.fail(404, "noSignature", "no such package or version")
    end
    context.targetType    = "package"
    context.targetId      = id
    context.targetVersion = version
    return { status = 200, headers = { ["Content-Type"] = "text/plain" }, body = row.repoSig }
end)


-- What is newer than the packages a DVX has:  {installed: [{id, version}]}.  The list is the
-- client's own and nothing of it is kept;  the answer names nothing private.
svc:route("POST", "/v1/updates", "updates.check", function(req, context)
    local body, bad = web.body(req)
    if not body then return bad end
    if type(body.installed) ~= "table" then
        return web.fail(400, "badRequest", "installed must be a list of {id, version}")
    end
    local newer, unknown = packages.updates(handle, body.installed)

    context.detail = { installed = #body.installed, newer = #newer, unknown = #unknown, dvx = body.dvx, abi = body.abi }
    return web.json(200, { updates = newer, unknown = unknown })
end)


-- An upload:  a .DPK as the body, from a signed-in account (or the operator), taken in as a
-- pending version for review once the package tool has checked it.
svc:route("POST", "/v1/upload", "package.upload", function(req, context)
    local account  = svc:signedIn(req, context)
    local operator = isOperator(req, account)

    if not account and not operator then
        context.result = "denied"
        return web.fail(401, "noSession", "sign in first")
    end
    if not req.bodyFile or (req.bodySize or 0) == 0 then
        return web.fail(400, "badRequest", "the body is the package (.dpk)")
    end
    local status, manifest, why = dvxpkg({ "check", req.bodyFile })
    if status ~= 0 then
        context.detail = { reason = toolReason(why) }
        return web.fail(400, "badPackage", toolReason(why))
    end
    local _, info = dvxpkg({ "info", req.bodyFile })
    local signer  = info:match("signed by key (%x+)")
    local blobId  = blobs.put(handle, settings.dataDir, req.bodyFile)
    local id, result, reason = packages.intake(handle, manifest, blobId, signer, (not operator) and account.id or nil)

    if not id then
        context.result = "denied"
        context.detail = { reason = reason }
        return web.fail(result, "refused", reason)
    end
    local version = result
    context.targetType    = "package"
    context.targetId      = id
    context.targetVersion = version
    context.description   = string.format("%s uploaded %s %s for review", account and account.email or "the operator", id, version)
    return web.json(202, { id = id, version = version, state = "pending" })
end, { spool = { dir = settings.dataDir .. "/work", max = settings.uploadMax } })


-- The uploader's own packages, every version with the review's word on it.
svc:route("GET", "/v1/mine", "catalogue.mine", function(req, context)
    local account = svc:signedIn(req, context)

    if not account then
        context.result = "denied"
        return web.fail(401, "noSession", "sign in first")
    end
    return web.json(200, { versions = packages.mine(handle, account.id) })
end)


-- Operator:  the versions waiting for review.
svc:route("GET", "/v1/admin/pending", "admin.pending", function(req, context)
    local ok, refusal = svc:operator(req, context)
    if not ok then return refusal end
    local rows = packages.pending(handle)

    context.detail = { count = #rows }
    return web.json(200, { pending = rows, count = #rows })
end)


-- Operator:  /v1/admin/download/<id>/<version> -- any version's bytes, to review and sign.
svc:route("GET", "/v1/admin/download/", "admin.download", function(req, context)
    local ok, refusal = svc:operator(req, context)
    if not ok then return refusal end
    local id, version = idVersion(req)
    local row = id and packages.servable(handle, id, version, true)

    if not row then
        return web.fail(404, "noVersion", "no such package or version")
    end
    context.targetType    = "package"
    context.targetId      = id
    context.targetVersion = version
    return (sendVersion(req, id, version, row))
end)


-- Operator:  {package, version, state: approved or rejected, note, signature}.  Approving takes the
-- repository's detached signature of the package (dvxpkg sign NAME.DPK REPO.SEC -d, on the
-- operator's own computer), which must check against the repository's public key.
svc:route("POST", "/v1/admin/version/state", "admin.versionState", function(req, context)
    local ok, refusal = svc:operator(req, context)
    if not ok then return refusal end
    local body, bad = web.body(req)
    if not body then return bad end
    local id      = web.field(body, "package")
    local version = packages.versionOf(web.field(body, "version"))
    local state   = web.field(body, "state")
    local row     = id and version and packages.servable(handle, id, version, true)

    if state ~= "approved" and state ~= "rejected" then
        return web.fail(400, "badState", "state is approved or rejected")
    end
    if not row then
        return web.fail(404, "noVersion", "no such package or version")
    end
    local signature = web.field(body, "signature")
    if state == "approved" then
        if not signature then
            return web.fail(400, "noSignature", "approving needs the repository's signature of the package")
        end
        local sigFile = blobs.workPath(settings.dataDir)
        fsWrite(sigFile, signature)
        local status, _, why = dvxpkg({ "verify", blobs.path(settings.dataDir, row.sha256), "-s", sigFile, settings.repoKey })
        fsRemove(sigFile)
        if status ~= 0 then
            context.result = "denied"
            context.detail = { reason = toolReason(why) }
            return web.fail(400, "badSignature", "that is not the repository's signature of this package:  " .. toolReason(why))
        end
    end
    packages.review(handle, id, version, state == "approved", signature, body.note)
    context.action        = (state == "approved") and "version.approve" or "version.reject"
    context.targetType    = "package"
    context.targetId      = id
    context.targetVersion = version
    context.description   = string.format("%s %s %s %s%s", svc:operatorName(context), state, id, version,
                                          body.note and (":  " .. tostring(body.note)) or "")
    context.detail        = { state = state, note = body.note }
    return web.json(200, { package = id, version = version, state = state })
end)


-- Operator:  {package, state: published or withdrawn}.  Withdrawn packages are not listed or handed
-- out;  the copies installed keep working, and stop being offered updates.
svc:route("POST", "/v1/admin/package/state", "admin.packageState", function(req, context)
    local ok, refusal = svc:operator(req, context)
    if not ok then return refusal end
    local body, bad = web.body(req)
    if not body then return bad end
    local id    = web.field(body, "package")
    local state = web.field(body, "state")

    if state ~= "published" and state ~= "withdrawn" then
        return web.fail(400, "badState", "state is published or withdrawn")
    end
    if not id or packages.setState(handle, id, state) == 0 then
        return web.fail(404, "noPackage", "no such package")
    end
    context.targetType  = "package"
    context.targetId    = id
    context.description = string.format("%s set %s to %s", svc:operatorName(context), id, state)
    return web.json(200, { package = id, state = state })
end)


-- Serving.  The main context listens and does the chores;  serveContexts serving contexts, each with
-- its own database handle and routes, accept on its listener, so a slow download to a DOS machine
-- holds up only the context sending it.  Each connection goes to one of them;  with every one busy,
-- new connections wait in the system's queue until one is free.  An accept that fails (a client gone
-- before it was taken, no descriptors to spare) is under pcall so it cannot end a serving context;
-- the pause after it is outside, where calog's stop (calogExit, Ctrl-C, SIGTERM) still reaches.
if worker then
    local listener = kvGet(LISTENER_KEY)

    while true do
        local ok, conn = pcall(tcpAccept, listener, STOP_MS)
        if not ok then
            timeSleep(RETRY_MS)
        elseif conn then
            server:handle(conn)
        end
    end
end

local listener = tcpListen(settings.bindPort, { host = settings.bindHost })
if not listener then
    calogPrint(string.format("dvxserver: cannot listen on %s:%d", settings.bindHost, settings.bindPort))
    calogExit(1)
end
kvSet(CONFIG_KEY, configPath)
kvSet(LISTENER_KEY, listener)

local script = HERE .. "/dvxserver.lua"
for n = 1, settings.serveContexts do
    taskSpawn("lua", string.format("DVX_WORKER = %d\nDVX_HERE = %q\nassert(load(fsRead(%q), %q))()", n, HERE, script, "@" .. script))
end

audit.write(log, { requestId = "-", action = "service.start", result = "ok",
                   description = string.format("the service started, schema %d, listening on %s:%d with %d serving contexts",
                                               schemaVersion, settings.bindHost, settings.bindPort, settings.serveContexts),
                   detail = { schema = schemaVersion, host = settings.bindHost, port = settings.bindPort, contexts = settings.serveContexts } })
calogPrint(string.format("dvx packages: schema %d, listening on %s:%d for %s (%d serving contexts)",
    schemaVersion, settings.bindHost, settings.bindPort, settings.publicOrigin, settings.serveContexts))


-- The chores (queued mail, expired accounts), every CHORE_SECONDS.  The pause is short so a stop
-- reaches this context promptly:  calog raises it at the next native call, and a sleep does not wake.
local lastChore = 0

while true do
    local now = math.floor(timeNow())

    if now - lastChore >= CHORE_SECONDS then
        lastChore = now
        svc:chores()
    end
    timeSleep(STOP_MS)
end