singe/util/zbstudioApi.py

# Builds ZeroBrane Studio's autocompletion and tooltips for Singe, zbstudio/packages/singe/api*.lua,
# from two sources that cannot drift apart:  the names a running Singe really defines, and what the
# Reference says about each one.  One file per language the Reference is kept in;  the package picks
# the one matching ZeroBrane Studio's own language setting.
#
# The names come from Singe itself, run headless on a script that lists its globals after
# Framework.singe has loaded, so nothing here has to know how the engine registers them.  The words
# come from each language's Reference.adoc:  an entry's signatures, its first paragraph, its Returns
# and Since lines, and, for the constants, the Engine Constants table.  Argument types come from the
# English Parameters lists whatever the language, since Lua's type names are English everywhere.
#
# Along the way it lists every engine function the Reference has no entry for, and every entry
# naming a function the engine does not define, which is how a missing entry gets found.  --strict
# makes either kind fatal.
#
# Usage:  python3 util/zbstudioApi.py [--singe BINARY] [--strict]
import glob
import os
import re
import subprocess
import sys
import tempfile
from singeBinary import singeBinary


REPO       = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
DOCS       = os.path.join(REPO, "docs")
OUT        = os.path.join(REPO, "zbstudio", "packages", "singe")
DUMP_TAG   = "ZBSTUDIO_API"
TIMEOUT    = 120
TYPE_WORDS = ("integer", "number", "string", "boolean", "table", "function", "nil")

# How each language writes a range of names:  `GAMEPAD_0` to `GAMEPAD_3`.
RANGE_WORDS = "to|\u00e0|bis|a"

# Globals whose value depends on the run that listed them, so the tooltip gives no value.
VARYING = ("DIR", "SINGE_DEAD_ZONE", "SINGE_DISC", "SINGE_LEGACY_SPRITE_ARGS", "SINGE_TRIGGER_THRESHOLD")

# The label words of an entry, per language, as the translations write them.  Non-ASCII letters are
# escaped so this file stays ASCII.
LABELS = {
	"en": {"Parameters": "params", "Returns": "returns", "Notes": "notes", "Since": "since", "See also": "seeAlso"},
	"fr": {"Param\u00e8tres": "params", "Renvoie": "returns", "Remarques": "notes", "Depuis": "since", "Voir aussi": "seeAlso"},
	"it": {"Parametri": "params", "Restituisce": "returns", "Note": "notes", "Disponibile da": "since", "Vedi anche": "seeAlso"},
	"de": {"Parameter": "params", "R\u00fcckgabe": "returns", "Hinweise": "notes", "Seit": "since", "Siehe auch": "seeAlso"},
	"es": {"Par\u00e1metros": "params", "Devuelve": "returns", "Notas": "notes", "Desde": "since", "V\u00e9ase tambi\u00e9n": "seeAlso"},
}

# The standard library, which ZeroBrane Studio already describes.
STANDARD = set("""
	_G _VERSION assert collectgarbage coroutine debug dofile error getmetatable io ipairs load loadfile
	math next os package pairs pcall print rawequal rawget rawlen rawset require select setmetatable
	string table tonumber tostring type utf8 warn xpcall
""".split())

# Lists the globals, one line each, and the members of every table that is not the standard
# library's.  A member that is itself a table (SCANCODE.LEFT) shows its own keys, so the tooltip can
# say what is in it.
DUMP_SCRIPT = """
dofile("Singe/Framework.singe")
local function scalar(v)
	if type(v) == "string" or type(v) == "number" or type(v) == "boolean" then
		return tostring(v):gsub("[\\t\\n]", " ")
	end
	return ""
end
local skip = {}
for name in ([[%s]]):gmatch("%%S+") do
	skip[name] = true
end
for name, value in pairs(_G) do
	if type(name) == "string" and not skip[name] then
		-- A function's identity, so an alias can be told apart from the function it names.
		debugPrint("%s\\tG\\t" .. name .. "\\t" .. type(value) .. "\\t" .. ((type(value) == "function") and tostring(value) or scalar(value)))
		if type(value) == "table" then
			for key, member in pairs(value) do
				if type(key) == "string" then
					local keys = ""
					if type(member) == "table" then
						local list = {}
						for k, v in pairs(member) do
							list[#list + 1] = tostring(k) .. ((scalar(v) ~= "") and (" = " .. ((type(v) == "string") and ("\\"" .. scalar(v) .. "\\"") or scalar(v))) or "")
						end
						table.sort(list)
						keys = table.concat(list, ", ")
					end
					debugPrint("%s\\tC\\t" .. name .. "\\t" .. key .. "\\t" .. type(member) .. "\\t" .. (keys ~= "" and keys or scalar(member)))
				end
			end
		end
	end
end
singeQuit()
"""


# Undocumented globals that are the very function a documented name is:  Framework.singe's
# discGetLanguageDescription is videoGetLanguageDescription, for one.
def findAliases(globalsFound, english):
	byIdentity = {}
	aliases    = {}
	for name in sorted(english):
		found = globalsFound.get(name)
		if found and (found["type"] == "function"):
			byIdentity.setdefault(found["value"], name)
	for name, found in sorted(globalsFound.items()):
		if (found["type"] == "function") and (name not in english) and (found["value"] in byIdentity):
			aliases[name] = byIdentity[found["value"]]
	return aliases


def asciiDocToText(text):
	text = re.sub(r"<<[^,>]+,([^>]+)>>", r"\1", text)
	text = re.sub(r"<<([^>]+)>>", r"\1", text)
	text = re.sub(r"xref:[^\[]+\[([^\]]*)\]", r"\1", text)
	text = re.sub(r"(https?://\S+?)\[([^\]]*)\]", r"\2 (\1)", text)
	text = re.sub(r"`\+?(.*?)\+?`", r"\1", text)
	text = re.sub(r"(?<![\w*])\*([^*\n]+)\*(?![\w*])", r"\1", text)
	text = text.replace("{nbsp}", " ").replace(" +\n", "\n")
	text = text.replace("\u00a0", " ").replace("\u202f", " ")
	return re.sub(r"[ \t]+", " ", text).strip()


def autoId(title):
	return "_" + re.sub(r"[^a-z0-9]+", "_", title.lower()).strip("_")


def dumpGlobals(singe):
	script = DUMP_SCRIPT % (" ".join(sorted(STANDARD)), DUMP_TAG, DUMP_TAG)
	with tempfile.TemporaryDirectory() as work:
		with open(os.path.join(work, "dump.singe"), "w") as f:
			f.write(script)
		env = dict(os.environ, SDL_VIDEO_DRIVER="offscreen", SDL_GPU_DRIVER="nothing", SDL_AUDIO_DRIVER="dummy")
		run = subprocess.run([singe, "-k", "-s", "-d", os.path.join(work, "data"), "dump.singe"], cwd=work, env=env, capture_output=True, text=True, timeout=TIMEOUT)
	globalsFound = {}
	for line in run.stdout.splitlines():
		parts = line.split("\t")
		if (len(parts) < 2) or (parts[0] != DUMP_TAG):
			continue
		if parts[1] == "G":
			globalsFound[parts[2]] = {"type": parts[3], "value": parts[4], "members": {}}
		elif parts[1] == "C":
			globalsFound[parts[2]]["members"][parts[3]] = {"type": parts[4], "value": parts[5]}
	if not globalsFound:
		sys.exit("Singe listed no globals:\n" + run.stdout[-2000:])
	return globalsFound


def languages():
	found = ["en"]
	for locale in sorted(glob.glob(os.path.join(DOCS, "*", "locale.adoc"))):
		found.append(os.path.basename(os.path.dirname(locale)))
	return found


def luaString(text):
	text = text.replace("\\", "\\\\").replace("\"", "\\\"").replace("\n", "\\n")
	return "\"" + text + "\""


# The first paragraph, the Returns and Since lines, and, when the forms differ in more than their
# trailing arguments, every form;  enough for a tooltip, with the Reference a click away for the rest.
def describe(entry, allForms):
	parts = []
	if allForms and (len(entry["signatures"]) > 1):
		parts.append("\n".join(entry["signatures"]))
	if entry["paragraphs"]:
		parts.append(asciiDocToText(entry["paragraphs"][0]))
	for warning in entry["warnings"]:
		parts.append(asciiDocToText(warning))
	for field in ("returns", "since"):
		if entry["fields"].get(field):
			parts.append(entry["fieldLabels"][field] + " " + asciiDocToText(entry["fields"][field]))
	return "\n".join(parts)


# The rows of every table in lines, as (cells, line number) pairs.  A cell starts at a '|';  a row is
# as many cells as the table has columns, whether they sit on one line or one a line.
def parseTables(lines):
	rows      = []
	cells     = None
	columns   = 0
	start     = 0
	inListing = False
	for number, line in enumerate(lines):
		if re.match(r"^-{4,}$", line):
			inListing = not inListing
			continue
		if inListing:
			continue
		attribute = re.match(r'^\[cols="([^"]+)"', line)
		if attribute:
			columns = len(attribute.group(1).split(","))
			continue
		if line.startswith("|==="):
			if cells is None:
				cells = []
				first = True
				start = number
			else:
				for i in range(0, len(cells) - columns + 1, max(columns, 1)):
					rows.append((cells[i:i + columns], start))
				cells   = None
				columns = 0
			continue
		if cells is None:
			continue
		if line.startswith("|"):
			parts = [part.strip() for part in line.split("|")[1:]]
			if first and (columns == 0):
				columns = len(parts)
			first = False
			cells.extend(parts)
		elif line.strip() and cells:
			cells[-1] = (cells[-1] + " " + line.strip()).strip()
	return rows


def parseEntry(lines, labels):
	entry      = {"signatures": [], "paragraphs": [], "warnings": [], "params": [], "fields": {}, "fieldLabels": {}}
	index      = 0
	paragraph  = []
	field      = None
	inListing  = False

	# The signature block is the first listing.
	while index < len(lines):
		if lines[index].startswith("[source"):
			index += 2
			while (index < len(lines)) and (lines[index] != "----"):
				if lines[index].strip() and (lines[index].strip() != "end"):
					entry["signatures"].append(lines[index].strip())
				index += 1
			index += 1
			break
		index += 1

	def flush():
		if paragraph:
			text = " ".join(paragraph).strip()
			if field == "params":
				pass
			elif field is not None:
				entry["fields"][field] = (entry["fields"].get(field, "") + " " + text).strip()
			elif re.match(r"^(WARNING|IMPORTANT|CAUTION):", text):
				entry["warnings"].append(text)
			else:
				entry["paragraphs"].append(text)
			paragraph.clear()

	while index < len(lines):
		line  = lines[index]
		index += 1
		if re.match(r"^-{4,}$", line) or re.match(r"^\.{4,}$", line):
			inListing = not inListing
			continue
		if inListing:
			continue
		if line.startswith(".Example") or line.startswith("[source") or line.startswith("|===") or line.startswith("[cols"):
			flush()
			field = "skip"
			continue
		label = re.match(r"^\*([^*]+?)\s*:\*\s*(.*)$", line.replace("\u00a0", " ").replace("\u202f", " "))
		if label and (label.group(1).strip() in labels):
			flush()
			field = labels[label.group(1).strip()]
			entry["fieldLabels"][field] = label.group(1).strip() + ":"
			if label.group(2):
				paragraph.append(label.group(2))
			continue
		if field == "params":
			bullet = re.match(r"^\*\s+(.*)$", line)
			if bullet:
				entry["params"].append(bullet.group(1))
			elif line.strip() and entry["params"]:
				entry["params"][-1] += " " + line.strip()
			elif not line.strip() and entry["params"]:
				field = "skip"
			continue
		if field == "skip":
			continue
		if not line.strip():
			flush()
			if field is not None:
				field = "skip"
			continue
		if line.startswith("[") and line.endswith("]"):
			continue
		paragraph.append(line.strip())
	flush()
	entry["fields"].pop("skip", None)
	return entry


# Each language's API chapter entries, keyed by the names in the heading, and the Engine
# Constants table as (names cell, meaning) rows.
def parseReference(path, lang):
	lines       = open(path, encoding="utf-8").read().split("\n")
	headings    = []
	pendingId   = None
	inListing   = False
	for number, line in enumerate(lines):
		if re.match(r"^-{4,}$", line):
			inListing = not inListing
			continue
		if inListing:
			continue
		anchor = re.match(r"^\[(?:#|\[)([^\],]+)", line)
		if anchor and (line.endswith("]") or line.endswith("]]")):
			pendingId = anchor.group(1)
			continue
		heading = re.match(r"^(=+) (.+)$", line)
		if heading:
			headings.append({"level": len(heading.group(1)), "title": heading.group(2).strip(), "id": pendingId or autoId(heading.group(2)), "line": number})
			pendingId = None
		elif line.strip():
			pendingId = None
	for i, heading in enumerate(headings):
		end = len(lines)
		for later in headings[i + 1:]:
			if later["level"] <= heading["level"]:
				end = later["line"]
				break
		heading["end"] = end
	chapter = next(h for h in headings if h["id"] == "reference")
	entries = {}
	for heading in headings:
		if (heading["level"] != 4) or not (chapter["line"] < heading["line"] < chapter["end"]):
			continue
		names = [n.strip() for n in heading["title"].split("/")]
		if not all(re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", n) for n in names):
			continue
		entry = parseEntry(lines[heading["line"] + 1:heading["end"]], LABELS[lang])
		entry["anchor"] = heading["id"]
		for name in names:
			entries[name] = entry
	# The calls the chapter's tables describe, the aliases and the score calls among them, for any
	# name without an entry of its own.
	for row, number in parseTables(lines[chapter["line"]:chapter["end"]]):
		if len(row) < 2:
			continue
		# The Reference opens at the section holding the table.
		number  += chapter["line"]
		section = max((h for h in headings if (h["line"] < number) and (h["level"] >= 3)), key=lambda h: h["line"])
		for token in re.findall(r"`([A-Za-z_][A-Za-z0-9_]*(?:\([^`]*\))?)`", row[0]):
			name = re.match(r"^[A-Za-z_][A-Za-z0-9_]*", token).group(0)
			# Constants are named in capitals and described by the Engine Constants table.
			if (name in entries) or (name.upper() == name):
				continue
			signature = token if "(" in token else ""
			text      = " ".join(cell for cell in row[1:] if cell not in ("--", ""))
			entries[name] = {"signatures": [signature] if signature else [], "paragraphs": [text], "warnings": [], "params": [], "fields": {}, "fieldLabels": {}, "anchor": section["id"]}
	# The Engine Constants rows;  the book's paragraphs cover the rest, what Framework.singe adds among them.
	constants = []
	section   = next(h for h in headings if h["id"] == "_engine_constants")
	for line in lines[section["line"] + 1:section["end"]]:
		row = re.match(r"^\|\s*(.+?)\s*\|\s*(.+)$", line)
		if row and ("`" in row.group(1)):
			constants.append((row.group(1), row.group(2)))
	return entries, {"table": constants, "paragraphs": bookParagraphs(lines)}


# Where a constant is described:  its Engine Constants row, a row naming its range or family, the
# first paragraph naming it, or the first naming its family.
def constantMeaning(name, constants):
	exact = re.compile(r"`%s[`.\[]" % re.escape(name))

	def family(cell):
		for token in re.findall(r"`([A-Z][A-Z0-9_]*\*?)`", cell):
			prefix = token[:-1] if token.endswith("*") else token.rsplit("_", 1)[0] + "_"
			ranged = token.endswith("*") or ("..." in cell) or re.search(r"`%s`\s+(?:%s)\s+`" % (re.escape(token), RANGE_WORDS), cell)
			if ranged and name.startswith(prefix):
				return True
		return False

	for test in (lambda cell: exact.search(cell), family):
		for cell, meaning in constants["table"]:
			if test(cell):
				return asciiDocToText(meaning)
	for test in (lambda cell: exact.search(cell), family):
		for paragraph in constants["paragraphs"]:
			if test(paragraph):
				return asciiDocToText(paragraph)
	return ""


# Every prose paragraph of a book, in order, without listings, tables or block attributes.
def bookParagraphs(lines):
	paragraphs = []
	paragraph  = []
	inListing  = False
	inTable    = False
	for line in lines + [""]:
		if re.match(r"^-{4,}$", line) or re.match(r"^\.{4,}$", line):
			inListing = not inListing
			continue
		if line.startswith("|==="):
			inTable = not inTable
			continue
		if inListing or inTable:
			continue
		if line.strip() and not line.startswith(("[", "=", "*", ".", "//")):
			paragraph.append(line.strip())
		elif paragraph:
			paragraphs.append(" ".join(paragraph))
			paragraph = []
	return paragraphs


def argList(entry):
	forms = []
	for signature in entry["signatures"]:
		call = re.match(r"^(?:function\s+)?(?:[\w, ]+?\s*=\s*)?[A-Za-z_][\w.:]*\s*\((.*)\)\s*$", signature)
		if call:
			forms.append([a.strip() for a in call.group(1).split(",") if a.strip()])
	if not forms:
		return "()", True
	types = {}
	for param in entry["params"]:
		names = re.findall(r"`([A-Za-z_][A-Za-z0-9_]*)`", param.split("--")[0])
		after = param.split("--", 1)[1].strip() if "--" in param else ""
		words = re.sub(r"^optional\s+", "", after).split(";")[0].split(",")[0].strip()
		kind  = words.split(" ")[0] if words.split(" ")[0] in TYPE_WORDS else ""
		for name in names:
			types[name] = kind
	longest  = max(forms, key=len)
	shortest = min(forms, key=len)
	prefixed = all(form == longest[:len(form)] for form in forms)

	def typed(name):
		return name + (": " + types[name] if types.get(name) else "")

	if not prefixed:
		return "(" + ", ".join(typed(n) for n in forms[0]) + ")", True
	required = ", ".join(typed(n) for n in shortest)
	optional = ", ".join(typed(n) for n in longest[len(shortest):])
	if optional:
		return "(" + required + (" [, " if required else "[") + optional + "])", False
	return "(" + required + ")", False


def returnList(entry):
	lhs = None
	for signature in entry["signatures"]:
		match = re.match(r"^([\w, ]+?)\s*=\s*[A-Za-z_]", signature)
		if match:
			lhs = [n.strip() for n in match.group(1).split(",")]
			break
	if not lhs:
		return "()"
	first = entry["fields"].get("returns", "").split(" ")[0].strip(".,;").lower()
	if (len(lhs) == 1) and (first in TYPE_WORDS):
		return "(" + first + ")"
	return "(" + ", ".join(lhs) + ")"


def build(globalsFound, english, entries, constants):
	api = {}
	for name, entry in sorted(entries.items()):
		englishEntry = english.get(name, entry)
		args, allForms = argList(englishEntry)
		api[name] = {"type": "function", "description": describe(entry, allForms), "args": args, "returns": returnList(englishEntry)}
		# Where the package's Look Up opens the Reference, when it is not the name itself.
		if englishEntry["anchor"] != name.lower():
			api[name]["anchor"] = englishEntry["anchor"]
	for name, found in sorted(globalsFound.items()):
		if name in api:
			continue
		if found["type"] in ("number", "string", "boolean"):
			meaning = constantMeaning(name, constants)
			value   = found["value"] if found["type"] != "string" else "\"" + found["value"] + "\""
			if name in VARYING:
				api[name] = {"type": "value", "description": meaning}
			else:
				api[name] = {"type": "value", "description": (meaning + "\n" if meaning else "") + "= " + value}
		elif (found["type"] == "table") and found["members"]:
			childs = {}
			for key, member in sorted(found["members"].items()):
				if member["type"] == "table":
					childs[key] = {"type": "value", "description": "{ " + member["value"] + " }"}
				elif member["type"] in ("number", "string", "boolean"):
					childs[key] = {"type": "value", "description": "= " + member["value"]}
				elif member["type"] == "function":
					childs[key] = {"type": "function", "description": "", "args": "(...)", "returns": "()"}
			meaning = constantMeaning(name, constants)
			api[name] = {"type": "lib", "description": meaning, "childs": childs}
	return api


def writeLua(path, api, lang):
	out = []
	out.append("-- Singe's API for ZeroBrane Studio's autocompletion and tooltips (%s)." % lang)
	out.append("-- Generated by util/zbstudioApi.py from the Reference and Singe's own globals;  do not edit.")
	out.append("return {")

	def emit(table, indent):
		for name, item in sorted(table.items()):
			out.append("%s%s = {" % (indent, name if re.match(r"^[A-Za-z_]\w*$", name) else "[" + luaString(name) + "]"))
			for field in ("type", "description", "args", "returns", "anchor"):
				if field in item:
					out.append("%s\t%s = %s," % (indent, field, luaString(item[field])))
			if "childs" in item:
				out.append("%s\tchilds = {" % indent)
				emit(item["childs"], indent + "\t\t")
				out.append("%s\t}," % indent)
			out.append("%s}," % indent)

	emit(api, "\t")
	out.append("}")
	with open(path, "w", encoding="utf-8", newline="\n") as f:
		f.write("\n".join(out) + "\n")


def main():
	strict = "--strict" in sys.argv
	if "--singe" in sys.argv:
		singe = sys.argv[sys.argv.index("--singe") + 1]
	else:
		singe = singeBinary()
	if not os.access(singe, os.X_OK):
		sys.exit("No Singe binary at %s to ask for its globals;  build one or pass --singe." % singe)
	globalsFound = dumpGlobals(singe)
	english, englishConstants = parseReference(os.path.join(DOCS, "Reference.adoc"), "en")

	functions    = set(n for n, g in globalsFound.items() if g["type"] == "function")
	callbacks    = set(n for n, e in english.items() if any(sig.startswith("function ") for sig in e["signatures"]))
	aliases      = findAliases(globalsFound, english)
	undocumented = sorted(functions - set(english) - set(aliases))
	unknown      = sorted(set(english) - functions - callbacks)
	for name in undocumented:
		print("Not in the Reference:  %s" % name)
	for name in unknown:
		print("In the Reference but not defined:  %s" % name)

	os.makedirs(OUT, exist_ok=True)
	for lang in languages():
		path              = os.path.join(DOCS, "Reference.adoc") if lang == "en" else os.path.join(DOCS, lang, "Reference.adoc")
		entries, constants = (english, englishConstants) if lang == "en" else parseReference(path, lang)
		api               = build(globalsFound, english, entries, constants)
		for alias, target in aliases.items():
			api[alias] = dict(api[target])
			api[alias]["description"] = "= " + target + "\n" + api[target]["description"]
			api[alias]["anchor"]      = api[target].get("anchor", target.lower())
		target            = os.path.join(OUT, "api.lua" if lang == "en" else "api-%s.lua" % lang)
		writeLua(target, api, lang)
		print("%s  %d names, %d bytes" % (os.path.relpath(target, REPO), len(api), os.path.getsize(target)))
	if strict and (undocumented or unknown):
		sys.exit(1)


main()