188 lines
9.7 KiB
Python
188 lines
9.7 KiB
Python
#!/usr/bin/env python3
|
|
# mergeSurvey.py - merge survey/*.json (phase 1 agent output) into annotation files and a knowledge base.
|
|
#
|
|
# annotations/10_survey.json unitLabels + routine header notes + text dataTypes
|
|
# survey/variableProposals.json every proposed variable name per address (for the naming arbiter)
|
|
# docs/knowledgeBase.md human readable routine / data / insight list for phase 2
|
|
import os, re, json, glob, sys
|
|
|
|
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
SURVEY = os.path.join(ROOT, "survey")
|
|
ANN = os.path.join(ROOT, "annotations")
|
|
DOCS = os.path.join(ROOT, "docs")
|
|
|
|
RESERVED = {"a", "x", "y", "s", "sp", "and", "or", "not", "xor", "mod", "div"}
|
|
MNEMONICS = {"adc","and","asl","bcc","bcs","beq","bit","bmi","bne","bpl","brk","bvc","bvs","clc","cld","cli","clv","cmp",
|
|
"cpx","cpy","dec","dex","dey","eor","inc","inx","iny","jmp","jsr","lda","ldx","ldy","lsr","nop","ora","pha",
|
|
"php","pla","plp","rol","ror","rti","rts","sbc","sec","sed","sei","sta","stx","sty","tax","tay","tsx","txa","txs","tya"}
|
|
|
|
sys.path.insert(0, os.path.join(ROOT, "tools"))
|
|
from buildAliases import UNIT_ALIASES
|
|
|
|
|
|
def canonicalUnit(u):
|
|
return UNIT_ALIASES.get(u, u)
|
|
|
|
|
|
def unitTag(unit):
|
|
base = unit.split("/")[-1]
|
|
m = re.search(r"_(A|B|T\d\d|T1S\d+|T1S18_20|8000|0200|0400|C000|FBB8|9300|F000|8800)$", base)
|
|
if base.startswith("ovl_6F00_"): return base[-1]
|
|
if base.startswith("ovl_E000_"): return base[-3:]
|
|
if base.startswith("ovl_EC00_") or base.startswith("ovl_EE00_"): return base[4:8] + base[-3:]
|
|
if base.startswith("boot/") or unit.startswith("boot/"): return "Boot"
|
|
if unit.startswith("drive/"): return "Drv"
|
|
return ""
|
|
|
|
SPLIT_HINTS = ("low byte", "high byte", "lo byte", "hi byte", "low bytes", "high bytes",
|
|
"lo bytes", "hi bytes", "split", "lsb table", "msb table")
|
|
|
|
|
|
def isSplitByteTable(name, description):
|
|
"""A 'lo'/'hi' half of a split address table is a byte table, not a table of words."""
|
|
if re.search(r"(Lo|Hi|Low|High|Lsb|Msb)$", name):
|
|
return True
|
|
text = description.lower()
|
|
return any(h in text for h in SPLIT_HINTS)
|
|
|
|
|
|
def validName(n):
|
|
return bool(re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", n)) and n.lower() not in RESERVED and n.lower() not in MNEMONICS
|
|
|
|
def main():
|
|
files = sorted(glob.glob(os.path.join(SURVEY, "*.json")))
|
|
files = [f for f in files if not f.endswith("variableProposals.json")]
|
|
unitLabels = {} # unit -> {addrHex: name}
|
|
notes = {} # "addr|unit" -> note
|
|
dataTypes = {}
|
|
hwNames = set()
|
|
try:
|
|
sys.path.insert(0, os.path.join(ROOT, "tools"))
|
|
from hwSymbols import c64Symbols, DRIVE
|
|
hwNames = set(c64Symbols().values()) | set(DRIVE.values())
|
|
except Exception:
|
|
pass
|
|
nameOwner = {} # name -> (unit, addr) first claim
|
|
varProposals = {} # addr -> list of {name, meaning, scope, confidence, chunk}
|
|
kb = {} # unit -> list of routine dicts
|
|
kbData = {}
|
|
insights = {}
|
|
problems = []
|
|
seed = json.load(open(os.path.join(ANN, "00_seed.json")))
|
|
for a, n in seed.get("labels", {}).items():
|
|
nameOwner[n] = ("seed", a.upper())
|
|
for unit, m in seed.get("unitLabels", {}).items():
|
|
for a, n in m.items():
|
|
nameOwner[n] = (unit, a.upper())
|
|
for z in seed.get("zp", {}).values():
|
|
nameOwner[z[0] if isinstance(z, list) else z] = ("seed-zp", None)
|
|
|
|
def claim(name, unit, addr):
|
|
"""return a globally unique variant of name for (unit, addr)"""
|
|
if not validName(name) or name in hwNames:
|
|
name = re.sub(r"[^A-Za-z0-9_]", "", name) or f"label{addr}"
|
|
if name[0].isdigit(): name = "l" + name
|
|
if name in hwNames or name.lower() in MNEMONICS or name.lower() in RESERVED: name += "Routine"
|
|
owner = nameOwner.get(name)
|
|
if owner is None or owner == (unit, addr):
|
|
nameOwner[name] = (unit, addr)
|
|
return name
|
|
# same address claimed from another unit / the seed: same thing, keep the name
|
|
if owner[1] == addr:
|
|
return name
|
|
tag = unitTag(unit)
|
|
cand = name + tag if tag and not name.endswith(tag) else name + "_" + addr
|
|
if nameOwner.get(cand) not in (None, (unit, addr)):
|
|
cand = name + "_" + addr
|
|
nameOwner[cand] = (unit, addr)
|
|
return cand
|
|
|
|
for f in files:
|
|
try:
|
|
j = json.load(open(f))
|
|
except Exception as e:
|
|
problems.append(f"{os.path.basename(f)}: invalid JSON ({e})")
|
|
continue
|
|
unit = canonicalUnit(j.get("unit") or "?")
|
|
chunk = j.get("chunk") or os.path.basename(f)
|
|
for r in j.get("routines", []):
|
|
addr = str(r.get("addr", "")).upper().replace("$", "").zfill(4)
|
|
if not re.match(r"^[0-9A-F]{4}$", addr):
|
|
problems.append(f"{chunk}: bad routine addr {r.get('addr')}"); continue
|
|
name = claim(str(r.get("name", f"sub_{addr}")), unit, addr)
|
|
unitLabels.setdefault(unit, {})[addr] = name
|
|
head = [f"{name} - {r.get('summary','').strip()}"]
|
|
if r.get("inputs"): head.append(f"In: {r['inputs'].strip()}")
|
|
if r.get("outputs"): head.append(f"Out: {r['outputs'].strip()}")
|
|
conf = r.get("confidence", "")
|
|
if conf and conf != "high": head.append(f"(confidence: {conf})")
|
|
notes[f"{addr}|{unit}"] = {"routine": head, "unit": unit}
|
|
kb.setdefault(unit, []).append({"addr": addr, "name": name, "summary": r.get("summary",""),
|
|
"inputs": r.get("inputs",""), "outputs": r.get("outputs",""), "confidence": conf})
|
|
for d in j.get("dataBlocks", []):
|
|
addr = str(d.get("addr", "")).upper().replace("$", "").zfill(4)
|
|
if not re.match(r"^[0-9A-F]{4}$", addr):
|
|
problems.append(f"{chunk}: bad data addr {d.get('addr')}"); continue
|
|
name = claim(str(d.get("name", f"data{addr}")), unit, addr)
|
|
unitLabels.setdefault(unit, {})[addr] = name
|
|
length = int(d.get("length", 0) or 0)
|
|
t = d.get("type", "unknown")
|
|
if t == "text" and 0 < length <= 256:
|
|
dataTypes[addr] = ["text", length]
|
|
elif t == "addrTable" and 0 < length <= 512 and length % 2 == 0 and not isSplitByteTable(name, d.get("description", "")):
|
|
dataTypes[addr] = ["addr", length]
|
|
desc = d.get("description", "")
|
|
key = f"{addr}|{unit}"
|
|
notes.setdefault(key, {"unit": unit})
|
|
notes[key]["block"] = [f"{name}: {t}, {length} bytes. {desc}".strip()]
|
|
kbData.setdefault(unit, []).append({"addr": addr, "name": name, "type": t, "length": length, "description": desc})
|
|
for v in j.get("variables", []):
|
|
addr = str(v.get("addr", "")).upper().replace("$", "")
|
|
if not re.match(r"^[0-9A-F]{2}$|^[0-9A-F]{4}$", addr):
|
|
continue
|
|
varProposals.setdefault(addr, []).append({"name": v.get("name"), "meaning": v.get("meaning"), "scope": v.get("scope"),
|
|
"confidence": v.get("confidence"), "chunk": chunk, "unit": unit})
|
|
for m in j.get("misclassified", []):
|
|
problems.append(f"{chunk}: misclassified {m}")
|
|
for s in j.get("insights", []):
|
|
insights.setdefault(unit, []).append(f"[{chunk}] {s}")
|
|
|
|
out = {"_comment": "generated by tools/mergeSurvey.py from survey/*.json - do not edit by hand",
|
|
"unitLabels": unitLabels,
|
|
"notes": {k.split("|")[0]: v for k, v in notes.items()} if False else {},
|
|
"dataTypes": dataTypes}
|
|
# notes keyed by address must stay unit scoped: build.py keys notes by address only, so emit one file per unit
|
|
perUnitNotes = {}
|
|
for k, v in notes.items():
|
|
addr, unit = k.split("|")
|
|
perUnitNotes.setdefault(unit, {})[addr] = v
|
|
json.dump({"_comment": out["_comment"], "unitLabels": unitLabels, "dataTypes": dataTypes}, open(os.path.join(ANN, "10_survey_labels.json"), "w"), indent=1)
|
|
for unit, m in perUnitNotes.items():
|
|
fn = "11_survey_notes_" + unit.replace("/", "_") + ".json"
|
|
json.dump({"_comment": out["_comment"], "notes": m}, open(os.path.join(ANN, fn), "w"), indent=1)
|
|
json.dump(varProposals, open(os.path.join(SURVEY, "variableProposals.json"), "w"), indent=1)
|
|
|
|
with open(os.path.join(DOCS, "knowledgeBase.md"), "w") as f:
|
|
f.write("# Modem Wars - routine and data knowledge base (phase 1 survey, auto-generated)\n\n")
|
|
f.write("Names are unit-scoped: overlay variants at the same address have different code.\n\n")
|
|
for unit in sorted(kb):
|
|
f.write(f"## {unit}\n\n")
|
|
f.write("| addr | name | summary | in | out | conf |\n|---|---|---|---|---|---|\n")
|
|
for r in sorted(kb[unit], key=lambda r: r["addr"]):
|
|
row = [r["addr"], r["name"], r["summary"], r["inputs"], r["outputs"], r["confidence"]]
|
|
f.write("| " + " | ".join(str(x).replace("|", "/").replace("\n", " ") for x in row) + " |\n")
|
|
if unit in kbData:
|
|
f.write("\nData:\n\n")
|
|
for d in sorted(kbData[unit], key=lambda d: d["addr"]):
|
|
f.write(f"- `${d['addr']}` **{d['name']}** ({d['type']}, {d['length']} bytes): {d['description']}\n")
|
|
if unit in insights:
|
|
f.write("\nInsights:\n\n")
|
|
for s in insights[unit]:
|
|
f.write(f"- {s}\n")
|
|
f.write("\n")
|
|
print(f"units: {len(unitLabels)}, labels: {sum(len(v) for v in unitLabels.values())}, notes: {len(notes)}, "
|
|
f"variable addresses: {len(varProposals)}, problems: {len(problems)}")
|
|
for p in problems[:40]:
|
|
print(" -", p)
|
|
|
|
main()
|