conco/OpenMaidEngine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Refactor opcode metadata ownership

Browse Source
This commit is contained in:
gamer147
2026-08-03 14:37:58 -04:00
parent 43f7c6c959
commit 9dd89bb0bf
14 changed files with 193 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
tools/opcode_context.py
View File

@@ -3,7 +3,7 @@
For an opcode (or the top-N unnamed ones) prints: frequency + share + cumulative coverage,
argc, operand-type signature histogram, most common predecessor/successor opcodes, a few
real disassembly snippets, and Kelebek's inline comment (from the upstream cpp). Read-only.
real disassembly snippets, and the canonical registry note. Read-only.
Usage:
py -3.11 -X utf8 tools/opcode_context.py --top 20 # ranked unnamed summary + coverage
@@ -16,14 +16,14 @@ import collections
from pathlib import Path
HERE = Path(__file__).resolve().parent
ROOT = HERE.parent
sys.path.insert(0, str(HERE))
import paths
import sys4load
import age_opcodes as ao
import opcodes_model as M
CORPUS = paths.DATA1
KELEBEK_CPP = ROOT / "vm-map" / "kelebek1-age-shared.cpp"
REGISTRY = M.load(paths.VM_MAP / "opcodes.toml")
def is_unnamed(op: int) -> bool:
@@ -35,13 +35,14 @@ def label(op: int) -> str:
return ao.OPCODES.get(op, (f"?{op:x}", 0))[0]
def kelebek_comments() -> dict[int, str]:
def registry_notes() -> dict[int, str]:
out = {}
if not KELEBEK_CPP.exists():
return out
for m in re.finditer(r"\{\s*(0x[0-9A-Fa-f]+)\s*,\s*\"[^\"]*\"\s*,\s*0x[0-9A-Fa-f]+\s*\}\s*,?\s*//\s*(.*)",
KELEBEK_CPP.read_text(encoding="utf-8")):
out[int(m.group(1), 16)] = m.group(2).strip()
for op, entry in REGISTRY.opcodes.items():
note = entry.abi_note
if not note and entry.semantics:
note = entry.semantics.summary
if note:
out[op] = note
return out
@@ -71,19 +72,19 @@ def main() -> int:
freq[ins.opcode] += 1
total += 1
named_vol = sum(c for op, c in freq.items() if not is_unnamed(op))
comments = kelebek_comments()
notes = registry_notes()
if args and args[0] == "--top":
n = int(args[1]) if len(args) > 1 else 20
unnamed = [(op, c) for op, c in freq.most_common() if is_unnamed(op)]
print(f"corpus {len(scrs)} scripts, {total} instructions; "
f"named coverage {100*named_vol/total:.2f}%; {len(unnamed)} distinct unnamed ops")
print(f"{'#':>3} {'op':<7}{'argc':>5}{'count':>9}{'share':>8}{'cum-cov':>9} kelebek-comment")
print(f"{'#':>3} {'op':<7}{'argc':>5}{'count':>9}{'share':>8}{'cum-cov':>9} registry-note")
cum = named_vol
for i, (op, c) in enumerate(unnamed[:n]):
cum += c
argc = ao.OPCODES.get(op, ("", 0))[1]
print(f"{i+1:>3} 0x{op:<5x}{argc:>5}{c:>9}{100*c/total:>7.2f}%{100*cum/total:>8.2f}% {comments.get(op,'')[:48]}")
print(f"{i+1:>3} 0x{op:<5x}{argc:>5}{c:>9}{100*c/total:>7.2f}%{100*cum/total:>8.2f}% {notes.get(op,'')[:48]}")
return 0
# detailed per-op evidence
@@ -111,8 +112,8 @@ def main() -> int:
argc = ao.OPCODES.get(op, ("", 0))[1]
print(f"\n{'='*72}\nopcode 0x{op:x} label={label(op)} argc={argc} "
f"count={c} ({100*c/total:.2f}% of instrs)")
if comments.get(op):
print(f" kelebek-comment: {comments[op]}")
if notes.get(op):
print(f" registry-note: {notes[op]}")
print(f" operand-type signatures: " +
", ".join(f"{'/'.join(s) if s else 'none'}×{n}" for s, n in sig[op].most_common(4)))
print(f" top predecessors: " + ", ".join(f"{k}×{v}" for k, v in pred[op].most_common(5)))

70
tools/opcodes_build.py
View File

@@ -1,8 +1,8 @@
#!/usr/bin/env python3
"""Generator + linter for the living opcode reference (vm-map/opcodes.toml).
--bootstrap seed skeletons for every used opcode (append-only; preserves hand edits)
--bootstrap-age seed compatibility stubs for every opcode in the broader AGE catalog
--build emit age_opcodes_himegari.py + build/opcodes.json + docs/opcode-reference.md + build/opcode-coverage.md
--bootstrap-age seed compatibility stubs from the canonical broader AGE catalog
--build emit age_opcodes.py + age_opcodes_himegari.py + JSON/Markdown views
--lint run the linter, print errors/warnings, exit nonzero on errors
See docs/superpowers/specs/2026-07-06-opcode-reference-design.md."""
from __future__ import annotations
@@ -12,7 +12,6 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import paths
import sys4load
import opcodes_model as M
from age_opcodes import OPCODES
TOML_DEFAULT = paths.VM_MAP / "opcodes.toml"
@@ -100,15 +99,17 @@ def skeleton_toml(op: int, label: str, argc: int, argtypes_for_op: dict,
f"observed_types = [{obs}]"]
return "\n".join(lines) + "\n"
def bootstrap(toml_path: Path, *, corpus_scan=None) -> None:
def bootstrap(toml_path: Path, *, corpus_scan=None, catalog_model: M.Model | None = None) -> None:
"""Append observed skeletons from a real scan or an injected synthetic scan fixture."""
used, argtypes = corpus_scan if corpus_scan is not None else scan_corpus()
catalog = catalog_model or M.load(TOML_DEFAULT)
present = set(M.load(toml_path).opcodes) if toml_path.exists() else set()
blocks = []
for op in sorted(used):
if op in present:
continue
label, argc = OPCODES.get(op, ("0x%x" % op, 0))
entry = catalog.opcodes.get(op)
label, argc = ((entry.label, entry.argc) if entry else ("0x%x" % op, 0))
blocks.append(skeleton_toml(op, label, argc, argtypes[op]))
if not toml_path.exists():
toml_path.parent.mkdir(parents=True, exist_ok=True)
@@ -117,12 +118,13 @@ def bootstrap(toml_path: Path, *, corpus_scan=None) -> None:
f.write("\n".join(blocks))
print(f"bootstrap: {len(used)} used opcodes; appended {len(blocks)} new skeletons -> {toml_path}")
def bootstrap_age(toml_path: Path) -> None:
def bootstrap_age(toml_path: Path, *, catalog_model: M.Model | None = None) -> None:
"""Append compatibility stubs for catalog opcodes absent from the canonical map."""
catalog = catalog_model or M.load(TOML_DEFAULT)
present = set(M.load(toml_path).opcodes) if toml_path.exists() else set()
blocks = [
skeleton_toml(op, label, argc, {}, observed_in_himegari=False)
for op, (label, argc) in sorted(OPCODES.items())
skeleton_toml(op, entry.label, entry.argc, {}, observed_in_himegari=False)
for op, entry in sorted(catalog.opcodes.items())
if op not in present
]
if not toml_path.exists():
@@ -130,11 +132,58 @@ def bootstrap_age(toml_path: Path) -> None:
toml_path.write_text(META_TOML + "\n", encoding="utf-8")
with toml_path.open("a", encoding="utf-8") as f:
f.write("\n".join(blocks))
print(f"bootstrap-age: {len(OPCODES)} catalog opcodes; "
print(f"bootstrap-age: {len(catalog.opcodes)} catalog opcodes; "
f"appended {len(blocks)} compatibility stubs -> {toml_path}")
GEN_HEADER = "# DO NOT EDIT -- generated from vm-map/opcodes.toml by tools/opcodes_build.py --build\n"
def emit_runtime_py(model: M.Model) -> str:
"""Emit the Python disassembler/runtime ABI view from the canonical registry."""
arg_types = {
int(key, 0): value
for key, value in model.meta.get("arg_types", {}).items()
}
targets = {
op: tuple(oc.code_target_args)
for op, oc in model.opcodes.items()
if oc.code_target_args
}
lines = [
GEN_HEADER,
'"""AGE opcode framing and operand metadata (generated canonical view)."""',
"from __future__ import annotations",
"",
"# opcode -> (historical/canonical label, argument count)",
"OPCODES: dict[int, tuple[str, int]] = {",
]
for op, oc in sorted(model.opcodes.items()):
lines.append(f" 0x{op:04x}: ({oc.label!r}, {oc.argc}),")
lines += ["}", "", "# argument type tag -> disassembly label", "ARG_TYPES: dict[int, str] = {"]
for tag, label in sorted(arg_types.items()):
lines.append(f" 0x{tag:04x}: {label!r},")
lines += [
"}",
"",
"# One-based operand indices whose raw values are code offsets.",
"CODE_TARGET_ARGS: dict[int, frozenset[int]] = {",
]
for op, indices in sorted(targets.items()):
values = ", ".join(str(i) for i in indices)
if len(indices) == 1:
values += ","
lines.append(f" 0x{op:04x}: frozenset(({values})),")
lines += [
"}",
"CONTROL_FLOW = frozenset(CODE_TARGET_ARGS)",
f"ARRAY_OPCODE = 0x{int(model.meta['inline_array_opcode']):x}",
"",
"def is_label_argument(op: int, arg_index: int, raw_value: int) -> bool:",
' """Return whether a zero-based operand is a non-fallthrough code target."""',
" return (raw_value != 0xFFFFFFFF",
" and arg_index + 1 in CODE_TARGET_ARGS.get(op, ()))",
]
return "\n".join(lines) + "\n"
def emit_inferred_py(model: M.Model) -> str:
lines = [GEN_HEADER,
'"""Inferred Himegari opcode semantics (generated). sys4load reads INFERRED[op][\'name\']."""',
@@ -248,8 +297,9 @@ def main(argv=None):
for m in errors:
print("error:", m)
return 1
(paths.REPO / "tools" / "age_opcodes.py").write_text(emit_runtime_py(model), encoding="utf-8")
(paths.REPO / "tools" / "age_opcodes_himegari.py").write_text(emit_inferred_py(model), encoding="utf-8")
print("build: wrote tools/age_opcodes_himegari.py")
print("build: wrote tools/age_opcodes.py, tools/age_opcodes_himegari.py")
paths.BUILD.mkdir(parents=True, exist_ok=True)
(paths.BUILD / "opcodes.json").write_text(emit_json(model), encoding="utf-8")
(paths.REPO / "docs" / "opcode-reference.md").write_text(emit_reference_md(model), encoding="utf-8")

1
tools/paths.py
View File

@@ -27,7 +27,6 @@ BUILD = REPO / "build" # derived corpora (rege
VM_MAP = REPO / "vm-map"
BIN = REPO / "bin" # 3rd-party tools (BinExtractALF, ...)
AGE_EXE = GAME_DIR / "AGE.EXE"
KELEBEK_CPP = VM_MAP / "kelebek1-age-shared.cpp"
def add_self_to_syspath():

16
tools/sys4load.py
View File

@@ -7,8 +7,8 @@ Parses the confirmed container format (see ../sys4-format-notes.md):
* strings: XOR-0xFF cp932, NUL-terminated, referenced by a `0x02 <dword-off>` pair
The container format is byte-verified across all 481 DATA1 scripts. Opcodes are now
DECODED using the AGE opcode table (age_opcodes.py, transcribed from Kelebek1's
decompiler and validated 476/476 clean on Himegari): the code section is a flat stream
DECODED using the canonical AGE opcode registry (vm-map/opcodes.toml, generated into
age_opcodes.py and validated 476/476 clean on Himegari): the code section is a flat stream
of `<opcode:u32> + argc*(<argtype:u32><value:u32>)` instructions, length 1+2*argc dwords.
Inline strings live after the code inside [0,F8), so decoding stops at the first string
(type-2) or array (op 0x64) operand offset.
@@ -38,8 +38,8 @@ except ImportError: # allow import from another cwd
from age_opcodes import (OPCODES, ARG_TYPES, CONTROL_FLOW, ARRAY_OPCODE,
is_label_argument)
# Himegari inference layer (optional): improves labels for unnamed opcodes. Keeps the
# verbatim Kelebek table (age_opcodes) pristine; see age_opcodes_himegari.py.
# Himegari inference layer (optional): improves labels for unnamed opcodes from the
# canonical generated ABI view; see age_opcodes_himegari.py.
try:
from age_opcodes_himegari import INFERRED
except ImportError:
@@ -96,7 +96,7 @@ CALLSCRIPT_OP = 0x03
def display_label(op: int) -> str:
"""Rendered mnemonic: Kelebek name if it has one, else the inferred name, else u00…."""
"""Rendered mnemonic: canonical ABI label, else the inferred name, else u00…."""
lbl = OPCODES.get(op, (f"?{op:x}", 0))[0]
is_unnamed = (lbl.startswith(("u00", "dev_ukn")) or lbl.lower() == f"{op:x}")
if is_unnamed and op in INFERRED:
@@ -422,10 +422,10 @@ def render_listing(scr: Sys4Script) -> str:
continue
ops = " ".join(_fmt_operand(ins.opcode, x, t, v, scr.strings)
for x, (t, v) in enumerate(ins.args))
mnem = display_label(ins.opcode) # prefers Kelebek name, else inferred, else u00…
mnem = display_label(ins.opcode) # prefers canonical label, else inferred, else u00…
# annotate unnamed/inferred ops with their raw value for grep-ability
kelebek = ins.label
unnamed = kelebek.startswith(("u00", "dev_ukn")) or kelebek[:1].isdigit()
raw_label = ins.label
unnamed = raw_label.startswith(("u00", "dev_ukn")) or raw_label[:1].isdigit()
raw = f" ; op 0x{ins.opcode:x}" + (" inferred" if unnamed and ins.opcode in INFERRED else "") \
if unnamed else ""
out.append(f" 0x{ins.offset:05x}: {mnem}{(' ' + ops) if ops else ''}{raw}")

24
tools/test_opcodes.py
View File

@@ -119,13 +119,12 @@ def test_lint():
def test_bootstrap():
import opcodes_build as B
from age_opcodes import OPCODES
from pathlib import Path
canonical = M.load(Path(__file__).resolve().parents[1] / "vm-map" / "opcodes.toml")
observed = sorted(op for op, entry in canonical.opcodes.items() if entry.observed_in_himegari)
synthetic_scan = (
Counter({op: 1 for op in observed}),
{op: {i: {0} for i in range(OPCODES[op][1])} for op in observed},
{op: {i: {0} for i in range(canonical.opcodes[op].argc)} for op in observed},
)
fd, p = tempfile.mkstemp(suffix=".toml"); os.close(fd); os.remove(p)
tp = Path(p)
@@ -139,9 +138,9 @@ def test_bootstrap():
check(len(M.load(tp).opcodes) == n1, "second bootstrap adds no duplicates")
e, w = M.lint(m)
check(e == [], f"bootstrapped file lints clean (errors: {e[:3]})")
B.bootstrap_age(tp)
B.bootstrap_age(tp, catalog_model=canonical)
full = M.load(tp)
check(len(full.opcodes) == len(OPCODES), "bootstrap-age seeds the complete AGE catalog")
check(len(full.opcodes) == len(canonical.opcodes), "bootstrap-age seeds the complete AGE catalog")
check(sum(o.observed_in_himegari for o in full.opcodes.values()) == len(m.opcodes),
"bootstrap-age marks only added catalog entries unobserved")
check(all(not o.semantics.noop_headless for o in full.opcodes.values()
@@ -160,6 +159,22 @@ def test_emit_inferred():
check(0x90 in inf and inf[0x90]["name"] == "hotspot-branch", "generated INFERRED[0x90]['name'] correct")
check(0x1f4 in inf, "named marker 0x1f4 (name != label) included")
def test_emit_runtime():
import opcodes_build as B
from pathlib import Path
canonical = M.load(Path(__file__).resolve().parents[1] / "vm-map" / "opcodes.toml")
src = B.emit_runtime_py(canonical)
ns = {}
exec(compile(src, "<runtime-gen>", "exec"), ns)
check(len(ns["OPCODES"]) == len(canonical.opcodes), "runtime view contains the complete catalog")
check(ns["OPCODES"][0x90] == (canonical.opcodes[0x90].label, 7),
"runtime view preserves opcode label and argc")
check(ns["ARG_TYPES"][0] == "imm", "runtime view emits canonical argument labels")
check(ns["is_label_argument"](0x90, 4, 0x123), "runtime view recognizes hotspot target arg")
check(not ns["is_label_argument"](0x90, 3, 0x123), "runtime view rejects hotspot data arg")
check(not ns["is_label_argument"](0x90, 4, 0xffffffff), "fallthrough sentinel is not a target")
check(ns["ARRAY_OPCODE"] == 0x64, "runtime view emits the inline-array opcode")
def test_emit_views():
import opcodes_build as B, json as _json
m = M.load(write_tmp(FIXTURE))
@@ -176,6 +191,7 @@ def main():
test_lint()
test_bootstrap()
test_emit_inferred()
test_emit_runtime()
test_emit_views()
print("FAILURES:", len(FAILS))
return 1 if FAILS else 0

14
tools/validate_opcode_table.py
View File

@@ -1,16 +1,12 @@
#!/usr/bin/env python3
"""Definitive test: replicate Kelebek's data_array_end shrinking (stop code at first
string/array offset), then measure clean decode rate over all Himegari scripts."""
import os, re, sys, collections
from pathlib import Path
"""Definitive raw-walker test for the canonical opcode registry and data boundary."""
import os, sys, collections
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import paths
import sys4load
from age_opcodes import OPCODES
CPP=paths.KELEBEK_CPP.read_text(encoding="utf-8")
TABLE={}; LABEL={}
for m in re.finditer(r'\{\s*(0x[0-9A-Fa-f]+)\s*,\s*"([^"]*)"\s*,\s*(0x[0-9A-Fa-f]+)\s*\}', CPP):
TABLE[int(m.group(1),16)]=int(m.group(3),16); LABEL[int(m.group(1),16)]=m.group(2)
TABLE={op: entry[1] for op, entry in OPCODES.items()}
files=paths.scripts()
@@ -48,7 +44,7 @@ for name,p in sorted(files.items()):
if len(examples)<20 and reason: pass
print(f"scripts: {len(files)} parsefail(container): {parsefail}")
print(f"CLEAN decode (Kelebek table + string-pool boundary): {clean}")
print(f"CLEAN decode (canonical table + string-pool boundary): {clean}")
print(f"DIRTY: {dirty}")
print(f"total instructions decoded in clean files: {instr_total}")
print(f"\nremaining small unknown opcodes (genuine gaps):")

19
tools/validate_opcode_table_naive.py
View File

@@ -1,27 +1,22 @@
#!/usr/bin/env python3
"""Validate Kelebek1's AGE opcode table against Himegari SYS4 scripts.
"""Validate the canonical AGE opcode table against Himegari SYS4 scripts.
Model (from Kelebek1 disassembler.cpp + age-shared.cpp):
Model recorded in vm-map/opcodes.toml:
code stream = sequence of instructions.
each instruction = <opcode:u32> then argument_count * <arg>, where each arg = <type:u32><value:u32>.
=> instruction length in dwords = 1 + 2*argument_count (uniform; type-2/0x64 args seek elsewhere, don't consume inline)
arg type 2 = inline string (value = dword offset into body). types: 0 imm,1 float,3 g-int,9 l-int, etc.
A clean decode consumes exactly code_len dwords with no unknown opcode and no arg overrun.
"""
import os, re, sys, collections
from pathlib import Path
import os, sys, collections
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import paths
import sys4load
from age_opcodes import OPCODES
CPP = paths.KELEBEK_CPP.read_text(encoding="utf-8")
# parse {0x1F4, "label", 0x0},
TABLE = {}
LABEL = {}
for m in re.finditer(r'\{\s*(0x[0-9A-Fa-f]+)\s*,\s*"([^"]*)"\s*,\s*(0x[0-9A-Fa-f]+)\s*\}', CPP):
op = int(m.group(1), 16); lbl = m.group(2); argc = int(m.group(3), 16)
TABLE[op] = argc; LABEL[op] = lbl
print(f"parsed {len(TABLE)} opcode defs from Kelebek1 table (max op 0x{max(TABLE):x})")
TABLE = {op: entry[1] for op, entry in OPCODES.items()}
LABEL = {op: entry[0] for op, entry in OPCODES.items()}
print(f"loaded {len(TABLE)} opcode defs from the canonical registry (max op 0x{max(TABLE):x})")
files = paths.scripts()

2
tools/vm0.py
View File

@@ -3,7 +3,7 @@
Executes one script's bytecode to validate the *execution model* before any C#/Godot work.
Reuses tools/sys4load.py for all parsing/decoding. Effectful ops and call-script are STUBBED;
show-text is captured. Named-op semantics come from Kelebek's table; the classified markers are
show-text is captured. Named-op semantics come from the generated canonical registry; the classified markers are
treated as no-ops (this run TESTS that assumption).
Purpose (see docs/phase-a-slice-plan.md):