diff --git a/README.md b/README.md index c3ca823..bf57ae2 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,7 @@ selected requirement that is unavailable is an error, not a silent skip. - [Tools reference](docs/tools-reference.md) — exact commands, prerequisites, inputs, and outputs. - [Phase B framework](docs/phase-b-framework.md) — boot, menu, session, and gameplay execution order. - [Engine reverse engineering](docs/engine-re.md) — native AGE findings and provenance. +- [Third-party notices](THIRD_PARTY_NOTICES.md) — incorporated-code licenses and research acknowledgments. Follow the repository's canonical-document map when recording new knowledge: extend the existing owner and cross-link it instead of duplicating facts here. diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md new file mode 100644 index 0000000..0c0ded1 --- /dev/null +++ b/THIRD_PARTY_NOTICES.md @@ -0,0 +1,37 @@ +# Third-party notices and research acknowledgments + +This file records incorporated third-party code and prior public research that materially informed this +project. It does not license Eushully game assets or code, and no original game files are distributed with +this repository. + +## Eushully-Decompiler research + +Early SYS4 instruction decoding and the initial opcode ABI/catalog were informed by Kelebek's public +[Eushully-Decompiler](https://github.com/Kelebek1/Eushully-Decompiler) project. That work established a +valuable starting point and accelerated this reimplementation. Historical and per-opcode provenance is +preserved in `vm-map/opcodes.toml`. + +This repository does not vendor the upstream source files. Its opcode registry combines that foundational +catalog with independent script-corpus analysis, native `AGE.EXE` investigation, runtime probes, and +reimplementation testing. This acknowledgment is not a claim that the project was developed under a formal +clean-room process. + +## GARbro AGF decoding algorithm + +`engine/Age.Engine/Sys4/AgfDecoder.cs` ports the AGF decoding algorithm from GARbro's +`ArcFormats/Eushully/ImageAGF.cs`, Copyright (c) 2014-2020 morkt, under the MIT License: + +> Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +> associated documentation files (the "Software"), to deal in the Software without restriction, including +> without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +> copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the +> following conditions: +> +> The above copyright notice and this permission notice shall be included in all copies or substantial +> portions of the Software. +> +> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT +> LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO +> EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +> IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +> USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/docs/PROJECT-STRUCTURE.md b/docs/PROJECT-STRUCTURE.md index e4733e2..afb1b6d 100644 --- a/docs/PROJECT-STRUCTURE.md +++ b/docs/PROJECT-STRUCTURE.md @@ -29,6 +29,7 @@ S:\Game Hacking\Eushully\Himegari\ ← workspace root (three siblings) └── age-reimpl/ ← OUR WORK (everything we made lives here) │ ├── README.md navigation-only repository front door; canonical facts stay in docs/ + ├── THIRD_PARTY_NOTICES.md incorporated-code licenses + prior-research acknowledgment ├── global.json pins the validated .NET 8 SDK feature band ├── .editorconfig / .gitattributes UTF-8, indentation, text/EOL, and binary-file policy; │ tracked source and generated references use canonical LF @@ -43,8 +44,8 @@ S:\Game Hacking\Eushully\Himegari\ ← workspace root (three siblings) │ ├── validate.py layered core/workspace/runtime/full validation driver │ ├── test_validate.py pure resolver + validation-plan regressions │ ├── sys4load.py loader + disassembler (opcode-decoding) - │ ├── age_opcodes.py 548-entry Kelebek AGE opcode/arg-type table (PRISTINE; never edit) - │ ├── opcodes_build.py generator/linter: vm-map/opcodes.toml -> the 4 artifacts below + │ ├── age_opcodes.py GENERATED complete Python ABI view (do not hand-edit) + │ ├── opcodes_build.py generator/linter: vm-map/opcodes.toml -> the 5 artifacts below │ ├── opcodes_model.py load + lint (dangling-ref, confidence-ceiling, vocab) + dependents │ ├── age_opcodes_himegari.py GENERATED from opcodes.toml (do not hand-edit) │ ├── globals_build.py generator/linter: vm-map/globals.toml (+auto map) -> build/globals.json, docs/global-reference.md @@ -69,7 +70,6 @@ S:\Game Hacking\Eushully\Himegari\ ← workspace root (three siblings) │ │ + provenance + depends_on). Single source of truth for opcodes. │ ├── globals.toml ★ CANONICAL global-variable registry (hand-edited: name + category │ │ + value_domain + provenance). Single source of truth for globals/story-flags. - │ ├── kelebek1-age-shared.cpp / -disassembler.cpp upstream opcode-table source │ └── opcode-leads.json, small-script-listings.md │ ├── docs/ all documentation @@ -288,9 +288,10 @@ have no repository output tree and write their automatic maps below `user://diag extractor. `bin/README.md` records the optional local convention; no extractor binary is tracked. Safe to delete and rebuild; do not hand-edit. - **Opcode knowledge is edited ONLY in `vm-map/opcodes.toml`** (ABI + semantics + provenance + - `depends_on`). Run `tools/opcodes_build.py --build` to regenerate the shim (`tools/age_opcodes_himegari.py`), - machine JSON (`build/opcodes.json`), reference (`docs/opcode-reference.md`), and coverage. `--lint` - checks dangling deps / confidence-ceiling / vocabulary. Kelebek's `tools/age_opcodes.py` stays pristine. + `depends_on`). Run `tools/opcodes_build.py --build` to regenerate both Python views + (`tools/age_opcodes.py`, `tools/age_opcodes_himegari.py`), machine JSON (`build/opcodes.json`), reference + (`docs/opcode-reference.md`), and coverage. `--lint` checks dangling deps / confidence-ceiling / vocabulary. + Both Python modules are generated and must not be hand-edited. - **Global-variable knowledge is edited ONLY in `vm-map/globals.toml`** (name + category + value_domain + provenance). Run `tools/globals_build.py --build` to regenerate `build/globals.json` (sys4load labels) and `docs/global-reference.md`; `--lint` checks vocabulary / auto-shape≠high / dangling deps. Curated entries diff --git a/docs/remake-architecture-and-roadmap.md b/docs/remake-architecture-and-roadmap.md index 2719c2f..b784eed 100644 --- a/docs/remake-architecture-and-roadmap.md +++ b/docs/remake-architecture-and-roadmap.md @@ -962,6 +962,16 @@ do not mix mechanical moves with semantic changes. branch; the workflow's push and pull-request filters target `develop`. The first actual hosted Linux core run succeeded on 2026-08-03 at `6ae75b3`, closing the CI execution gate. + **Kelebek source-redistribution boundary (2026-08-03):** the user selected a deliberately narrow policy: + retain useful factual opcode ABI data, established labels, and explicit per-entry provenance, while no longer + vendoring Kelebek's authored source snapshots or a separately maintained verbatim Python transcription. The + canonical `vm-map/opcodes.toml` registry now generates both Python opcode views as well as the JSON/reference + outputs; consumers and validators no longer parse the removed snapshots. `THIRD_PARTY_NOTICES.md` credits the + foundational Eushully-Decompiler research without claiming a formal clean-room process. This resolves the + source-tree redistribution concern at the current tip without gratuitous renaming or discarding independently + verified work. The removed files remain in existing Git history, so history sanitization is still required + before any future public publication; the project-license choice remains user-owned and unresolved. + **Not cleanup targets:** generated `build/` output, the two intentional solution files, historical `docs/superpowers/` plans/specifications, and fidelity-specific complexity that is directly covered by the native ABI. Reorganization is successful when ownership and reproduction become clearer, not when the raw @@ -1308,9 +1318,10 @@ layer's rendering diverges from ADV; save layout. --- ## 8. Immediate next step -Continue step 5 of the **codebase consolidation** maintenance slice by making the user-owned project-license and -Kelebek-derived-material decisions required before wider distribution. The private remote, rewritten-history -backup, and hosted Linux core gate are now established. Do not infer a license choice or change remote policy +Continue step 5 of the **codebase consolidation** maintenance slice by sanitizing the removed Kelebek source +snapshots/verbatim transcription from public-facing Git history before any public publication, then obtain the +user-owned project-license choice. The private remote, rewritten-history backup, hosted Linux core gate, and +source-tree redistribution boundary are now established. Do not infer a license choice or change remote policy without the user's explicit direction. Concrete playthrough blockers may still preempt this bounded maintenance work; the consolidation effort does not replace Phase B gameplay validation or the open cross-platform gates. diff --git a/docs/tools-reference.md b/docs/tools-reference.md index c5f9e3d..1fbb662 100644 --- a/docs/tools-reference.md +++ b/docs/tools-reference.md @@ -58,7 +58,7 @@ are recorded in `bin/README.md`. PE-sieve is obsolete and is not retained; its h | Tool | Purpose | Run | Reads → Writes | |---|---|---|---| | `sys4load.py` | Loader + opcode-decoding disassembler for SYS4 `.BIN` scripts (the container-format core every other tool builds on). Annotates global operands (`build/globals.json`) and **`call-script` targets by name** (`build/callscript-names.json`, e.g. `call-script 0x1ab =ADDITEM.BIN`). | `sys4load.py ` · `--summary` · `--strings` · `--json` · `sys4load.py --validate` (corpus check) | `.BIN` + `age_opcodes*.py` + `build/globals.json` + `build/callscript-names.json` → stdout listing, or `build/scripts-json/` with `--json` | -| `age_opcodes.py` | 548-entry Kelebek AGE opcode/arg-type table. **PRISTINE upstream data — never edit.** | *Imported.* | — | +| `age_opcodes.py` | ⚙ Generated Python ABI view: the complete opcode label/argument-count catalog, operand-type labels, control-flow target operands, and inline-array opcode. **Do not hand-edit.** | *Imported.* | `vm-map/opcodes.toml` → generated module | ## Opcode reference toolchain — single source of truth = `vm-map/opcodes.toml` @@ -67,11 +67,11 @@ All opcode knowledge (ABI, semantics, provenance, `depends_on`) is hand-edited * | Tool | Purpose | Run | Reads → Writes | |---|---|---|---| -| `opcodes_build.py` | Generator + linter for the opcode reference. `--bootstrap` appends entries observed in the Himegari corpus; `--bootstrap-age` appends every missing opcode from the pristine 548-entry AGE catalog as an unobserved compatibility stub. Catalog stubs retain Kelebek's ABI label/argument count so other AGE scripts decode past them, but remain `noop_headless=false`: the VM currently traces/skips them, while coverage continues to report them as unresolved rather than semantically safe. | `--build` · `--lint` · `--bootstrap` · `--bootstrap-age` | `vm-map/opcodes.toml` → ⚙ `tools/age_opcodes_himegari.py`, ⚙ `build/opcodes.json`, ⚙ `docs/opcode-reference.md`, ⚙ `build/opcode-coverage.md` | +| `opcodes_build.py` | Generator + linter for the opcode reference. `--bootstrap` appends corpus-observed entries to a new/alternate registry; `--bootstrap-age` copies every missing canonical catalog entry into one as an unobserved compatibility stub. Stubs retain the registry's ABI label/argument count so other AGE scripts decode past them, but remain `noop_headless=false`: the VM traces/skips them while coverage reports them as unresolved rather than semantically safe. | `--build` · `--lint` · `--bootstrap` · `--bootstrap-age` | `vm-map/opcodes.toml` → ⚙ `tools/age_opcodes.py`, ⚙ `tools/age_opcodes_himegari.py`, ⚙ `build/opcodes.json`, ⚙ `docs/opcode-reference.md`, ⚙ `build/opcode-coverage.md` | | `opcodes_model.py` | In-memory model + loader + linter (dangling-ref / confidence-ceiling / vocabulary / dependents). | *Imported by `opcodes_build.py`.* | `vm-map/opcodes.toml` → — | | `test_opcodes.py` | Unit tests for the opcode tooling. Bootstrap coverage injects 248 synthetic observations derived from the canonical observed-opcode set; it does not read the private script corpus. Production `--bootstrap` still scans the real corpus by default. | `test_opcodes.py` | `vm-map/opcodes.toml` → temporary files only | -| `opcode_context.py` | Read-only evidence gatherer for classifying unnamed opcodes (frequency, argc, operand-type signature, neighbours, disasm snippets, Kelebek comment). | `--top 20` · `opcode_context.py 0x1f4 0x71 …` | corpus → stdout | -| `validate_opcode_table.py` | Definitive decode-coverage validator (replicates Kelebek's `data_array_end` code/data split). | `validate_opcode_table.py` | corpus → stdout | +| `opcode_context.py` | Read-only evidence gatherer for classifying unnamed opcodes (frequency, argc, operand-type signature, neighbours, disassembly snippets, canonical registry note). | `--top 20` · `opcode_context.py 0x1f4 0x71 …` | `vm-map/opcodes.toml` + corpus → stdout | +| `validate_opcode_table.py` | Definitive decode-coverage validator using the canonical registry's generated ABI and the SYS4 code/data boundary. | `validate_opcode_table.py` | corpus → stdout | | `validate_opcode_table_naive.py` | Naïve variant of the above (baseline comparison). | `validate_opcode_table_naive.py` | corpus → stdout | | `age_opcodes_himegari.py` | ⚙ Inferred Himegari opcode semantics — **generated; do not hand-edit.** | *Imported by `sys4load.py`.* | — | | `globals_build.py` | Merge curated `globals.toml` over the auto shape map, preserve optional machine-readable row-table `columns`, and generate the global registry + linter. | `--build` · `--lint` | `vm-map/globals.toml`, `build/global-var-map.json` → ⚙ `build/globals.json`, ⚙ `docs/global-reference.md` | diff --git a/tools/opcode_context.py b/tools/opcode_context.py index 5dcbcfa..64373b3 100644 --- a/tools/opcode_context.py +++ b/tools/opcode_context.py @@ -3,7 +3,7 @@ For an opcode (or the top-N unnamed ones) prints: frequency + share + cumulative coverage, argc, operand-type signature histogram, most common predecessor/successor opcodes, a few -real disassembly snippets, and Kelebek's inline comment (from the upstream cpp). Read-only. +real disassembly snippets, and the canonical registry note. Read-only. Usage: py -3.11 -X utf8 tools/opcode_context.py --top 20 # ranked unnamed summary + coverage @@ -16,14 +16,14 @@ import collections from pathlib import Path HERE = Path(__file__).resolve().parent -ROOT = HERE.parent sys.path.insert(0, str(HERE)) import paths import sys4load import age_opcodes as ao +import opcodes_model as M CORPUS = paths.DATA1 -KELEBEK_CPP = ROOT / "vm-map" / "kelebek1-age-shared.cpp" +REGISTRY = M.load(paths.VM_MAP / "opcodes.toml") def is_unnamed(op: int) -> bool: @@ -35,13 +35,14 @@ def label(op: int) -> str: return ao.OPCODES.get(op, (f"?{op:x}", 0))[0] -def kelebek_comments() -> dict[int, str]: +def registry_notes() -> dict[int, str]: out = {} - if not KELEBEK_CPP.exists(): - return out - for m in re.finditer(r"\{\s*(0x[0-9A-Fa-f]+)\s*,\s*\"[^\"]*\"\s*,\s*0x[0-9A-Fa-f]+\s*\}\s*,?\s*//\s*(.*)", - KELEBEK_CPP.read_text(encoding="utf-8")): - out[int(m.group(1), 16)] = m.group(2).strip() + for op, entry in REGISTRY.opcodes.items(): + note = entry.abi_note + if not note and entry.semantics: + note = entry.semantics.summary + if note: + out[op] = note return out @@ -71,19 +72,19 @@ def main() -> int: freq[ins.opcode] += 1 total += 1 named_vol = sum(c for op, c in freq.items() if not is_unnamed(op)) - comments = kelebek_comments() + notes = registry_notes() if args and args[0] == "--top": n = int(args[1]) if len(args) > 1 else 20 unnamed = [(op, c) for op, c in freq.most_common() if is_unnamed(op)] print(f"corpus {len(scrs)} scripts, {total} instructions; " f"named coverage {100*named_vol/total:.2f}%; {len(unnamed)} distinct unnamed ops") - print(f"{'#':>3} {'op':<7}{'argc':>5}{'count':>9}{'share':>8}{'cum-cov':>9} kelebek-comment") + print(f"{'#':>3} {'op':<7}{'argc':>5}{'count':>9}{'share':>8}{'cum-cov':>9} registry-note") cum = named_vol for i, (op, c) in enumerate(unnamed[:n]): cum += c argc = ao.OPCODES.get(op, ("", 0))[1] - print(f"{i+1:>3} 0x{op:<5x}{argc:>5}{c:>9}{100*c/total:>7.2f}%{100*cum/total:>8.2f}% {comments.get(op,'')[:48]}") + print(f"{i+1:>3} 0x{op:<5x}{argc:>5}{c:>9}{100*c/total:>7.2f}%{100*cum/total:>8.2f}% {notes.get(op,'')[:48]}") return 0 # detailed per-op evidence @@ -111,8 +112,8 @@ def main() -> int: argc = ao.OPCODES.get(op, ("", 0))[1] print(f"\n{'='*72}\nopcode 0x{op:x} label={label(op)} argc={argc} " f"count={c} ({100*c/total:.2f}% of instrs)") - if comments.get(op): - print(f" kelebek-comment: {comments[op]}") + if notes.get(op): + print(f" registry-note: {notes[op]}") print(f" operand-type signatures: " + ", ".join(f"{'/'.join(s) if s else 'none'}×{n}" for s, n in sig[op].most_common(4))) print(f" top predecessors: " + ", ".join(f"{k}×{v}" for k, v in pred[op].most_common(5))) diff --git a/tools/opcodes_build.py b/tools/opcodes_build.py index e7aeb17..afdb428 100644 --- a/tools/opcodes_build.py +++ b/tools/opcodes_build.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 """Generator + linter for the living opcode reference (vm-map/opcodes.toml). --bootstrap seed skeletons for every used opcode (append-only; preserves hand edits) - --bootstrap-age seed compatibility stubs for every opcode in the broader AGE catalog - --build emit age_opcodes_himegari.py + build/opcodes.json + docs/opcode-reference.md + build/opcode-coverage.md + --bootstrap-age seed compatibility stubs from the canonical broader AGE catalog + --build emit age_opcodes.py + age_opcodes_himegari.py + JSON/Markdown views --lint run the linter, print errors/warnings, exit nonzero on errors See docs/superpowers/specs/2026-07-06-opcode-reference-design.md.""" from __future__ import annotations @@ -12,7 +12,6 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import paths import sys4load import opcodes_model as M -from age_opcodes import OPCODES TOML_DEFAULT = paths.VM_MAP / "opcodes.toml" @@ -100,15 +99,17 @@ def skeleton_toml(op: int, label: str, argc: int, argtypes_for_op: dict, f"observed_types = [{obs}]"] return "\n".join(lines) + "\n" -def bootstrap(toml_path: Path, *, corpus_scan=None) -> None: +def bootstrap(toml_path: Path, *, corpus_scan=None, catalog_model: M.Model | None = None) -> None: """Append observed skeletons from a real scan or an injected synthetic scan fixture.""" used, argtypes = corpus_scan if corpus_scan is not None else scan_corpus() + catalog = catalog_model or M.load(TOML_DEFAULT) present = set(M.load(toml_path).opcodes) if toml_path.exists() else set() blocks = [] for op in sorted(used): if op in present: continue - label, argc = OPCODES.get(op, ("0x%x" % op, 0)) + entry = catalog.opcodes.get(op) + label, argc = ((entry.label, entry.argc) if entry else ("0x%x" % op, 0)) blocks.append(skeleton_toml(op, label, argc, argtypes[op])) if not toml_path.exists(): toml_path.parent.mkdir(parents=True, exist_ok=True) @@ -117,12 +118,13 @@ def bootstrap(toml_path: Path, *, corpus_scan=None) -> None: f.write("\n".join(blocks)) print(f"bootstrap: {len(used)} used opcodes; appended {len(blocks)} new skeletons -> {toml_path}") -def bootstrap_age(toml_path: Path) -> None: +def bootstrap_age(toml_path: Path, *, catalog_model: M.Model | None = None) -> None: """Append compatibility stubs for catalog opcodes absent from the canonical map.""" + catalog = catalog_model or M.load(TOML_DEFAULT) present = set(M.load(toml_path).opcodes) if toml_path.exists() else set() blocks = [ - skeleton_toml(op, label, argc, {}, observed_in_himegari=False) - for op, (label, argc) in sorted(OPCODES.items()) + skeleton_toml(op, entry.label, entry.argc, {}, observed_in_himegari=False) + for op, entry in sorted(catalog.opcodes.items()) if op not in present ] if not toml_path.exists(): @@ -130,11 +132,58 @@ def bootstrap_age(toml_path: Path) -> None: toml_path.write_text(META_TOML + "\n", encoding="utf-8") with toml_path.open("a", encoding="utf-8") as f: f.write("\n".join(blocks)) - print(f"bootstrap-age: {len(OPCODES)} catalog opcodes; " + print(f"bootstrap-age: {len(catalog.opcodes)} catalog opcodes; " f"appended {len(blocks)} compatibility stubs -> {toml_path}") GEN_HEADER = "# DO NOT EDIT -- generated from vm-map/opcodes.toml by tools/opcodes_build.py --build\n" +def emit_runtime_py(model: M.Model) -> str: + """Emit the Python disassembler/runtime ABI view from the canonical registry.""" + arg_types = { + int(key, 0): value + for key, value in model.meta.get("arg_types", {}).items() + } + targets = { + op: tuple(oc.code_target_args) + for op, oc in model.opcodes.items() + if oc.code_target_args + } + lines = [ + GEN_HEADER, + '"""AGE opcode framing and operand metadata (generated canonical view)."""', + "from __future__ import annotations", + "", + "# opcode -> (historical/canonical label, argument count)", + "OPCODES: dict[int, tuple[str, int]] = {", + ] + for op, oc in sorted(model.opcodes.items()): + lines.append(f" 0x{op:04x}: ({oc.label!r}, {oc.argc}),") + lines += ["}", "", "# argument type tag -> disassembly label", "ARG_TYPES: dict[int, str] = {"] + for tag, label in sorted(arg_types.items()): + lines.append(f" 0x{tag:04x}: {label!r},") + lines += [ + "}", + "", + "# One-based operand indices whose raw values are code offsets.", + "CODE_TARGET_ARGS: dict[int, frozenset[int]] = {", + ] + for op, indices in sorted(targets.items()): + values = ", ".join(str(i) for i in indices) + if len(indices) == 1: + values += "," + lines.append(f" 0x{op:04x}: frozenset(({values})),") + lines += [ + "}", + "CONTROL_FLOW = frozenset(CODE_TARGET_ARGS)", + f"ARRAY_OPCODE = 0x{int(model.meta['inline_array_opcode']):x}", + "", + "def is_label_argument(op: int, arg_index: int, raw_value: int) -> bool:", + ' """Return whether a zero-based operand is a non-fallthrough code target."""', + " return (raw_value != 0xFFFFFFFF", + " and arg_index + 1 in CODE_TARGET_ARGS.get(op, ()))", + ] + return "\n".join(lines) + "\n" + def emit_inferred_py(model: M.Model) -> str: lines = [GEN_HEADER, '"""Inferred Himegari opcode semantics (generated). sys4load reads INFERRED[op][\'name\']."""', @@ -248,8 +297,9 @@ def main(argv=None): for m in errors: print("error:", m) return 1 + (paths.REPO / "tools" / "age_opcodes.py").write_text(emit_runtime_py(model), encoding="utf-8") (paths.REPO / "tools" / "age_opcodes_himegari.py").write_text(emit_inferred_py(model), encoding="utf-8") - print("build: wrote tools/age_opcodes_himegari.py") + print("build: wrote tools/age_opcodes.py, tools/age_opcodes_himegari.py") paths.BUILD.mkdir(parents=True, exist_ok=True) (paths.BUILD / "opcodes.json").write_text(emit_json(model), encoding="utf-8") (paths.REPO / "docs" / "opcode-reference.md").write_text(emit_reference_md(model), encoding="utf-8") diff --git a/tools/paths.py b/tools/paths.py index 99b3a36..eefafe4 100644 --- a/tools/paths.py +++ b/tools/paths.py @@ -27,7 +27,6 @@ BUILD = REPO / "build" # derived corpora (rege VM_MAP = REPO / "vm-map" BIN = REPO / "bin" # 3rd-party tools (BinExtractALF, ...) AGE_EXE = GAME_DIR / "AGE.EXE" -KELEBEK_CPP = VM_MAP / "kelebek1-age-shared.cpp" def add_self_to_syspath(): diff --git a/tools/sys4load.py b/tools/sys4load.py index 3b562dc..f9a0edd 100644 --- a/tools/sys4load.py +++ b/tools/sys4load.py @@ -7,8 +7,8 @@ Parses the confirmed container format (see ../sys4-format-notes.md): * strings: XOR-0xFF cp932, NUL-terminated, referenced by a `0x02 ` pair The container format is byte-verified across all 481 DATA1 scripts. Opcodes are now -DECODED using the AGE opcode table (age_opcodes.py, transcribed from Kelebek1's -decompiler and validated 476/476 clean on Himegari): the code section is a flat stream +DECODED using the canonical AGE opcode registry (vm-map/opcodes.toml, generated into +age_opcodes.py and validated 476/476 clean on Himegari): the code section is a flat stream of ` + argc*()` instructions, length 1+2*argc dwords. Inline strings live after the code inside [0,F8), so decoding stops at the first string (type-2) or array (op 0x64) operand offset. @@ -38,8 +38,8 @@ except ImportError: # allow import from another cwd from age_opcodes import (OPCODES, ARG_TYPES, CONTROL_FLOW, ARRAY_OPCODE, is_label_argument) -# Himegari inference layer (optional): improves labels for unnamed opcodes. Keeps the -# verbatim Kelebek table (age_opcodes) pristine; see age_opcodes_himegari.py. +# Himegari inference layer (optional): improves labels for unnamed opcodes from the +# canonical generated ABI view; see age_opcodes_himegari.py. try: from age_opcodes_himegari import INFERRED except ImportError: @@ -96,7 +96,7 @@ CALLSCRIPT_OP = 0x03 def display_label(op: int) -> str: - """Rendered mnemonic: Kelebek name if it has one, else the inferred name, else u00….""" + """Rendered mnemonic: canonical ABI label, else the inferred name, else u00….""" lbl = OPCODES.get(op, (f"?{op:x}", 0))[0] is_unnamed = (lbl.startswith(("u00", "dev_ukn")) or lbl.lower() == f"{op:x}") if is_unnamed and op in INFERRED: @@ -422,10 +422,10 @@ def render_listing(scr: Sys4Script) -> str: continue ops = " ".join(_fmt_operand(ins.opcode, x, t, v, scr.strings) for x, (t, v) in enumerate(ins.args)) - mnem = display_label(ins.opcode) # prefers Kelebek name, else inferred, else u00… + mnem = display_label(ins.opcode) # prefers canonical label, else inferred, else u00… # annotate unnamed/inferred ops with their raw value for grep-ability - kelebek = ins.label - unnamed = kelebek.startswith(("u00", "dev_ukn")) or kelebek[:1].isdigit() + raw_label = ins.label + unnamed = raw_label.startswith(("u00", "dev_ukn")) or raw_label[:1].isdigit() raw = f" ; op 0x{ins.opcode:x}" + (" inferred" if unnamed and ins.opcode in INFERRED else "") \ if unnamed else "" out.append(f" 0x{ins.offset:05x}: {mnem}{(' ' + ops) if ops else ''}{raw}") diff --git a/tools/test_opcodes.py b/tools/test_opcodes.py index 0ed5f0f..9ecb6dc 100644 --- a/tools/test_opcodes.py +++ b/tools/test_opcodes.py @@ -119,13 +119,12 @@ def test_lint(): def test_bootstrap(): import opcodes_build as B - from age_opcodes import OPCODES from pathlib import Path canonical = M.load(Path(__file__).resolve().parents[1] / "vm-map" / "opcodes.toml") observed = sorted(op for op, entry in canonical.opcodes.items() if entry.observed_in_himegari) synthetic_scan = ( Counter({op: 1 for op in observed}), - {op: {i: {0} for i in range(OPCODES[op][1])} for op in observed}, + {op: {i: {0} for i in range(canonical.opcodes[op].argc)} for op in observed}, ) fd, p = tempfile.mkstemp(suffix=".toml"); os.close(fd); os.remove(p) tp = Path(p) @@ -139,9 +138,9 @@ def test_bootstrap(): check(len(M.load(tp).opcodes) == n1, "second bootstrap adds no duplicates") e, w = M.lint(m) check(e == [], f"bootstrapped file lints clean (errors: {e[:3]})") - B.bootstrap_age(tp) + B.bootstrap_age(tp, catalog_model=canonical) full = M.load(tp) - check(len(full.opcodes) == len(OPCODES), "bootstrap-age seeds the complete AGE catalog") + check(len(full.opcodes) == len(canonical.opcodes), "bootstrap-age seeds the complete AGE catalog") check(sum(o.observed_in_himegari for o in full.opcodes.values()) == len(m.opcodes), "bootstrap-age marks only added catalog entries unobserved") check(all(not o.semantics.noop_headless for o in full.opcodes.values() @@ -160,6 +159,22 @@ def test_emit_inferred(): check(0x90 in inf and inf[0x90]["name"] == "hotspot-branch", "generated INFERRED[0x90]['name'] correct") check(0x1f4 in inf, "named marker 0x1f4 (name != label) included") +def test_emit_runtime(): + import opcodes_build as B + from pathlib import Path + canonical = M.load(Path(__file__).resolve().parents[1] / "vm-map" / "opcodes.toml") + src = B.emit_runtime_py(canonical) + ns = {} + exec(compile(src, "", "exec"), ns) + check(len(ns["OPCODES"]) == len(canonical.opcodes), "runtime view contains the complete catalog") + check(ns["OPCODES"][0x90] == (canonical.opcodes[0x90].label, 7), + "runtime view preserves opcode label and argc") + check(ns["ARG_TYPES"][0] == "imm", "runtime view emits canonical argument labels") + check(ns["is_label_argument"](0x90, 4, 0x123), "runtime view recognizes hotspot target arg") + check(not ns["is_label_argument"](0x90, 3, 0x123), "runtime view rejects hotspot data arg") + check(not ns["is_label_argument"](0x90, 4, 0xffffffff), "fallthrough sentinel is not a target") + check(ns["ARRAY_OPCODE"] == 0x64, "runtime view emits the inline-array opcode") + def test_emit_views(): import opcodes_build as B, json as _json m = M.load(write_tmp(FIXTURE)) @@ -176,6 +191,7 @@ def main(): test_lint() test_bootstrap() test_emit_inferred() + test_emit_runtime() test_emit_views() print("FAILURES:", len(FAILS)) return 1 if FAILS else 0 diff --git a/tools/validate_opcode_table.py b/tools/validate_opcode_table.py index d96f794..937048f 100644 --- a/tools/validate_opcode_table.py +++ b/tools/validate_opcode_table.py @@ -1,16 +1,12 @@ #!/usr/bin/env python3 -"""Definitive test: replicate Kelebek's data_array_end shrinking (stop code at first -string/array offset), then measure clean decode rate over all Himegari scripts.""" -import os, re, sys, collections -from pathlib import Path +"""Definitive raw-walker test for the canonical opcode registry and data boundary.""" +import os, sys, collections sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import paths import sys4load +from age_opcodes import OPCODES -CPP=paths.KELEBEK_CPP.read_text(encoding="utf-8") -TABLE={}; LABEL={} -for m in re.finditer(r'\{\s*(0x[0-9A-Fa-f]+)\s*,\s*"([^"]*)"\s*,\s*(0x[0-9A-Fa-f]+)\s*\}', CPP): - TABLE[int(m.group(1),16)]=int(m.group(3),16); LABEL[int(m.group(1),16)]=m.group(2) +TABLE={op: entry[1] for op, entry in OPCODES.items()} files=paths.scripts() @@ -48,7 +44,7 @@ for name,p in sorted(files.items()): if len(examples)<20 and reason: pass print(f"scripts: {len(files)} parsefail(container): {parsefail}") -print(f"CLEAN decode (Kelebek table + string-pool boundary): {clean}") +print(f"CLEAN decode (canonical table + string-pool boundary): {clean}") print(f"DIRTY: {dirty}") print(f"total instructions decoded in clean files: {instr_total}") print(f"\nremaining small unknown opcodes (genuine gaps):") diff --git a/tools/validate_opcode_table_naive.py b/tools/validate_opcode_table_naive.py index 31d726c..15ee7ec 100644 --- a/tools/validate_opcode_table_naive.py +++ b/tools/validate_opcode_table_naive.py @@ -1,27 +1,22 @@ #!/usr/bin/env python3 -"""Validate Kelebek1's AGE opcode table against Himegari SYS4 scripts. +"""Validate the canonical AGE opcode table against Himegari SYS4 scripts. -Model (from Kelebek1 disassembler.cpp + age-shared.cpp): +Model recorded in vm-map/opcodes.toml: code stream = sequence of instructions. each instruction = then argument_count * , where each arg = . => instruction length in dwords = 1 + 2*argument_count (uniform; type-2/0x64 args seek elsewhere, don't consume inline) arg type 2 = inline string (value = dword offset into body). types: 0 imm,1 float,3 g-int,9 l-int, etc. A clean decode consumes exactly code_len dwords with no unknown opcode and no arg overrun. """ -import os, re, sys, collections -from pathlib import Path +import os, sys, collections sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import paths import sys4load +from age_opcodes import OPCODES -CPP = paths.KELEBEK_CPP.read_text(encoding="utf-8") -# parse {0x1F4, "label", 0x0}, -TABLE = {} -LABEL = {} -for m in re.finditer(r'\{\s*(0x[0-9A-Fa-f]+)\s*,\s*"([^"]*)"\s*,\s*(0x[0-9A-Fa-f]+)\s*\}', CPP): - op = int(m.group(1), 16); lbl = m.group(2); argc = int(m.group(3), 16) - TABLE[op] = argc; LABEL[op] = lbl -print(f"parsed {len(TABLE)} opcode defs from Kelebek1 table (max op 0x{max(TABLE):x})") +TABLE = {op: entry[1] for op, entry in OPCODES.items()} +LABEL = {op: entry[0] for op, entry in OPCODES.items()} +print(f"loaded {len(TABLE)} opcode defs from the canonical registry (max op 0x{max(TABLE):x})") files = paths.scripts() diff --git a/tools/vm0.py b/tools/vm0.py index 7a0c2a2..814b88b 100644 --- a/tools/vm0.py +++ b/tools/vm0.py @@ -3,7 +3,7 @@ Executes one script's bytecode to validate the *execution model* before any C#/Godot work. Reuses tools/sys4load.py for all parsing/decoding. Effectful ops and call-script are STUBBED; -show-text is captured. Named-op semantics come from Kelebek's table; the classified markers are +show-text is captured. Named-op semantics come from the generated canonical registry; the classified markers are treated as no-ops (this run TESTS that assumption). Purpose (see docs/phase-a-slice-plan.md): diff --git a/vm-map/opcodes.toml b/vm-map/opcodes.toml index 68c08b5..39be22c 100644 --- a/vm-map/opcodes.toml +++ b/vm-map/opcodes.toml @@ -5,9 +5,10 @@ [meta] instruction_model = "code = seq of then argc*(); len_dwords = 1 + 2*argc" opcodes_used_by_himegari = 248 +inline_array_opcode = 0x64 [meta.arg_types] -"0x0" = "immediate" +"0x0" = "imm" "0x1" = "float" "0x2" = "string" "0x3" = "global-int" @@ -21,6 +22,10 @@ opcodes_used_by_himegari = 248 "0xc" = "local-ptr" "0xd" = "local-float-ptr" "0xe" = "local-string-ptr" +"0x8003" = "type-0x8003" +"0x8005" = "type-0x8005" +"0x8009" = "type-0x8009" +"0x800b" = "type-0x800B" [meta.header_fields] "F0" = "local_integer_1" @@ -1256,6 +1261,7 @@ observed_types = ["imm", "l-int"] op = 0x7b label = "u0041ADB0" argc = 2 +code_target_args = [1, 2] abi_source = "kelebek+decode-validated" [opcode.semantics] @@ -1435,6 +1441,7 @@ observed_types = ["imm"] op = 0x8c label = "jmp" argc = 1 +code_target_args = [1] abi_source = "kelebek+decode-validated" [opcode.semantics] @@ -1456,6 +1463,7 @@ observed_types = ["imm"] op = 0x8f label = "call" argc = 1 +code_target_args = [1] abi_source = "kelebek+decode-validated" [opcode.semantics] @@ -1478,6 +1486,7 @@ observed_types = ["imm", "g-int"] op = 0x90 label = "u0041BEB0" argc = 7 +code_target_args = [5, 6, 7] abi_source = "kelebek+decode-validated" [opcode.semantics] @@ -1623,6 +1632,7 @@ observed_types = ["imm"] op = 0xa0 label = "jcc" argc = 3 +code_target_args = [2, 3] abi_source = "kelebek+decode-validated" [opcode.semantics] @@ -2082,6 +2092,7 @@ observed_types = ["imm"] op = 0xcc label = "mouse_callback" argc = 2 +code_target_args = [2] abi_source = "kelebek+decode-validated" [opcode.semantics] @@ -2161,6 +2172,7 @@ evidence = "Ghidra /v2: op_0xd3_handler@0x41a430 clears the 16-byte-entry vector op = 0xd4 label = "u004266F0" argc = 4 +code_target_args = [3, 4] abi_source = "kelebek+decode-validated" [opcode.semantics] @@ -2235,6 +2247,7 @@ evidence = "Ghidra op 0xd9 handler 0x416da0: ctx->run_state_flags &= ~0x1000; wh op = 0xfb label = "joy_callback" argc = 2 +code_target_args = [2] abi_source = "kelebek+decode-validated" [opcode.semantics]