From 463877773cd5a8d6459b26d6f58da0d2fc2ceb9a Mon Sep 17 00:00:00 2001 From: gamer147 Date: Mon, 6 Jul 2026 12:58:30 -0400 Subject: [PATCH] chore: initialize age-reimpl repo Reverse-engineering + open reimplementation workspace for Eushully's AGE/SYS4 engine (first target: Himegari). The repo root is age-reimpl/; the original game install and the extracted ALF data are siblings outside the repo and are never tracked. build/ (derived corpora) is gitignored and regenerated by the tools. Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitignore | 28 + docs/PROJECT-STRUCTURE.md | 96 + docs/himegari-port-reference.md | 180 ++ docs/name-resolution.md | 130 ++ docs/phase-a-slice-plan.md | 182 ++ docs/remake-architecture-and-roadmap.md | 267 +++ docs/script-inventory.md | 110 + .../plans/2026-07-06-opcode-reference.md | 911 ++++++++ .../2026-07-06-opcode-reference-design.md | 181 ++ docs/sys4-format-notes.md | 158 ++ docs/vm-mapping-plan.md | 153 ++ tools/age_opcodes.py | 597 +++++ tools/age_opcodes_himegari.py | 98 + tools/extract_init.py | 186 ++ tools/extract_phase2.py | 112 + tools/global_map.py | 199 ++ tools/opcode_context.py | 131 ++ tools/pack_check.py | 41 + tools/paths.py | 50 + tools/probe_header.py | 69 + tools/probe_leads.py | 74 + tools/probe_refs.py | 76 + tools/probe_tables.py | 76 + tools/probe_tags.py | 76 + tools/probe_types.py | 101 + tools/probe_xref.py | 64 + tools/sys4load.py | 521 +++++ tools/validate_opcode_table.py | 58 + tools/validate_opcode_table_naive.py | 80 + tools/vm0.py | 425 ++++ vm-map/himegari-opcode-notes.md | 111 + vm-map/opcode-leads.json | 1194 ++++++++++ vm-map/opcodes-himegari.json | 2023 +++++++++++++++++ vm-map/small-script-listings.md | 140 ++ 34 files changed, 8898 insertions(+) create mode 100644 .gitignore create mode 100644 docs/PROJECT-STRUCTURE.md create mode 100644 docs/himegari-port-reference.md create mode 100644 docs/name-resolution.md create mode 100644 docs/phase-a-slice-plan.md create mode 100644 docs/remake-architecture-and-roadmap.md create mode 100644 docs/script-inventory.md create mode 100644 docs/superpowers/plans/2026-07-06-opcode-reference.md create mode 100644 docs/superpowers/specs/2026-07-06-opcode-reference-design.md create mode 100644 docs/sys4-format-notes.md create mode 100644 docs/vm-mapping-plan.md create mode 100644 tools/age_opcodes.py create mode 100644 tools/age_opcodes_himegari.py create mode 100644 tools/extract_init.py create mode 100644 tools/extract_phase2.py create mode 100644 tools/global_map.py create mode 100644 tools/opcode_context.py create mode 100644 tools/pack_check.py create mode 100644 tools/paths.py create mode 100644 tools/probe_header.py create mode 100644 tools/probe_leads.py create mode 100644 tools/probe_refs.py create mode 100644 tools/probe_tables.py create mode 100644 tools/probe_tags.py create mode 100644 tools/probe_types.py create mode 100644 tools/probe_xref.py create mode 100644 tools/sys4load.py create mode 100644 tools/validate_opcode_table.py create mode 100644 tools/validate_opcode_table_naive.py create mode 100644 tools/vm0.py create mode 100644 vm-map/himegari-opcode-notes.md create mode 100644 vm-map/opcode-leads.json create mode 100644 vm-map/opcodes-himegari.json create mode 100644 vm-map/small-script-listings.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bf29764 --- /dev/null +++ b/.gitignore @@ -0,0 +1,28 @@ +# age-reimpl repo. NOTE: the repo root IS age-reimpl/. The original game install +# (姫狩りダンジョンマイスター/) and the extracted ALF data (extracted/) are SIBLINGS +# of this directory, outside the repo entirely — they can never be committed. + +# Derived / regenerable corpora (rebuild via tools/extract_phase2.py, opcodes_build.py, etc.). +# "build/ is disposable" — see docs/PROJECT-STRUCTURE.md. Machine JSON + bulk disasm/text +# corpora live here and are reproducible from tools/ + ../extracted/. +build/ + +# Python +__pycache__/ +*.py[cod] +*.egg-info/ +.venv/ + +# Godot / C# (future) +.godot/ +*.import +.mono/ +mono_crash.*.json +[Bb]in/obj/ +[Oo]bj/ + +# OS / editor cruft +Thumbs.db +Desktop.ini +.DS_Store +*.swp diff --git a/docs/PROJECT-STRUCTURE.md b/docs/PROJECT-STRUCTURE.md new file mode 100644 index 0000000..819b45a --- /dev/null +++ b/docs/PROJECT-STRUCTURE.md @@ -0,0 +1,96 @@ +# Project Structure + +Layout for the *姫狩りダンジョンマイスター* (Himegari) → open AGE-engine reimplementation. +The guiding rule is **source vs. derived vs. our work**: the shipped game is read-only +input, the extracted archives and everything our tools generate are disposable/reproducible, +and our code + docs live entirely apart from the game install. Nothing we produce is ever +written back into the game folder. + +Workspace root: `S:\Game Hacking\Eushully\Himegari\` + +``` +S:\Game Hacking\Eushully\Himegari\ ← workspace root (three siblings) +│ +├── 姫狩りダンジョンマイスター/ ← SOURCE — pristine game install (read-only) +│ │ Never edit, move, or add to this folder. It holds ORIGINALS ONLY. +│ ├── AGE.EXE, AGERC.DLL, *.dll shipped engine (packed). Stays intact and +│ │ runnable in place — Frida launches it if needed. +│ ├── DATA1-5.ALF, APPEND01.ALF/.AAI shipped archives (~2.3 GB). +│ ├── *.BIN 52 loose patch-override scripts (v1.03) — +│ │ AUTHORITATIVE over their DATA1 copies. Plus +│ │ non-script indices (SYS4INI=S4IC, SYS4AB=S4AB). +│ └── *.exe (uninstallers), SAS0099.OGG … other shipped files. +│ +├── extracted/ ← DERIVED (game-side) — extracted ALF contents, +│ │ ~3.9 GB, regenerable via age-reimpl/bin/BinExtractALF. +│ └── DATA1/ … DATA5/ DATA1 = 481 .BIN scripts (the corpus we parse) +│ + AGF/BMP/WAV in the others. +│ +└── age-reimpl/ ← OUR WORK (everything we made lives here) + │ + ├── tools/ Python tooling (parser/disassembler + extractors + VM) + │ ├── paths.py ★ central path anchor — the ONLY place that knows + │ │ where the game / extracted / build dirs are. All + │ │ tools import it; relocatable with no other edits. + │ ├── sys4load.py loader + disassembler (opcode-decoding) + │ ├── age_opcodes.py 548-entry AGE opcode/arg-type table (pristine) + │ ├── age_opcodes_himegari.py inferred Himegari opcode-name overlay + │ ├── vm0.py headless Python VM (Phase A0); `--test` = RECOVER unit test + │ ├── extract_phase2.py batch: disasm + text + data extraction + │ ├── extract_init.py, global_map.py … *INIT parsers, global-var map builder + │ ├── validate_opcode_table*.py decode-coverage validators + │ └── probe_*.py format reverse-engineering probes (historical) + │ + ├── bin/ 3rd-party binaries we use (not ours, not the game's) + │ ├── BinExtractALF.exe ALF archive extractor → produces extracted/ + │ └── LzssCpp.dll its LZSS codec dependency + │ + ├── vm-map/ VM / reverse-engineering reference artifacts + │ ├── opcodes-himegari.json validated opcode table (what this game uses) + │ ├── kelebek1-age-shared.cpp / -disassembler.cpp upstream opcode-table source + │ └── opcode-leads.json, small-script-listings.md, himegari-opcode-notes.md + │ + ├── docs/ all documentation + │ ├── PROJECT-STRUCTURE.md this file + │ ├── remake-architecture-and-roadmap.md THE direction doc (phases A–E) + │ ├── phase-a-slice-plan.md the current slice (A0/A1/A2) + │ ├── vm-mapping-plan.md the phased decode plan + │ ├── himegari-port-reference.md master reference + engine background + │ ├── name-resolution.md call-script + global-var name recovery + │ ├── sys4-format-notes.md byte-level container format + │ └── script-inventory.md what the 481 scripts are + │ + ├── build/ DERIVED (our-work-side) — generated by tools/; disposable + │ ├── disasm/ .asm — human-readable disassembly, one per script + │ ├── text/ extracted text: + │ │ ├── .strings.txt all inline strings in a script + │ │ ├── dialogue.jsonl show-text lines only (the translation corpus) + │ │ └── strings.jsonl every string, tagged by source opcode + │ ├── data/ parsed data tables (*INIT → JSON) + │ ├── scripts-json/ machine-readable full dumps (on demand via --json) + │ ├── global-var-map.{json,md} partial global-variable name map + │ └── manifest.json, opcode-coverage.md + │ + └── godot/ DELIVERABLE — the Godot/C# engine project (built in Phase A2+) +``` + +## Conventions + +- **Three-way separation.** `姫狩りダンジョンマイスター/` = untouched originals; `extracted/` = + game-derived data (regenerable, game-side); `age-reimpl/` = everything we authored. The first two + are consumed, never modified. +- **Tools never hard-code paths.** `tools/paths.py` derives `GAME_DIR`, `EXTRACTED`, `DATA1`, + `BUILD`, etc. from its own location. To point the tools at a different install, edit that one file. + The whole tree can be relocated without touching any other tool. +- **Path references in docs** are `age-reimpl/`-relative (e.g. `tools/sys4load.py`, + `build/text/dialogue.jsonl`) unless they name a game/extracted path explicitly. +- **Authoritative script copies:** where a script exists both as a loose `.BIN` in the game folder + and under `extracted/DATA1/`, the game-folder copy (patch v1.03) wins. `paths.scripts()` resolves + this automatically (overrides win). +- **`build/` and `extracted/` are disposable.** `build/` regenerates via `tools/extract_phase2.py` + (or `sys4load.py`); `extracted/` regenerates via `bin/BinExtractALF.exe` on the `.ALF` files. + Safe to delete and rebuild; do not hand-edit. +- **Encoding:** all generated text is UTF-8 (source strings are cp932/Shift-JIS, decoded on + extraction). Run Python as `py -3.11 -X utf8`. +- **The game install is a runnable unit** — do not relocate `AGE.EXE`/`*.ALF`/DLLs relative to each + other, or the game (and any Frida work) breaks. diff --git a/docs/himegari-port-reference.md b/docs/himegari-port-reference.md new file mode 100644 index 0000000..1d1542f --- /dev/null +++ b/docs/himegari-port-reference.md @@ -0,0 +1,180 @@ +# Princess Hunting Dungeon Meister — Godot Port Reference + +Working reference for porting *姫狩りダンジョンマイスター* (Eushully, 2009) to Godot. + +Source install: `C:\Program Files (x86)\Eushully\姫狩りダンジョンマイスター\` +Workspace: `S:\Game Hacking\Eushully\Himegari\` — three siblings: `姫狩りダンジョンマイスター\` (pristine game), `extracted\` (extracted ALF data), `age-reimpl\` (our work). See `docs/PROJECT-STRUCTURE.md`. + +--- + +## Engine background + +Eushully built their own engine, called **AGE**, with three major generations distinguished by their config/index files and archive magic bytes: + +| Generation | Index file | ALF magic | Era (rough) | +|---|---|---|---| +| SYS3 | `sys3ini.bin` | `S3IC` / `S3IN` | mid-2000s | +| SYS4 | `sys4ini.bin` / `.AAI` | `S4AC` / `S4IC` | late-2000s — early-2010s | +| SYS5 | `sys5ini.bin` | `S5IN` / `S5IC` / `S5A` | ~2013+ | + +**Confirmed SYS4** for this game — `extracted\DATA1\SYSTEM4.BIN` is the smoking gun, and the `APPEND01.AAI` sidecar is the SYS4 signature pattern. + +Install dir contents: `AGE.EXE`, `DATA1-5.ALF`, `APPEND01.ALF`, `APPEND01.AAI`, plus `BinExtractALF.exe` and `AGE Patch.exe`. + +--- + +## Tier 1 — Primary references + +### [Kelebek1/Eushully-Decompiler](https://github.com/Kelebek1/Eushully-Decompiler) +C++ decompiler + recompiler for `.bin` scripts, with `extract_alf.py` for archive extraction. + +- Targets **SYS5** specifically (its extractor scans for `SYS5INI.BIN` and `S5*` magic). Will need adaptation for SYS4, but the structure is parallel and the opcode work is gold as a starting framework. +- Most actively maintained (latest release Aug 2024). 22 stars but the most serious decompiler effort in the wild. +- Known limitation: opcode tables are hand-built per engine version — community has open requests to add more opcodes. +- See [extract_alf.py](https://github.com/Kelebek1/Eushully-Decompiler/blob/master/extract_alf.py) for the cleanest reference implementation of ALF parsing. + +### [morkt/GARbro](https://github.com/morkt/GARbro) +Swiss-army VN asset extractor. Has [`ArcFormats/Eushully/ArcALF.cs`](https://github.com/morkt/GARbro/blob/master/ArcFormats/Eushully/ArcALF.cs) covering **all three engine generations** (SYS3/4/5), plus contained formats AGF (graphics), AOG (audio), SCR. + +- The more directly applicable extractor for this game — use it as the authoritative reference for the SYS4 ALF format and LZSS decompression of the index. +- C# / .NET. Source is readable and well-organized per format. + +## Tier 2 — Supporting tools + +### [marcussacana/EushullyEditor](https://github.com/marcussacana/EushullyEditor) +C# library for string-level edits to `.bin` scripts (translation-focused, not full decompilation). + +- Pre-configured for *Kamidori Alchemy Meister* and *Kami no Rhapsody*. +- Useful as a second reference for the script string-table structure, especially since Kamidori is the same SYS4 era as Dungeon Meister. + +### Existing tools already on disk +- `BinExtractALF.exe` — pre-built ALF extractor someone already ran here. Worth confirming origin (likely from a Japanese tools site or HongFire / Mikocon thread). +- `extracted\DATA1\AGF2BMP2AGF.exe` + `LzssCpp.dll` — bidirectional AGF↔BMP converter for the proprietary graphics format. Original by **asmodean** ([asmodean.reverse.net](http://asmodean.reverse.net/) — canonical source of older Japanese game tooling, worth bookmarking). +- `AGE Patch.exe` — likely an English localization or no-DVD patch, not a tool per se. + +--- + +## Current extraction state + +| Archive | Files | Size | Contents | +|---|---|---|---| +| DATA1 | 3,749 | 2.2 GB | **Mixed core data**: 1511 AGF + 1508 BMP (graphics, ~all already converted), **481 .BIN scripts**, 238 WAV (system sfx), 9 cursors, the AGF2BMP2AGF tool + LzssCpp.dll | +| DATA2 | 985 | 791 MB | Event CGs (`.AGF` only) — pure graphics archive | +| DATA3 | 39 | 124 MB | BGM tracks (`.OGG` only) | +| DATA4 | 9,733 | 471 MB | Voice files (`.OGG` only) — character lines | +| DATA5 | 210 | 291 MB | More `.AGF` graphics — likely appendix/extra content | + +### Notable scripts already extracted (DATA1) +- `SYSTEM4.BIN` — engine config/setup (the SYS4 index) +- `HISTORY.BIN`, `MENU.BIN`, `HIDEWIN.BIN` — UI/system scripts +- `SC0000.BIN` and presumably hundreds of `SC####.BIN` — scene scripts, the actual game logic + +### Status by category +- **Graphics**: 1508/1511 in DATA1 already converted to BMP. ~1,195 AGFs remaining in DATA2 and DATA5. +- **Audio**: Fully extracted as standard OGG Vorbis (9,772 files). Godot ingests natively — no further work. +- **Scripts**: 481 `.BIN` files extracted but still in AGE bytecode form. This is the real porting work. + +--- + +## Next steps + +### Where things stand (read this first) +Container format is **fully reversed and machine-verified** (header, 4 sections, 3 +typed pointer tables, inline string encoding — all 481 scripts parse clean). Tooling +exists: `tools/sys4load.py` (loader + disassembler-ish dumper + `--validate`) and the +`tools/probe_*.py` analysis scripts. Companion docs: [script-inventory.md](script-inventory.md) +(what the 481 scripts are) and [sys4-format-notes.md](sys4-format-notes.md) (the byte format). + +**The one blocker for everything downstream is opcode semantics** — the code stream is +a tagged-dword format whose instruction meanings are unknown. That needs the VM dispatch +loop in `AGE.EXE`, which is the Ghidra task below. Graphics conversion (DATA2/5 AGFs) and +save-format work remain deferred. + +### Immediate (no tools needed beyond what's on disk) +1. ~~**Relocate the `Output\` tree**~~ **DONE** — workspace now at `S:\Game Hacking\Eushully\Himegari\姫狩りダンジョンマイスター\`. +2. **Convert remaining AGFs** in DATA2 (985 files) and DATA5 (210 files) with `AGF2BMP2AGF.exe`. *(Deferred — graphics not needed yet.)* The 3-file DATA1 gap is `CHAPTER.AGF`, `LOGO.AGF`, `TEST.AGF`. +3. ~~**Inventory the script files**~~ **DONE** — see [script-inventory.md](script-inventory.md). Key findings: all 481 scripts share magic `SYS4422 `; 52 loose root-dir `.BIN` files are patch overrides that shadow DATA1 copies (use those as authoritative); heavy game logic (damage calc, dungeon loop, battle flow) lives in bytecode, favoring a VM re-implementation in Godot. +1. ~~**Relocate the `Output\` tree**~~ **DONE** — workspace now at `S:\Game Hacking\Eushully\Himegari\姫狩りダンジョンマイスター\`. +2. **Convert remaining AGFs** in DATA2 (985 files) and DATA5 (210 files) with `AGF2BMP2AGF.exe`. *(Deferred — graphics not needed yet.)* The 3-file DATA1 gap is `CHAPTER.AGF`, `LOGO.AGF`, `TEST.AGF`. +3. ~~**Inventory the script files**~~ **DONE** — see [script-inventory.md](script-inventory.md). Key findings: all 481 scripts share magic `SYS4422 `; 52 loose root-dir `.BIN` files are patch overrides that shadow DATA1 copies (use those as authoritative); heavy game logic (damage calc, dungeon loop, battle flow) lives in bytecode, favoring a VM re-implementation in Godot. + +### Header structure — DONE (hex-first, pre-Ghidra) +See [sys4-format-notes.md](sys4-format-notes.md). Confirmed across all 481 files: 60-byte header (magic `SYS4422 ` + 13 u32 fields, all offsets in dwords), body split into CODE + 3 typed pointer tables (tags 0x71/0x03/0x8F, 1 dword each, 100% pure) + inline strings. Strings are XOR-0xFF cp932, referenced by a `0x02 ` tagged operand — verified by decoding real dialogue out of `SC0030.BIN`. Remaining unknowns (opcode dispatch, flag fields F0/F2/F3/F5) need the VM. + +### NEXT ACTION — port the opcode table (see [vm-mapping-plan.md](vm-mapping-plan.md)) +**The full phased playbook lives in [vm-mapping-plan.md](vm-mapping-plan.md)** — start there. + +> ✅ **BREAKTHROUGH (verified 2026-07-05): the opcode set is already solved.** +> Kelebek1's decompiler (`age-shared.cpp`) ships an AGE opcode table that decodes +> this game directly — **476/476 scripts decode 100% clean, 1.46M instructions, 0 +> unknown opcodes, 37,392/0 string args resolved.** Model: code = instructions of +> ` + argc*()`, length `1+2*argc`; stop code at the first inline +> string offset. Himegari uses 248 opcodes, 52 named (see `vm-map/opcodes-himegari.json`). +> Header fields F0–F5 are now known = local-variable counts (Kelebek's `BinaryHeader`). +> **Unpacking `AGE.EXE` is no longer the blocker** — it's demoted to optional Phase 3 +> enrichment (prefer Frida hooking). Reproduce: `tools/validate_opcode_table.py`. +> +> ⚠️ **Coverage nuance (measured 2026-07-06):** "solved" means every instruction *decodes* +> (structure/length known, 481/481 clean). It does **not** mean every instruction is +> *understood*: the 52 named ops are only **72.6% of instruction volume**; the unnamed +> `u004xxxx` 27.4% is concentrated in the highest-frequency opcodes and can't be fully +> deferred before the Godot VM. Also the 52 semantic *labels* come from a later AGE title — +> numbers+argc are validated for Himegari, semantics are not (text ops confirmed by the +> dialogue corpus; effectful ops need Frida confirmation). See `vm-mapping-plan.md` Phase 3. + +> ⚠️ **Note on `AGE.EXE`:** still packed (max-entropy sections, IAT RVA 0). Only +> relevant if you later need to name the 196 unnamed opcodes statically — see the +> plan's appendix. `SYS4AB.BIN` (magic `S4AB`) is a 2nd encrypted engine image. + +Original Ghidra sketch (superseded; kept only for the appendix unpack route): + +4. **Find the dispatch loop.** Look for where `AGE.EXE` reads a script's first body + dword (`0x259` in 301/481 files — likely the prologue/entry opcode) and switches on + dword tag values. Expect a large switch or jump table. Diff against Kelebek1's SYS5 + opcode-handler addresses to map SYS4 equivalents (same engine family, parallel structure). +5. **Seed the opcode map from known anchors** (from `sys4load.py` dumps, already observed): + - `0x02` = string-pointer operand tag (confirmed). + - `0x1A7` / `0x1A5` = opcodes that immediately precede string refs in `MENU.BIN` → + candidate text/message-display instructions. Start here; they're the easiest to confirm. + - `0x8F` / `0x71` / `0x03` = the tags at T3-line / T1-label / T2-data table targets. + - `0x55` = most frequent code lead (likely statement/expr separator); `0x09` = recurring + operand-type prefix (register/var ref?). + Cross-reference with marcussacana's Kamidori (same SYS4 era) as a second opcode source. +6. **Encode the opcode table into `sys4load.py`.** As each opcode's length + operand + grammar is confirmed in Ghidra, add it so the dumper decodes real instructions instead + of chunking by the T3 line-index. `MENU.BIN` (3 strings, small control flow) is the + validation target — decode it fully first, then a mid-size `SC####` scene end-to-end. +7. **(Optional sanity check)** Locate the ALF mount/decrypt code to confirm GARbro's parser + matches this build. Low priority — extraction already succeeded, so this is only if an + archive anomaly shows up. + +### Data-table extraction (unblocks in parallel once a few opcodes are known) +8. The `*INIT` giants (`STINIT` 579 KB stages, `EBINIT` 338 KB enemies, `MPINIT` 330 KB + maps, `ITINIT` items, `SKINIT` skills, `CGINIT` gallery) are static data tables. Once + the T2/data-entry grammar is understood they can be dumped to JSON/CSV **without** a + complete opcode set — an early, high-value win for the game database. +9. **Bulk-extract all dialogue** — `sys4load.py --strings` already pulls clean cp932 text + from every scene today. A batch run over all `SC####`/`SP####` yields the full script + corpus for translation, independent of the VM work. + +### Deferred tracks +10. **Save-file format** — reverse `SAVE.BIN` / the save layout only if the port must read + existing saves. Likely a small struct dump; low priority until gameplay runs. +11. **Godot representation** — decide: re-implement the AGE VM in GDScript/C#, or transpile + `.BIN` → native Godot scenes. The inventory already tilts toward **re-implementing the + VM** (damage calc, dungeon loop, battle flow all live in bytecode, so a transpiler would + have to cover nearly the whole opcode set anyway). Ghidra's view of how much logic sits + in `AGE.EXE` vs. bytecode makes the final call. + +--- + +## Sources + +- [Kelebek1/Eushully-Decompiler](https://github.com/Kelebek1/Eushully-Decompiler) +- [Kelebek1/Eushully-Decompiler/extract_alf.py](https://github.com/Kelebek1/Eushully-Decompiler/blob/master/extract_alf.py) +- [Kelebek1issue #2 — AGE engine version discussion](https://github.com/Kelebek1/Eushully-Decompiler/issues/2) +- [marcussacana/EushullyEditor](https://github.com/marcussacana/EushullyEditor) +- [morkt/GARbro](https://github.com/morkt/GARbro) +- [GARbro ArcFormats/Eushully/ArcALF.cs](https://github.com/morkt/GARbro/blob/master/ArcFormats/Eushully/ArcALF.cs) +- [Eushully Fandom Wiki — Princess Hunting Dungeon Meister notes](https://eushully.fandom.com/wiki/Princess_Hunting_Dungeon_Meister:Notes) +- [asmodean.reverse.net](http://asmodean.reverse.net/) — canonical home of older Japanese game tooling diff --git a/docs/name-resolution.md b/docs/name-resolution.md new file mode 100644 index 0000000..3f962ca --- /dev/null +++ b/docs/name-resolution.md @@ -0,0 +1,130 @@ +# Name resolution — recovering what the compiler stripped + +The disassembler reads the SYS4 bytecode's **operations and control flow** cleanly (see any +`build/disasm/*.asm`). What it can't show is the two kinds of *names* the AGE compiler +discarded: **which function a call targets** (#1) and **what a global variable means** (#2). +Both are data-labeling problems, not decoding problems. This note records what each is, what +we found, and how tractable it is. + +Motivating example: `RECOVER.BIN` translates to correct pseudocode today, but reads as +`call-script 0x329d` (#1) and `C[unit][s] = E[unit][s]` over raw addresses (#2). Naming those +would make it read like source. + +--- + +## #1 — `call-script` target resolution (naming the call graph) + +**What it is.** `call-script N` (Kelebek opcode 0x03) carries a bare number — `0x329d`, +`0x2ade` — the id of an engine entry point. To render `call RECOVER` instead of +`call-script 0x329d` you need a table `id → (script, entry)`. + +**Findings (inspected 2026-07-06):** +- `SYSTEM4.BIN` is **not** an index — it's a small SYS4 script (375 instrs) titled + "SYSTEM4 INIT", the engine boot/init routine (ADV mode, fonts, error text). +- `SYS4INI.BIN` (`S4IC422`) is the **ALF asset index** — archive filenames for extraction + (`SYSTEM4.BIN`, `M002.OGG`, `EV049A.AGF`…), not a script-call registry. +- The ids are large and sparse (`0x329d` = 12,957 ≫ 481 scripts), so the number is an index + into a global **entry-point registry** the engine builds, not a script-file index. +- Even Kelebek's reference decompiler leaves these numeric (its comment only says "param = + SYSTEM4.bin index"). So this is genuinely **unresolved upstream**, not merely unfinished. + +**Why it's engine-level (harder than a file lookup).** There is no `id → name` table sitting +on disk to read. Resolving it needs one of: +- ~~Decode `SCJUMP.BIN`~~ **RULED OUT as the registry (recon 2026-07-06).** `SCJUMP.BIN` (29,796 + instrs) is a **progression state machine**, not an id→code table: it switches on `global 0x3234` + (mode 1–9) then nested `eq`/`ne`/`and`/`jcc` on flags, ending in `mov`s to output globals. It + decides *what comes next* via state; it barely uses `call-script`. Useful for game-flow logic, not + for resolving `call-script` ids. So the id→code registry is genuinely engine-level. +- **Watch the engine resolve one (Frida)** — breakpoint the `call-script` handler in the running + game, log `id → resolved address/script`. Ground truth; Phase-3 (live-tools) work. +- **Find the registration path** — if a boot script assigns ids to entry points, extract it + statically (SYSTEM4.BIN is far too small to hold ~13k, so it's cumulative or lives in AGE.EXE). + +**Status: deferred.** Not the quick win first assumed. Belongs with the engine/dispatch work +(Phase 3), or a dedicated `SCJUMP.BIN` reverse. Until then `call-script` stays numeric. + +--- + +## #2 — The global-variable map (naming the data) + +**What it is.** The VM has one flat **global memory bank**; the bytecode addresses it by raw +offset (`global-int 0x152616`, `global-int 0x52383`). Each offset is a specific piece of game +state (a unit's HP, the current-unit index, a stat table). The map we want is +`offset → (name, type, structure)`. + +**Why it's opaque.** No symbol table exists anywhere; meaning lives in how AGE.EXE and the +scripts *use* each global. Nothing declares "0x152616 is the current unit." + +**Why a big chunk is recoverable statically (the tractable one).** Unlike #1, #2 has strong +free handholds — several of which we've already built: + +1. **The `*INIT` scripts are the writers, and we already extracted them.** `EBINIT`/`ITINIT`/ + `SKINIT`/`CGINIT`/`MPINIT` populate global arrays with names and data (`build/data/*.json`). + The base address `EBINIT` writes 277 unit names into *is* the unit-name table. Each JSON's + `name_array_base`, `desc_array_bases`, and `field_columns` are literally global addresses we + can label by which table wrote them. +2. **Strings anchor the string side for free.** `set-string` writes skill names to + `global-string 0x23a3…` → that array is the skill-name table. `*MES` tables likewise. +3. **Access shape reveals structure without names.** A global read as `base[unit*stride + col]` + exposes a per-unit record and its width (RECOVER showed 14-, 3-, 30-column tables). A global + used as the loop-invariant row index everywhere (`0x152616`) is a "current X" pointer. + Constants-compared → mode/flag; only-incremented → counter. +4. **Frida for the ambiguous ones (heavy, ground truth).** Do a known action in-game (take + damage, gain a level), watch which global changes → definitive labels. Reserve for leftovers. + +**Feasibility.** A *partial* map — enough to make most gameplay scripts readable — is achievable +now, statically, from methods 1–3. A *complete* map needs Frida for the tail. It's incremental: +label the ~dozen hottest globals first (biggest readability payoff), grow the rest on demand. + +**Partial map — BUILT (v1, 2026-07-06).** `tools/global_map.py` → `build/global-var-map.json` +(all evidence) + `build/global-var-map.md` (labelled subset). It ingests `build/data/*.json` +(name/desc/field bases), scans the 481-script corpus for each global's **access shape** +(2D-table base + stride, 1D-array base, row-index, scalar), and ranks "current entity" index +pointers by purity. **First result: 16,354 of 49,435 distinct globals labelled** — + +| kind | count | example | +|---|---|---| +| string tables (names/descs/messages) | 3,199 | `0x23a3` = skill-name table | +| per-entity data-field arrays (from *INIT) | 12,700 | dense = shared fields, `?` = sparse per-entity | +| row-major record tables (from access shape) | 122 | `0x52383` = record-table[stride 30] | +| 1D arrays | 307 | | +| index / "current entity" pointers | 26 | `0x152616` (purity 0.51), `0xeff75` (0.95) | + +**Validated against `RECOVER`:** the map independently reproduces its hand-traced layout — +`0x4e11b`→stride 14, `0x52383`→stride 30, `0xaacb4`→1D array, `0x152616`→current-entity index. + +**Wired into the disassembler.** `sys4load` annotates global operands with the map's high/medium +-confidence labels (low-confidence tail omitted for readability), e.g. RECOVER now renders +`lookup-array-2d p0 (global-int 0x4e11b =rec[s14]) (global-int 0x152616 =current-entity-index?) …`. +Labels are prefixed `=` to mark them as inferred aliases. Regenerate the `.asm` corpus with +`tools/extract_phase2.py` after refreshing the map. Turn it off by deleting/renaming +`build/global-var-map.json` (the loader degrades gracefully). + +Confidence is marked per entry; labels ending `?` are low-confidence guesses. + +### Future step — growing the map (planned, not yet done) + +The v1 map labels *shapes and tables*; the next increments add *meaning*, cheapest first: + +1. **Fold in the `*MES` message-table writers** (`ITMES`, `SKMES`, `VIMES`, …) and any other + `set-string`/`copy-to-global` writers not covered by the `*INIT` set — pure static win, + extends the string/data labels. (Also: most name-table bases are *read* rarely — reads + likely go through `*MES`/an indirection; tracing that would connect names to their readers.) +2. **Label 2D record tables by their readers** — cross-reference which scripts read each + `rec[sN]` table and infer purpose from context (e.g. RECOVER's 30-wide tables ↔ a + status/recovery system). Static, medium effort. +3. **Name *which stat* each field is (Frida).** The one step needing live tools: change a + known value in-game (take damage, gain XP), watch which global moves → definitive + `field@X = "HP"`. Reserve for the fields that matter; this is the last mile. + +Re-run `tools/global_map.py` after each increment; `sys4load` picks up the new labels +automatically (it reads `build/global-var-map.json` at load). + +--- + +## How the two relate +#1 names **functions** (the call graph); #2 names **data** (game state). In `RECOVER`, #1 turns +`call-script 0x329d` into a name; #2 turns `C[unit][s] = E[unit][s]` into `unit.hp[s] = +unit.maxHp[s]`. Priority reversal from the first guess: **#2 is the tractable readability lever +now** (static handholds already half-built via the `*INIT` extraction); **#1 needs the engine** +(dispatch reverse or Frida) and is deferred. diff --git a/docs/phase-a-slice-plan.md b/docs/phase-a-slice-plan.md new file mode 100644 index 0000000..1a65d69 --- /dev/null +++ b/docs/phase-a-slice-plan.md @@ -0,0 +1,182 @@ +# Phase A — Vertical Slice Plan (the first build step) + +Concrete execution plan for Phase A of `remake-architecture-and-roadmap.md`. Decided over the +alternative (fully decoding `SCJUMP.BIN`) after recon showed SCJUMP is not the gating unknown. + +## Why the slice, and why headless-first + +**SCJUMP recon (2026-07-06):** `SCJUMP.BIN` is a 29,796-instruction **progression state machine**, +not the `call-script` registry. Top level switches on `global 0x3234` (mode 1–9 → big blocks); each +block is nested `eq`/`ne`/`and`/`jcc` on flags, ending in `mov`s to output globals. Almost no +`call-script`. So it decides *what scene/branch comes next* via state, and does **not** resolve +`call-script id → code`. Consequence: the id→code registry stays engine-level (deferred), **but the +slice can stub `call-script`** — it is not gating for running one scene's dialogue. + +**Correctness bootstrap (roadmap §5) drives the ordering:** the VM must be *validated-correct* before +it is trustworthy. Our strongest oracle is `build/text/dialogue.jsonl` (the `show-text` lines per +script). So the very first slice is **headless and text-only, validated by that oracle** — no Godot, +no AGF, no audio, no dispatch registry. Only once the VM reproduces dialogue do we add rendering. + +Phase A therefore splits: +- **A0 — headless VM, dialogue-validated (Python prototype).** ← immediate, executable now. +- **A1 — port the validated model to C#** (the runtime's VM core). +- **A2 — Godot ADV backend** (render one scene with visuals + voice). + +--- + +## A0 — Headless VM validated by the dialogue oracle + +**Goal:** a Python interpreter that executes one ADV scene's bytecode and emits its `show-text` +sequence; that sequence is a coherent, in-order subsequence of the script's static `dialogue.jsonl` +lines. This proves the execution model — control flow, operand/pointer semantics, string handling, +and the no-op-marker assumptions — *before* any C#/Godot investment. Reuses `tools/sys4load.py` for +all parsing/decoding (no new parser). + +### Execution model to implement +- **Memory:** one flat **global bank** = `dict[int,int]` (globals are raw offsets into one space; + `global-int A` ⇒ `G[A]`, default 0). Per-call **local frame** with typed banks sized by header + F0–F5 (`local_int[F0]`, `local_float[F1]`, `local_string[F2]`, …). +- **PC / control flow:** build `offset→instruction-index` map from `sys4load` instructions (each has + `.offset` = dword index; jump targets are dword indices). `jmp t` → pc = map[t]. `jcc(cond, A, B)` + → cond truthy ? goto A : goto B, where `0xffffffff` = fall through (confirmed model from RECOVER). +- **Operand resolution by type:** imm→value; global-int→`G[value]`; local-int→`frame.int[value]`; + string(2)→decoded string at dword offset; float/global-string/etc. analogous. +- **⚠ Pointer/lvalue semantics — the key modeling task.** RECOVER proves `-ptr` operands are + *lvalues*: `lookup-array(dst_ptr, base, idx)` yields a *reference* to `G[base+idx]`; `mov` through a + ptr writes to the referenced cell; reading a ptr rvalue dereferences it. Model a ptr slot as holding + an address into the global bank; nail this so the RECOVER array-copy produces correct results (unit + test it directly). +- **Opcode handlers (~52 named ops):** + - arithmetic/bit `add sub mul div mod and or sar shl` → `p1 = p2 ⊙ p3`. + - compares `eq ne lt lte gr gre` → 0/1. + - `mov` (incl. through ptr), `lookup-array` (`p1=mem[base+idx]`), `lookup-array-2d` + (`p1=mem[base + i*stride + col]`), `copy-to-global`, `set-array-to`, `bit-set/reset`, `check-bit`. + - control `jmp call jcc ret exit exit-script`. + - string `set-string concat strlen toString`. + - **ADV capture:** `show-text` → append (arg text) to the emitted list; `end-text-line`, + `wait-for-input`, `set-font`, `comment` → capture/skip (no visible state). +- **Markers → no-op (this TESTS the classification):** `0x1f4 0x1f5 0x1d5 0x1bc 0x1bf` skip; + tentative `0x21b 0x1d2 0x258` skip — if dialogue stays correct, the no-op assumption is validated. +- **`call-script` → STUB:** log `(id)`, return immediately. (Its dialogue belongs to other scripts; + stubbing keeps the emitted set = this script's own lines.) +- **Effectful (draw/texture/audio/ui/input) → STUB:** log and ignore. +- **Unknown/other opcodes → log + no-op**, so a rare op doesn't halt the run (record coverage). + +### Oracle & scene choice +- **Oracle:** with calls stubbed and default state, every emitted `show-text` line must be a real + decoded string from the script's pool, and the sequence must be an **in-order subsequence** of that + script's `dialogue.jsonl` lines (≈ equality for a linear scene). Catches: garbage strings (bad + operand/ptr handling), impossible ordering (bad control flow), missing/extra lines. +- **Scene pick:** choose a **short, mostly-linear ADV scene** — high `show-text` count, low `jcc` + density, few `call-script`. Selection step: rank `SC####`/`SP####` by + `(show-text count) / (jcc + call-script count)`, small size. Known-good fallback: `SC0030.BIN` + (dialogue verified). Also run a **RECOVER unit test** to validate pointer/array semantics independent + of dialogue. + +### Steps +1. `tools/vm0.py`: load a script via `sys4load`, build offset→index map, frame + global bank. +2. Implement operand resolution + the arithmetic/compare/mov/lookup/control handlers; unit-test on + `RECOVER.BIN` (array copy + both loops must produce correct global writes). +3. Add ADV capture + markers-as-noop + call/effectful stubs; add opcode-coverage logging. +4. Run on the chosen linear scene; diff emitted `show-text` vs `dialogue.jsonl` (subsequence check); + eyeball the first ~15 lines for coherence. +5. Iterate until several scenes pass; record which ops/markers were exercised and any surprises + (esp. whether the tentative-no-op markers hold). + +### Success criteria (A0 done) +- RECOVER unit test passes (pointer/array model correct). +- ≥3 ADV scenes: emitted `show-text` is a coherent in-order subsequence of their `dialogue.jsonl`, + no garbage strings. +- Coverage report of which opcodes actually executed (drives A1/A2 priorities). +- The no-op-marker assumption is confirmed or corrected with evidence. + +--- + +### A0 result (2026-07-06) — execution model VALIDATED + +`tools/vm0.py` built (reuses `sys4load`; ~250 lines). Results: +- **RECOVER unit test PASSES** — all 7 checks (block-1 3-field copy, block-2 restore + flag, both + skip-guards). The pointer/lvalue model, 2D stride indexing, both loops, and two-way `jcc` all + execute correctly. **The core execution model is proven.** +- **Full SC/SP oracle sweep (`vm0.py --sweep`): 282 / 294 scenes DIALOGUE-VALID = 95.9%.** Every + emitted `show-text` line is checked (by string offset) as an in-order subsequence of the script's + static `dialogue.jsonl` lines. **Zero STRAY and zero ORDER violations across all 294 scenes** — the + model never emits a garbage string and never emits dialogue out of order. 279 CLEAN (valid + natural + `exit`); 3 OK/LOOP (valid subsequence, halted by the loop-guard); 12 EMPTY; 3 skipped (no static + show-text). SC0000 = 326 static / clean; SP0062 = 220/220 CLEAN. + +**A0-remainder work done (2026-07-06, session 2):** +- **Loop-guard added** (`EMIT_CAP=2`): halt a run once any single line is re-emitted a 3rd time — + a semantic guard tied to the oracle (vs. a blind step limit), and it *classifies* the scene LOOPED + instead of spewing garbage. The 3 zero-state spinners (SC0010/SC0600/SC0200) now terminate cleanly + in <12k steps and their emitted lines are all valid. +- **SP0062 "stray" was a measurement artifact**, not a bug — the precise offset-based oracle shows it + CLEAN (220/220, natural exit). Offset-match ⟹ text-match (VM decodes each string at the same offset + the extractor did), so CLEAN is trustworthy. +- **`0x71` (label-def) folded into the no-op marker set** — structural, no runtime effect. +- **Sweep + single-scene diff harness** added to `vm0.py`: `--sweep [N]` (coverage table over all + SC/SP), `--scene NAME` (detailed diff for one script), plus `load_oracle`/`subsequence_status`. + +**op 0x90 investigated in depth — it is input chrome, NOT a correctness hole** (full evidence: +`vm-map/himegari-opcode-notes.md` §F). Kelebek left it "ukn"; corpus analysis resolves it: +`0x90 x y w h tgt_a tgt_b tgt_c` (argc 7) is a **cursor/input hotspot hit-test** that branches per +interaction outcome and **falls through to pc+1 when nothing matches** (design-confirmed: enc.len 15 +lands the next instr on the fall-through statement). It occurs ONLY in a shared ADV-chrome subroutine +that is byte-identical in all 301 ADV scripts — **exactly 8 sites each** (5 immediate-rect buttons at +`(684..772, 572)` toggling `G[0x6c9..0x6cd]` + 3 local-operand keyed forms), **zero scene-specific +use**. Headless (no cursor/input) ⇒ fall through ⇒ **vm0's stub is already correct**, proven safe by +all 279 CLEAN scenes (which contain these same 8 sites). `op 0x97` (argc 5, no targets) is its +companion register-hotspot call. **So 0x90 stays as fall-through in A1 with confidence; it is modelled +as a live hotspot test only in A2** (Godot input backend), confirming target→state mapping via Frida. + +**The 12 EMPTY scenes — state-gated interactive screens, not a model failure.** Traced SC0830: it +exits early because `G[0xaba5c]==1` gates the content; past that gate the dialogue sits behind the ADV +input-wait loop (the hotspot-polling chrome above), so with no seeded state and no input the scene +exits or spins before reaching text. Unlocking them = seed per-scene state + supply input → +**Phase A2/B**, not an A0 model fix. + +**⚠ Honest scope of the 95.9%:** the subsequence oracle proves **no-garbage / in-order**, not a +*complete* path — inherent to a subsequence oracle run headlessly (interactive/state-gated branches +take the no-input path by design). That anti-garbage guarantee is exactly what A0 set out to prove. + +**Confirmed by this run:** the classified no-op markers (`0x1f4/0x1f5/0x1d5/0x1bc/0x1bf` + tentative +`0x21b/0x1d2/0x258`, now + `0x71`) are safe as no-ops for ADV flow; `call-script` is stubbable; +effectful ops (`draw-texture`/`create-texture`/`play-voice`/`0x1f7`/`0x202`/`0x203`/…) stub cleanly. + +**✅ A0 COMPLETE.** Success criteria met: RECOVER unit test green (pointer/array/control-flow model +proven); 282 ADV scenes emit clean in-order subsequences with zero garbage; coverage number recorded; +no-op-marker assumption confirmed at scale; `op 0x90` (the last big control-flow unknown) resolved as +input chrome whose fall-through stub is correct headless. Next = **A1** — port the model to the C# VM +core, differential-test against `vm0.py`. 0x90/0x97 stay stubbed (correct headless); the interactive +input path + per-scene state seeding land in **A2** (Godot backend) alongside the real hotspot model. + +## A1 — Port the validated model to C# +Reimplement the A0 execution model as the runtime VM core in C# (the language decision from the +roadmap; GDScript is too slow for the loop). A0 is the reference: differential-test C# against the +Python prototype's traces on the same scenes. Port the container parser too (or load via a shared +spec). Deliverable: headless C# VM reproducing A0's results. + +## A2 — Godot ADV backend (one scene, with visuals) +Wire the C# VM's effectful ops to Godot: `show-text`/message window (+ furigana via `display-furigana`), +`set-font`, `wait-for-input`, choices, `play-voice`/`play-bgm`, and `create-texture`/`set-texture`/ +`draw-texture`/`draw-string` for the background + sprites. Convert the scene's AGF art with the +on-disk `AGF2BMP2AGF.exe`. Resolve just-enough `call-script`/state so the scene's setup runs (or +hand-set the preconditions). Deliverable: **the chosen scene playable in Godot** — bg + dialogue + +a choice + voice — matching A0's text. + +--- + +## Risks / open questions for A0 +- **Pointer/lvalue semantics** — the main modeling risk; RECOVER is the litmus test. +- **Initial global state** — a scene may assume preconditions from earlier flow (`SCJUMP`/prior + scenes). Mitigation: default-zero globals + set the few a scene reads early; the subsequence oracle + tolerates a shortened path. +- **Runtime vs static dialogue order** — static `dialogue.jsonl` is file-order (all lines); runtime is + execution-order (branch taken). Hence *subsequence*, not equality; pick linear scenes to tighten it. +- **Hidden effect in a "stub"** — a stubbed effectful op that actually gates control flow could skew + output. Watch for divergence; promote a stub to a real handler if a scene needs it. + +## Immediate next action +Build `tools/vm0.py` and get the **RECOVER unit test** green (pointer/array/control-flow correctness), +then run the first linear ADV scene against the dialogue oracle. That single result tells us whether +the whole VM approach executes correctly — the load-bearing question behind option 3. diff --git a/docs/remake-architecture-and-roadmap.md b/docs/remake-architecture-and-roadmap.md new file mode 100644 index 0000000..0bec93f --- /dev/null +++ b/docs/remake-architecture-and-roadmap.md @@ -0,0 +1,267 @@ +# AGE Remake — Architecture & Roadmap (option 3: remake / enhance / mod) + +**Decided goal (2026-07-06):** not a translation patch (a translated build is already playable), +not a bare cross-platform port — but an **open reimplementation of the AGE engine that runs the +original games and makes modding a first-class feature.** Himegari (SYS4) is the first target; +the design must extend to other AGE games and engine versions (SYS3/SYS5). + +The right mental model is **ScummVM / OpenMW for Eushully's AGE engine**: we ship an *engine*; +the user supplies the *original game data* they own; mods layer on top. Those projects prove this +shape is feasible — and that it's a large, long-lived effort. Our decoding work (done) is the +foundation; the runtime + backends + mod system is the bulk of the remaining work. + +--- + +## 1. Guiding principles + +1. **Bytecode-faithful logic, pragmatic presentation.** Run the original `.BIN` scripts on a + reimplemented VM — you get every gameplay rule (damage, dungeon, battle, recovery) correct by + construction, without re-deriving them. Reimplement the *effectful* ops (draw/text/audio/input) + against a clean modern backend that looks right, not byte-identical to D3D9. +2. **One engine, many profiles.** Do NOT fork per game or per version. A shared VM core, with + *version front-ends* (parser/codec/opcode table) and *per-game profiles* (maps, data schemas, + asset rules) selected by a manifest. +3. **Modding is architecture, not an afterthought.** The data model, content loading, and script + dispatch are designed so mods can override assets, edit data, patch scripts, and inject + host-language hooks. The engine already hints at this: 52 loose root `.BIN` files natively + shadow their archived copies — a built-in override mechanism we generalize. +4. **The original owns the content; we own the engine.** Users provide their AGE install; the + runtime imports/loads it. This keeps us on the right side of distribution and mirrors ScummVM. +5. **De-risk with vertical slices.** Prove "run one scene end-to-end" before breadth. Nothing is + real until a script executes and renders. + +--- + +## 2. Target architecture — the split + +Three layers, cleanly separated: + +``` +┌──────────────────────────────────────────────────────────────────────┐ +│ RUNTIME (Godot + C#) — ships to players & modders │ +│ ├─ AGE VM core (C#) fetch/execute, typed var banks, control flow │ +│ ├─ Version front-ends SYS3 / SYS4 / SYS5: header, string codec, │ +│ │ opcode table, archive format │ +│ ├─ Backend adapters render(Godot 2D) · text/msg · audio · input │ +│ │ — the effectful opcodes call into these │ +│ ├─ Content loader ALF archives + loose overrides + mod folders │ +│ ├─ Data layer game data from moddable files (bootstrapped │ +│ │ from *INIT extraction) │ +│ └─ Mod system override resolution · hook API · patch loader │ +├──────────────────────────────────────────────────────────────────────┤ +│ PROFILE / MANIFEST (per game) — data, not code │ +│ engine_version, archives, string_codec, opcode_table_ref, │ +│ global_var_map, callscript_map, data_schemas, boot_entry, │ +│ asset_conversion rules │ +├──────────────────────────────────────────────────────────────────────┤ +│ TOOLCHAIN (Python — what we've already built) — offline, for modders │ +│ disassembler · assembler · extractors · global-map builder · │ +│ data exporters · mod packaging │ +└──────────────────────────────────────────────────────────────────────┘ +``` + +- **Runtime language.** VM core in **C#** (Godot's C# support) — a 1.5M-instruction fetch/execute + loop is too hot for GDScript. Presentation, UI, mod tooling, and export targets use Godot. This + is why **Godot now fits**: under the earlier "faithful port" framing it was overkill (you'd use + ~10% of it); under *remake/enhance/mod* its editor, UI toolkit, asset pipeline, GDScript modding, + and multi-platform export all earn their keep. +- **Toolchain vs runtime.** The runtime owns the canonical parser+VM (C#). The Python tools remain + the offline analysis/authoring chain; they were the reference implementation and stay useful for + modders. They share the *format spec* (documented), not code — acceptable for a small, stable + container format. +- **Profile = manifest.** Adding a game = a new profile + its maps. Adding an engine version = a new + front-end plugin + profiles that reference it. See §5. + +--- + +## 3. How modding works with a bytecode VM + +Modding is tiered from trivial to deep. The first two tiers cover the large majority of "proper +modding" and need **no decompilation**. + +**Tier 1 — assets & data (easy, no tools needed beyond a text/image editor).** +- *Asset overrides:* drop replacement textures/CGs/voices/BGM into a mod folder; the content loader + resolves mod → loose-override → archive (generalizing the engine's native override behavior). +- *Data edits:* game data (skills/items/units/maps/stages) is **externalized to editable files** + (JSON) that the runtime loads, bootstrapped from our `*INIT` extraction. Rebalancing, new items, + new skills = editing JSON. No bytecode involved. + +**Tier 2 — logic (medium; asm-level or host-language).** +- *Script patches:* disassemble → edit the `.age-asm` → reassemble to `.BIN` (Kelebek's project has + a reassembler to adapt). Mods ship patched/replacement scripts; the VM runs them unmodified. +- *Host hooks:* a mod API lets mods register callbacks in **GDScript/C#** — fire before/after a + script, intercept an opcode, replace a script by id, react to events, add UI. Original scripts run + as-is; mods augment. (This is the BepInEx/script-extender model and avoids a bytecode compiler for + most behavioral mods.) + +**Tier 3 — a friendly modding language (stretch, later).** +- A high-level decompiled DSL + a compiler back to bytecode, so mods are written in readable source. + This is a real compiler project and its quality is bounded by how complete the global-var map and + call-script resolution are. Realistic as a *later* milestone, not near-term. + +**Readability, concretely:** annotated disassembly is achievable today (opcode names + global-var +aliases + eventually call-script names). Pseudo-decompilation for *reading* is feasible (demonstrated +on RECOVER). Clean round-trippable *source* is Tier 3. So near-term "how readable" = well-annotated +assembly + external data/assets + host hooks; the read-like-C dream is a stretch goal. + +**Two enablers become load-bearing under this goal** (they were "polish" for a port): +- **Global-var map** — modders must know what game state a global is to touch it safely. +- **Call-script resolution** — needed both to *run* scripts and to *add/replace* scenes. This is now + on the critical path, not deferred. + +--- + +## 4. Roadmap — high-level progression + +Each phase ends with something demonstrable. The documented side-tasks map into these phases (noted). + +### Phase A — Prove the VM (vertical slice) ⟵ the immediate priority +Goal: **run one ADV scene end-to-end** in the new runtime — background image + dialogue + a choice + +a voice line — and match its `show-text` sequence to `build/text/dialogue.jsonl`. +Forces, and thereby de-risks, every core unknown at once: +- Port the container parser + VM core to C#. +- Implement the ADV effectful ops against Godot: `show-text`, `end-text-line`, `wait-for-input`, + `set-font`, `play-voice`, `play-bgm`, `draw-texture`/`create-texture`/`draw-string`, choices. +- Resolve **just enough `call-script`** to enter/leave a scene (side-task: `SCJUMP.BIN` decode or a + targeted Frida capture — now critical-path). +- **AGF → texture** for the one scene's art (side-task; `AGF2BMP2AGF.exe` already on disk). +- Treat the classified no-op markers as skips; validate the tentative-no-op ops via the dialogue diff. + +### Phase B — Broaden coverage (playable ADV, then systems) +- Implement the remaining effectful ops; Frida sessions for the opaque ones (the shortlist in + `build/opcode-coverage.md`); Unicorn for `0x215`-style computational ops. +- Grow the **global-var map** (side-task 2.5: `*MES` writers → record-table readers → Frida field + naming) — now a core enabler, not polish. +- Get a full chapter of ADV playable; then the dungeon/battle/menu systems (they run as bytecode — + we implement the ops they use, not the rules). +- Side-tasks absorbed here: `STINIT` parser, remaining data schemas, save-file format (needed for a + real playthrough — reversible struct work). + +### Phase C — Externalize & modding foundation +- Move game data from bytecode-embedded tables to **editable external files** the runtime loads. +- Generalize the **override/mod-loading** (mod folders, load order) from the engine's native + loose-file mechanism. +- Asset pipeline: AGF↔PNG, audio, packaging. → Tier-1 modding works. + +### Phase D — Logic modding +- Integrate the **assembler** (Tier-2 bytecode-patch mods) and ship the **host hook API** + (GDScript/C#). → Tier-2 modding works. +- Optionally invest in decompiler quality toward Tier 3. + +### Phase E — Enhance, polish, productize +- Enhancements the VM unlocks: higher/wide resolution, faster text, QoL, save-anywhere, new-content + mods. Modding docs + tools. Save/UX polish. + +--- + +## 5. The VM as our analysis instrument — and the correctness bootstrap + +Beyond being the runtime, the VM is the best analysis tool we can build — **but only once it is +validated-correct, and that ordering is load-bearing.** + +**Upside: owning the VM turns static RE into dynamic observation.** Instrumenting the original +packed `AGE.EXE` means fighting an anti-debug binary with Frida; instrumenting *our* interpreter is +one line in a handler. So a class of documented side-tasks become built-in debugger views instead of +separate investigations: +- **Live named-global watch** — with the global-var map, watch state change in real time; take damage + → see which global moved → *that is* the "name which stat" step (`name-resolution.md` → Future), + now a debugger feature, not a Frida session. +- **Call-graph / dispatch trace** — watch `call-script` resolve live, helping crack the entry-point + registry dynamically. +- **Breakpoints, single-step, var-bank inspection, opcode/script/global coverage**; and because VM + state is just variable banks + a program counter, cheap **snapshot / rewind** (time-travel + debugging nearly falls out of the design). + +**The hard caveat: the instrument is only as trustworthy as the VM is correct.** A VM that executes +*wrong* produces *wrong* observations — and circularly so: you would "learn" false facts about game +state from a broken interpreter and bake them into the global map, the dispatch model, and everything +downstream. **The analysis power is unlocked by correctness, not a substitute for achieving it.** You +cannot debug the unknown with an instrument you have not first validated. Until the VM is running +mostly correctly, using it for analysis is meaningless. + +**Therefore the bootstrap order matters:** +1. Build the VM. +2. **Validate it against ground truth we already hold, by *independent* means** — chiefly the dialogue + oracle (`build/text/dialogue.jsonl`: the VM's `show-text` sequence per scene must match), plus + differential checks against known-correct behavior. This *external* oracle certifies the instrument. +3. Only then use the validated core's observability to understand the *adjacent unknown* — which + globals mean what, dispatch, effectful-op behavior. **Correctness propagates outward from validated + anchors.** + +**Two refinements that bound the trust:** +- **Trust is per-subsystem, and only as strong as the oracle covering it.** The dialogue diff strongly + validates the ADV/text layer. But computational/battle logic has *weaker* oracles — a wrong damage + number can look plausible and pass unnoticed. So "mostly working" must mean *demonstrated correct per + subsystem*; where oracles are weak (battle math), the VM-as-instrument is correspondingly less + trustworthy, and **Frida/Unicorn cross-checks retain their value there** (exactly why Unicorn stayed + on the list for computational ops). A green ADV oracle does not imply the battle math is right. +- **Coverage-limited, plus a chicken-and-egg.** Runtime tools only observe what a playthrough + exercises (rare branches stay dark), so they complement rather than replace static analysis. And + bootstrapping the VM needs *just enough* `call-script` dispatch to run before observation can help + refine dispatch — hence Phase A hardcodes a minimal dispatch first. + +**Net effect on sequencing:** the runtime debugger makes *most* of the Frida/Unicorn side-tasks +cheaper or obsolete — but only after the VM earns trust on a validated core. So the priority is to +reach *validated* correctness on the ADV layer first (via the dialogue oracle); that trusted anchor is +what makes the instrument usable for everything else. The debugger is a force multiplier on a correct +VM and dead weight on an incorrect one. + +--- + +## 6. Extending to other AGE games and versions + +### Other AGE games, same version (e.g. Kamidori, also SYS4) +**Reused for free:** container parser, VM core, backend adapters, the opcode table (the engine ABI +is shared across the family), disassembler/assembler, the whole extraction methodology. +**Per-game (inherent content work):** the **global-var map** (globals are game-specific), the +**call-script registry**, the **data-table layouts** (each game's `*INIT` differs), assets, and any +game-specific effectful behavior. Process: point the toolchain at the new game's archives, re-run +extraction, rebuild its global map, resolve its call-script registry, author a profile. The long +poles are exactly the two enablers (global map + call-script). **This is the core payoff of the VM +approach:** the *engine* cost amortizes across all AGE games; only content-mapping recurs — far less +than re-coding each game's logic bespoke. + +### Other engine versions (SYS3 / SYS5) — one app, not many +Versions differ in: header (SYS4 `0x3C` vs SYS5 `0x44`), string codec (SYS4 cp932^0xFF vs SYS5 +UTF-16^0xFFFF), opcode set (overlapping, version-specific; Kelebek's table already spans the family +and notes version-gated ops), operand types (SYS5 adds `0x8003+`), and archive magic (`S3IC`/`S4IC`/ +`S5IN`). **Architecture answer: version-parameterize, don't fork.** One runtime with: +- a **version-detection** step (magic → SYS3/4/5), +- **pluggable front-ends** (header parser, string decoder, opcode table, archive reader per version), +- the **shared VM core** (the execution model — opcode+typed operands, var banks, control flow — is + the same engine evolving), +- **per-game profiles** that name the version + game-specific maps. + +So: **not a separate application per version — a manifest/profile selects the front-end + game data.** +SYS5 is well-covered (it's Kelebek's target); SYS3 is older and less documented and would need more +front-end work (Kelebek's parser only does SYS4/SYS5), but the plug-in shape accommodates it. Result: +one "AGE Engine" app that, given a profile, runs Himegari, Kamidori, a SYS5 title, etc., each moddable +through the same system. + +--- + +## 7. Feasibility, risks, open questions + +**Feasible? Yes — but it is the largest phase of the whole effort**, on the scale of a small ScummVM +target. The decoding groundwork substantially de-risks it (we understand the format, 97% of opcodes, +the data, a partial global map). Biggest risks, with mitigations: +- **`call-script` dispatch entangled in the packed AGE.EXE** → try `SCJUMP.BIN` static decode first; + fall back to a targeted Frida capture. Gating for *anything* running. +- **Effectful-op surface is large and quirk-laden** (esp. SRPG battle/dungeon UI) → ADV-first; defer + SRPG; lean on the Frida shortlist. +- **AGF graphics** → low risk; `AGF2BMP2AGF.exe` (asmodean) already present. +- **Save format** → reversible struct work; needed before a full playthrough. +- **Toolchain (Python) vs runtime (C#) drift** → share the documented format spec; runtime is + canonical. + +**Open questions to resolve early:** exact `call-script` mechanism; how scenes register (needed to +*add* content); how much the SRPG layer's rendering diverges from ADV; save layout. + +--- + +## 8. Immediate next step +Start **Phase A, the vertical slice** — it converts all of the above from architecture into evidence +and tells us fast whether option 3 is as feasible as it looks. Concretely: pick one small ADV scene, +stand up the C# VM core + Godot ADV backend, resolve just-enough `call-script`, convert that scene's +AGF art, and get its dialogue rendering and matching `build/text/dialogue.jsonl`. Everything else in +this roadmap is sequenced behind that proof. diff --git a/docs/script-inventory.md b/docs/script-inventory.md new file mode 100644 index 0000000..09ef826 --- /dev/null +++ b/docs/script-inventory.md @@ -0,0 +1,110 @@ +# DATA1 Script Inventory — 481 `.BIN` files + +All scripts share the magic header `SYS4422 ` (8 bytes), confirming a uniform SYS4 +bytecode format (engine version 4.4.2.2) across the entire set. Header is followed +by what appear to be little-endian u32 fields (version/section table — to be mapped +during disassembler work). + +Source: `extracted\DATA1\` (extracted from `DATA1.ALF`). + +**Patch overrides:** 52 loose `.BIN` files sit in the game root directory and shadow +their DATA1 counterparts at runtime (sizes differ slightly — e.g. `FIELD.BIN` root +200,536 vs archive 200,224). These are the v1.03 / append-patch versions and should be +treated as **authoritative** over the archive copies. Two engine files exist only in +the root: `SYS4INI.BIN` (272 KB) and `SYS4AB.BIN` (1.08 MB). + +--- + +## Breakdown by series + +| Series | Count | Total size | Role (inferred) | +|---|---|---|---| +| `SC####` | 136 | 15.6 MB | Scenario/event scripts (numbered 0000–1690, step 10) | +| `SP####` | 163 | 10.6 MB | Secondary scene series (0010–1369; incl. `SP0051A/B` split) — likely character/H-events | +| Named scripts | 175 | ~7.5 MB | Engine subsystems, data tables, UI, battle logic | +| `DEBUG*` | 5 | 1.4 MB | Debug tools (`DEBUGADV.BIN` alone is 1.4 MB — a scene viewer/jump menu) | +| `RTN_M###` / `RTN_B###` | 26 | ~70 KB | Small routine scripts (map routines M001–M061, battle routines B001–B004) | + +### SC series notes +- Numbered `SC0000`–`SC0880` (main chapters, largest files — up to 700 KB) and + `SC1000`–`SC1690` (smaller; likely sub-events, endings, appendix content). +- `SCJUMP.BIN` (778 KB) is almost certainly the master scene-dispatch/jump table. +- `SCINIT.BIN` (88 KB) initializes scenario state. + +--- + +## Named scripts by subsystem (inferred from names) + +### Engine core / boot flow +`SYSTEM4.BIN` (config root), `INIT.BIN` (64 bytes — smallest script, ideal first +disassembly target), `INITCONFIG`, `LOADCONFIG`, `CONFIG`, `TUNE`, `LOGO`, `OP`, +`ED`, `TITLE`, `GAMESTART`, `GAMECLEAR`, `STAGECLEAR` + +### Data-table INIT scripts (likely static game data, not logic) +Large, table-like scripts — prime candidates for data extraction: +- `STINIT` (579 KB) — stages/scenario tables +- `EBINIT` (338 KB) — enemy battle data +- `MPINIT` (330 KB) — maps +- `SCINIT` (88 KB), `CGINIT` (79 KB — CG gallery), `ITINIT` (70 KB — items), + `RTINIT` (67 KB), `CCINIT` (41 KB), `SKINIT` (37 KB — skills), `CDINIT` (31 KB), + `BTANINIT` (105 KB — battle animations) +- Smaller: `AFINIT`, `ALINIT`, `CIINIT`, `CNINIT`, `CTINIT`, `CVINIT`, `ILINIT`, + `LAINIT`, `MAINIT`, `OBINIT`, `SPINIT`, `TRINIT`, `VIINIT` + +### Message/string tables (`*MES`) +`ITMES` (64 KB — item text), `VIMES` (43 KB), `EIMES` (37 KB), `SKMES` (31 KB — skill +text), `CIMES` (15 KB), `MAMES`, `INFOMES`, `MES` — where most translatable text +outside scenes lives. + +### Battle system +`BTL` (61 KB — main battle loop), `BTRTN`, `ROUND`, `AIM`, `ATSEEK`, `MVSEEK`, +`MVRTN`, `MAGIC`, `USEMAGIC`, `SUMMON`, `EXILE`, `DISARM`, `RECOVER`, `COUNTUNIT`, +`SETOCC`, and the `CALC*` family: `CALCBTPARAM`, `CALCDMG` (16 KB — damage formula!), +`CALCSCOPE`, `CALCOCC`, `CALCREVISE`, `CALCCC`, `CALCILL`, `CALCARR` + +### Dungeon/map engine +`FIELD` (200 KB — the core dungeon-crawl loop), `DRAWMAP`, `RENDERMAP`, +`DRAWMINIMAP`, `DRAWCH`/`DRAWCHP`/`DRAWENP`/`DRAWOBJ`/`DRAWTIP`/`DRAWVOL`, +`SETCH`/`SETEN`/`SETLAND`/`SETOBJ`/`SETROUTE`/`SETMVWORK`, +`DELCH`/`DELEN`/`DELENMASS`/`DELLAND`, `RESETLAND`, `WARPU`/`WARPD`, `LOOK`, +`READICON` + +### Unit/party management +`ADDEXP`, `ADDSKILL`, `ADDEN`, `ADDITEM`, `ADDRANDOMITEM`, `LOSTRANDOMITEM`, +`ADDILL`/`ADDILLSUB`, `EVOLVE`, `IMPROVE` (37 KB), `TRAIN`, `STUDY`, `UNITECH`, +`REMOVECH`, `SHOWGROW`, `STATUS`, `USEITEM`, `SETCH` + +### Base/facility gameplay +`CAMP`, `ROOM`, `FORT`, `ALCHEMY` (40 KB), `SALLY` (40 KB — sortie/deployment), +`READY` (39 KB — pre-battle prep), `SELSTAGE` (34 KB), `SELACT` (30 KB) + +### Menus / UI / meta +`MENU`, `CHMENU` (69 KB — character menu), `INFO`/`INFOAF`/`INFOCH`/`INFOEN`/ +`INFOIT`/`INFOVO` (info panels: characters, enemies, items, voices), `SAVE` (40 KB), +`HISTORY`, `HIDEWIN`, `CLOSE`, `INPUTNAME` (28 KB), `CGMODE` (24 KB — gallery), +`MMODE` (music mode), `HMODE` (22 KB — scene replay) + +### Flow control / branching +`BUNKI` (分岐 = branch, 15 KB), `SBUNKI`, `BUNKIMOVE`, `SBUNKIMOVE`, `SCJUMP` + +### Callbacks (engine → script hooks) +`CALLBACK_LOAD`, `CALLBACK_LOST`, `CALLBACK_SETTING`, `CALLBACK_WINDOW` + +### Debug +`DEBUG`, `DEBUGADV` (1.4 MB), `DEBUGANIME`, `DEBUGBTL`, `DEBUGMAP` (+2 numbered) + +--- + +## Implications for the port + +1. **Much more game logic lives in bytecode than expected.** Damage formulas + (`CALCDMG`), the dungeon loop (`FIELD`), battle flow (`BTL`, `ROUND`), and unit + progression (`ADDEXP`, `EVOLVE`) are all scripts — `AGE.EXE` is closer to a pure + VM/renderer. This strengthens the case for **re-implementing the AGE VM in Godot** + rather than transpiling every script by hand (the doc's open question #12). +2. **Disassembler bootstrapping order** (small → large, system → scene): + `INIT.BIN` (64 B) → `ED.BIN`/`OP.BIN`/`LOGO.BIN` (~230 B) → `CALLBACK_LOST` + (176 B) → `MENU.BIN` (3 KB) → `CALCDMG` → a mid-size `SC####`. +3. **The `*INIT` giants are likely data tables**, decodable early even with a + partial opcode map — instant win for extracting item/skill/enemy/stage databases. +4. **Use root-directory overrides, not archive copies**, for the 52 patched scripts. diff --git a/docs/superpowers/plans/2026-07-06-opcode-reference.md b/docs/superpowers/plans/2026-07-06-opcode-reference.md new file mode 100644 index 0000000..07b7614 --- /dev/null +++ b/docs/superpowers/plans/2026-07-06-opcode-reference.md @@ -0,0 +1,911 @@ +# Living Opcode Reference Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make `vm-map/opcodes.toml` the single hand-edited source of truth for opcode knowledge (ABI + semantics + provenance + dependencies), from which we generate the Python shim the tooling imports, a machine JSON, a human Markdown reference, and coverage. + +**Architecture:** One canonical TOML file. A generator/linter (`tools/opcodes_build.py`) reads it (stdlib `tomllib`) through a small data model (`tools/opcodes_model.py`) and emits four artifacts. Bootstrap seeds all 248 used opcodes from the pristine Kelebek table (`tools/age_opcodes.py`) by *appending* skeleton text (no TOML writer dependency). Emitters are pure `(model) -> str` functions so the real files are only rewritten in the final migration task. + +**Tech Stack:** Python 3.11 (`py -3.11 -X utf8`), stdlib only (`tomllib`, `json`, `dataclasses`). Reuses `tools/sys4load.py` + `tools/paths.py`. No new dependencies. + +## Global Constraints + +- Run all Python as `py -3.11 -X utf8` (Shift-JIS strings need utf8 mode on Windows). +- Stdlib only — do NOT add `pyyaml`/`tomli_w`/`pytest`. Tests are plain scripts run with `py -3.11`. +- `tools/age_opcodes.py` (Kelebek table) is PRISTINE — never edit it. +- `tools/paths.py` is the only place that knows filesystem locations; import paths from it, never hardcode. +- The generated `tools/age_opcodes_himegari.py` MUST keep exposing `INFERRED: dict[int, dict]` where each entry has a `name` key (the only field `sys4load` reads: `sys4load.py:84`). Do not change `sys4load.py`. +- **This workspace is not a git repo.** Treat every **Checkpoint** step as: if `git` is initialized, run the shown `git add/commit`; otherwise just confirm the named outputs exist and continue. Do not run `git init` unless the user asks. +- Controlled vocabularies (the linter enforces these): + - `category ∈ {marker, structural, control, adv, draw, audio, input, compute, unknown}` + - `source ∈ {kelebek, harness, investigation, frida, unicorn, inference}` + - `confidence ∈ {low, med, high}` (ordered low Model`; `Model(meta: dict, opcodes: dict[int, Opcode])`; `Opcode(op, label, argc, code_target_args, abi_source, abi_note, semantics)`; `Semantics(name, category, summary, noop_headless, source, confidence, depends_on: list[int], evidence, details, confirm_by, args: list[dict])`; `dependents(model) -> dict[int, list[int]]`; constants `CATEGORIES`, `SOURCES`, `CONFIDENCE`. + +- [ ] **Step 1: Write the failing test** + +Create `tools/test_opcodes.py`: + +```python +#!/usr/bin/env python3 +"""Standalone tests for the opcode reference tooling. Run: py -3.11 -X utf8 tools/test_opcodes.py""" +import os, sys, tempfile +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import opcodes_model as M + +FAILS = [] +def check(cond, msg): + print((" ok " if cond else " FAIL ") + msg) + if not cond: FAILS.append(msg) + +FIXTURE = ''' +[meta] +opcodes_used_by_himegari = 2 +[[opcode]] +op = 0x90 +label = "u0041BEB0" +argc = 7 +code_target_args = [5, 6, 7] +[opcode.semantics] +name = "hotspot-branch" +category = "input" +summary = "cursor hotspot hit-test" +noop_headless = true +source = "investigation" +confidence = "high" +depends_on = [0x1f4] +evidence = "301/301 uniform" +[[opcode.semantics.args]] +i = 1 +role = "x" +observed_types = ["imm"] +[[opcode]] +op = 0x1f4 +label = "u004160D0" +argc = 0 +[opcode.semantics] +name = "stmt-begin" +category = "marker" +source = "investigation" +confidence = "high" +''' + +def write_tmp(text): + fd, p = tempfile.mkstemp(suffix=".toml"); os.close(fd) + open(p, "w", encoding="utf-8").write(text) + return p + +def test_load(): + m = M.load(write_tmp(FIXTURE)) + check(set(m.opcodes) == {0x90, 0x1f4}, "loads both opcodes keyed by int") + o = m.opcodes[0x90] + check(o.argc == 7, "0x90 argc == 7") + check(o.code_target_args == [5, 6, 7], "0x90 code_target_args parsed") + check(o.semantics.name == "hotspot-branch", "0x90 semantics.name") + check(o.semantics.depends_on == [0x1f4], "depends_on parsed as int list") + check(o.semantics.args[0]["role"] == "x", "arg role parsed") + rev = M.dependents(m) + check(rev.get(0x1f4) == [0x90], "dependents: 0x1f4 depended on by 0x90") + +def main(): + test_load() + print("FAILURES:", len(FAILS)) + return 1 if FAILS else 0 + +if __name__ == "__main__": + sys.exit(main()) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `py -3.11 -X utf8 tools/test_opcodes.py` +Expected: FAIL — `ModuleNotFoundError: No module named 'opcodes_model'`. + +- [ ] **Step 3: Write minimal implementation** + +Create `tools/opcodes_model.py`: + +```python +#!/usr/bin/env python3 +"""In-memory model + loader + linter for vm-map/opcodes.toml (the canonical opcode reference). +Read-only: uses stdlib tomllib. See docs/superpowers/specs/2026-07-06-opcode-reference-design.md.""" +from __future__ import annotations +import tomllib +from dataclasses import dataclass, field +from pathlib import Path + +CATEGORIES = {"marker", "structural", "control", "adv", "draw", "audio", "input", "compute", "unknown"} +SOURCES = {"kelebek", "harness", "investigation", "frida", "unicorn", "inference"} +CONFIDENCE = {"low": 1, "med": 2, "high": 3} + +@dataclass +class Semantics: + name: str + category: str = "unknown" + summary: str = "" + noop_headless: bool = False + source: str = "kelebek" + confidence: str = "low" + depends_on: list[int] = field(default_factory=list) + evidence: str = "" + details: str = "" + confirm_by: str = "" + args: list[dict] = field(default_factory=list) + +@dataclass +class Opcode: + op: int + label: str + argc: int + code_target_args: list[int] = field(default_factory=list) + abi_source: str = "kelebek+decode-validated" + abi_note: str = "" + semantics: Semantics | None = None + +@dataclass +class Model: + meta: dict + opcodes: dict[int, Opcode] + +def load(path) -> Model: + data = tomllib.loads(Path(path).read_text(encoding="utf-8")) + ops: dict[int, Opcode] = {} + for e in data.get("opcode", []): + sem = None + s = e.get("semantics") + if s is not None: + sem = Semantics( + name=s.get("name", e.get("label", "")), + category=s.get("category", "unknown"), + summary=s.get("summary", ""), + noop_headless=bool(s.get("noop_headless", False)), + source=s.get("source", "kelebek"), + confidence=s.get("confidence", "low"), + depends_on=[int(x) for x in s.get("depends_on", [])], + evidence=s.get("evidence", ""), + details=s.get("details", ""), + confirm_by=s.get("confirm_by", ""), + args=list(s.get("args", [])), + ) + ops[int(e["op"])] = Opcode( + op=int(e["op"]), label=e.get("label", ""), argc=int(e["argc"]), + code_target_args=[int(x) for x in e.get("code_target_args", [])], + abi_source=e.get("abi_source", "kelebek+decode-validated"), + abi_note=e.get("abi_note", ""), semantics=sem, + ) + return Model(meta=data.get("meta", {}), opcodes=ops) + +def dependents(model: Model) -> dict[int, list[int]]: + """Reverse of depends_on: op -> [ops whose semantics depend on it].""" + rev: dict[int, list[int]] = {op: [] for op in model.opcodes} + for op, oc in model.opcodes.items(): + if oc.semantics: + for dep in oc.semantics.depends_on: + rev.setdefault(dep, []).append(op) + for k in rev: + rev[k].sort() + return rev +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `py -3.11 -X utf8 tools/test_opcodes.py` +Expected: all `test_load` lines `ok`, `FAILURES: 0`, exit 0. + +- [ ] **Step 5: Checkpoint** + +If git initialized: `git add tools/opcodes_model.py tools/test_opcodes.py && git commit -m "feat(opcodes): data model + loader for opcodes.toml"` +Else: confirm `tools/opcodes_model.py` and `tools/test_opcodes.py` exist; continue. + +--- + +## Task 2: Linter (`lint` in `opcodes_model.py`) + +**Files:** +- Modify: `tools/opcodes_model.py` (add `lint`) +- Test: `tools/test_opcodes.py` (add `test_lint`) + +**Interfaces:** +- Produces: `lint(model) -> tuple[list[str], list[str]]` returning `(errors, warnings)`. +- Rules: (1) `category`/`source`/`confidence` must be in the controlled vocab — else **error**. (2) every `depends_on` id must exist — else **error** (dangling-ref). (3) an entry's confidence may not exceed the min confidence among its dependencies — else **warning** (confidence-ceiling). + +- [ ] **Step 1: Write the failing test** + +Add to `tools/test_opcodes.py` (call `test_lint()` from `main` before the summary): + +```python +DANGLING = ''' +[[opcode]] +op = 0x10 +label = "x" +argc = 0 +[opcode.semantics] +name = "a" +category = "compute" +source = "inference" +confidence = "low" +depends_on = [0x99] +''' + +CEILING = ''' +[[opcode]] +op = 0x10 +label = "x" +argc = 0 +[opcode.semantics] +name = "low-op" +category = "compute" +source = "kelebek" +confidence = "low" +[[opcode]] +op = 0x11 +label = "y" +argc = 0 +[opcode.semantics] +name = "high-op" +category = "compute" +source = "inference" +confidence = "high" +depends_on = [0x10] +''' + +BADVOCAB = ''' +[[opcode]] +op = 0x10 +label = "x" +argc = 0 +[opcode.semantics] +name = "a" +category = "bogus" +source = "inference" +confidence = "low" +''' + +def test_lint(): + e, w = M.lint(M.load(write_tmp(DANGLING))) + check(any("0x99" in m for m in e), "dangling depends_on is an error") + e, w = M.lint(M.load(write_tmp(CEILING))) + check(any("0x11" in m for m in w), "confidence-ceiling violation is a warning") + check(e == [], "confidence-ceiling case has no errors") + e, w = M.lint(M.load(write_tmp(BADVOCAB))) + check(any("category" in m for m in e), "unknown category is an error") + e, w = M.lint(M.load(write_tmp(FIXTURE))) + check(e == [], "clean fixture has no lint errors") +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `py -3.11 -X utf8 tools/test_opcodes.py` +Expected: FAIL — `AttributeError: module 'opcodes_model' has no attribute 'lint'`. + +- [ ] **Step 3: Write minimal implementation** + +Add to `tools/opcodes_model.py`: + +```python +def lint(model: Model) -> tuple[list[str], list[str]]: + errors: list[str] = [] + warnings: list[str] = [] + ops = model.opcodes + for op, oc in sorted(ops.items()): + s = oc.semantics + if not s: + continue + tag = f"0x{op:x}" + if s.category not in CATEGORIES: + errors.append(f"{tag}: bad category {s.category!r}") + if s.source not in SOURCES: + errors.append(f"{tag}: bad source {s.source!r}") + if s.confidence not in CONFIDENCE: + errors.append(f"{tag}: bad confidence {s.confidence!r}") + for dep in s.depends_on: + if dep not in ops: + errors.append(f"{tag}: depends_on missing opcode 0x{dep:x}") + if s.confidence in CONFIDENCE: + dep_confs = [CONFIDENCE[ops[d].semantics.confidence] + for d in s.depends_on + if d in ops and ops[d].semantics + and ops[d].semantics.confidence in CONFIDENCE] + if dep_confs and CONFIDENCE[s.confidence] > min(dep_confs): + warnings.append(f"{tag}: confidence {s.confidence!r} exceeds dependency ceiling") + return errors, warnings +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `py -3.11 -X utf8 tools/test_opcodes.py` +Expected: `test_load` + `test_lint` all `ok`, `FAILURES: 0`. + +- [ ] **Step 5: Checkpoint** + +If git: `git add tools/opcodes_model.py tools/test_opcodes.py && git commit -m "feat(opcodes): linter (dangling-ref, confidence-ceiling, vocabulary)"` + +--- + +## Task 3: Bootstrap (`opcodes_build.py --bootstrap`) + +**Files:** +- Create: `tools/opcodes_build.py` +- Test: `tools/test_opcodes.py` (add `test_bootstrap`) + +**Interfaces:** +- Consumes: `paths.scripts()`, `sys4load.load`, `age_opcodes.OPCODES`, `opcodes_model`. +- Produces: `scan_corpus() -> (used: Counter, argtypes: dict[int, dict[int, set[int]]])`; `skeleton_toml(op, label, argc, argtypes_for_op) -> str`; `bootstrap(toml_path: Path) -> None` (creates file with `[meta]` on first run, then appends a skeleton block for each used opcode not already present). CLI: `py -3.11 -X utf8 tools/opcodes_build.py --bootstrap [--toml PATH]`. + +- [ ] **Step 1: Write the failing test** + +Add to `tools/test_opcodes.py`: + +```python +def test_bootstrap(): + import opcodes_build as B + fd, p = tempfile.mkstemp(suffix=".toml"); os.close(fd); os.remove(p) + from pathlib import Path + tp = Path(p) + B.bootstrap(tp) # first run: meta + all skeletons + m = M.load(tp) + check(len(m.opcodes) >= 240, f"bootstrap seeded ~248 opcodes (got {len(m.opcodes)})") + check(0x90 in m.opcodes and m.opcodes[0x90].argc == 7, "0x90 seeded with argc 7") + n1 = len(m.opcodes) + B.bootstrap(tp) # idempotent: appends nothing new + check(len(M.load(tp).opcodes) == n1, "second bootstrap adds no duplicates") + e, w = M.lint(m) + check(e == [], f"bootstrapped file lints clean (errors: {e[:3]})") +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `py -3.11 -X utf8 tools/test_opcodes.py` +Expected: FAIL — `ModuleNotFoundError: No module named 'opcodes_build'`. + +- [ ] **Step 3: Write minimal implementation** + +Create `tools/opcodes_build.py`: + +```python +#!/usr/bin/env python3 +"""Generator + linter for the living opcode reference (vm-map/opcodes.toml). + --bootstrap seed skeletons for every used opcode (append-only; preserves hand edits) + --build emit age_opcodes_himegari.py + build/opcodes.json + docs/opcode-reference.md + build/opcode-coverage.md + --lint run the linter, print errors/warnings, exit nonzero on errors +See docs/superpowers/specs/2026-07-06-opcode-reference-design.md.""" +from __future__ import annotations +import os, sys, json, argparse, collections +from pathlib import Path +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import paths +import sys4load +import opcodes_model as M +from age_opcodes import OPCODES + +TOML_DEFAULT = paths.VM_MAP / "opcodes.toml" + +TYPE_NAMES = {0x0: "imm", 0x1: "float", 0x2: "string", 0x3: "g-int", 0x4: "g-float", + 0x5: "g-str", 0x6: "g-ptr", 0x8: "g-str-ptr", 0x9: "l-int", 0xa: "l-float", + 0xb: "l-str", 0xc: "l-ptr", 0xd: "l-float-ptr", 0xe: "l-str-ptr"} + +META_TOML = '''# vm-map/opcodes.toml -- CANONICAL living opcode reference (hand-edited). +# Generated artifacts (age_opcodes_himegari.py, build/opcodes.json, docs/opcode-reference.md, +# build/opcode-coverage.md) come from this file via tools/opcodes_build.py --build. Do not edit those. +# Skeletons are appended by --bootstrap; enrich each [opcode.semantics] as we investigate. +[meta] +instruction_model = "code = seq of then argc*(); len_dwords = 1 + 2*argc" +opcodes_used_by_himegari = 248 + +[meta.arg_types] +"0x0" = "immediate" +"0x1" = "float" +"0x2" = "string" +"0x3" = "global-int" +"0x4" = "global-float" +"0x5" = "global-string" +"0x6" = "global-ptr" +"0x8" = "global-string-ptr" +"0x9" = "local-int" +"0xa" = "local-float" +"0xb" = "local-string" +"0xc" = "local-ptr" +"0xd" = "local-float-ptr" +"0xe" = "local-string-ptr" + +[meta.header_fields] +"F0" = "local_integer_1" +"F1" = "local_floats" +"F2" = "local_strings_1" +"F3" = "local_integer_2" +"F4" = "unknown_data" +"F5" = "local_strings_2" +"F6" = "sub_header_length(=0x1C)" +"F7" = "table_1_length" +"F8" = "table_1_offset(=code end)" +"F9" = "table_2_length" +"F10" = "table_2_offset" +"F11" = "table_3_length" +"F12" = "table_3_offset" +''' + +def scan_corpus(): + """used[op] = count; argtypes[op][arg_index] = set(type-codes) across the corpus.""" + used = collections.Counter() + argtypes: dict[int, dict[int, set]] = collections.defaultdict(lambda: collections.defaultdict(set)) + for name, path in paths.scripts().items(): + try: + scr = sys4load.load(path) + except Exception: + continue + for ins in scr.instructions: + used[ins.opcode] += 1 + for i, (t, v) in enumerate(ins.args): + argtypes[ins.opcode][i].add(t) + return used, argtypes + +def _is_named(label: str) -> bool: + return not (label.startswith("u00") or label == "dev_ukn" or label.startswith("?")) + +def skeleton_toml(op: int, label: str, argc: int, argtypes_for_op: dict) -> str: + conf = "med" if _is_named(label) else "low" + lines = ["[[opcode]]", f"op = 0x{op:x}", f'label = "{label}"', f"argc = {argc}", + 'abi_source = "kelebek+decode-validated"', "", "[opcode.semantics]", + f'name = "{label}"', 'category = "unknown"', 'summary = ""', + "noop_headless = false", 'source = "kelebek"', f'confidence = "{conf}"', + "depends_on = []", 'evidence = ""'] + for i in range(argc): + tnames = [TYPE_NAMES.get(t, "t%#x" % t) for t in sorted(argtypes_for_op.get(i, ()))] + obs = ", ".join('"%s"' % n for n in tnames) + lines += ["", "[[opcode.semantics.args]]", f"i = {i + 1}", 'role = ""', + f"observed_types = [{obs}]"] + return "\n".join(lines) + "\n" + +def bootstrap(toml_path: Path) -> None: + used, argtypes = scan_corpus() + present = set(M.load(toml_path).opcodes) if toml_path.exists() else set() + blocks = [] + for op in sorted(used): + if op in present: + continue + label, argc = OPCODES.get(op, ("0x%x" % op, 0)) + blocks.append(skeleton_toml(op, label, argc, argtypes[op])) + if not toml_path.exists(): + toml_path.parent.mkdir(parents=True, exist_ok=True) + toml_path.write_text(META_TOML + "\n", encoding="utf-8") + with toml_path.open("a", encoding="utf-8") as f: + f.write("\n".join(blocks)) + print(f"bootstrap: {len(used)} used opcodes; appended {len(blocks)} new skeletons -> {toml_path}") + +def main(argv=None): + ap = argparse.ArgumentParser() + ap.add_argument("--bootstrap", action="store_true") + ap.add_argument("--build", action="store_true") + ap.add_argument("--lint", action="store_true") + ap.add_argument("--toml", default=str(TOML_DEFAULT)) + args = ap.parse_args(argv) + tp = Path(args.toml) + if args.bootstrap: + bootstrap(tp) + return 0 + ap.error("no action (expected --bootstrap/--build/--lint)") + +if __name__ == "__main__": + sys.exit(main()) +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `py -3.11 -X utf8 tools/test_opcodes.py` +Expected: `test_bootstrap` lines `ok`, `FAILURES: 0`. + +- [ ] **Step 5: Checkpoint** + +If git: `git add tools/opcodes_build.py tools/test_opcodes.py && git commit -m "feat(opcodes): bootstrap seeds 248 skeletons from Kelebek + corpus arg-types"` + +--- + +## Task 4: Emit the Python shim (`--build` → `age_opcodes_himegari.py`) + +**Files:** +- Modify: `tools/opcodes_build.py` (add `emit_inferred_py`, wire `--build`) +- Test: `tools/test_opcodes.py` (add `test_emit_inferred`) + +**Interfaces:** +- Produces: `emit_inferred_py(model) -> str`. Pure. Emits `INFERRED: dict[int, dict]` containing an entry ONLY for opcodes whose `semantics.name != label` (i.e., ops we've given a distinct mnemonic) — this reproduces the current `sys4load` behavior exactly (bare Kelebek skeletons add nothing, so they are omitted and untouched unnamed ops keep rendering from `OPCODES`). Each entry carries `name` (required by sys4load) plus `category/noop/confidence/source/summary`. + +- [ ] **Step 1: Write the failing test** + +Add to `tools/test_opcodes.py`: + +```python +def test_emit_inferred(): + import opcodes_build as B + src = B.emit_inferred_py(M.load(write_tmp(FIXTURE))) + check("INFERRED" in src and "hotspot-branch" in src, "shim contains INFERRED + our mnemonic") + ns = {} + exec(compile(src, "", "exec"), ns) + inf = ns["INFERRED"] + check(0x90 in inf and inf[0x90]["name"] == "hotspot-branch", "generated INFERRED[0x90]['name'] correct") + check(0x1f4 in inf, "named marker 0x1f4 (name != label) included") +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `py -3.11 -X utf8 tools/test_opcodes.py` +Expected: FAIL — `AttributeError: module 'opcodes_build' has no attribute 'emit_inferred_py'`. + +- [ ] **Step 3: Write minimal implementation** + +Add to `tools/opcodes_build.py` (above `main`): + +```python +GEN_HEADER = "# DO NOT EDIT -- generated from vm-map/opcodes.toml by tools/opcodes_build.py --build\n" + +def emit_inferred_py(model: M.Model) -> str: + lines = [GEN_HEADER, '"""Inferred Himegari opcode semantics (generated). sys4load reads INFERRED[op][\'name\']."""', + "from __future__ import annotations", "", "INFERRED: dict[int, dict] = {"] + for op, oc in sorted(model.opcodes.items()): + s = oc.semantics + if not s or s.name == oc.label: # only ops we've given a distinct mnemonic + continue + lines.append(" 0x%x: dict(name=%r, category=%r, noop=%r, confidence=%r, source=%r, summary=%r)," + % (op, s.name, s.category, s.noop_headless, s.confidence, s.source, s.summary)) + lines.append("}") + return "\n".join(lines) + "\n" +``` + +And wire `--build` in `main` (replace the final `ap.error(...)` line): + +```python + if args.build: + model = M.load(tp) + errors, warnings = M.lint(model) + for m in warnings: + print("warn:", m) + if errors: + for m in errors: + print("error:", m) + return 1 + (paths.REPO / "tools" / "age_opcodes_himegari.py").write_text(emit_inferred_py(model), encoding="utf-8") + print("build: wrote tools/age_opcodes_himegari.py") + return 0 + if args.lint: + errors, warnings = M.lint(M.load(tp)) + for m in warnings: + print("warn:", m) + for m in errors: + print("error:", m) + print(f"lint: {len(errors)} errors, {len(warnings)} warnings") + return 1 if errors else 0 + ap.error("no action (expected --bootstrap/--build/--lint)") +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `py -3.11 -X utf8 tools/test_opcodes.py` +Expected: `test_emit_inferred` lines `ok`, `FAILURES: 0`. + +- [ ] **Step 5: Checkpoint** + +If git: `git add tools/opcodes_build.py tools/test_opcodes.py && git commit -m "feat(opcodes): emit drop-in age_opcodes_himegari.py shim; wire --build/--lint"` + +--- + +## Task 5: Emit JSON + Markdown reference + coverage (`--build`) + +**Files:** +- Modify: `tools/opcodes_build.py` (add `emit_json`, `emit_reference_md`, `emit_coverage_md`; wire into `--build`) +- Test: `tools/test_opcodes.py` (add `test_emit_views`) + +**Interfaces:** +- Produces: `emit_json(model) -> str` (includes a `dependents` map), `emit_reference_md(model) -> str` (per-opcode section with a "depended on by" line), `emit_coverage_md(model) -> str` (counts by source/confidence/category). All pure. + +- [ ] **Step 1: Write the failing test** + +Add to `tools/test_opcodes.py`: + +```python +def test_emit_views(): + import opcodes_build as B, json as _json + m = M.load(write_tmp(FIXTURE)) + j = _json.loads(B.emit_json(m)) + check(j["dependents"]["0x1f4"] == ["0x90"], "json dependents index correct") + check(any(o["op"] == "0x90" for o in j["opcodes"]), "json lists opcode 0x90") + md = B.emit_reference_md(m) + check("hotspot-branch" in md and "depended on by" in md.lower(), "reference md has entry + dependents line") + cov = B.emit_coverage_md(m) + check("investigation" in cov, "coverage md breaks down by source") +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `py -3.11 -X utf8 tools/test_opcodes.py` +Expected: FAIL — `AttributeError: ... 'emit_json'`. + +- [ ] **Step 3: Write minimal implementation** + +Add to `tools/opcodes_build.py`: + +```python +def emit_json(model: M.Model) -> str: + rev = M.dependents(model) + out = {"meta": model.meta, "opcodes": [], + "dependents": {"0x%x" % k: ["0x%x" % d for d in v] for k, v in rev.items() if v}} + for op, oc in sorted(model.opcodes.items()): + e = {"op": "0x%x" % op, "label": oc.label, "argc": oc.argc, + "code_target_args": oc.code_target_args, "abi_source": oc.abi_source} + s = oc.semantics + if s: + e["semantics"] = {"name": s.name, "category": s.category, "summary": s.summary, + "noop_headless": s.noop_headless, "source": s.source, + "confidence": s.confidence, "depends_on": ["0x%x" % d for d in s.depends_on], + "evidence": s.evidence, "details": s.details, "args": s.args} + out["opcodes"].append(e) + return json.dumps(out, ensure_ascii=False, indent=2) + "\n" + +def emit_reference_md(model: M.Model) -> str: + rev = M.dependents(model) + L = ["", + "# Opcode Reference (generated)", "", + f"{len(model.opcodes)} opcodes used by Himegari. Source of truth: `vm-map/opcodes.toml`.", ""] + by_cat = collections.defaultdict(list) + for op, oc in model.opcodes.items(): + cat = oc.semantics.category if oc.semantics else "unknown" + by_cat[cat].append(op) + for cat in sorted(by_cat): + L += [f"## {cat}", ""] + for op in sorted(by_cat[cat]): + oc = model.opcodes[op] + s = oc.semantics + name = s.name if s else oc.label + L.append(f"### 0x{op:x} `{name}` ({oc.label}, argc {oc.argc})") + if s: + L.append(f"- **summary:** {s.summary}" if s.summary else "- **summary:** —") + L.append(f"- **grounding:** source={s.source}, confidence={s.confidence}" + + (f", noop_headless={s.noop_headless}" if s.noop_headless else "")) + if s.depends_on: + L.append("- **depends on:** " + ", ".join("0x%x" % d for d in s.depends_on)) + if rev.get(op): + L.append("- **depended on by:** " + ", ".join("0x%x" % d for d in rev[op])) + if s.evidence: + L.append(f"- **evidence:** {s.evidence}") + if s.details: + L += ["", s.details] + L.append("") + return "\n".join(L) + "\n" + +def emit_coverage_md(model: M.Model) -> str: + by_src = collections.Counter() + by_conf = collections.Counter() + by_cat = collections.Counter() + named = 0 + for oc in model.opcodes.values(): + s = oc.semantics + if s: + by_src[s.source] += 1 + by_conf[s.confidence] += 1 + by_cat[s.category] += 1 + if s.name != oc.label: + named += 1 + L = ["", "# Opcode Coverage (generated)", "", + f"- opcodes: {len(model.opcodes)}", f"- given a distinct mnemonic: {named}", "", + "## by source", ""] + L += [f"- {k}: {v}" for k, v in sorted(by_src.items())] + L += ["", "## by confidence", ""] + [f"- {k}: {by_conf[k]}" for k in ("high", "med", "low")] + L += ["", "## by category", ""] + [f"- {k}: {v}" for k, v in sorted(by_cat.items())] + return "\n".join(L) + "\n" +``` + +Extend the `--build` block in `main` (after writing the shim, before `return 0`): + +```python + (paths.BUILD).mkdir(parents=True, exist_ok=True) + (paths.BUILD / "opcodes.json").write_text(emit_json(model), encoding="utf-8") + (paths.REPO / "docs" / "opcode-reference.md").write_text(emit_reference_md(model), encoding="utf-8") + (paths.BUILD / "opcode-coverage.md").write_text(emit_coverage_md(model), encoding="utf-8") + print("build: wrote build/opcodes.json, docs/opcode-reference.md, build/opcode-coverage.md") +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `py -3.11 -X utf8 tools/test_opcodes.py` +Expected: `test_emit_views` lines `ok`, `FAILURES: 0`. + +- [ ] **Step 5: Checkpoint** + +If git: `git add tools/opcodes_build.py tools/test_opcodes.py && git commit -m "feat(opcodes): emit opcodes.json, opcode-reference.md, coverage"` + +--- + +## Task 6: Real bootstrap + migrate legacy inferences + differential-verify + +**Files:** +- Create: `vm-map/opcodes.toml` (via bootstrap, then hand-edit) +- Regenerate: `tools/age_opcodes_himegari.py`, `build/opcodes.json`, `docs/opcode-reference.md`, `build/opcode-coverage.md` + +**Interfaces:** +- Consumes: everything above. No new code except a one-time migration helper (shown below; not committed as tooling). + +- [ ] **Step 1: Snapshot current disassembly (regression baseline)** + +Run (captures pre-change mnemonics for two representative scripts): +```bash +py -3.11 -X utf8 tools/sys4load.py ../../extracted/DATA1/SC0830.BIN > /tmp/sc0830.before.asm +py -3.11 -X utf8 tools/sys4load.py ../../extracted/DATA1/MENU.BIN > /tmp/menu.before.asm +``` +Expected: two files written (they contain `hotspot-branch`, `stmt-begin`, etc. from the current hand-written overlay). + +- [ ] **Step 2: Bootstrap the real canonical file** + +Run: `py -3.11 -X utf8 tools/opcodes_build.py --bootstrap` +Expected: `bootstrap: 248 used opcodes; appended 248 new skeletons -> ...opcodes.toml`. Confirm `vm-map/opcodes.toml` exists with a `[meta]` block and 248 `[[opcode]]` blocks. + +- [ ] **Step 3: Generate migration suggestions from the legacy overlay** + +Run this one-time helper (reads the CURRENT hand-written `age_opcodes_himegari.py` before it gets overwritten, and prints TOML `[opcode.semantics]` blocks to paste): +```bash +py -3.11 -X utf8 - <<'PY' +import sys, os +sys.path.insert(0, "tools") +from age_opcodes_himegari import INFERRED +MAP = {"structure": "investigation", "context": "inference", "harness": "harness", + "frida": "frida", "unicorn": "unicorn"} +for op, e in sorted(INFERRED.items()): + src = MAP.get(e.get("method", ""), "investigation") + print(f"# --- 0x{op:x}: replace the seeded [opcode.semantics] with: ---") + print("[opcode.semantics]") + print(f'name = {e["name"]!r}') + print(f'category = {e.get("category","unknown")!r}') + print(f'summary = {e.get("note","")!r}') + print(f'noop_headless = {str(bool(e.get("noop", False))).lower()}') + print(f'source = {src!r}') + print(f'confidence = {e.get("confidence","low")!r}') + print("depends_on = [] # FILL: opcodes this reading rests on") + print(f'evidence = {e.get("note","")!r}') + print() +PY +``` +Expected: ~26 TOML blocks printed (0x71, 0x7a, 0x90, 0x97, 0xb6, 0x1a2, 0x1bc, 0x1bf, 0x1d2, 0x1d5, 0x1f4, 0x1f5, 0x1f7, 0x1fa, 0x1ff, 0x202, 0x203, 0x215, 0x217, 0x218, 0x21a, 0x21b, 0x258). + +- [ ] **Step 4: Hand-migrate into `vm-map/opcodes.toml`** + +For each printed block, find that opcode's `[opcode.semantics]` in `vm-map/opcodes.toml` and replace the seeded fields with the printed ones. Then add `depends_on`. + +**`depends_on` principle:** it tracks *inference-on-inference* chains — list an op here ONLY when our reading rests on another op whose meaning is itself uncertain (our inference), so a later correction cascades. Reliance on a **validated core op** (e.g. `jcc 0xa0`, `call 0x8f`, `mov 0x55` — Kelebek-named and harness/RECOVER-proven) is a solid root: put that reasoning in `evidence` text, NOT in `depends_on` (adding it would also trip a spurious confidence-ceiling warning, since core ops seed at `med`). Applying this: +- `0x90` (`hotspot-branch`): `depends_on = [0x1f4, 0x1f5]` (rests on our *inferred* stmt markers); set `details` to the multi-line evidence from `vm-map/himegari-opcode-notes.md` §F (paste the section body into a TOML `details = """ ... """`). +- `0x97` (`hotspot-reg?`): `depends_on = [0x90]` (its role was inferred from interleaving with our inferred 0x90). +- `0x1d5`/`0x1bc`/`0x1bf` (markers inferred from following `jcc`/`call`): `depends_on = []`; put "always follows jcc 0xa0" / "call 0x8f → 0x1bf" in `evidence` (jcc/call are validated roots). +- Leave `depends_on = []` for ops grounded directly (`0x1f4`/`0x1f5`/`0x71` structural, harness-confirmed ADV ops). + +Also fold the cross-cutting evidence from `himegari-opcode-notes.md` (bucket intros, the coverage narrative) that you want to keep into the relevant entries' `details` or the `[meta]` block — everything that must survive the retirement of that file in Task 7. + +- [ ] **Step 5: Lint, then build** + +Run: +```bash +py -3.11 -X utf8 tools/opcodes_build.py --lint +py -3.11 -X utf8 tools/opcodes_build.py --build +``` +Expected: lint prints `0 errors` (confidence-ceiling warnings are acceptable — review each; downgrade confidence or fix a dependency if a warning is legitimate). Build writes all four artifacts. + +- [ ] **Step 6: Differential verification (the proof it's a faithful drop-in)** + +Run: +```bash +py -3.11 -X utf8 tools/vm0.py --test # RECOVER unit test +py -3.11 -X utf8 tools/vm0.py --sweep | tail -2 # coverage number +py -3.11 -X utf8 tools/sys4load.py ../../extracted/DATA1/SC0830.BIN > /tmp/sc0830.after.asm +py -3.11 -X utf8 tools/sys4load.py ../../extracted/DATA1/MENU.BIN > /tmp/menu.after.asm +diff /tmp/sc0830.before.asm /tmp/sc0830.after.asm && echo "SC0830 identical" +diff /tmp/menu.before.asm /tmp/menu.after.asm && echo "MENU identical" +``` +Expected: `RECOVER unit test: PASS`; sweep still `282/294 = 95.9%`; both `diff`s empty (`... identical`). If a diff is non-empty, an opcode's `name` was migrated wrong — fix that entry in `opcodes.toml`, rebuild, re-diff. + +- [ ] **Step 7: Checkpoint** + +If git: `git add vm-map/opcodes.toml tools/age_opcodes_himegari.py build/opcodes.json docs/opcode-reference.md build/opcode-coverage.md && git commit -m "feat(opcodes): migrate to opcodes.toml as single source of truth; regenerate artifacts"` + +--- + +## Task 7: Retire superseded files + update docs/memory + +**Files:** +- Delete: `vm-map/opcodes-himegari.json`, `vm-map/himegari-opcode-notes.md` +- Modify: `docs/PROJECT-STRUCTURE.md`, `C:\Users\m\.claude\projects\S--Game-Hacking-Eushully-Himegari\memory\himegari-port-status.md`, `...\memory\MEMORY.md` + +**Interfaces:** none (documentation). + +- [ ] **Step 1: Confirm content is preserved before deleting** + +Verify the retiring files' load-bearing content now lives in `vm-map/opcodes.toml` / `docs/opcode-reference.md`: +```bash +grep -c "hotspot-branch" docs/opcode-reference.md # >=1 +grep -c "instruction_model" vm-map/opcodes.toml # ==1 (meta migrated from opcodes-himegari.json) +grep -ci "hotspot" vm-map/opcodes.toml # 0x90/0x97 details migrated from notes §F +``` +Expected: all nonzero. Only proceed if the §F evidence and the JSON's meta really made it into `opcodes.toml`. + +- [ ] **Step 2: Delete the superseded files** + +```bash +rm vm-map/opcodes-himegari.json vm-map/himegari-opcode-notes.md +``` +(The hand-maintained `build/opcode-coverage.md` is now overwritten by `--build`, so no delete needed — it's generated.) + +- [ ] **Step 3: Update `docs/PROJECT-STRUCTURE.md`** + +In the `vm-map/` and `tools/` sections, replace mentions of `opcodes-himegari.json` / `himegari-opcode-notes.md` and the hand-written `age_opcodes_himegari.py` with the new model: +``` + ├── vm-map/ + │ ├── opcodes.toml ★ CANONICAL opcode reference (hand-edited: ABI + semantics + │ │ + provenance + depends_on). Source of truth for the opcode layer. + │ ├── kelebek1-age-shared.cpp upstream opcode-table source + │ └── opcode-leads.json, small-script-listings.md + ├── tools/ + │ ├── opcodes_build.py generator/linter: opcodes.toml -> {age_opcodes_himegari.py, + │ │ build/opcodes.json, docs/opcode-reference.md, build/opcode-coverage.md} + │ ├── opcodes_model.py load + lint (dangling-ref, confidence-ceiling, vocab) + dependents + │ ├── age_opcodes.py Kelebek table, PRISTINE (ABI baseline; never edit) + │ ├── age_opcodes_himegari.py GENERATED from opcodes.toml (do not hand-edit) +``` +Add a bullet under Conventions: *"Opcode knowledge is edited ONLY in `vm-map/opcodes.toml`; run `tools/opcodes_build.py --build` to regenerate the shim/JSON/reference/coverage. `docs/opcode-reference.md` and `build/opcodes.json` are generated."* + +- [ ] **Step 4: Update memory** + +In `himegari-port-status.md`, update the tooling/opcode paragraph: opcode work is now a single source of truth at `vm-map/opcodes.toml` (+ `opcodes_build.py`/`opcodes_model.py`), generating `age_opcodes_himegari.py` + `build/opcodes.json` + `docs/opcode-reference.md` + coverage; `himegari-opcode-notes.md` and `opcodes-himegari.json` retired (content folded in). In `MEMORY.md`, adjust the `[SYS4 script format]` / status hooks that referenced those files. + +- [ ] **Step 5: Final verification** + +```bash +py -3.11 -X utf8 tools/test_opcodes.py # FAILURES: 0 +py -3.11 -X utf8 tools/opcodes_build.py --lint # 0 errors +py -3.11 -X utf8 tools/vm0.py --test # PASS +``` +Expected: all green. No remaining references to the deleted files in `tools/` or `docs/`: +```bash +grep -rl "opcodes-himegari.json\|himegari-opcode-notes" tools docs || echo "no stale references" +``` +Expected: `no stale references`. + +- [ ] **Step 6: Checkpoint** + +If git: `git add -A && git commit -m "docs(opcodes): retire superseded opcode files; update structure + memory"` + +--- + +## Self-Review + +**Spec coverage** (each spec section → task): +- Single source of truth / data flow → Tasks 3–6 (bootstrap, build, migrate). ✓ +- `[meta]` + `[[opcode]]` schema (ABI vs semantics) → Task 1 model + Task 3 skeleton. ✓ +- Source vocabulary + confidence → Task 2 lint (vocabulary) + Task 6 migration mapping. ✓ +- Generator subcommands (`--bootstrap/--build/--lint`) → Tasks 3, 4, 5. ✓ +- Four generated artifacts → Task 4 (shim) + Task 5 (json/md/coverage). ✓ +- Three+ lint checks (dangling-ref, dependents index, confidence-ceiling, vocab) → Task 2 (+ dependents in Task 1, rendered in Task 5). ✓ +- Bootstrap auto-fills observed_types from corpus → Task 3 `scan_corpus`/`skeleton_toml`. ✓ +- Migration of ~26 inferences + notes evidence → Task 6. ✓ +- Retire 3 files; keep Kelebek pristine → Task 7 (+ Global Constraint). ✓ +- Zero disruption to sys4load/vm0 → Task 4 emit rule (name != label) + Task 6 diff regression. ✓ +- Testing: regression (disasm diff, --test, --sweep), lint fixtures, round-trip-ish load → Tasks 1,2,6. ✓ + +**Placeholder scan:** no "TBD/handle edge cases"; the only intentionally-manual step is Task 6 Step 4 (paste migration blocks + assign `depends_on`), which is inherent to a human judgement task and is spelled out per-opcode. + +**Type consistency:** `Model`/`Opcode`/`Semantics` fields are used identically across `load`, `lint`, `dependents`, and every `emit_*`. `emit_inferred_py` writes `dict(name=...)` → `INFERRED[op]["name"]`, matching `sys4load.py:84`. CLI flags `--bootstrap/--build/--lint/--toml` consistent between Task 3 and Tasks 4–5. diff --git a/docs/superpowers/specs/2026-07-06-opcode-reference-design.md b/docs/superpowers/specs/2026-07-06-opcode-reference-design.md new file mode 100644 index 0000000..edb28cf --- /dev/null +++ b/docs/superpowers/specs/2026-07-06-opcode-reference-design.md @@ -0,0 +1,181 @@ +# Design: Living Opcode Reference (single-source-of-truth + provenance) + +Status: **approved (design)** · Date: 2026-07-06 · Author: session work +Related: `docs/phase-a-slice-plan.md`, `vm-map/himegari-opcode-notes.md`, `docs/PROJECT-STRUCTURE.md` + +## Problem + +Opcode knowledge for the AGE/SYS4 VM is currently spread across five artifacts: + +| File | Role today | Consumed by | +|------|-----------|-------------| +| `tools/age_opcodes.py` | Kelebek table verbatim: `{op: (label, argc)}` + arg-types | sys4load, vm0 | +| `tools/age_opcodes_himegari.py` | our `INFERRED` dict (name/category/noop/confidence/method/note) | sys4load | +| `vm-map/opcodes-himegari.json` | data snapshot (arg-types, header fields, per-op array) | — (reference) | +| `vm-map/himegari-opcode-notes.md` | prose evidence, buckets A–F | — (humans) | +| `build/opcode-coverage.md` | coverage tiers | — (humans) | + +The same fact lives in several places, so they drift (this session already found stale "6/8 scenes" +wording, and `age_opcodes_himegari.py`'s `method` field duplicates prose in the notes doc). None of +them record **why** we believe a claim or **what other claims it rests on** — so when a reversal proves +one opcode wrong, there is no way to find the downstream inferences that depended on it. + +## Goals + +1. **One canonical, hand-edited file** as the single source of truth; every other opcode artifact is + generated from it and can never disagree with it or with the VM. +2. **Provenance per claim** — record the source (Kelebek / our harness / our investigation / Frida / + Unicorn) and confidence, with the near-certain ABI facts kept separate from the fallible semantics. +3. **Dependency tracking** — a semantic claim can declare the opcodes whose interpretation it rests on, + so a correction's blast radius is visible and mechanically checkable. +4. **Zero disruption to working tooling** — sys4load/vm0 keep importing the same Python interface, + now generated. +5. **Complete map** — an entry for every one of the 248 opcodes Himegari uses, so coverage is measurable + and any opcode can be named as a dependency. + +## Non-goals + +- Not touching `tools/age_opcodes.py` (the Kelebek table): it stays pristine as the ABI baseline and the + bootstrap input. +- No per-call-site type modelling (arg types are per-instruction in the bytecode, not fixed per opcode). +- No live-capture work (Frida/Unicorn) here — this is the reference structure; those remain sources a + future entry can cite. +- Not covering the ~300 Kelebek opcodes Himegari never uses. + +## Architecture + +One canonical file; everything else is generated: + +``` +vm-map/opcodes.toml ← CANONICAL, hand-edited. [meta] + 248 [[opcode]] tables. + │ + ▼ tools/opcodes_build.py (generator + linter; stdlib tomllib read, no new dep) + ├─ tools/age_opcodes_himegari.py GENERATED — exposes INFERRED (sys4load imports unchanged) + ├─ build/opcodes.json GENERATED — language-neutral machine view (future C# VM) + ├─ docs/opcode-reference.md GENERATED — human reference incl. reverse dependents index + └─ build/opcode-coverage.md GENERATED — counts by source / confidence / category +``` + +- `tools/age_opcodes.py` (Kelebek) **stays**: ABI baseline + bootstrap input, never hand-edited. +- **Retired** (content migrates into `opcodes.toml`, then re-emitted): hand-maintained + `age_opcodes_himegari.py` (now generated), `vm-map/opcodes-himegari.json` (→ `build/opcodes.json`), + `vm-map/himegari-opcode-notes.md` (per-op evidence → each entry's `details`; rendered into + `docs/opcode-reference.md`), and hand-maintained `build/opcode-coverage.md`. +- Editing loop: edit `opcodes.toml` → run `opcodes_build.py` → tooling, machine view, human doc, and + coverage all update together. + +## Canonical file format (`vm-map/opcodes.toml`) + +### `[meta]` (migrated from `opcodes-himegari.json`) +- `instruction_model` — the `1 + 2*argc` decode description (string). +- `[meta.arg_types]` — type-code → name (`0x0`→immediate, `0x2`→string, `0x9`→local-int, …). +- `[meta.header_fields]` — F0–F12 meanings. +- `[meta.sources]`, `[meta.confidence]`, `[meta.categories]` — controlled vocabularies (documented + below), so the linter can reject unknown values. + +### `[[opcode]]` — ABI block (high trust) +| field | meaning | +|-------|---------| +| `op` | opcode number (TOML `0x..`) | +| `label` | Kelebek engine address (e.g. `"u0041BEB0"`) | +| `argc` | argument count — **validated by our 481/481 clean decode** (`len = 1 + 2*argc`) | +| `code_target_args` | 1-based arg indices that are jump targets (from Kelebek's notes; optional) | +| `abi_source` | e.g. `"kelebek+decode-validated"` | +| `abi_note` | optional caveat | + +### `[opcode.semantics]` — our fallible layer (where cascades live) +| field | meaning | +|-------|---------| +| `name` | short mnemonic for the disassembler (**the one field sys4load requires**) | +| `category` | `marker\|structural\|control\|adv\|draw\|audio\|input\|compute\|unknown` | +| `summary` | one-line description | +| `noop_headless` | bool — safe for the headless VM v1 to skip/fall-through (no state/visible effect) | +| `source` | `kelebek\|harness\|investigation\|frida\|unicorn\|inference` | +| `confidence` | `high\|med\|low` | +| `depends_on` | list of op ids whose interpretation this claim rests on (default `[]`) | +| `evidence` | concise grounding (one line) | +| `details` | optional multi-line long-form evidence (migrated from the notes doc) | +| `confirm_by` | optional — what would raise confidence or falsify this | + +### `[[opcode.semantics.args]]` — per-arg roles (array of tables, optional) +| field | meaning | +|-------|---------| +| `i` | 1-based arg index | +| `role` | semantic role (`"x"`, `"y"`, `"target:click"`, `"count"`, …) | +| `observed_types` | list of type names actually seen at this position (**auto-filled from corpus**) | +| `note` | optional | + +### Source vocabulary (grounding roots vs. fallible) +- `kelebek` — from the Kelebek table (ABI trusted; *semantics* from a later AGE title → treat as a lead, + not truth). A root. +- `harness` — confirmed by our dialogue-oracle diff (`build/text/dialogue.jsonl`). Empirically grounded root. +- `investigation` — our own static/corpus analysis (may `depends_on` other ops). +- `frida` / `unicorn` — live runtime capture / micro-execution. Roots (direct observation). +- `inference` — reasoned from neighbours/context; weakest, usually has `depends_on`. + +## Generator + linter (`tools/opcodes_build.py`) + +Subcommands: +- `--bootstrap` — create `opcodes.toml`: the used-opcode set is computed by scanning the corpus with + `sys4load` (authoritative, reproducible); seed each with `op/label/argc` from `age_opcodes.py`, + `abi_source="kelebek+decode-validated"` (argc is validated for all 248 by our 481/481 decode), + `semantics.source="kelebek"`, `confidence` = `med` if Kelebek names it else `low` (bare + `u004xxx`/`dev_ukn` label → `category="unknown"`); **auto-fill `observed_types` per arg position** + from the corpus scan. Idempotent: + re-running preserves hand-edited entries (only fills missing). +- `--build` (default) — read `opcodes.toml`, run lint, then emit the four generated artifacts. The + generated `age_opcodes_himegari.py` exposes `INFERRED: dict[int, dict]` with at least `name` (plus + category/noop/confidence/source for future consumers) — a drop-in for the current interface. +- `--lint` — run the three checks; nonzero exit on any error. + +Generated-file guard: each emitted file carries a `DO NOT EDIT — generated from vm-map/opcodes.toml` +header. + +## Lint checks (the cascade mechanism) + +1. **Dangling-ref (error):** every id in a `depends_on` must be an existing `op`. No orphan dependencies. +2. **Reverse index / dependents (report):** build `A → [ops whose semantics depend on A]`; render it in + `docs/opcode-reference.md` as a "depended on by: 0x…" line per opcode, so a revision's blast radius is + visible. Also emitted to the lint output. +3. **Confidence-ceiling (warning):** an entry's `confidence` may not exceed the minimum confidence among + its `depends_on` targets (can't be `high` while resting on a `low`). Surfaces shaky foundations. +4. **Vocabulary (error):** `category`/`source`/`confidence` must be in the `[meta]` controlled lists. + +Chains terminate at a `kelebek`/`harness`/`frida`/`unicorn` root or at direct `evidence`. + +## Bootstrap / migration procedure + +1. Implement `opcodes_build.py`; run `--bootstrap` → `opcodes.toml` with 248 seeded skeletons + observed + arg-type histograms. +2. Hand-migrate our real inferences into their entries: the ~26 in `age_opcodes_himegari.py` plus the + evidence in `himegari-opcode-notes.md` (buckets A–F, incl. the `0x90`/`0x97` deep-dive), each with + proper `source`, `depends_on`, `evidence`, `details`. +3. Run `--build`; confirm generated `age_opcodes_himegari.py` reproduces the current names, and + `sys4load … --validate` + `vm0.py --test` + `vm0.py --sweep` are unchanged (differential check). +4. Delete the retired hand-maintained files; update `docs/PROJECT-STRUCTURE.md` and the memory index. + +## Testing / verification + +- **Regression (must be byte-for-byte where it matters):** after migration, `sys4load` disassembly of a + few scripts (MENU, SC0830) shows the same mnemonics; `vm0.py --test` PASS and `--sweep` still + 282/294. This proves the generated shim is a true drop-in. +- **Generator round-trip:** `--build` is deterministic; re-running produces no diff. +- **Lint unit checks:** craft a tiny fixture TOML exercising dangling-ref (error), a confidence-ceiling + violation (warning), and an unknown category (error). +- **Coverage sanity:** generated `opcode-coverage.md` totals reconcile with 248 used opcodes. + +## Risks / open questions + +- **TOML verbosity for 248 entries** — acceptable; entries are mostly skeletons and diffs read cleanly. +- **`observed_types` staleness** — it's derived; re-run `--bootstrap --refresh-observed` (fills only that + field) if the corpus set changes. Not load-bearing (evidence, not truth). +- **Deciding `confidence` for Kelebek-named ops** — Kelebek names are ABI-labels from another title; + default seed = `med` for named, and we downgrade/confirm as the VM exercises them (harness). +- **git** — workspace isn't a git repo, so the design/spec aren't committed; offer `git init` later. + +## Out of scope (future) + +- Emitting a C#-native binding from `build/opcodes.json` (A1 will consume the JSON directly). +- A GUI/queryable browser over the reference. +- Automated "review-needed" flags on git-diff of `opcodes.toml` (the dependents index already gives the + manual signal). diff --git a/docs/sys4-format-notes.md b/docs/sys4-format-notes.md new file mode 100644 index 0000000..5f3bd85 --- /dev/null +++ b/docs/sys4-format-notes.md @@ -0,0 +1,158 @@ +# SYS4 Script Format — Reverse-Engineering Notes (hex-first) + +Derived purely from byte-pattern analysis of the 481 DATA1 `.BIN` scripts, before +any Ghidra work on `AGE.EXE`. Confidence levels flagged per finding. Probe scripts +live in `tools/probe_*.py`. + +> **UPDATE 2026-07-05 — opcode set solved via Kelebek1's table.** Everything below +> under "Instruction stream — PARTIAL" is now resolved: code = instructions of +> ` + argc*()`, length `1+2*argc`; inline strings +> live after code inside `[0,F8)`, so stop decoding at the first type-2/`0x64` arg +> offset. 476/476 scripts decode clean (0 unknown opcodes). Header fields F0–F5 are +> **local-variable counts** (F0=local_integer_1, F1=local_floats, F2=local_strings_1, +> F3=local_integer_2, F4=unknown, F5=local_strings_2). See `vm-mapping-plan.md` and +> `vm-map/opcodes-himegari.json`. The tag values below (0x71/0x03/0x8F etc.) are the +> *opcodes at table targets*: 0x8F=`call`, 0x03=`call-script`, 0x71=`u0041A7B0`. + +## Header — CONFIRMED + +Fixed 60-byte (0x3C) header: 8-byte magic + thirteen little-endian u32 fields. +Verified across all 481 files. + +``` +off field meaning evidence +0x00 magic "SYS4422 " (0x53 59 53 34 34 32 32 20) 481/481 identical +0x08 F0 scenario/script id or flags 98 distinct; 0x5 dominant (252×) +0x0C F1 = 1 always 481/481 == 1 +0x10 F2 = 1 (457×) or 2 (20×); 4 outliers format/feature flag +0x14 F3 0x06 (268×),0x04,0x01,0x05... 15 distinct — minor version? +0x18 F4 = 1 always 481/481 == 1 +0x1C F5 = 2 (319×) or 1 (144×); rarely 3/5 flag +0x20 F6 = 0x1C always 481/481 == 0x1C (header-size marker) +0x24 F7 table-1 entry count see below +0x28 F8 table-1 offset == code-section length ordering F8<=F10<=F12<=EOF +0x2C F9 table-2 entry count +0x30 F10 table-2 offset +0x34 F11 table-3 entry count +0x38 F12 table-3 offset +0x3C body dword stream (code + 3 tables + strings) +``` + +**All offsets/counts are in DWORDS (×4 bytes), relative to body start (0x3C).** +The `0x1C` in F6 is the only *byte* count — it's the offset from F6's own position +(0x20) back-referenced, i.e. a self-describing "28 bytes of descriptor follow" marker +consistent with the SYS4/SYS5 family. + +### Section layout (CONFIRMED — 0 ordering violations, 481/481) + +``` +body[0 .. F8) CODE bytecode instruction stream +body[F8 .. F10) TABLE-1 (F7 entries, 1 dword each) -> targets of type 0x71 +body[F10 .. F12) TABLE-2 (F9 entries, 1 dword each) -> targets of type 0x03 +body[F12 .. EOF) TABLE-3 (F11 entries, 1 dword each) -> targets of type 0x8F +``` + +Every table entry is exactly **1 dword** — a pointer (dword index into body). +Solved algebraically across the whole corpus: `(F10-F8)/F7 == (F12-F10)/F9 == +(EOF-F12)/F11 == 1` with zero non-integer results. + +### The three tables are typed pointer indexes (CONFIRMED) + +Each table points at body locations, and the dword *at* every target is a constant +tag identifying the pointed-to construct: + +| Table | count/off | Target dword tag | Hits | Meaning (inferred) | +|---|---|---|---|---| +| T1 | F7 / F8 | **0x71** | 26,445/26,445 | labels / call targets (operand at +2 is small: mostly 1) | +| T2 | F9 / F10 | **0x03** | 3,018/3,018 | data/variable entries (operand at +2 large, e.g. addresses) | +| T3 | F11 / F12| **0x8F** | 72,941/72,941 | instruction/line entries (largest table; operand at +2 huge) | + +100% type purity — not a single target had a different tag. T3 is the big one +(~73k entries corpus-wide), consistent with it being a per-instruction or +per-source-line index (a debug/line table). T1 ≈ labels, T2 ≈ a smaller symbol set. + +## Instruction stream — PARTIAL + +The code section is a flat dword stream. Recurring "type/opcode" dwords observed: +`0x03, 0x55, 0x6E, 0x6F, 0x71, 0x72, 0x8F`. These read as **operand-type tags** +in a tagged-operand VM rather than raw opcodes, e.g. the repeating shape: + +``` +... ... tag 0x02 => string pointer (see below) +... 0x71 0x00 0x01 0x55 ... label marker + following instruction +``` + +- **First body dword is 0x259 (601) in 301/481 files** — likely a standard + "script entry" / prologue opcode. Second-most-common openers are small ints. +- `0x55` appears pervasively as an instruction lead — probably the most common + opcode (statement / expression separator). + +*Full opcode semantics need the VM dispatch loop in `AGE.EXE` — that's the Ghidra +task. These tags give a head start on labeling the disassembly.* + +## Strings — CONFIRMED + +- Stored inline in the body as **byte-complement (XOR 0xFF) cp932 / Shift-JIS**, + packed 4 bytes per dword, NUL-terminated (a `\0` byte, i.e. `0xFF` after XOR, + ends the string), then padded to the next dword. +- **Referenced by a tagged operand: the dword `0x02` immediately followed by the + dword-offset of the string.** Confirmed directly by xref: + - `MENU.BIN`: `...2 30b...` @0x2A9 → offset 0x30B = `"ＭＳ 明朝"`; `...2 30e...` → `"loadmesskip menu"` + - `SC0030.BIN`: `...2 ef80...` → `"▼G0030　２章マップ０２１クリア"`; `...2 ef89...` → `"「よし、素晴らしい成果だな」"` +- Decoder (validated — pulls clean Japanese dialogue): + ```python + raw = bytes(b ^ 0xFF for b in body[off*4:]) # until a 0x00 appears + text = raw.split(b"\0")[0].decode("cp932") + ``` +- Scene scripts hold the full dialogue; e.g. `SC0030.BIN` decodes to readable + story text, choice-branch labels ("本来の分岐", "チェック用"), font names + ("ＭＳ 明朝"), and engine directives ("loadmesskip advset", "ＡＤＶパート"). +- Non-scene scripts (`MENU`, `ADDEXP`) contain only a handful of control strings — + consistent with the inventory's subsystem/data-table categorization. + +## Patch-override caveat (re-confirmed) + +52 loose `.BIN` in the game root shadow their DATA1 copies at runtime and differ +slightly in size. The disassembler should target the **root** copies where present. +The header format is identical (same magic/layout) so tooling is copy-agnostic. + +## What's solid vs. what needs Ghidra + +**Solid (byte-verified, build a loader now):** +- 60-byte header, all 13 fields, dword units, section boundaries +- 3 typed pointer tables (0x71 / 0x03 / 0x8F), 1 dword each, 100% pure +- String encoding (XOR-0xFF cp932) + reference mechanism (tag 0x02 + offset) + +**Needs the VM (Ghidra on `AGE.EXE`):** +- Opcode dispatch — confirm tagged-operand model, enumerate opcodes +- Meaning of F0/F2/F3/F5 flag fields +- Exact operand grammar per instruction (how many dwords each opcode consumes) +- Semantics of T1/T2/T3 beyond "label/data/line" guesses + +## Loader — DONE + +`tools/sys4load.py` parses the header, splits the 4 sections, resolves the 3 +tables, decodes inline strings, and emits an assembly-ish listing with strings +inlined at their `2 ` refs (opcodes not yet named — code is chunked by the +T3 line-index). Importable API (`load()` → `Sys4Script`) plus CLI: + +``` +sys4load.py full listing +sys4load.py --summary header + section sizes + table/string counts +sys4load.py --strings decoded string pool +sys4load.py --json machine-readable structure +sys4load.py --validate re-check invariants across a folder +``` + +`--validate` over all 481 DATA1 scripts: **481 parsed clean, 0 failures, 0 impure +table tags** — the format spec above is fully machine-verified. This listing is the +artifact to diff against Ghidra output once the VM dispatch loop is mapped. + +### Observations surfaced by the listing (leads for the VM work) +- String-display sites look like ` 0x02 ` — e.g. opcodes `0x1A7` + and `0x1A5` immediately precede string refs in `MENU.BIN`. Candidate text/message ops. +- T3 entries are 3-dword records `[0x8F, 0x00, value]`; T1 labels are `[0x71, 0x00, + value]`. T3 acts as a per-statement line index (editor metadata) — note scene + scripts even carry editor annotation strings like `"LABEL"`, `"ループ開始"` (loop start). +- `0x55` is the most frequent code lead (likely statement/expr separator); `0x09` + recurs as an operand-type prefix (register/var reference?). diff --git a/docs/vm-mapping-plan.md b/docs/vm-mapping-plan.md new file mode 100644 index 0000000..34316ba --- /dev/null +++ b/docs/vm-mapping-plan.md @@ -0,0 +1,153 @@ +# SYS4 VM Mapping — Plan of Action + +> **For the executing agent:** This is a reverse-engineering playbook. Work it phase-by-phase; each phase ends with a concrete, checkable deliverable. Verify claims against bytes before recording them. Validated seed data lives in `vm-map/`. + +**Goal:** Decode the SYS4 bytecode into named instructions so the game logic can be re-implemented in Godot. + +--- + +## ✅ BREAKTHROUGH (2026-07-05): the opcode set is already solved + +**The prior "unpack AGE.EXE in Ghidra" critical path is no longer needed to disassemble scripts.** Kelebek1's decompiler ships a complete AGE opcode table that decodes this game directly. + +**What was verified this session** (see `tools/validate_opcode_table.py`, run it to reproduce): + +- Kelebek1/Eushully-Decompiler's `age-shared.cpp` contains an opcode table (`{op_code, label, argument_count}`) and a header parser that **explicitly handles the SYS4 signature** (`"SYS4"`, header length `0x3C`, cp932 XOR-0xFF strings) — this exact game's format. +- The instruction model: **code = a flat sequence of instructions; each instruction = `` followed by `argument_count` arguments, where every argument is a `` pair. Instruction length in dwords = `1 + 2*argc`.** Inline strings sit *after* the code inside the `[0,F8)` region; stop decoding at the lowest string offset referenced (a type-2 arg, or op `0x64` arg 1). +- Applying that table to Himegari's scripts: **476 of 476 parseable scripts decode 100% clean — 1,463,788 instructions, 0 unknown opcodes, and all 37,392 inline-string arguments resolve to valid decoded strings.** (The 7 non-decoding `.BIN` are container-level non-scripts like `SYS4AB`/`SYS4INI`, different magic.) +- Himegari uses **248 distinct opcodes; 52 have semantic names** (in `vm-map/opcodes-himegari.json`). The other 196 decode perfectly (known length) but have engine-internal names only (`u004xxxx`). **Caveat (measured 2026-07-06):** the named 52 are the dialogue/ADV core but cover only **72.6% of instruction volume**, not "the entire core" — the unnamed 27.4% is concentrated in the highest-frequency opcodes and must be partly addressed before Phase 4. See Phase 3's coverage correction. + +**This resolves the header unknowns too.** Kelebek's `BinaryHeader` struct maps my F0–F12 exactly: `F0`=local_integer_1, `F1`=local_floats, `F2`=local_strings_1, `F3`=local_integer_2, `F4`=unknown, `F5`=local_strings_2, `F6`=sub_header_length(0x1C), then the three (length, offset) table pairs. The "flag fields" were **local-variable counts**. Arg `type` codes: 0=immediate, 1=float, 2=string, 3=global-int, 4=global-float, 5=global-string, 6=global-ptr, 8=global-string-ptr, 9=local-int, A=local-float, B=local-string, C=local-ptr, D=local-float-ptr, E=local-string-ptr. + +**Consequence:** Unpacking `AGE.EXE` (still packed — see appendix) drops from *the blocker* to an *optional enrichment* used only to name the 196 unnamed opcodes' fine semantics, and even that has a cheaper dynamic alternative. + +**Provenance / sources in `vm-map/`:** `kelebek1-age-shared.cpp` (the opcode table), `kelebek1-disassembler.cpp` (the parser), `opcodes-himegari.json` (validated table filtered to what this game uses), `opcode-leads.json` + `small-script-listings.md` (this session's static analysis, now confirmed). + +--- + +## Global constraints + +- **Python:** `py -3.11 -X utf8 …` always (Shift-JIS output needs utf8 mode on Windows). +- **Authoritative copies:** the 52 loose game-folder `.BIN` shadow their `extracted/DATA1/` copies at runtime — target the game-folder copy where both exist. `sys4load.load()` is copy-agnostic; `paths.scripts()` resolves the override. +- **Units:** all script offsets/counts are DWORDS (×4 bytes), relative to body start `0x3C`. +- **Instruction rule:** `len_dwords = 1 + 2*argc`; args are `(type,value)`; **stop code decode at the first inline-string/array offset**, not blindly at `F8`. +- Record confidence per finding (confirmed-by-bytes / confirmed-by-runtime / hypothesis). + +--- + +## Phase 1 — Port the opcode table into `sys4load.py`, disassemble everything *(✅ DONE 2026-07-05)* + +**Deliverable:** `sys4load.py` emits real named instructions; every script disassembles with zero unknown opcodes. **Achieved: 481/481 DATA1 scripts decode fully clean; MENU.BIN and SC0030.BIN verified by hand.** + +- [x] **1.1 — Embed the opcode table.** Full Kelebek table (548 entries) transcribed to `tools/age_opcodes.py` (`OPCODES`, `ARG_TYPES`, `CONTROL_FLOW`, `is_label_argument`), generated from `vm-map/kelebek1-age-shared.cpp`. +- [x] **1.2 — Replace the T3-chunking stub.** `sys4load.py` now has `decode_code()` (the `1+2*argc` walker with shrinking `code_end`) and a rewritten `render_listing()` that prints mnemonics, typed operands, inline strings, and `label_xxxx:` control-flow anchors. +- [x] **1.3 — Validate.** `tools/sys4load.py ../../extracted/DATA1 --validate` → **481/481 parsed clean, 481/481 opcode-decode clean.** MENU.BIN: 148 instrs, `set-font "ＭＳ 明朝"` + `comment` strings correct. SC0030.BIN: 11,951 instrs, `show-text` shows dialogue inline. (`tools/validate_opcode_table.py` still reproduces the standalone 476/476 over the merged root+DATA1 set.) +- [x] **1.4 — Regression-guard:** container `--validate` still reports 481 clean, 0 failures, 0 impure tags. +- [x] **1.5 — Disassembler is the artifact.** `sys4load.py ` prints the full listing; `--json` now includes decoded `code` (with `--json` + `to_dict(with_code=True)`), instruction counts, and decode-clean flag. + +**Note:** 7 root `.BIN` are non-script engine indices with different magic (`SYS4INI` = `S4IC422`, `SYS4AB` = `S4AB`, etc.) — correctly rejected by the container parser, not scripts. + +## Phase 2 — Extract data tables + dialogue *(✅ mostly DONE 2026-07-06)* + +**Deliverable:** game database as JSON + full translatable dialogue corpus. Structure spec: `docs/PROJECT-STRUCTURE.md`. Extractors: `tools/extract_phase2.py`, `tools/extract_init.py`. + +- [x] **2.0 — Project structure.** Established `docs/`, `build/{disasm,text,data,scripts-json}/`, `godot/`; game install stays read-only in place. Also relaxed the loader magic check to the `SYS4` family (`SYS4424` patch scripts now parse — was silently skipping 5 scripts). +- [x] **2.1 — Text corpora.** `tools/extract_phase2.py` → 481/481 scripts: full disassembly (`build/disasm/*.asm`), per-script strings, `build/text/dialogue.jsonl` (**30,057 show-text lines** — the translation corpus), `build/text/strings.jsonl` (38,449 strings tagged by source opcode), `build/manifest.json`. +- [x] **2.2 — `*INIT` data tables → JSON.** `tools/extract_init.py` auto-detects table shape (`name`/`numeric`/`footer`) → **SKINIT (129 skills), ITINIT (189 items), EBINIT (277 units)** [name: name+desc+fields], **CGINIT (379 CG entries)** [numeric: index-keyed columns], **MPINIT (1472 map records)** [footer: 50-value arrays from the file footer]. Validated; see `build/data/README.md`. Column addresses are raw engine globals — naming them (attack/cost/…) needs the global-var map (Phase 3-adjacent). +- [ ] **2.3 — `STINIT` (74 stages) needs a bespoke parser.** Heterogeneous per-stage `copy-to-global` param blocks + a variable number of clear-condition strings per stage — fits none of the three auto modes. Its clear-condition text is already in `build/text/STINIT.strings.txt`; only the per-stage numeric params await a custom extractor. Low priority (stages are also encoded in the SC-scene scripts). +- [x] **2.4 — Partial global-var map BUILT + wired into the disassembler.** `tools/global_map.py` → `build/global-var-map.{json,md}` (16,354/49,435 globals labelled: string tables, `*INIT` field arrays, 122 record tables w/ strides, current-entity index pointers). `sys4load` renders the labels inline (`=rec[s30]`, `=current-entity-index?`). See `docs/name-resolution.md`. +- [ ] **2.5 — Grow the global-var map (future, incremental).** Static first: fold in `*MES` writers; label 2D record tables by their reader scripts. Then Frida to name *which stat* each field is. Full detail: `docs/name-resolution.md` → "Future step — growing the map". Also deferred: `call-script` id→name resolution (engine-level — SCJUMP.BIN decode or Frida; see `docs/name-resolution.md` #1). + +## Phase 3 — Name the unnamed opcodes *(top ~20 BEFORE Phase 4; the rest on demand)* + +> **⚠️ Coverage correction (measured 2026-07-06).** The earlier framing — "52 named ops +> cover the entire core, name the other 196 lazily" — is **overstated**. Across the full +> corpus (1,503,166 instructions, all 481 scripts), **named opcodes are only 72.6% of +> instructions; the 195 unnamed `u004xxxx` ops are 27.4%** — and that 27% is front-loaded +> into the *most common* opcodes, not a deferrable long tail. The top unnamed ops by +> frequency: `0x1f4`/`0x1f5` (**60,297 each** — equal counts → a begin/end or push/pop +> pair, both zero-arg), `0x1d5` (34k), `0x1bc` (27k), `0x71` (26,445 — *exactly* the +> corpus T1 label-table entry count, so it's the **label-definition pseudo-op**, nameable +> by structure for free), `0x1a2` (18k), `0x7a` (17k, argc 3, follows arithmetic → +> computational), `0x1d2` (17k). **A Godot VM hits these in the first few instructions of +> any script.** So naming the top ~20 is a *prerequisite* for Phase 4, not a lazy +> follow-on. Only the genuine long tail (rare ops) is deferrable. Reproduce the measurement +> by iterating `sys4load.load` over the corpus and bucketing `ins.opcode` against +> `age_opcodes.OPCODES` (label starting `u00`/`dev_ukn` = unnamed). + +> **⚠️ Named labels are from a *different* AGE title.** The 52 semantic labels are +> transcribed from Kelebek's table for a *later* AGE game. The opcode **number + argc** are +> validated for Himegari (481/481 clean decode proves structure), but the **semantics are +> not independently verified**. The ADV/text core is empirically safe — the 30,057-line +> `build/text/dialogue.jsonl` is proof that `show-text`/`end-text-line`/the string +> mechanism are right, and arithmetic/control-flow labels are corroborated by operand-type +> and jump-target consistency. The exposure is the **effectful named ops you can't see in +> text output** (`play-voice 0xc4`, `draw-texture 0x1fb`, sound/UI/draw ops) — Frida-confirm +> those against Himegari before the VM relies on them; don't assume them. + +**Do this before Phase 4:** name/classify the ~20 highest-frequency unnamed opcodes. +Most fall to free inference (3.0); a few opaque effectful ones want a Frida session; +computational ones suit Unicorn. Everything below still applies — it's the *ordering* that +changes, not the toolkit. The genuine rare tail stays lazy (name on demand). + +- [x] **3.0 — Inference pass DONE (2026-07-06).** Classified the top 21 unnamed opcodes → + **instruction coverage 72.62% (named) → 96.94% (classified)**; ~90.5% is VM-handleable by + inference alone. Tooling: `tools/opcode_context.py` (evidence gatherer). Results: + `vm-map/himegari-opcode-notes.md` (per-op evidence), `tools/age_opcodes_himegari.py` + (`INFERRED` dict consumed by the disassembler + future VM), `build/opcode-coverage.md` + (tiers + Frida/Unicorn shortlist). `sys4load` now renders inferred names (verified: MENU's + `label-def 0x71` land exactly on its T1 targets). Key findings: `0x1f4`/`0x1f5` = stmt + begin/end brackets, `0x1d5`/`0x1bc`/`0x1bf` = block markers (all zero-arg no-ops); `0x71` + = label-def (count == T1 size); `0x21b`/`0x1d2`/`0x258` = tentative-no-op statement metadata + (harness-verify); `0x7a` = ADV text param, `0x202/0x203/0x1f7/0x1fa/0x217/0x218/0x21a/0x1ff` + = draw/UI, `0xb6` = audio, `0x215` = count/search — the effectful/computational Frida/Unicorn + shortlist. Reserve live tools for those; rare tail (3%) stays lazy. + +### 3.1 — Frida: dynamic observation *(primary tool for effectful opcodes)* + +Frida injects a JS engine into the **running** game and hooks functions live. It sidesteps the packer (memory is already decrypted by the time you attach), gives ground-truth behavior, and lets you correlate an opcode with its on-screen/audible effect — the only reliable way to name rendering/audio/input/save/UI handlers. Two stages: + +- [ ] **3.1a — Locate the dispatch loop.** Kelebek's `u004xxxx` addresses are from a *different* AGE title and will NOT match Himegari's `AGE.EXE`, so find Himegari's dispatch first. Best anchor: search process memory for a known script's opening opcode sequence (you have every script decoded), set a **hardware read breakpoint / `MemoryAccessMonitor` guard page** on its first opcode dword; when the VM fetches it, the instruction pointer is inside the dispatch fetch. Alternate anchors: breakpoint a winmm/DirectSound call and trigger `play-voice` (0xC4), then walk the stack back; or pattern-scan for the bounds-check + `call [table + opcode*4]`. **Payoff:** read the jump-table base → you get the handler address for all 548 opcodes in Himegari at once. +- [ ] **3.1b — Instrument + correlate.** `Interceptor.attach` the dispatch (or a specific handler); log opcode + operand `(type,value)` pairs (read from the bytecode pointer — layout known) + effect. Three correlation techniques: **API** (hook a basket of D3D9/winmm/user32/file APIs; see which an unknown handler calls), **behavioral** (trigger one in-game action, diff the opcode trace vs. baseline to attribute ops to subsystems), **memory** (log which global/local var-bank slots — sized by header F0–F5 — the handler reads/writes). +- **Setup / gotchas:** `pip install frida-tools`; **attach to the already-running game** (`frida AGE.EXE`) after the title screen rather than spawning — this skips the packer's startup anti-debug. 32-bit x86 target. Japanese locale required to run. Eushully's protector *may* detect Frida's injected thread; if it trips, quiet it (ScyllaHide-style hooks or `frida-gadget`). + +### 3.2 — Unicorn: microexecution *(complement for computational opcodes)* + +Unicorn is a bare CPU emulator (no OS). It is the **better** tool for the *pure-computation* handlers — arithmetic/bit/string/array helpers and especially the `CALC*` damage/stat formulas — where you want the *exact* operation, not a label. It is **blind** to effectful handlers: the instant one calls D3D9/winmm/file APIs it runs into unmapped code and stubbing tells you nothing (the effect *is* the meaning). Do not use it as a Frida replacement. + +- [ ] **3.2a — Microexecute a handler.** Map the handler's code + a synthetic VM state (variable bank + operand), run from entry to `ret`, read back what changed; sweep inputs to recover the formula deterministically, offline. +- [ ] **3.2b — Preferred combo: Frida-snapshot → Unicorn-replay.** Use Frida (3.1a) to find handler addresses and dump the relevant memory (code + var banks + globals) at a known-good moment (e.g. mid-battle); load that snapshot into Unicorn and microexecute individual handlers with input sweeps. Gets Frida's context-setup for free + Unicorn's determinism. **Caveat:** microexecution only recovers behavior that's a pure function of the captured state — if a handler reads a global you didn't snapshot, results are wrong silently. Fine for pure ops; a rabbit hole for stateful ones (leave those to live Frida). Needs a decrypted image to feed (a dump, or bytes pulled via Frida) since `AGE.EXE` is packed. + +### 3.3 — Cross-reference siblings *(free, do alongside 3.0)* +- [ ] Kelebek's labels come from a later AGE title; marcussacana/EushullyEditor targets *Kamidori* (same SYS4 era). Diff their handler notes for the specific opcodes you need. + +### 3.4 — Static unpack + Ghidra *(last resort)* +- [ ] Only if the above stall. See appendix — dump the decrypted image, load in Ghidra, read the handler at its address. High effort; reserve for genuinely opaque ops that Frida/Unicorn can't pin down. + +## Phase 4 — Godot re-implementation + +**Deliverable:** the AGE VM running Himegari scripts in Godot. + +> **Prerequisite:** don't start Phase 4 against a blank opcode set — the top ~20 unnamed ops +> (Phase 3 preamble) are hit in the first few instructions of any script. Do that thin +> naming slice first, or bring-up stalls immediately on `0x1f4`/`0x71`/etc. + +- [ ] **4.0 — Stand up the validation harness *first* (before writing VM opcodes).** The + strongest correctness oracle already exists in `build/`: for a given scene script, the VM's + emitted `show-text` sequence must match that file's lines in `build/text/dialogue.jsonl`. + Wire this as an automated diff (drive one `SC####` script → collect show-text → compare to + the 30k-line corpus filtered by `file`). This turns "is the VM right?" into a per-scene + regression test and catches control-flow/branch bugs (wrong jcc → wrong dialogue order) + early. Extend later to assert extracted-table reads (SKINIT/ITINIT/EBINIT JSON) once + data-driven opcodes come online. +- [ ] **4.1 — Re-implement the VM** (GDScript/C#): a dword-fetch loop, the core opcodes (arithmetic, comparisons, `jmp`/`call`/`jcc`, `mov`, string ops) **plus the high-frequency unnamed ops named in Phase 3**, the global/local variable banks (sized by the header's F0–F5 counts), and the ADV layer (`show-text`/`end-text-line`/`wait-for-input`/`set-font`/`play-voice`/`draw-*`). Note the named set is only ~73% of instruction volume — budget for the unnamed remainder. The bytecode-heavy design (damage calc, dungeon loop, battle flow are all scripts) makes re-implementation the right call over transpilation. +- [ ] **4.2 — Fill opcodes on demand** from Phase 3's genuine long tail as scripts exercise them (the top ~20 are already done as a Phase 3/4 prerequisite). +- [ ] **4.3 — Deferred:** save-file format (reverse `SAVE.BIN` only if the port must read existing saves). + +--- + +## Appendix — `AGE.EXE` is packed (relevant to Phase 3.4, and as the image source for 3.2) + +Verified this session (`tools/pack_check.py`): 32-bit PE, code sections at max entropy (8.00), blank section names, IAT RVA 0, no plaintext anchors. `SYS4AB.BIN` (magic `S4AB`, entropy 7.94) is a second encrypted engine image whose header stores `AGE.EXE`'s exact size (`0x0010E000`) — likely the patched VM the loader maps. + +Static analysis therefore requires a **runtime dump first** — you're dumping for *analysis* not redistribution, so don't chase OEP: launch to the title screen (Japanese locale required), then dump the decrypted image and load it in Ghidra. Tools: **PE-sieve** (CLI, agent-drivable: `pe-sieve.exe /pid /imp 3`) or **x32dbg + Scylla + ScyllaHide** (GUI, handles the anti-debug). Validate the dump by confirming `SYS4422`/`.BIN`/`DATA1` now appear in plaintext. **But prefer Frida dynamic hooking (Phase 3.1) — it avoids the unpack entirely.** diff --git a/tools/age_opcodes.py b/tools/age_opcodes.py new file mode 100644 index 0000000..d3df6fa --- /dev/null +++ b/tools/age_opcodes.py @@ -0,0 +1,597 @@ +# DO NOT EDIT -- generated from vm-map/opcodes.toml by tools/opcodes_build.py --build + +"""AGE opcode framing and operand metadata (generated canonical view).""" +from __future__ import annotations + +# opcode -> (historical/canonical label, argument count) +OPCODES: dict[int, tuple[str, int]] = { + 0x0001: ('throw-exit-request', 0), + 0x0002: ('exit', 0), + 0x0003: ('call-script', 1), + 0x0004: ('u00417E30', 2), + 0x0005: ('ret', 0), + 0x0006: ('preload-script-slot', 2), + 0x0007: ('u00417F90', 1), + 0x0008: ('call-preloaded-script-slot', 1), + 0x0009: ('exit-script', 0), + 0x000a: ('u00424170', 2), + 0x000b: ('u00418090', 11), + 0x000c: ('u004149E0', 0), + 0x000d: ('u004181A0', 4), + 0x000e: ('u00418200', 12), + 0x000f: ('u00418300', 1), + 0x0010: ('u00414A00', 4), + 0x0011: ('u00418330', 9), + 0x0012: ('u004183F0', 1), + 0x0013: ('u00418420', 4), + 0x0014: ('u00414A20', 0), + 0x0015: ('u00418490', 5), + 0x0016: ('u00418520', 2), + 0x0017: ('u00418560', 2), + 0x001e: ('u004185B0', 8), + 0x001f: ('u00418690', 12), + 0x0020: ('u004187C0', 6), + 0x0021: ('u00418860', 2), + 0x0022: ('u00418920', 2), + 0x0023: ('u004189D0', 2), + 0x0024: ('u00418A90', 2), + 0x0025: ('u00418B40', 3), + 0x0026: ('u00418C00', 4), + 0x0027: ('u00418CC0', 4), + 0x0028: ('u00418D90', 4), + 0x002a: ('u00418E60', 4), + 0x002b: ('u00418F30', 5), + 0x002c: ('u00419010', 5), + 0x002d: ('u004190A0', 12), + 0x002e: ('u004194B0', 5), + 0x002f: ('u004195A0', 4), + 0x0030: ('u00419670', 5), + 0x0031: ('u00419750', 4), + 0x0032: ('u004197C0', 10), + 0x0033: ('u00419900', 6), + 0x0034: ('u004199C0', 12), + 0x0035: ('u00419AF0', 11), + 0x0036: ('u00419C00', 3), + 0x0037: ('u00419C90', 11), + 0x0038: ('u00419DA0', 12), + 0x0050: ('add', 3), + 0x0051: ('sub', 3), + 0x0052: ('mul', 3), + 0x0053: ('div', 3), + 0x0054: ('mod', 3), + 0x0055: ('mov', 2), + 0x0056: ('and', 3), + 0x0057: ('or', 3), + 0x0058: ('sar', 3), + 0x0059: ('shl', 3), + 0x005a: ('eq', 3), + 0x005b: ('ne', 3), + 0x005c: ('lt', 3), + 0x005d: ('lte', 3), + 0x005e: ('gr', 3), + 0x005f: ('gre', 3), + 0x0060: ('u0041A270', 2), + 0x0061: ('lookup-array', 3), + 0x0062: ('u0041A360', 3), + 0x0063: ('take-address', 2), + 0x0064: ('copy-inline-int-array', 2), + 0x0065: ('u00414AA0', 2), + 0x0066: ('u00414AE0', 3), + 0x0067: ('u00414B20', 3), + 0x0068: ('u00414B60', 3), + 0x0069: ('u00414BA0', 3), + 0x006a: ('u00414BE0', 3), + 0x006b: ('u00414C20', 3), + 0x006c: ('copy-to-global', 2), + 0x006d: ('u00416960', 0), + 0x006e: ('show-text', 2), + 0x006f: ('end-text-line', 1), + 0x0070: ('define-adv-text-layout', 5), + 0x0071: ('reset-adv-text-layout', 1), + 0x0072: ('wait-for-input', 1), + 0x0073: ('configure-adv-wait-indicator', 10), + 0x0074: ('u0041AC00', 1), + 0x0075: ('set-font-size', 1), + 0x0076: ('set-text-color', 1), + 0x0077: ('set-text-effect-color', 1), + 0x0078: ('set-text-render-mode', 1), + 0x0079: ('set-adv-text-reset-cursor', 3), + 0x007a: ('set-adv-text-cursor', 3), + 0x007b: ('u0041ADB0', 2), + 0x007c: ('u00416A90', 0), + 0x007d: ('u0041AE00', 2), + 0x007e: ('u0041AEA0', 1), + 0x007f: ('u00414C60', 1), + 0x0080: ('u0041AF00', 1), + 0x0081: ('u0041AF30', 1), + 0x0082: ('u0041AF80', 5), + 0x0083: ('u00414C90', 3), + 0x0084: ('u0041AFE0', 1), + 0x0085: ('clear-text-history', 0), + 0x0086: ('u0041B210', 1), + 0x0087: ('u00414D10', 0), + 0x0088: ('u0041B290', 1), + 0x0089: ('u0041B2E0', 4), + 0x008a: ('u0041B330', 6), + 0x008b: ('u0041B3D0', 1), + 0x008c: ('jmp', 1), + 0x008d: ('u0041BCE0', 2), + 0x008e: ('u0041BD60', 1), + 0x008f: ('call', 1), + 0x0090: ('u0041BEB0', 7), + 0x0091: ('u0041BFB0', 1), + 0x0092: ('u0041C030', 2), + 0x0093: ('u00415040', 0), + 0x0094: ('u00415090', 0), + 0x0095: ('u0041C0C0', 2), + 0x0096: ('u004150C0', 0), + 0x0097: ('u0041C150', 5), + 0x00a0: ('jcc', 3), + 0x00a1: ('begin-value-switch', 0), + 0x00a2: ('add-value-switch-case', 2), + 0x00a3: ('value-switch-jump', 2), + 0x00aa: ('u0041C270', 2), + 0x00ab: ('u0041C330', 2), + 0x00ac: ('u0041C3E0', 9), + 0x00ad: ('u00415110', 0), + 0x00ae: ('continue-save-load-stack-restore', 0), + 0x00af: ('u00415480', 0), + 0x00b0: ('u0041C530', 1), + 0x00b1: ('u0041C560', 1), + 0x00b2: ('u0041C590', 2), + 0x00b3: ('u004154B0', 0), + 0x00b4: ('play-sound-effect', 2), + 0x00b5: ('u0041D050', 1), + 0x00b6: ('u0041D080', 1), + 0x00b7: ('restart-bgm-loop', 1), + 0x00b8: ('stop-bgm', 0), + 0x00b9: ('restart-bgm-once', 1), + 0x00ba: ('sfx-start-loop', 1), + 0x00bb: ('u0041D250', 1), + 0x00bc: ('u0041D280', 1), + 0x00bd: ('u00415570', 1), + 0x00be: ('u004155E0', 1), + 0x00bf: ('play-bgm', 1), + 0x00c0: ('get-current-bgm-track', 1), + 0x00c1: ('u00415650', 0), + 0x00c2: ('u0041D2B0', 2), + 0x00c3: ('u0041D390', 1), + 0x00c4: ('play-voice', 1), + 0x00c5: ('get-audio-volume', 2), + 0x00c6: ('set-audio-volume', 2), + 0x00c7: ('get-audio-route-enabled', 2), + 0x00c8: ('sleep', 1), + 0x00c9: ('u00415770', 0), + 0x00ca: ('u004157A0', 0), + 0x00cb: ('u00415800', 1), + 0x00cc: ('mouse_callback', 2), + 0x00cd: ('get-input-type', 0), + 0x00ce: ('u0041E0B0', 3), + 0x00cf: ('u00416D40', 0), + 0x00d0: ('get-monotonic-time-ms', 1), + 0x00d1: ('u00415860', 0), + 0x00d2: ('u0041E110', 1), + 0x00d3: ('u00425960', 0), + 0x00d4: ('u004266F0', 4), + 0x00d5: ('u004262C0', 1), + 0x00d6: ('u004267D0', 6), + 0x00d7: ('u0041E1A0', 1), + 0x00d8: ('u0041E150', 2), + 0x00d9: ('u00415880', 0), + 0x00da: ('u004158B0', 6), + 0x00fa: ('u00415940', 0), + 0x00fb: ('joy_callback', 2), + 0x00fc: ('u004159F0', 0), + 0x00fd: ('u0041E2D0', 2), + 0x00fe: ('u0041E360', 1), + 0x00ff: ('u00415A10', 0), + 0x0100: ('u00415A60', 0), + 0x0101: ('u00415BF0', 0), + 0x0102: ('u0041E3C0', 3), + 0x0103: ('u0041E4A0', 1), + 0x0104: ('u00415C50', 0), + 0x0105: ('u0041E4D0', 1), + 0x0106: ('u00415E40', 1), + 0x0107: ('u0041E500', 2), + 0x0108: ('u00415E70', 1), + 0x0109: ('u00415EC0', 2), + 0x010a: ('u0041E540', 2), + 0x010b: ('u0041E5A0', 2), + 0x010c: ('u0041E5E0', 2), + 0x010d: ('u00415F10', 1), + 0x010e: ('u0041E650', 2), + 0x010f: ('u0041E690', 1), + 0x012c: ('lookup-array-2d', 5), + 0x012d: ('u0041E720', 7), + 0x012e: ('find-hit-rectangle', 8), + 0x012f: ('sort-indices-by-key-sum', 4), + 0x0130: ('get-initial-root-run', 1), + 0x0131: ('get-message-window-alpha', 1), + 0x0132: ('reset-int-queue', 1), + 0x0133: ('enqueue-int', 2), + 0x0134: ('try-dequeue-int', 3), + 0x0135: ('bit-set', 2), + 0x0136: ('bit-reset', 2), + 0x0137: ('reset-int-stack', 1), + 0x0138: ('push-int-stack', 2), + 0x0139: ('try-pop-int-stack', 3), + 0x013a: ('u0041F3A0', 6), + 0x013b: ('u0041F440', 7), + 0x013c: ('u0041F7E0', 1), + 0x013d: ('u0041F840', 3), + 0x013e: ('u0041F8D0', 2), + 0x013f: ('check-bit', 3), + 0x0140: ('u0041F9C0', 4), + 0x0141: ('set-message-window-alpha', 1), + 0x0142: ('set-system-menu-enabled', 1), + 0x0143: ('u00415FB0', 0), + 0x0144: ('edit-fullwidth-string-dialog', 2), + 0x0145: ('u00416040', 1), + 0x0146: ('u0041FB40', 1), + 0x0147: ('u0041FB80', 6), + 0x0148: ('get-system-menu-show-delay', 1), + 0x0149: ('set-system-menu-show-delay', 1), + 0x014a: ('u0041FD10', 7), + 0x014b: ('u0041FF50', 1), + 0x014c: ('set-agerc-export', 2), + 0x014d: ('call-agerc-export', 6), + 0x0190: ('u0041C5E0', 2), + 0x0191: ('absolute-value', 2), + 0x0192: ('set-string', 2), + 0x0193: ('concat', 3), + 0x0194: ('string-equals', 3), + 0x0195: ('string-not-equals', 3), + 0x0196: ('display-furigana', 3), + 0x0197: ('set-ruby-font-size', 1), + 0x0198: ('set-adv-text-layout-origin', 3), + 0x0199: ('u00414D50', 0), + 0x019a: ('u00414E50', 1), + 0x019b: ('u00414E80', 0), + 0x019c: ('u00414EC0', 0), + 0x019d: ('is-catalog-resource-unlocked', 2), + 0x019e: ('save-numbered-slot', 2), + 0x019f: ('load-numbered-slot-data-only', 2), + 0x01a0: ('query-numbered-save-metadata', 9), + 0x01a1: ('load-numbered-slot-and-resume', 2), + 0x01a2: ('store-shared-profile-int', 1), + 0x01a3: ('load-shared-profile-int', 1), + 0x01a4: ('set-text-effect-offset', 2), + 0x01a5: ('set-font', 1), + 0x01a6: ('halve-strlen', 2), + 0x01a7: ('comment', 1), + 0x01a8: ('dev_ukn', 0), + 0x01a9: ('store-shared-profile-string', 1), + 0x01aa: ('load-shared-profile-string', 1), + 0x01ab: ('delete-numbered-save', 2), + 0x01ac: ('copy-numbered-save', 3), + 0x01ad: ('mark-save-resume-frame', 0), + 0x01ae: ('write-numbered-save-thumbnail', 3), + 0x01af: ('load-numbered-save-thumbnail', 3), + 0x01b0: ('copy-dwords', 3), + 0x01b1: ('u0041B5C0', 1), + 0x01b2: ('u00425790', 1), + 0x01b3: ('u004257D0', 0), + 0x01b4: ('u004237C0', 0), + 0x01b5: ('u0041B5F0', 1), + 0x01b6: ('u00414F60', 1), + 0x01b7: ('u0041B640', 1), + 0x01b8: ('u0041B670', 2), + 0x01b9: ('u0041B710', 2), + 0x01ba: ('set-audio-route-enabled', 2), + 0x01bb: ('set-text-history-recording', 1), + 0x01bc: ('u00415670', 0), + 0x01bd: ('u0041D910', 1), + 0x01be: ('u0041D9D0', 2), + 0x01bf: ('u004156C0', 0), + 0x01c0: ('u0041DB70', 1), + 0x01c1: ('set-adv-text-bounds', 3), + 0x01c2: ('u0041B860', 2), + 0x01c3: ('u0041B8A0', 2), + 0x01c4: ('u00415720', 1), + 0x01c5: ('u00425800', 4), + 0x01c6: ('u0041DD80', 2), + 0x01c7: ('get-message-skip', 1), + 0x01c8: ('toString', 2), + 0x01c9: ('u0041B8E0', 3), + 0x01ca: ('u0041B9B0', 1), + 0x01cb: ('u00414FD0', 1), + 0x01cc: ('get-adv-read-skip-state', 1), + 0x01cd: ('u0041A560', 2), + 0x01ce: ('u0041B9F0', 1), + 0x01cf: ('set-voice-bgm-duck-control', 1), + 0x01d0: ('step-text-history', 3), + 0x01d1: ('u0041BAE0', 5), + 0x01d2: ('append-text-history-metadata', 2), + 0x01d3: ('find-text-history-value', 5), + 0x01d4: ('find-text-history-pair', 4), + 0x01d5: ('u00415700', 0), + 0x01d6: ('u0041DA40', 2), + 0x01d7: ('u0041DA80', 2), + 0x01d8: ('u0041DAD0', 3), + 0x01d9: ('u0041DB20', 2), + 0x01f4: ('u004160D0', 0), + 0x01f5: ('u00416120', 0), + 0x01f6: ('clear-retained-gfx-objects', 0), + 0x01f7: ('gfx-elem-erase', 2), + 0x01f8: ('create-texture', 4), + 0x01f9: ('set-texture', 3), + 0x01fa: ('gfx-elem-release', 1), + 0x01fb: ('draw-texture', 8), + 0x01fc: ('u004205F0', 1), + 0x01fd: ('u00420620', 4), + 0x01fe: ('set-current-rotation-axis-angle', 5), + 0x01ff: ('set-gfx-geom3-c', 4), + 0x0200: ('u00420800', 1), + 0x0201: ('u00416190', 1), + 0x0202: ('gfx-blit-color', 5), + 0x0203: ('gfx-draw-color', 4), + 0x0204: ('draw-string', 4), + 0x0205: ('u00420A60', 6), + 0x0206: ('u004161C0', 7), + 0x0207: ('copy-surface-rect', 8), + 0x0208: ('get-texture-size', 3), + 0x0209: ('u00420C50', 5), + 0x020a: ('u00420CE0', 1), + 0x020b: ('fill-surface-rect', 7), + 0x020c: ('present-frame', 0), + 0x020d: ('select-render-target', 1), + 0x020e: ('clear-render-target', 0), + 0x020f: ('play-modal-movie-to-surface', 3), + 0x0210: ('u00420FF0', 1), + 0x0211: ('u00421060', 1), + 0x0212: ('set-adv-wait-indicator-handle', 2), + 0x0213: ('set-adv-text-object-range', 3), + 0x0214: ('u00421120', 2), + 0x0215: ('query-gfx-object?', 2), + 0x0216: ('query-gfx-field?', 2), + 0x0217: ('set-gfx-geom3', 4), + 0x0218: ('get-gfx-geom3?', 4), + 0x0219: ('set-gfx-geom3-b', 4), + 0x021a: ('get-gfx-geom3-b?', 4), + 0x021b: ('u004213E0', 1), + 0x021c: ('mark-frame-yield', 0), + 0x021d: ('clone-gfx-object', 2), + 0x021e: ('set-anim-transform-norm', 6), + 0x021f: ('set-anim-rotation-axis-angle', 7), + 0x0220: ('set-anim-transform-abs', 6), + 0x0221: ('u00421670', 4), + 0x0222: ('present-gfx-object-range', 2), + 0x0223: ('queue-surface-alpha-transition', 8), + 0x0224: ('clear-gfx-command-queue', 0), + 0x0225: ('u00421780', 2), + 0x0226: ('u004217D0', 5), + 0x0227: ('u00421880', 6), + 0x0228: ('u00421940', 5), + 0x0229: ('set-gfx-range-transform', 5), + 0x022a: ('set-gfx-range-scale-current', 3), + 0x022b: ('u00421B30', 4), + 0x022c: ('set-gfx-range-translation-current', 3), + 0x022d: ('set-gfx-range-scale-target', 5), + 0x022e: ('u00421D10', 6), + 0x022f: ('u00421DD0', 5), + 0x0230: ('u00421E70', 1), + 0x0231: ('u00421EA0', 4), + 0x0232: ('u00421EF0', 4), + 0x0233: ('set-scale-cycle', 5), + 0x0234: ('anim-start', 5), + 0x0235: ('u00422100', 5), + 0x0236: ('play-movie-to-surface', 4), + 0x0237: ('u00422350', 2), + 0x0238: ('set-anim-clock', 1), + 0x0239: ('u004223C0', 6), + 0x023a: ('query-movie-surface-active', 2), + 0x023b: ('u00422460', 7), + 0x023c: ('sample-frame-time', 0), + 0x023d: ('release-transient-surfaces', 0), + 0x023e: ('u004228C0', 2), + 0x023f: ('u00422930', 2), + 0x0240: ('u004229A0', 4), + 0x0241: ('play-movie-to-surface-at-position', 5), + 0x0242: ('set-object-animation-detached', 2), + 0x0243: ('reset-anim-clock', 0), + 0x0244: ('u00416360', 0), + 0x0245: ('u00422DA0', 2), + 0x0246: ('u00422E10', 2), + 0x0247: ('u00416390', 1), + 0x0248: ('set-tiled-surface-edge-length', 1), + 0x0249: ('load-raw-texture-surface', 3), + 0x024a: ('u004163C0', 3), + 0x024d: ('play-movie-mask-transition', 12), + 0x024e: ('set-gfx-animation-service-flags', 1), + 0x024f: ('u00422ED0', 10), + 0x0250: ('u00422F60', 10), + 0x0251: ('u00422FF0', 12), + 0x0252: ('u00423000', 1), + 0x0253: ('u00423019', 2), + 0x0254: ('u00423049', 5), + 0x0256: ('u00423050', 5), + 0x0257: ('257', 5), + 0x0258: ('set-surface-persistence-flags', 2), + 0x0259: ('script-entry', 0), + 0x025a: ('u00423120', 1), + 0x025b: ('25B', 1), + 0x025c: ('u00423122', 8), + 0x025d: ('u00423123', 3), + 0x025e: ('u00423124', 5), + 0x025f: ('u00423125', 4), + 0x0260: ('u00423126', 4), + 0x0261: ('u00423127', 1), + 0x0262: ('262', 1), + 0x0263: ('263', 1), + 0x0264: ('264', 5), + 0x02bc: ('u00423020', 11), + 0x02bd: ('set-font-bold', 1), + 0x02be: ('u00423140', 1), + 0x02bf: ('schedule-sfx-start', 3), + 0x02c0: ('schedule-voice-playback', 3), + 0x02c1: ('u00425BC0', 1), + 0x02c2: ('u00425CD0', 6), + 0x02c3: ('u00423200', 2), + 0x02c4: ('u00416450', 0), + 0x02c5: ('strlen', 2), + 0x02c6: ('cp932-character-length', 2), + 0x02c7: ('u0042B5F0', 4), + 0x02c8: ('cp932-substring', 4), + 0x02c9: ('2C9', 3), + 0x02cc: ('2CC', 1), + 0x02cd: ('2CD', 1), + 0x02ce: ('u0042B616', 1), + 0x02cf: ('u0042B617', 1), + 0x02d0: ('u0042B940', 3), + 0x02d1: ('u0042B950', 3), + 0x02d2: ('u0042B960', 3), + 0x02d3: ('u0042B970', 3), + 0x02d5: ('u0042B990', 2), + 0x02d7: ('u0042B9B0', 2), + 0x02d8: ('set-array-to', 3), + 0x02d9: ('u0042BA30', 2), + 0x02da: ('u004234E0', 8), + 0x02db: ('u004235C0', 1), + 0x02dc: ('u0042BA80', 1), + 0x02dd: ('u0042D880', 2), + 0x02de: ('u0042BAC0', 2), + 0x02df: ('u0042BAC1', 3), + 0x02e0: ('u0042CE0F', 3), + 0x02e1: ('u0042CE10', 3), + 0x02e2: ('u0042CE11', 3), + 0x02e3: ('u0042CE30', 3), + 0x02e4: ('u0042CE31', 3), + 0x02e5: ('u0042CE50', 1), + 0x02e6: ('u0042CE60', 2), + 0x02e7: ('u0042CE70', 2), + 0x02e8: ('u0042CE80', 1), + 0x02e9: ('u0042CE90', 1), + 0x02ea: ('u0042CEA0', 1), + 0x02eb: ('u0042CEB0', 1), + 0x02ec: ('u0042CEC0', 2), + 0x02ee: ('u0042CEC2', 1), + 0x02ef: ('u0042CEC3', 11), + 0x02f0: ('u0042CEC4', 9), + 0x02f1: ('u0042CEC5', 7), + 0x02f2: ('u0042CEC6', 6), + 0x02f3: ('2F3', 6), + 0x02f4: ('2F4', 3), + 0x02f5: ('2F5', 4), + 0x02f6: ('2F6', 1), + 0x02f7: ('2F7', 1), + 0x02f8: ('2F8', 2), + 0x02f9: ('2F9', 7), + 0x02fa: ('2FA', 1), + 0x02fb: ('2FB', 1), + 0x02fc: ('2FC', 5), + 0x02fd: ('2FD', 6), + 0x02fe: ('2FE', 1), + 0x02ff: ('2FF', 2), + 0x0300: ('300', 3), + 0x0301: ('301', 1), + 0x0302: ('302', 2), + 0x0303: ('303', 3), + 0x0304: ('304', 0), + 0x0305: ('305', 0), + 0x0306: ('306', 1), + 0x0307: ('307', 1), + 0x0308: ('308', 1), + 0x030a: ('30A', 2), + 0x030c: ('30C', 1), + 0x0320: ('u0043AA20', 10), + 0x0321: ('u0043AA30', 3), + 0x0322: ('u0043AA40', 4), + 0x0323: ('u0043AA50', 5), + 0x0324: ('u0043AA60', 0), + 0x0325: ('u0043AA70', 2), + 0x0326: ('u0043AA80', 4), + 0x0327: ('u0043AA90', 1), + 0x0328: ('u0043AAA0', 3), + 0x0329: ('u0043AAB0', 2), + 0x032a: ('32A', 1), + 0x032b: ('u0043AAD0', 0), + 0x032c: ('u0043AAE0', 6), + 0x032d: ('u0043AAF0', 2), + 0x032e: ('u0043AB10', 11), + 0x032f: ('u0043AB11', 1), + 0x0330: ('u0043AB12', 2), + 0x0332: ('u0043AB14', 4), + 0x0334: ('u0043AB16', 1), + 0x0335: ('u0043AB17', 4), + 0x0337: ('u0043AB19', 4), + 0x033b: ('u0043AB1D', 4), + 0x033d: ('u0043AB1E', 3), + 0x033e: ('u0043AB1F', 5), + 0x033f: ('u0043AB20', 3), + 0x0340: ('340', 1), + 0x0341: ('341', 2), + 0x0342: ('342', 1), + 0x0344: ('344', 2), + 0x0345: ('345', 3), + 0x0349: ('349', 4), + 0x034d: ('34D', 6), + 0x034e: ('34E', 4), + 0x0352: ('352', 3), + 0x0353: ('353', 2), + 0x0354: ('354', 2), + 0x0358: ('358', 5), + 0x035a: ('35A', 5), + 0x035b: ('35B', 2), + 0x035c: ('35C', 2), + 0x035d: ('35D', 3), + 0x035f: ('35F', 3), + 0x0360: ('360', 3), + 0x0361: ('361', 2), + 0x0363: ('363', 3), + 0x0364: ('364', 3), + 0x0384: ('384', 3), + 0x0386: ('386', 11), + 0x0387: ('387', 8), + 0x0388: ('388', 3), + 0x0389: ('389', 6), + 0x038f: ('38F', 6), + 0x0390: ('390', 7), + 0x0391: ('391', 2), + 0x0392: ('392', 1), + 0x0393: ('393', 6), + 0x0396: ('396', 5), + 0x0398: ('398', 3), + 0x0399: ('399', 7), + 0x039b: ('39B', 5), +} + +# argument type tag -> disassembly label +ARG_TYPES: dict[int, str] = { + 0x0000: 'imm', + 0x0001: 'float', + 0x0002: 'string', + 0x0003: 'global-int', + 0x0004: 'global-float', + 0x0005: 'global-string', + 0x0006: 'global-ptr', + 0x0008: 'global-string-ptr', + 0x0009: 'local-int', + 0x000a: 'local-float', + 0x000b: 'local-string', + 0x000c: 'local-ptr', + 0x000d: 'local-float-ptr', + 0x000e: 'local-string-ptr', + 0x8003: 'type-0x8003', + 0x8005: 'type-0x8005', + 0x8009: 'type-0x8009', + 0x800b: 'type-0x800B', +} + +# One-based operand indices whose raw values are code offsets. +CODE_TARGET_ARGS: dict[int, frozenset[int]] = { + 0x007b: frozenset((1, 2)), + 0x008c: frozenset((1,)), + 0x008f: frozenset((1,)), + 0x0090: frozenset((5, 6, 7)), + 0x00a0: frozenset((2, 3)), + 0x00cc: frozenset((2,)), + 0x00d4: frozenset((3, 4)), + 0x00fb: frozenset((2,)), +} +CONTROL_FLOW = frozenset(CODE_TARGET_ARGS) +ARRAY_OPCODE = 0x64 + +def is_label_argument(op: int, arg_index: int, raw_value: int) -> bool: + """Return whether a zero-based operand is a non-fallthrough code target.""" + return (raw_value != 0xFFFFFFFF + and arg_index + 1 in CODE_TARGET_ARGS.get(op, ())) diff --git a/tools/age_opcodes_himegari.py b/tools/age_opcodes_himegari.py new file mode 100644 index 0000000..4b2818d --- /dev/null +++ b/tools/age_opcodes_himegari.py @@ -0,0 +1,98 @@ +"""Himegari-specific opcode INFERENCE layer (sits on top of age_opcodes.py). + +`age_opcodes.py` is the verbatim Kelebek table (opcode number + argc, validated 481/481). +Many opcodes have only engine-address labels (`u004xxxx`). This module records what the +top-frequency unnamed opcodes *mean*, inferred from operand types, disassembly context, +and neighbouring named ops (see `vm-map/himegari-opcode-notes.md` for the evidence). + +These are INFERENCES, not ground truth. Fields: + name short mnemonic for the disassembler + category marker | structural | computational | draw | audio | adv | control + noop True = safe for the Godot VM v1 to skip (no state/visible effect expected) + False = has an effect; must be implemented (or knowingly stubbed) + confidence high | med | low + method how the inference was reached / how to confirm: + structure | context | harness (confirm via dialogue diff in Phase 4) | + frida (needs live capture) | unicorn (micro-exec) + note one-line rationale / caveat + +Consumed by `sys4load.render_listing` (nicer disassembly) and, later, the Godot VM. +""" +from __future__ import annotations + +INFERRED: dict[int, dict] = { + # ---- statement / scope scaffolding: zero-arg, no operands, bracket statements ---- + 0x1f4: dict(name="stmt-begin", category="marker", noop=True, confidence="high", + method="structure", note="zero-arg; opens scripts, pairs with stmt-end 0x1f5"), + 0x1f5: dict(name="stmt-end", category="marker", noop=True, confidence="high", + method="structure", note="zero-arg; precedes exit/next-stmt, pairs with 0x1f4"), + 0x1d5: dict(name="cond-block", category="marker", noop=True, confidence="high", + method="context", note="zero-arg; ALWAYS follows jcc — marks conditional body entry"), + 0x1bc: dict(name="block-mark", category="marker", noop=True, confidence="high", + method="context", note="zero-arg; follows jcc/mov, precedes mov/ret — block boundary"), + 0x1bf: dict(name="call-end", category="marker", noop=True, confidence="med", + method="context", note="zero-arg; call->0x1bf->stmt-end — end-of-call-statement marker"), + + # ---- statement metadata carrying an id (tentative no-op, confirm via harness) ---- + 0x21b: dict(name="line-id?", category="marker", noop=True, confidence="med", + method="harness", note="1 imm; mov->0x21b->stmt-end; near save/load-messkip — likely line/stmt id, verify not msg-control"), + 0x1d2: dict(name="stmt-desc?", category="marker", noop=True, confidence="med", + method="harness", note="2 imm; immediately after stmt-begin 0x1f4 — statement descriptor?"), + 0x258: dict(name="decl?", category="marker", noop=True, confidence="low", + method="harness", note="2 imm; runs in a chain right after script-entry 0x259, enumerating ids — prologue declaration/registration?"), + + # ---- structural ---- + 0x71: dict(name="label-def", category="structural", noop=True, confidence="high", + method="structure", note="1 imm; count == T1 table size -> the label/anchor T1 indexes. v1 no-op; revisit if menu/callback dispatch looks up by id"), + + # ---- input / UI hotspot (u0041Bxxx/Cxxx widget module; see himegari-opcode-notes.md) ---- + 0x90: dict(name="hotspot-branch", category="input", noop=False, confidence="high", + method="structure", note="argc7: x y w h + 3 code targets (0xffffffff=unused). " + "Cursor/input hotspot hit-test; branches per interaction (inferred hover-enter->flag=1 / " + "hover-leave->flag=0 / click->run-action), FALLS THROUGH to pc+1 on no-match " + "(design-confirmed: enc.len 15 lands on the next stmt). HEADLESS: fall through = correct " + "no-input behaviour (proven safe by 279 CLEAN scenes). Occurs ONLY in a shared ADV-chrome " + "subroutine, identical in all 301 ADV scripts (8/script = 5 imm-rect buttons @y=572 " + "x=684..772 20x20 toggling G[0x6c9..0x6cd] + 3 local-operand keyed forms). Model live in A2; " + "confirm target->state mapping via input capture/Frida."), + 0x97: dict(name="hotspot-reg?", category="input", noop=False, confidence="med", + method="frida", note="argc5: v1 v2 1 1 ; NO code targets. Interleaves with " + "0x90 in the ADV-chrome subroutine -> companion register-hotspot / set-widget-action " + "(trailing imm = action id 0x0/0x7/0x8). u0041C150, same widget cluster as 0x90/0x91/0x92/0x95."), + + # ---- computational (exact behaviour via Unicorn micro-exec) ---- + 0x215: dict(name="count?", category="computational", noop=False, confidence="med", + method="unicorn", note="2 args -> writes global then result tested >0 (gre/lt) — count/search-returns-index helper"), + 0x1a2: dict(name="resolve-handle?", category="computational", noop=False, confidence="low", + method="frida", note="1 local-ptr from lookup-array, then create-texture — resolves a looked-up resource/handle"), + + # ---- ADV / text-display (effectful; confirm via Frida) ---- + 0x7a: dict(name="text-param?", category="adv", noop=False, confidence="med", + method="frida", note="3 args (imm/computed/imm); sub computes a value then 0x7a then show-text — text speed/wait/window param"), + + # ---- draw / UI (0x420-0x421 graphics family; effectful; Frida) ---- + 0x202: dict(name="draw-blit?", category="draw", noop=False, confidence="med", + method="frida", note="5 args (coords/sizes); preceded by coord arithmetic, near draw ops"), + 0x203: dict(name="draw?", category="draw", noop=False, confidence="med", + method="frida", note="4 args; chains with 0x202/draw-texture"), + 0x1f7: dict(name="ui-elem?", category="draw", noop=False, confidence="med", + method="frida", note="2 args; 0x420 family, pairs with 0x1fa — create/begin a UI element"), + 0x1fa: dict(name="ui-clear?", category="draw", noop=False, confidence="med", + method="frida", note="1 arg (element id); follows 0x1f7 — show/hide/clear UI element by id"), + 0x217: dict(name="gfx-geom?", category="draw", noop=False, confidence="low", + method="frida", note="4 global-ints; part of a 0x217/0x218/0x21a geometry chain"), + 0x218: dict(name="gfx-geom?", category="draw", noop=False, confidence="low", + method="frida", note="4 global-ints; chains with 0x21a/0x217"), + 0x21a: dict(name="gfx-geom?", category="draw", noop=False, confidence="low", + method="frida", note="4 global-ints; chains with 0x218/0x217"), + 0x1ff: dict(name="draw?", category="draw", noop=False, confidence="low", + method="frida", note="4 args (global+imms); follows 0x217, then call"), + + # ---- audio (0x41D family, near play-sound-effect 0xb4) ---- + 0xb6: dict(name="snd-ctrl?", category="audio", noop=False, confidence="low", + method="frida", note="1 imm; self-chains, 0x41D family near play-sound-effect/0xb5 — sound channel/volume/stop control"), +} + +# Names ending in "?" are low/medium-confidence guesses; the disassembler shows them +# so listings read better than `u004xxxx`, but the VM must not treat non-noop ops as +# no-ops without Frida/Unicorn/harness confirmation (see `method`). diff --git a/tools/extract_init.py b/tools/extract_init.py new file mode 100644 index 0000000..bb86649 --- /dev/null +++ b/tools/extract_init.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +"""Extract a *INIT data table to JSON. Auto-detects the table's shape. + +*INIT scripts populate parallel global arrays with static game data. Three shapes seen: + + name — records keyed by a name string. Each record: set-string(name), mov(fields..), + set-string(desc). Arrays indexed by record id in lockstep (+1/record). + (SKINIT skills, ITINIT items, EBINIT units) + numeric— column table with NO names: mov/copy-to-global into parallel int arrays, keyed + by an incrementing index column. (CGINIT gallery) + footer — copy-local-array (op 0x64) bulk-loads length-prefixed arrays from the file + footer into per-record global arrays. The data lives in the footer. (MPINIT maps) + +Records are {id, name?, desc?, fields:{"0x": value}} or, for footer tables, +{id, global_addr, footer_off, values:[...]}. Column addresses are raw engine globals; +naming them (attack, cost, …) needs the engine global-var map — later work. + +Usage: py -3.11 -X utf8 tools/extract_init.py [OUTNAME] [--mode name|numeric|footer] +""" +from __future__ import annotations +import json +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import paths +import sys4load + +SET_STRING = 0x192 +MOV = 0x55 +COPY_TO_GLOBAL = 0x6C +COPY_LOCAL_ARRAY = 0x64 +T_GLOBAL_INT = 3 +T_GLOBAL_STRING = 5 +T_IMM = 0 + + +def resolve(name: str) -> Path: + for cand in (paths.GAME_DIR / f"{name}.BIN", paths.DATA1 / f"{name}.BIN"): + if cand.exists(): + return cand + raise SystemExit(f"not found: {name}.BIN") + + +def _val(arg): + """Render an operand as an int (immediate) or a {type,value} ref.""" + t, v = arg + return v if t == T_IMM else {"type": f"0x{t:x}", "value": f"0x{v:x}"} + + +def read_footer_array(scr, off): + """Read a length-prefixed Data_Array at dword `off`: [length][v0..v_{length-1}].""" + dw = scr.dwords + if not (0 <= off < scr.nbody): + return None + length = dw[off] + if length > scr.nbody or off + 1 + length > scr.nbody: + return None + return list(dw[off + 1: off + 1 + length]) + + +def detect_mode(scr): + ops = [ins.opcode for ins in scr.instructions] + has_str = any(ins.opcode == SET_STRING and ins.args and ins.args[0][0] == T_GLOBAL_STRING + for ins in scr.instructions) + if has_str: + return "name" + n_footer = ops.count(COPY_LOCAL_ARRAY) + n_int = ops.count(MOV) + ops.count(COPY_TO_GLOBAL) + return "footer" if n_footer >= max(4, n_int) else "numeric" + + +def extract_name(scr): + name_base = None + for ins in scr.instructions: + if ins.opcode == SET_STRING and ins.args and ins.args[0][0] == T_GLOBAL_STRING: + name_base = ins.args[0][1]; break + records, cur, prev_gstr, desc_slot, desc_bases = [], None, None, 0, {} + for ins in scr.instructions: + if ins.opcode == SET_STRING and ins.args and ins.args[0][0] == T_GLOBAL_STRING: + addr = ins.args[0][1] + txt = scr.strings.get(ins.args[1][1], (None,))[0] if len(ins.args) > 1 else None + if prev_gstr is None or addr < prev_gstr: + cur = {"id": addr - name_base, "name": txt, "fields": {}} + records.append(cur); desc_slot = 0 + elif cur is not None: + key = "desc" if desc_slot == 0 else f"desc{desc_slot}" + cur[key] = txt; desc_bases.setdefault(key, addr - cur["id"]); desc_slot += 1 + prev_gstr = addr + elif ins.opcode == MOV and cur is not None and ins.args and ins.args[0][0] == T_GLOBAL_INT: + cur["fields"][f"0x{ins.args[0][1] - cur['id']:x}"] = _val(ins.args[1]) + return records, {"name_array_base": f"0x{name_base:x}", + "desc_array_bases": {k: f"0x{v:x}" for k, v in sorted(desc_bases.items())}} + + +def _int_writes(scr): + """Ordered (addr, value_arg) for global-int mov / copy-to-global.""" + out = [] + for ins in scr.instructions: + if ins.opcode in (MOV, COPY_TO_GLOBAL) and ins.args and ins.args[0][0] == T_GLOBAL_INT: + out.append((ins.args[0][1], ins.args[1])) + return out + + +def _longest_stride1_column(addrs): + """Pick the primary index array: the stride-1 arithmetic run covering the most records.""" + seen = set(addrs) + best_base, best_len = None, 0 + for a in sorted(seen): + if a - 1 in seen: + continue # only start at a run's base + n = 0 + while a + n in seen: + n += 1 + if n > best_len: + best_base, best_len = a, n + return best_base, best_len + + +def extract_numeric(scr): + writes = _int_writes(scr) + base, n = _longest_stride1_column([a for a, _ in writes]) + if base is None: + return [], {} + primary = set(range(base, base + n)) + records, buf = [], [] + for addr, varg in writes: + buf.append((addr, varg)) + if addr in primary: # primary write closes the record + rid = addr - base + fields = {f"0x{a - rid:x}": _val(v) for a, v in buf} + records.append({"id": rid, "fields": fields}) + buf = [] + return records, {"primary_index_base": f"0x{base:x}", "record_span": n} + + +def extract_footer(scr): + records = [] + for i, ins in enumerate(scr.instructions): + if ins.opcode == COPY_LOCAL_ARRAY and ins.args and ins.args[0][0] == T_GLOBAL_INT: + addr = ins.args[0][1] + foff = ins.args[1][1] + vals = read_footer_array(scr, foff) + records.append({"id": i, "global_addr": f"0x{addr:x}", + "footer_off": f"0x{foff:x}", + "length": len(vals) if vals else 0, + "values": vals if vals else []}) + return records, {} + + +def main() -> int: + argv = [a for a in sys.argv[1:] if not a.startswith("--")] + mode_arg = next((sys.argv[i + 1] for i, a in enumerate(sys.argv) if a == "--mode"), None) + if not argv: + raise SystemExit(__doc__) + name = argv[0].upper().removesuffix(".BIN") + outname = argv[1] if len(argv) > 1 else name + scr = sys4load.load(resolve(name)) + + mode = mode_arg or detect_mode(scr) + extractor = {"name": extract_name, "numeric": extract_numeric, "footer": extract_footer}[mode] + recs, meta = extractor(scr) + + cols = sorted({c for r in recs for c in r.get("fields", {})}, key=lambda h: int(h, 16)) + out = {"table": name, "source": scr.path.name, "magic": scr.magic, "mode": mode, + "record_count": len(recs), **meta, + "field_columns": cols if mode != "footer" else None, "records": recs} + outpath = paths.BUILD / "data" / f"{outname}.json" + outpath.parent.mkdir(parents=True, exist_ok=True) + outpath.write_text(json.dumps(out, ensure_ascii=False, indent=2), encoding="utf-8") + print(f"{name}: mode={mode}, {len(recs)} records" + + (f", {len(cols)} field-columns" if mode != 'footer' else "") + + f" -> build/data/{outname}.json") + for r in recs[:4]: + if mode == "footer": + print(f" id {r['id']:>4} {r['global_addr']} <- footer {r['footer_off']} " + f"len {r['length']} head={r['values'][:8]}") + else: + f4 = {k: r['fields'][k] for k in list(r['fields'])[:4]} + print(f" id {r['id']:>4} {r.get('name','')!r:12} desc={r.get('desc','')!r} {f4}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/extract_phase2.py b/tools/extract_phase2.py new file mode 100644 index 0000000..6f76835 --- /dev/null +++ b/tools/extract_phase2.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""Phase 2 batch extraction: disassembly + text corpora from every SYS4 script. + +Outputs (all under build/, workspace-root relative): + build/disasm/.asm full disassembly listing (one per script) + build/text/.strings.txt all inline strings in that script + build/text/dialogue.jsonl show-text (0x6E) lines only — the translation corpus + build/text/strings.jsonl every inline string, tagged by the opcode that references it + build/manifest.json per-script stats (instructions, strings, dialogue, decode-clean) + +Authoritative copies: a game-dir override shadows its extracted/DATA1 copy. Run: + py -3.11 -X utf8 tools/extract_phase2.py +""" +from __future__ import annotations +import json +import os +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import paths +import sys4load + +DATA1 = paths.DATA1 +BUILD = paths.BUILD +DISASM = BUILD / "disasm" +TEXT = BUILD / "text" + +SHOW_TEXT = 0x6E # dialogue opcode; type-2 arg = displayed line + + +def authoritative_scripts() -> dict[str, Path]: + """name -> path, game-dir overrides winning over extracted/DATA1.""" + return paths.scripts() + + +def opcode_for_string_ref(scr, value_dword_index: int) -> tuple[int | None, str | None]: + """Given the body index of a string's *value* operand, find the owning instruction.""" + for ins in scr.instructions: + base = ins.offset + 1 + for a in range(len(ins.args)): + if base + 2 * a + 1 == value_dword_index: + return ins.opcode, ins.label + return None, None + + +def main() -> int: + for d in (DISASM, TEXT, BUILD / "data", BUILD / "scripts-json"): + d.mkdir(parents=True, exist_ok=True) + + files = authoritative_scripts() + manifest = [] + n_dialogue = n_strings = 0 + skipped = [] + + with (TEXT / "dialogue.jsonl").open("w", encoding="utf-8") as dlg, \ + (TEXT / "strings.jsonl").open("w", encoding="utf-8") as allstr: + for name, p in sorted(files.items()): + try: + scr = sys4load.load(p) + except sys4load.Sys4Error as e: + skipped.append((name, str(e))) + continue + + # 1) disassembly listing + (DISASM / f"{Path(name).stem}.asm").write_text( + sys4load.render_listing(scr), encoding="utf-8") + + # 2) per-script strings file + if scr.strings: + lines = [f"0x{off:05x}\t{txt}" for off, (txt, _) in sorted(scr.strings.items())] + (TEXT / f"{Path(name).stem}.strings.txt").write_text( + "\n".join(lines) + "\n", encoding="utf-8") + + # 3) combined corpora, tagged by referencing opcode + for val_idx, off in sorted(scr.string_refs.items(), key=lambda kv: kv[1]): + txt = scr.strings.get(off, (None,))[0] + if txt is None: + continue + op, label = opcode_for_string_ref(scr, val_idx) + rec = {"file": name, "off": f"0x{off:x}", + "op": f"0x{op:x}" if op is not None else None, + "label": label, "text": txt} + allstr.write(json.dumps(rec, ensure_ascii=False) + "\n") + n_strings += 1 + if op == SHOW_TEXT: + dlg.write(json.dumps({"file": name, "off": f"0x{off:x}", "text": txt}, + ensure_ascii=False) + "\n") + n_dialogue += 1 + + n, unk, trunc, clean = sys4load.decode_stats(scr) + manifest.append({"file": name, "instructions": n, "strings": len(scr.strings), + "clean": clean, "unknown": unk, "truncated": trunc}) + + (BUILD / "manifest.json").write_text( + json.dumps({"scripts": len(manifest), "skipped": skipped, + "total_dialogue_lines": n_dialogue, "total_strings": n_strings, + "per_script": manifest}, ensure_ascii=False, indent=2), encoding="utf-8") + + clean = sum(1 for m in manifest if m["clean"]) + print(f"scripts processed: {len(manifest)} (decode-clean: {clean}/{len(manifest)})") + print(f"skipped (non-script magic): {len(skipped)} -> {[s[0] for s in skipped]}") + print(f"disasm listings: build/disasm/*.asm") + print(f"dialogue lines (show-text): {n_dialogue} -> build/text/dialogue.jsonl") + print(f"all inline strings: {n_strings} -> build/text/strings.jsonl") + print(f"manifest: build/manifest.json") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/global_map.py b/tools/global_map.py new file mode 100644 index 0000000..4c345de --- /dev/null +++ b/tools/global_map.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +"""Partial global-variable map — label raw global offsets by evidence (static, read-only). + +Combines three static signals (see docs/name-resolution.md #2): + 1. *INIT writers — build/data/*.json name/desc/field bases ARE labelable global addresses. + 2. string anchors — set-string targets across the corpus = string tables. + 3. access shape — how each global is used: 2D-table base (+stride), 1D-array base, + row-index (=> "current entity" pointer), or scalar. + +Emits build/global-var-map.json (all evidence) + build/global-var-map.md (labelled subset). +Usage: py -3.11 -X utf8 tools/global_map.py +""" +from __future__ import annotations +import json +import sys +import collections +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parent +sys.path.insert(0, str(HERE)) +import paths +import sys4load + +CORPUS = paths.DATA1 +DATA = ROOT / "build" / "data" +OUT = ROOT / "build" + +LOOKUP = 0x61 # lookup-array: (dst, base1d, idx) +LOOKUP2D = 0x12c # lookup-array-2d: (dst, base2d, rowidx, stride, col) +SET_STRING = 0x192 # (strbase, text) +GLOBAL_TYPES = {3: "int", 4: "float", 5: "string", 6: "ptr", 8: "string-ptr"} + +INIT_IDENTITY = { + "SKINIT": "skill", "ITINIT": "item", "EBINIT": "unit", "CGINIT": "cg-gallery", + "MPINIT": "map", "STINIT": "stage", +} + + +def from_init_tables(): + """Label name/desc/field base addresses from the extracted *INIT JSONs.""" + labels = {} # addr -> (label, table, confidence, kind) + for jf in sorted(DATA.glob("*.json")): + try: + d = json.loads(jf.read_text(encoding="utf-8")) + except Exception: + continue + tbl = d.get("table", jf.stem) + ent = INIT_IDENTITY.get(tbl, tbl.lower()) + if d.get("name_array_base"): + a = int(d["name_array_base"], 16) + labels[a] = (f"{ent}-name-table", tbl, "high", "string-table") + for k, v in (d.get("desc_array_bases") or {}).items(): + labels[int(v, 16)] = (f"{ent}-{k}-table", tbl, "high", "string-table") + # field columns: classify dense (shared) vs sparse (per-entity extras) + recs = d.get("records", []) + freq = collections.Counter() + for r in recs: + for c in r.get("fields", {}): + freq[c] += 1 + n = max(1, len(recs)) + for col, c in freq.items(): + a = int(col, 16) + if a in labels: + continue + dense = c >= 0.5 * n + labels[a] = (f"{ent}-field" + ("" if dense else "?"), tbl, + "med" if dense else "low", "entity-field-array") + return labels + + +def scan_corpus(): + scrs = [] + for p in sorted(CORPUS.glob("*.BIN")): + try: + scrs.append(sys4load.load(p)) + except Exception: + pass + roles = collections.defaultdict(collections.Counter) # addr -> role -> count + strides = collections.defaultdict(collections.Counter) # addr -> stride -> count + gtype = {} # addr -> global type name + rowidx = collections.Counter() # addr -> times used as 2D row index + strtable_writers = collections.defaultdict(collections.Counter) # straddr -> script -> count + for scr in scrs: + sname = scr.path.stem + for ins in scr.instructions: + op = ins.opcode + for pos, (t, v) in enumerate(ins.args): + if t not in GLOBAL_TYPES: + continue + gtype[v] = GLOBAL_TYPES[t] + if op == LOOKUP2D and pos == 1: + roles[v]["2d-base"] += 1 + if len(ins.args) > 3 and ins.args[3][0] == 0: # stride immediate + strides[v][ins.args[3][1]] += 1 + elif op == LOOKUP2D and pos == 2: + roles[v]["row-index"] += 1 + rowidx[v] += 1 + elif op == LOOKUP and pos == 1: + roles[v]["1d-base"] += 1 + elif op == SET_STRING and pos == 0: + roles[v]["string-table"] += 1 + strtable_writers[v][sname] += 1 + else: + roles[v]["scalar"] += 1 + return roles, strides, gtype, rowidx, strtable_writers, len(scrs) + + +def main() -> int: + init_labels = from_init_tables() + roles, strides, gtype, rowidx, strwriters, nscr = scan_corpus() + all_addrs = set(roles) | set(init_labels) + + # merge into per-address records + entries = {} + for a in sorted(all_addrs): + r = dict(roles.get(a, {})) + total = sum(r.values()) + st = sorted(strides.get(a, {}), key=lambda s: -strides[a][s]) + label = conf = kind = None + if a in init_labels: + label, tbl, conf, kind = init_labels[a] + elif r.get("string-table"): + w = strwriters.get(a, {}) + top = max(w, key=w.get) if w else "?" + label, conf, kind = f"string-table (written by {top})", "med", "string-table" + elif r.get("2d-base"): + stride = st[0] if st else "?" + label, conf, kind = f"record-table[stride {stride}]", "med", "record-table-2d" + elif r.get("1d-base"): + label, conf, kind = "array", "low", "array-1d" + elif r.get("row-index"): # used as a 2D row index but never a base -> an index var + ri, sc = r["row-index"], r.get("scalar", 0) + pure = ri / (ri + sc) if (ri + sc) else 0 + if pure >= 0.3: + label, conf, kind = "current-entity-index?", "med", "index" + else: + label, conf, kind = "index/counter?", "low", "index" + entries[a] = dict(addr=f"0x{a:x}", type=gtype.get(a, "?"), uses=total, + roles=r, stride_candidates=[f"0x{s:x}" for s in st[:3]] or None, + label=label, confidence=conf, kind=kind, table=(init_labels.get(a, (None, None))[1])) + + # "current entity" pointers: globals used as 2D row index, ranked by purity + def purity(a): + ri, sc = rowidx[a], roles[a].get("scalar", 0) + return ri / (ri + sc) if (ri + sc) else 0 + current_idx = [dict(addr=f"0x{a:x}", used_as_row_index=rowidx[a], + also_scalar=roles[a].get("scalar", 0), purity=round(purity(a), 2)) + for a in sorted(rowidx, key=lambda a: (-round(purity(a), 2), -rowidx[a])) + if rowidx[a] >= 10][:15] + + labeled = {k: v for k, v in entries.items() if v["label"]} + out = { + "generated_from": f"extracted/DATA1 ({nscr} scripts) + build/data/*.json", + "note": "Static partial map. Labels ending '?' are low-confidence. Column addresses " + "are raw engine globals; Frida can confirm the ambiguous ones (see docs/name-resolution.md).", + "totals": {"distinct_globals_seen": len(all_addrs), "labelled": len(labeled), + "from_init_tables": len(init_labels)}, + "current_entity_index_candidates": current_idx, + "globals": {v["addr"]: {k: val for k, val in v.items() if k != "addr"} + for v in entries.values()}, + } + (OUT / "global-var-map.json").write_text(json.dumps(out, ensure_ascii=False, indent=2), encoding="utf-8") + + # readable markdown of the labelled subset + md = ["# Partial global-variable map", "", + f"Static map from {nscr} scripts + `build/data/*.json`. " + f"**{len(labeled)} of {len(all_addrs)} distinct globals labelled.** " + "Regenerate: `py -3.11 -X utf8 tools/global_map.py`. See `docs/name-resolution.md`.", "", + "## 'Current entity' index globals (dominant 2D row-index)", "", + "The row index into per-entity tables — the VM's \"which unit/entity are we on\" pointers. " + "Ranked by purity (fraction of uses that are row-index vs. general scalar); high purity = " + "a dedicated index pointer, low = a general-purpose var reused as an index.", "", + "| global | row-index uses | also scalar | purity |", "|---|---|---|---|"] + for c in current_idx: + md.append(f"| `{c['addr']}` | {c['used_as_row_index']} | {c['also_scalar']} | {c['purity']} |") + for kind, title in [("string-table", "String tables (names / descriptions / messages)"), + ("entity-field-array", "Per-entity data-field arrays (from *INIT)"), + ("record-table-2d", "Row-major record tables (from access shape)")]: + rows = sorted((v for v in labeled.values() if v["kind"] == kind), key=lambda v: -v["uses"]) + md += ["", f"## {title} ({len(rows)})", "", "| global | type | label | conf | uses | stride |", + "|---|---|---|---|---|---|"] + for v in rows[:40]: + md.append(f"| `{v['addr']}` | {v['type']} | {v['label']} | {v['confidence']} | " + f"{v['uses']} | {(v['stride_candidates'] or ['—'])[0]} |") + if len(rows) > 40: + md.append(f"| … | | +{len(rows)-40} more (see JSON) | | | |") + (OUT / "global-var-map.md").write_text("\n".join(md), encoding="utf-8") + + print(f"distinct globals seen: {len(all_addrs)}; labelled: {len(labeled)} " + f"({len(init_labels)} from *INIT)") + print(f"top 'current entity' index globals: " + + ", ".join(f"{c['addr']}({c['used_as_row_index']})" for c in current_idx[:5])) + print(f"-> build/global-var-map.json, build/global-var-map.md") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/opcode_context.py b/tools/opcode_context.py new file mode 100644 index 0000000..5dcbcfa --- /dev/null +++ b/tools/opcode_context.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +"""Evidence gatherer for classifying unnamed AGE opcodes. + +For an opcode (or the top-N unnamed ones) prints: frequency + share + cumulative coverage, +argc, operand-type signature histogram, most common predecessor/successor opcodes, a few +real disassembly snippets, and Kelebek's inline comment (from the upstream cpp). Read-only. + +Usage: + py -3.11 -X utf8 tools/opcode_context.py --top 20 # ranked unnamed summary + coverage + py -3.11 -X utf8 tools/opcode_context.py 0x1f4 0x71 0x7a # detailed evidence per opcode +""" +from __future__ import annotations +import re +import sys +import collections +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parent +sys.path.insert(0, str(HERE)) +import paths +import sys4load +import age_opcodes as ao + +CORPUS = paths.DATA1 +KELEBEK_CPP = ROOT / "vm-map" / "kelebek1-age-shared.cpp" + + +def is_unnamed(op: int) -> bool: + lbl = ao.OPCODES.get(op, ("?", 0))[0] + return bool(re.fullmatch(r"u[0-9A-Fa-f]{8}", lbl)) or lbl.lower() == f"{op:x}" or lbl.startswith("dev_ukn") + + +def label(op: int) -> str: + return ao.OPCODES.get(op, (f"?{op:x}", 0))[0] + + +def kelebek_comments() -> dict[int, str]: + out = {} + if not KELEBEK_CPP.exists(): + return out + for m in re.finditer(r"\{\s*(0x[0-9A-Fa-f]+)\s*,\s*\"[^\"]*\"\s*,\s*0x[0-9A-Fa-f]+\s*\}\s*,?\s*//\s*(.*)", + KELEBEK_CPP.read_text(encoding="utf-8")): + out[int(m.group(1), 16)] = m.group(2).strip() + return out + + +def load_corpus(): + scrs = [] + for p in sorted(CORPUS.glob("*.BIN")): + try: + scrs.append(sys4load.load(p)) + except Exception: + pass + return scrs + + +def fmt_instr(scr, ins) -> str: + ops = " ".join(sys4load._fmt_operand(ins.opcode, i, t, v, scr.strings) + for i, (t, v) in enumerate(ins.args)) + return f"{label(ins.opcode)}{(' ' + ops) if ops else ''}" + + +def main() -> int: + args = sys.argv[1:] + scrs = load_corpus() + freq = collections.Counter() + total = 0 + for scr in scrs: + for ins in scr.instructions: + freq[ins.opcode] += 1 + total += 1 + named_vol = sum(c for op, c in freq.items() if not is_unnamed(op)) + comments = kelebek_comments() + + if args and args[0] == "--top": + n = int(args[1]) if len(args) > 1 else 20 + unnamed = [(op, c) for op, c in freq.most_common() if is_unnamed(op)] + print(f"corpus {len(scrs)} scripts, {total} instructions; " + f"named coverage {100*named_vol/total:.2f}%; {len(unnamed)} distinct unnamed ops") + print(f"{'#':>3} {'op':<7}{'argc':>5}{'count':>9}{'share':>8}{'cum-cov':>9} kelebek-comment") + cum = named_vol + for i, (op, c) in enumerate(unnamed[:n]): + cum += c + argc = ao.OPCODES.get(op, ("", 0))[1] + print(f"{i+1:>3} 0x{op:<5x}{argc:>5}{c:>9}{100*c/total:>7.2f}%{100*cum/total:>8.2f}% {comments.get(op,'')[:48]}") + return 0 + + # detailed per-op evidence + targets = [int(a, 16) for a in args] if args else [op for op, _ in + [(op, c) for op, c in freq.most_common() if is_unnamed(op)][:8]] + # precompute neighbour + signature stats + pred = collections.defaultdict(collections.Counter) + succ = collections.defaultdict(collections.Counter) + sig = collections.defaultdict(collections.Counter) + locs = collections.defaultdict(list) # op -> [(scr, index)] + for scr in scrs: + ins = scr.instructions + for i, x in enumerate(ins): + if x.opcode in targets: + if i > 0: + pred[x.opcode][label(ins[i-1].opcode)] += 1 + if i+1 < len(ins): + succ[x.opcode][label(ins[i+1].opcode)] += 1 + sig[x.opcode][tuple(ao.ARG_TYPES.get(t, hex(t)) for t, _ in x.args)] += 1 + if len(locs[x.opcode]) < 4: + locs[x.opcode].append((scr, i)) + + for op in targets: + c = freq.get(op, 0) + argc = ao.OPCODES.get(op, ("", 0))[1] + print(f"\n{'='*72}\nopcode 0x{op:x} label={label(op)} argc={argc} " + f"count={c} ({100*c/total:.2f}% of instrs)") + if comments.get(op): + print(f" kelebek-comment: {comments[op]}") + print(f" operand-type signatures: " + + ", ".join(f"{'/'.join(s) if s else 'none'}×{n}" for s, n in sig[op].most_common(4))) + print(f" top predecessors: " + ", ".join(f"{k}×{v}" for k, v in pred[op].most_common(5))) + print(f" top successors: " + ", ".join(f"{k}×{v}" for k, v in succ[op].most_common(5))) + print(f" snippets:") + for scr, i in locs[op]: + lo, hi = max(0, i-2), min(len(scr.instructions), i+3) + for j in range(lo, hi): + mark = ">>" if j == i else " " + print(f" {mark} [{scr.path.name}] {fmt_instr(scr, scr.instructions[j])}") + print() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/pack_check.py b/tools/pack_check.py new file mode 100644 index 0000000..946e744 --- /dev/null +++ b/tools/pack_check.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +import os, sys, struct, math, re +from pathlib import Path +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import paths +data = paths.AGE_EXE.read_bytes() +pe = struct.unpack_from("" + vsize,va,rsize,raw=struct.unpack_from("/ e.g. S:\\Game Hacking\\Eushully\\Himegari + 姫狩りダンジョンマイスター/ pristine game install (AGE.EXE, *.ALF, + loose *.BIN patch-overrides, DLLs) + extracted/ extracted ALF data: DATA1 .. DATA5 + age-reimpl/ our work (this repo) + tools/ build/ docs/ vm-map/ godot/ bin/ + +To point the tools at a different install, change GAME_DIR / EXTRACTED here only. +""" +import os +import sys +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent # age-reimpl/ +WORKSPACE = REPO.parent # workspace root +GAME_DIR = WORKSPACE / "姫狩りダンジョンマイスター" # pristine game install +EXTRACTED = WORKSPACE / "extracted" # extracted ALF archives +DATA1 = EXTRACTED / "DATA1" # the .BIN script corpus +BUILD = REPO / "build" # derived corpora (regenerable) +VM_MAP = REPO / "vm-map" +BIN = REPO / "bin" # 3rd-party tools (BinExtractALF, ...) +AGE_EXE = GAME_DIR / "AGE.EXE" +KELEBEK_CPP = VM_MAP / "kelebek1-age-shared.cpp" + + +def add_self_to_syspath(): + """Let a standalone script `import paths` / `import sys4load` from tools/.""" + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + + +def scripts(): + """Authoritative {UPPERNAME.BIN -> path} map for the script corpus. + + Loose *.BIN patch-overrides in the game dir shadow their extracted/DATA1 + copies (runtime behaviour), so they win on name collision. + """ + files = {} + if DATA1.is_dir(): + for p in sorted(DATA1.glob("*.BIN")): + files[p.name.upper()] = p + for p in sorted(GAME_DIR.glob("*.BIN")): # overrides win + files[p.name.upper()] = p + return files diff --git a/tools/probe_header.py b/tools/probe_header.py new file mode 100644 index 0000000..f9f73c3 --- /dev/null +++ b/tools/probe_header.py @@ -0,0 +1,69 @@ +"""Probe SYS4 script headers across the DATA1 corpus. + +Header hypothesis (offsets in bytes): + 0x00 char magic[8] "SYS4422 " + 0x08 u32 F0..F12 13 fields + 0x3C body (dword stream) + +Tests invariants and prints field stats. +""" +import struct +import sys +from pathlib import Path +from collections import Counter + +ROOT = Path(__file__).resolve().parents[2] / "extracted" / "DATA1" + +def parse(path): + data = path.read_bytes() + magic = data[:8] + fields = struct.unpack_from("<13I", data, 8) + return data, magic, fields + +def main(): + files = sorted(ROOT.glob("*.BIN")) + print(f"files: {len(files)}") + + magics = Counter() + distinct = [Counter() for _ in range(13)] + fail_f12 = [] # F12 != filedwords - 15 + fail_f6 = [] # F6 != 0x1C + fail_f11 = [] # F11 != 0 + rel_a = Counter() # F8 + F9 vs F12 + rel_b = Counter() # F10 + F11 vs F12 + body_first = Counter() + + for p in files: + data, magic, f = parse(p) + magics[magic] += 1 + nd = len(data) // 4 + for i, v in enumerate(f): + distinct[i][v] += 1 + if f[12] != nd - 15: + fail_f12.append((p.name, f[12], nd - 15)) + if f[6] != 0x1C: + fail_f6.append((p.name, f[6])) + if f[11] != 0: + fail_f11.append((p.name, f[10], f[11], f[12])) + rel_a["F8+F9==F12" if f[8] + f[9] == f[12] else + ("F8+F9F12")] += 1 + rel_b["F10+F11==F12" if f[10] + f[11] == f[12] else + ("F10+F11F12")] += 1 + if len(data) >= 0x40: + body_first[struct.unpack_from(" candidate +opcode. Histogram those leads over all root-override + DATA1 scripts. +Also: histogram the dword immediately preceding each `0x02 ` string ref +(candidate "takes-a-string" opcodes), with example files. +""" +import os, sys, collections +from pathlib import Path +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import paths +import sys4load + +# authoritative set: game-dir overrides shadow extracted/DATA1 +files = paths.scripts() +print(f"{len(files)} scripts (overrides win)") + +lead_hist = collections.Counter() # dword after each T3 record +lead_examples = collections.defaultdict(set) +pre_str = collections.Counter() # dword before each 0x02 ref +pre_str_ex = collections.defaultdict(set) +first_dword = collections.Counter() +t3_rec_shape_ok = 0 +t3_rec_total = 0 +bad = 0 + +for name, p in sorted(files.items()): + try: + scr = sys4load.load(p) + except Exception as e: + bad += 1 + continue + dw = scr.dwords + code_len = scr.code_len + first_dword[dw[0]] += 1 + # statement leads via T3 records + for e in scr.table_entries("T3"): + if not (0 <= e < code_len): + continue + t3_rec_total += 1 + if e + 2 < code_len and dw[e] == 0x8F and dw[e+1] == 0: + t3_rec_shape_ok += 1 + nxt = e + 3 + if nxt < code_len: + v = dw[nxt] + lead_hist[v] += 1 + if len(lead_examples[v]) < 3: + lead_examples[v].add(name) + # dword preceding string refs (ref index i+1 holds offset; tag at i) + for ref_idx in scr.string_refs: # ref_idx points at the offset dword + tag_idx = ref_idx - 1 # the 0x02 + if tag_idx - 1 >= 0: + v = dw[tag_idx - 1] + pre_str[v] += 1 + if len(pre_str_ex[v]) < 3: + pre_str_ex[v].add(name) + +print(f"parse failures: {bad}") +print(f"T3 records with shape [0x8F,0,*]: {t3_rec_shape_ok}/{t3_rec_total}") +print("\n== first body dword ==") +for v, c in first_dword.most_common(8): + print(f" {v:#06x} {c}") +print(f"\n== statement leads after T3 records (top 40 of {len(lead_hist)}) ==") +total = sum(lead_hist.values()) +for v, c in lead_hist.most_common(40): + ex = ",".join(sorted(lead_examples[v])) + print(f" {v:#06x} {c:>7} ({100*c/total:5.2f}%) e.g. {ex}") +print(f"\n== dword preceding `0x02 ` string refs (top 25 of {len(pre_str)}) ==") +tot2 = sum(pre_str.values()) +for v, c in pre_str.most_common(25): + ex = ",".join(sorted(pre_str_ex[v])) + print(f" {v:#06x} {c:>7} ({100*c/tot2:5.2f}%) e.g. {ex}") diff --git a/tools/probe_refs.py b/tools/probe_refs.py new file mode 100644 index 0000000..6053eca --- /dev/null +++ b/tools/probe_refs.py @@ -0,0 +1,76 @@ +"""What do T1/T2/T3 entries point at? Plus: extract XOR-0xFF strings from scripts. + +Prints dword windows around table-entry targets, and scans for complement-encoded +Shift-JIS strings. +""" +import struct +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] / "extracted" / "DATA1" + +def parse(path): + data = path.read_bytes() + f = struct.unpack_from("<13I", data, 8) + body = data[0x3C:] + return data, f, body + +def dw(body, i): + return struct.unpack_from(" {[hex(e) for e in entries]}") + for e in entries[:4]: + ctx = [dw(body, i) for i in range(max(0, e - 2), min(nd, e + 5))] + s, _ = try_string(body, e) + tag = f" str@target: {s!r}" if s else "" + print(f" target {e:#x}: [-2..+4] = {[hex(v) for v in ctx]}{tag}") + +def scan_strings(name, limit=15, min_len=4): + data, f, body = parse(ROOT / name) + nd = len(body) // 4 + print(f"\n=== strings in {name} ===") + found = 0 + i = 0 + while i < nd and found < limit: + s, n = try_string(body, i) + if s and len(s) >= min_len: + print(f" [{i:#x}] {s!r}") + found += 1 + i += n + else: + i += 1 + +if __name__ == "__main__": + for n in ("MENU.BIN", "ADDEXP.BIN", "CALLBACK_LOST.BIN", "ALCHEMY.BIN"): + windows(n) + scan_strings("MENU.BIN") + scan_strings("SC0030.BIN", limit=20, min_len=6) diff --git a/tools/probe_tables.py b/tools/probe_tables.py new file mode 100644 index 0000000..62693b9 --- /dev/null +++ b/tools/probe_tables.py @@ -0,0 +1,76 @@ +"""Verify the three (count, offset) table descriptors and inspect table contents. + +Layout hypothesis (all offsets in dwords, relative to body start at 0x3C): + code: body[0 .. F8) + T1: count=F7, offset=F8, entry size s1 (solve: F8 + s1*F7 == F10) + T2: count=F9, offset=F10, entry size s2 (solve: F10 + s2*F9 == F12) + T3: count=F11, offset=F12, entry size s3 (solve: F12 + s3*F11 == EOF) +""" +import struct +from pathlib import Path +from collections import Counter + +ROOT = Path(__file__).resolve().parents[2] / "extracted" / "DATA1" + +def parse(path): + data = path.read_bytes() + f = struct.unpack_from("<13I", data, 8) + return data, f + +def solve_entry_size(gap_pairs): + """gap_pairs: list of (count, gap_dwords). Return consistent entry size or None.""" + sizes = set() + for count, gap in gap_pairs: + if count: + if gap % count: + return f"non-integer ({count},{gap})" + sizes.add(gap // count) + return sizes + +def main(): + files = sorted(ROOT.glob("*.BIN")) + t1_pairs, t2_pairs, t3_pairs = [], [], [] + bad = [] + for p in files: + data, f = parse(p) + nd = len(data) // 4 - 15 # body dwords + F7, F8, F9, F10, F11, F12 = f[7], f[8], f[9], f[10], f[11], f[12] + if not (F8 <= F10 <= F12 <= nd): + bad.append((p.name, F8, F10, F12, nd)) + continue + t1_pairs.append((F7, F10 - F8)) + t2_pairs.append((F9, F12 - F10)) + t3_pairs.append((F11, nd - F12)) + print(f"ordering violations: {len(bad)} {bad[:5]}") + print("T1 entry sizes:", solve_entry_size(t1_pairs)) + print("T2 entry sizes:", solve_entry_size(t2_pairs)) + print("T3 entry sizes:", solve_entry_size(t3_pairs)) + + # zero-count but nonzero-gap sanity + for name, pairs in (("T1", t1_pairs), ("T2", t2_pairs), ("T3", t3_pairs)): + odd = sum(1 for c, g in pairs if c == 0 and g != 0) + print(f"{name}: count==0 but gap!=0 in {odd} files") + + # F2 outliers + print("\nF2 outliers:") + for p in files: + _, f = parse(p) + if f[2] not in (1, 2): + print(f" {p.name}: F2={f[2]:#x}") + + # dump tables for a few files + for name in ("MENU.BIN", "ADDEXP.BIN", "ADDILL.BIN", "ALCHEMY.BIN"): + data, f = parse(ROOT / name) + nd = len(data) // 4 - 15 + print(f"\n=== {name}: body={nd} F7-12: cnt/off T1={f[7]}/{f[8]} " + f"T2={f[9]}/{f[10]} T3={f[11]}/{f[12]}") + for label, cnt, off, end in (("T1", f[7], f[8], f[10]), + ("T2", f[9], f[10], f[12]), + ("T3", f[11], f[12], nd)): + vals = struct.unpack_from(f"<{end-off}I", data, 0x3C + off*4) + show = ", ".join(f"{v:#x}" for v in vals[:16]) + print(f" {label} ({cnt} entries, {end-off} dwords): {show}" + + (" ..." if end-off > 16 else "")) + +if __name__ == "__main__": + main() diff --git a/tools/probe_tags.py b/tools/probe_tags.py new file mode 100644 index 0000000..5e2ec03 --- /dev/null +++ b/tools/probe_tags.py @@ -0,0 +1,76 @@ +"""Confirm operand tag hypothesis: string pointers are preceded by tag dword 2. +Also profile the first body dword and the recurring 'type' dwords 0x55/0x6E/0x6F/0x71/0x72/0x8F. +""" +import struct +from pathlib import Path +from collections import Counter + +ROOT = Path(__file__).resolve().parents[2] / "extracted" / "DATA1" + +def parse(p): + data = p.read_bytes() + f = struct.unpack_from("<13I", data, 8) + body = data[0x3C:] + n = len(body) // 4 + dws = struct.unpack_from(f"<{n}I", body, 0) + return f, dws, body + +def is_string_at(body, i): + n = len(body) // 4 + if i >= n: + return False + raw = bytearray() + for j in range(i, min(i + 60, n)): + chunk = bytes(b ^ 0xFF for b in body[j*4:(j+1)*4]) + raw += chunk + if 0 in chunk: + break + else: + return False + s = bytes(raw).split(b"\0")[0] + if len(s) < 2: + return False + i2, chars = 0, 0 + while i2 < len(s): + b = s[i2] + if 0x20 <= b <= 0x7E: + i2 += 1; chars += 1 + elif 0x81 <= b <= 0x9F or 0xE0 <= b <= 0xEA: + if i2 + 1 < len(s) and 0x40 <= s[i2+1] <= 0xFC and s[i2+1] != 0x7F: + i2 += 2; chars += 1 + else: + return False + else: + return False + return chars >= 2 + +def main(): + files = sorted(ROOT.glob("*.BIN")) + preceding = Counter() # dword right before a string-region start + first_dword = Counter() + total_strings = 0 + for p in files: + f, dws, body = parse(p) + n = len(dws) + first_dword[dws[0]] += 1 + # find string regions in the string area (>= T3 end is where strings live typically) + i = 1 + while i < n: + if is_string_at(body, i) and not is_string_at(body, i - 1): + total_strings += 1 + preceding[dws[i - 1]] += 1 + # skip the string + while i < n and is_string_at(body, i): + i += 1 + else: + i += 1 + print(f"total string regions: {total_strings}") + print("dword immediately preceding a string (top 10):") + for v, c in preceding.most_common(10): + print(f" {v:#x}: {c} ({100*c/total_strings:.1f}%)") + print("\nfirst body dword (top 8):") + for v, c in first_dword.most_common(8): + print(f" {v:#x}: {c}") + +if __name__ == "__main__": + main() diff --git a/tools/probe_types.py b/tools/probe_types.py new file mode 100644 index 0000000..0a7a031 --- /dev/null +++ b/tools/probe_types.py @@ -0,0 +1,101 @@ +"""Corpus-wide: verify T1/T2/T3 target-dword types, test table completeness, +and do a stricter string scan. +""" +import struct +from pathlib import Path +from collections import Counter + +ROOT = Path(__file__).resolve().parents[2] / "extracted" / "DATA1" + +def parse(path): + data = path.read_bytes() + f = struct.unpack_from("<13I", data, 8) + body = data[0x3C:] + n = len(body) // 4 + dws = struct.unpack_from(f"<{n}I", body, 0) + return f, dws, body + +def main(): + files = sorted(ROOT.glob("*.BIN")) + type_hist = {1: Counter(), 2: Counter(), 3: Counter()} + complete = {1: Counter(), 2: Counter(), 3: Counter()} + for p in files: + f, dws, body = parse(p) + code_end = f[8] + specs = {1: (f[7], f[8], 0x71), 2: (f[9], f[10], 0x03), 3: (f[11], f[12], 0x8F)} + for t, (cnt, off, expect) in specs.items(): + targets = dws[off:off + cnt] + for e in targets: + type_hist[t][dws[e] if e < len(dws) else "OOB"] += 1 + # naive count of (expect, 0, X) triples in code, at any alignment + naive = sum(1 for i in range(code_end - 2) + if dws[i] == expect and dws[i + 1] == 0) + if cnt == naive: + complete[t]["exact"] += 1 + elif cnt < naive: + complete[t]["tablenaive"] += 1 + for t in (1, 2, 3): + print(f"T{t} target-dword histogram: " + + ", ".join(f"{v:#x}x{n}" for v, n in type_hist[t].most_common(6))) + print(f"T{t} completeness vs naive scan: {dict(complete[t])}") + + # value ranges of the operands referenced by each table (sample corpus-wide) + for t, expect in ((1, 0x71), (2, 0x03), (3, 0x8F)): + lo, hi = None, None + vals = Counter() + for p in files: + f, dws, body = parse(p) + cnt, off = (f[7], f[8]) if t == 1 else (f[9], f[10]) if t == 2 else (f[11], f[12]) + for e in dws[off:off + cnt]: + if e + 2 < len(dws): + v = dws[e + 2] + vals[v] += 1 + lo = v if lo is None else min(lo, v) + hi = v if hi is None else max(hi, v) + print(f"T{t} operand values: min={lo and hex(lo)}, max={hi and hex(hi)}, " + f"top: {', '.join(f'{v:#x}x{n}' for v, n in vals.most_common(8))}") + +def good_string(bs): + """bs = decoded-candidate raw bytes (already XOR'd). Strict cp932 validity.""" + i, chars = 0, 0 + while i < len(bs): + b = bs[i] + if 0x20 <= b <= 0x7E: + i += 1; chars += 1 + elif 0x81 <= b <= 0x9F or 0xE0 <= b <= 0xEA: + if i + 1 < len(bs) and (0x40 <= bs[i+1] <= 0xFC) and bs[i+1] != 0x7F: + i += 2; chars += 1 + else: + return 0 + else: + return 0 + return chars + +def scan(name, limit=25, min_chars=3): + f, dws, body = parse(ROOT / name) + n = len(body) // 4 + print(f"\n=== strings in {name} (strict) ===") + found, i = 0, 0 + while i < n and found < limit: + raw = bytearray() + j = i + while j < n: + chunk = bytes(b ^ 0xFF for b in body[j*4:(j+1)*4]) + raw += chunk + j += 1 + if 0 in chunk: + break + s = bytes(raw).split(b"\0")[0] + if len(s) >= 2 and good_string(s) >= min_chars: + print(f" [{i:#x}] {s.decode('cp932')!r}") + found += 1 + i = j + else: + i += 1 + +if __name__ == "__main__": + main() + scan("SC0030.BIN") + scan("MENU.BIN", limit=12) diff --git a/tools/probe_xref.py b/tools/probe_xref.py new file mode 100644 index 0000000..4ca0539 --- /dev/null +++ b/tools/probe_xref.py @@ -0,0 +1,64 @@ +"""Find how instructions reference string offsets, and inspect 0x71 operands +in context. Uses T3 jump-target values as confirmed instruction starts. +""" +import struct +from pathlib import Path +from collections import Counter + +ROOT = Path(__file__).resolve().parents[2] / "extracted" / "DATA1" + +def parse(name): + data = (ROOT / name).read_bytes() + f = struct.unpack_from("<13I", data, 8) + body = data[0x3C:] + n = len(body) // 4 + dws = struct.unpack_from(f"<{n}I", body, 0) + return f, dws, body + +def decode_str(body, i): + raw = bytearray() + n = len(body) // 4 + for j in range(i, min(i + 60, n)): + chunk = bytes(b ^ 0xFF for b in body[j*4:(j+1)*4]) + raw += chunk + if 0 in chunk: + break + try: + return bytes(raw).split(b"\0")[0].decode("cp932") + except UnicodeDecodeError: + return None + +def xref(name, targets): + f, dws, body = parse(name) + print(f"\n=== {name}: xrefs to {[hex(t) for t in targets]} ===") + for t in targets: + hits = [i for i, v in enumerate(dws[:f[8]]) if v == t] + for i in hits: + lo = max(0, i - 6) + ctx = " ".join(f"{v:x}" for v in dws[lo:i + 3]) + print(f" {t:#x} referenced at [{i:#x}]: ...{ctx}...") + +def sample_71(name, k=6): + f, dws, body = parse(name) + print(f"\n=== {name}: T1 (0x71) operands in context ===") + off, cnt = f[8], f[7] + for e in list(dws[off:off + cnt])[:k]: + lo = max(0, e - 8) + ctx = " ".join(f"{v:x}" for v in dws[lo:e + 4]) + print(f" target [{e:#x}]: ...{ctx}...") + +def first_instrs(name, count=20): + """Dump first N dwords raw, plus known instruction starts from T3 values.""" + f, dws, body = parse(name) + starts = sorted(set(dws[f[12]:f[12] + f[11]])) + print(f"\n=== {name}: first dwords ===") + print(" " + " ".join(f"{v:x}" for v in dws[:count])) + print(f" T3-confirmed instruction starts (first 12): {[hex(s) for s in starts[:12]]}") + +if __name__ == "__main__": + xref("MENU.BIN", [0x303, 0x30B, 0x30E]) + xref("SC0030.BIN", [0xEF80, 0xEF89, 0xEF91]) + sample_71("SC0030.BIN") + sample_71("MENU.BIN", k=2) + first_instrs("MENU.BIN") + first_instrs("SC0030.BIN") diff --git a/tools/sys4load.py b/tools/sys4load.py new file mode 100644 index 0000000..b009ce3 --- /dev/null +++ b/tools/sys4load.py @@ -0,0 +1,521 @@ +#!/usr/bin/env python3 +"""sys4load — loader / disassembler-ish dumper for Eushully SYS4 `.BIN` scripts. + +Parses the confirmed container format (see ../sys4-format-notes.md): + * 60-byte header: magic "SYS4422 " + 13 little-endian u32 fields + * body (dword stream) = CODE + three 1-dword pointer tables + inline strings + * strings: XOR-0xFF cp932, NUL-terminated, referenced by a `0x02 ` pair + +The container format is byte-verified across all 481 DATA1 scripts. Opcodes are now +DECODED using the AGE opcode table (age_opcodes.py, transcribed from Kelebek1's +decompiler and validated 476/476 clean on Himegari): the code section is a flat stream +of ` + argc*()` instructions, length 1+2*argc dwords. +Inline strings live after the code inside [0,F8), so decoding stops at the first string +(type-2) or array (op 0x64) operand offset. + +Usage: + sys4load.py full disassembly listing + sys4load.py --summary header + section sizes + table/string counts + sys4load.py --strings decoded string pool only + sys4load.py --json machine-readable structure (no code dump) + sys4load.py --validate re-check format invariants across a folder +""" +from __future__ import annotations + +import argparse +import json +import os +import struct +import sys +from dataclasses import dataclass, field +from pathlib import Path + +try: + from age_opcodes import (OPCODES, ARG_TYPES, CONTROL_FLOW, ARRAY_OPCODE, + is_label_argument) +except ImportError: # allow import from another cwd + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + from age_opcodes import (OPCODES, ARG_TYPES, CONTROL_FLOW, ARRAY_OPCODE, + is_label_argument) + +# Himegari inference layer (optional): improves labels for unnamed opcodes. Keeps the +# verbatim Kelebek table (age_opcodes) pristine; see age_opcodes_himegari.py. +try: + from age_opcodes_himegari import INFERRED +except ImportError: + INFERRED = {} + +# Global-variable labels (optional): annotate global operands with the partial global-var +# map (build/global-var-map.json, produced by tools/global_map.py). High/medium confidence +# only — the low-confidence tail (~12k sparse guesses) is left out to keep listings readable. +GLOBAL_ATYPES = {3, 4, 5, 6, 8} # global-int/float/string/ptr/string-ptr + + +def _short_global_label(lbl: str) -> str: + if lbl.startswith("record-table[stride "): + return "rec[s" + lbl[len("record-table[stride "):-1] + "]" + if lbl.startswith("string-table (written by "): + return "str<" + lbl[len("string-table (written by "):-1] + ">" + return lbl + + +def _load_global_labels() -> dict: + try: + p = Path(__file__).resolve().parent.parent / "build" / "global-var-map.json" + data = json.loads(p.read_text(encoding="utf-8")) + except Exception: + return {} + out = {} + for addr_s, e in data.get("globals", {}).items(): + lbl, conf = e.get("label"), e.get("confidence") + if lbl and conf in ("high", "med"): + out[int(addr_s, 16)] = _short_global_label(lbl) + return out + + +GLOBAL_LABELS = _load_global_labels() + + +def display_label(op: int) -> str: + """Rendered mnemonic: Kelebek name if it has one, else the inferred name, else u00….""" + lbl = OPCODES.get(op, (f"?{op:x}", 0))[0] + is_unnamed = (lbl.startswith(("u00", "dev_ukn")) or lbl.lower() == f"{op:x}") + if is_unnamed and op in INFERRED: + return INFERRED[op]["name"] + return lbl + +MAGIC = b"SYS4422 " # canonical; also seen: SYS4424 (patch scripts). Both 0x3C headers. +MAGIC_PREFIX = b"SYS4" # accept the whole SYS4 script family (SYS4422 / SYS4424 / ...) +HEADER_SIZE = 0x3C # 8 magic + 13*4 +NUM_FIELDS = 13 +BODY_OFF = HEADER_SIZE + +# Tag dword found at the target of each pointer table (100% pure across corpus). +TABLE_TAGS = {"T1": 0x71, "T2": 0x03, "T3": 0x8F} +STRING_REF_TAG = 0x02 # dword preceding an inline string offset + + +class Sys4Error(ValueError): + pass + + +@dataclass +class Instruction: + offset: int # dword index of the opcode within the body + opcode: int + label: str | None # mnemonic from the opcode table, or None if unknown + args: list # list of (type, value) operand pairs + unknown: bool = False # opcode not in the table (decode desynced/stopped) + truncated: bool = False # not enough dwords left for the declared arg count + + @property + def size(self): # length in dwords + return 1 + 2 * len(self.args) + + +@dataclass +class Sys4Script: + path: Path + fields: tuple # 13 header u32s (F0..F12) + dwords: tuple # body as tuple[int], length = nbody + magic: str = "SYS4422 " # actual 8-byte magic (SYS4422 / SYS4424 / ...) + strings: dict = field(default_factory=dict) # start_dword -> (text, ndwords) + string_refs: dict = field(default_factory=dict) # value-operand dword index -> string start + instructions: list = field(default_factory=list) # decoded Instruction list + code_end: int = 0 # dword where code stops and inline strings begin (<= code_len) + + # ---- section geometry (all in dword units, relative to body start) ---- + @property + def nbody(self): + return len(self.dwords) + + @property + def code_len(self): + return self.fields[8] # F8 + + @property + def t1(self): + return (self.fields[7], self.fields[8], self.fields[10]) # count, off, end + + @property + def t2(self): + return (self.fields[9], self.fields[10], self.fields[12]) + + @property + def t3(self): + return (self.fields[11], self.fields[12], self.nbody) + + def table_entries(self, which): + count, off, _ = getattr(self, which.lower()) + return self.dwords[off:off + count] + + # ---- byte offsets for reporting ---- + @staticmethod + def dword_to_file_off(idx): + return BODY_OFF + idx * 4 + + +# --------------------------------------------------------------------------- # +# parsing +# --------------------------------------------------------------------------- # +def _decode_string(dwords, start, limit=4096): + """Decode a XOR-0xFF cp932 string beginning at dword `start`. + + Returns (text, ndwords) or (None, 0) if it isn't a clean string. + """ + raw = bytearray() + n = len(dwords) + end = min(start + limit, n) + for j in range(start, end): + raw += struct.pack(" Sys4Script: + path = Path(path) + data = path.read_bytes() + if len(data) < HEADER_SIZE: + raise Sys4Error(f"{path.name}: too small ({len(data)} bytes)") + if data[:4] != MAGIC_PREFIX: + raise Sys4Error(f"{path.name}: bad magic {data[:8]!r}") + if len(data) % 4: + raise Sys4Error(f"{path.name}: length {len(data)} not dword-aligned") + fields = struct.unpack_from(f"<{NUM_FIELDS}I", data, 8) + nbody = (len(data) - BODY_OFF) // 4 + dwords = struct.unpack_from(f"<{nbody}I", data, BODY_OFF) + + scr = Sys4Script(path=path, fields=fields, dwords=dwords, + magic=data[:8].decode("ascii", "replace")) + _check_invariants(scr) + decode_code(scr) + return scr + + +def decode_code(scr: Sys4Script): + """Walk the code section into instructions, resolving inline strings. + + Model (age_opcodes.py): instruction = + argc*(), length + 1+2*argc dwords. Inline strings sit after the code inside [0,F8); a type-2 (string) + or op-0x64 array operand offset marks where code ends, so we lower `code_end` to the + smallest such offset seen and stop there. Populates scr.instructions / .strings / + .string_refs / .code_end. + """ + dw = scr.dwords + nbody = scr.nbody + code_end = scr.code_len # F8; shrinks to first inline-string/array offset + instrs, strings, refs = [], {}, {} + i = 0 + while i < code_end: + op = dw[i] + info = OPCODES.get(op) + if info is None: + instrs.append(Instruction(i, op, None, [], unknown=True)) + break + label, argc = info + base = i + 1 + if base + 2 * argc > code_end: + instrs.append(Instruction(i, op, label, [], truncated=True)) + break + args = [] + for a in range(argc): + atype = dw[base + 2 * a] + aval = dw[base + 2 * a + 1] + args.append((atype, aval)) + if atype == 2 and 0 <= aval < nbody: # inline string operand + code_end = min(code_end, aval) + if aval not in strings: + text, nd = _decode_string(dw, aval) + if text is not None: + strings[aval] = (text, nd) + refs[base + 2 * a + 1] = aval + elif op == ARRAY_OPCODE and a == 1 and 0 <= aval < nbody: # footer array ref + code_end = min(code_end, aval) + instrs.append(Instruction(i, op, label, args)) + i = base + 2 * argc + + scr.instructions = instrs + scr.strings = strings + scr.string_refs = refs + scr.code_end = code_end + return scr + + +def _check_invariants(scr: Sys4Script): + f = scr.fields + nbody = scr.nbody + F7, F8, F9, F10, F11, F12 = f[7], f[8], f[9], f[10], f[11], f[12] + problems = [] + if f[6] != 0x1C: + problems.append(f"F6={f[6]:#x} != 0x1C") + if not (0 <= F8 <= F10 <= F12 <= nbody): + problems.append(f"section ordering 0<=F8({F8})<=F10({F10})<=F12({F12})<=nbody({nbody})") + else: + if F10 - F8 != F7: + problems.append(f"T1 size {(F10 - F8)} != count {F7}") + if F12 - F10 != F9: + problems.append(f"T2 size {(F12 - F10)} != count {F9}") + if nbody - F12 != F11: + problems.append(f"T3 size {(nbody - F12)} != count {F11}") + if problems: + raise Sys4Error(f"{scr.path.name}: " + "; ".join(problems)) + + +def check_table_tags(scr: Sys4Script): + """Return dict which -> (ok_count, total) for target-tag purity.""" + out = {} + for which, tag in TABLE_TAGS.items(): + entries = scr.table_entries(which) + ok = sum(1 for e in entries if 0 <= e < scr.nbody and scr.dwords[e] == tag) + out[which] = (ok, len(entries)) + return out + + +# --------------------------------------------------------------------------- # +# rendering +# --------------------------------------------------------------------------- # +def _fmt_dwords(dwords, lo, hi, per_line=8): + out = [] + for i in range(lo, hi, per_line): + chunk = dwords[i:min(i + per_line, hi)] + out.append(" " + " ".join(f"{v:08x}" for v in chunk)) + return out + + +def decode_stats(scr: Sys4Script): + """(n_instructions, n_unknown, n_truncated, clean) for the decoded code.""" + n = len(scr.instructions) + unk = sum(1 for ins in scr.instructions if ins.unknown) + trunc = sum(1 for ins in scr.instructions if ins.truncated) + clean = unk == 0 and trunc == 0 and scr.code_end == ( + scr.instructions[-1].offset + scr.instructions[-1].size if scr.instructions else 0) + return n, unk, trunc, clean + + +def render_summary(scr: Sys4Script) -> str: + f = scr.fields + tags = check_table_tags(scr) + n, unk, trunc, clean = decode_stats(scr) + lines = [ + f"file {scr.path.name}", + f"body {scr.nbody} dwords ({scr.nbody * 4} bytes)", + f"header F0={f[0]:#x} F1={f[1]} F2={f[2]:#x} F3={f[3]:#x} " + f"F4={f[4]} F5={f[5]:#x} F6={f[6]:#x} (F0-F5 = local var counts)", + f"code [0x00000 .. 0x{scr.code_end:05x}) {scr.code_end} dwords, " + f"{n} instructions" + + (f", strings 0x{scr.code_end:05x}..0x{scr.code_len:05x}" if scr.code_end < scr.code_len else ""), + f"decode {'CLEAN' if clean else 'INCOMPLETE'}" + + (f" ({unk} unknown, {trunc} truncated)" if (unk or trunc) else ""), + f"table T1 off 0x{f[8]:05x} count {f[7]:<6} tag 0x71 " + f"purity {tags['T1'][0]}/{tags['T1'][1]}", + f"table T2 off 0x{f[10]:05x} count {f[9]:<6} tag 0x03 " + f"purity {tags['T2'][0]}/{tags['T2'][1]}", + f"table T3 off 0x{f[12]:05x} count {f[11]:<6} tag 0x8f " + f"purity {tags['T3'][0]}/{tags['T3'][1]}", + f"strings {len(scr.strings)} inline, {len(scr.string_refs)} references", + ] + return "\n".join(lines) + + +def render_strings(scr: Sys4Script) -> str: + out = [] + for off in sorted(scr.strings): + text, nd = scr.strings[off] + fo = scr.dword_to_file_off(off) + out.append(f" [0x{off:05x} @file 0x{fo:06x}] ({nd}dw) {text!r}") + return "\n".join(out) if out else " (no inline strings)" + + +def _fmt_operand(op: int, arg_index: int, atype: int, aval: int, strings: dict) -> str: + """Render one (type, value) operand the way the disassembler labels them.""" + if atype == 2: # inline string + text = strings.get(aval, ("?",))[0] + return f'"{text}"' + if is_label_argument(op, arg_index, aval): # code-offset jump/call target + return f"label_{aval:x}" + tlabel = ARG_TYPES.get(atype) + if atype == 0 or tlabel is None: # immediate / unknown-tag: raw value + if atype == 0: + return f"{aval:#x}" + return f"" + if tlabel == "float": + return f"(float {aval:#x})" + ann = "" + if atype in GLOBAL_ATYPES and aval in GLOBAL_LABELS: + ann = f" ={GLOBAL_LABELS[aval]}" # inferred global-var-map alias + return f"({tlabel} {aval:#x}{ann})" # e.g. (global-int 17a =skill-name-table) + + +def render_listing(scr: Sys4Script) -> str: + """Disassembly of the code section using the AGE opcode table.""" + out = [render_summary(scr), "", "; ---- CODE ----"] + + # collect jump/call label targets so we can print label_XXXX: anchors + label_targets = set() + for ins in scr.instructions: + for x, (atype, aval) in enumerate(ins.args): + if is_label_argument(ins.opcode, x, aval): + label_targets.add(aval) + + for ins in scr.instructions: + if ins.offset in label_targets: + out.append(f"label_{ins.offset:x}:") + if ins.unknown: + out.append(f" 0x{ins.offset:05x}: ??? 0x{ins.opcode:x} ; unknown opcode — decode stopped") + continue + if ins.truncated: + out.append(f" 0x{ins.offset:05x}: {ins.label} ") + continue + ops = " ".join(_fmt_operand(ins.opcode, x, t, v, scr.strings) + for x, (t, v) in enumerate(ins.args)) + mnem = display_label(ins.opcode) # prefers Kelebek name, else inferred, else u00… + # annotate unnamed/inferred ops with their raw value for grep-ability + kelebek = ins.label + unnamed = kelebek.startswith(("u00", "dev_ukn")) or kelebek[:1].isdigit() + raw = f" ; op 0x{ins.opcode:x}" + (" inferred" if unnamed and ins.opcode in INFERRED else "") \ + if unnamed else "" + out.append(f" 0x{ins.offset:05x}: {mnem}{(' ' + ops) if ops else ''}{raw}") + + out.append("") + out.append("; ---- TABLES ----") + for which in ("T1", "T2", "T3"): + count, off, end = getattr(scr, which.lower()) + entries = scr.table_entries(which) + preview = " ".join(f"{e:x}" for e in entries[:16]) + more = " ..." if count > 16 else "" + out.append(f"{which} (count {count}, off 0x{off:05x}): {preview}{more}") + out.append("") + out.append("; ---- STRINGS ----") + out.append(render_strings(scr)) + return "\n".join(out) + + +def to_dict(scr: Sys4Script, with_code=False) -> dict: + f = scr.fields + n, unk, trunc, clean = decode_stats(scr) + d = { + "file": scr.path.name, + "magic": scr.magic, + "fields": {f"F{i}": f[i] for i in range(NUM_FIELDS)}, + "body_dwords": scr.nbody, + "sections": { + "code": {"off": 0, "len": scr.code_len, "code_end": scr.code_end}, + "T1": {"count": f[7], "off": f[8], "tag": 0x71}, + "T2": {"count": f[9], "off": f[10], "tag": 0x03}, + "T3": {"count": f[11], "off": f[12], "tag": 0x8F}, + }, + "decode": {"instructions": n, "unknown": unk, "truncated": trunc, "clean": clean}, + "table_tag_purity": {k: v for k, v in check_table_tags(scr).items()}, + "strings": {f"0x{off:x}": txt for off, (txt, _) in sorted(scr.strings.items())}, + } + if with_code: + d["code"] = [ + {"off": ins.offset, "op": f"0x{ins.opcode:x}", "label": ins.label, + "args": [[f"0x{t:x}", f"0x{v:x}"] for t, v in ins.args]} + for ins in scr.instructions + ] + return d + + +# --------------------------------------------------------------------------- # +# validation pass over a directory +# --------------------------------------------------------------------------- # +def validate_dir(root: Path) -> int: + files = sorted(root.glob("*.BIN")) + if not files: + print(f"no .BIN files in {root}") + return 1 + ok, bad = 0, 0 + tag_fail = 0 + decode_clean = 0 + decode_dirty = [] + for p in files: + try: + scr = load(p) + except Sys4Error as e: + print(f" FAIL {e}") + bad += 1 + continue + purity = check_table_tags(scr) + impure = [k for k, (o, t) in purity.items() if o != t] + if impure: + tag_fail += 1 + print(f" TAG {p.name}: impure tables {impure} {purity}") + _, unk, trunc, clean = decode_stats(scr) + if clean: + decode_clean += 1 + else: + decode_dirty.append((p.name, unk, trunc)) + ok += 1 + print(f"\n{len(files)} files: {ok} parsed clean, {bad} header/section failures, " + f"{tag_fail} with impure table tags") + print(f"opcode decode: {decode_clean}/{ok} fully clean (0 unknown/truncated)") + for name, unk, trunc in decode_dirty[:20]: + print(f" DECODE {name}: {unk} unknown, {trunc} truncated") + if len(decode_dirty) > 20: + print(f" ... and {len(decode_dirty) - 20} more") + return 0 if bad == 0 and tag_fail == 0 else 2 + + +# --------------------------------------------------------------------------- # +def main(argv=None): + ap = argparse.ArgumentParser(description="SYS4 .BIN loader / dumper") + ap.add_argument("target", help="a .BIN file, or a directory with --validate") + g = ap.add_mutually_exclusive_group() + g.add_argument("--summary", action="store_true", help="header + section sizes only") + g.add_argument("--strings", action="store_true", help="decoded string pool only") + g.add_argument("--json", action="store_true", help="machine-readable structure") + g.add_argument("--validate", action="store_true", help="check invariants over a folder") + args = ap.parse_args(argv) + + target = Path(args.target) + if args.validate or target.is_dir(): + return validate_dir(target) + + try: + scr = load(target) + except Sys4Error as e: + print(f"error: {e}", file=sys.stderr) + return 1 + + if args.summary: + print(render_summary(scr)) + elif args.strings: + print(render_strings(scr)) + elif args.json: + print(json.dumps(to_dict(scr), ensure_ascii=False, indent=2)) + else: + print(render_listing(scr)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/validate_opcode_table.py b/tools/validate_opcode_table.py new file mode 100644 index 0000000..d96f794 --- /dev/null +++ b/tools/validate_opcode_table.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Definitive test: replicate Kelebek's data_array_end shrinking (stop code at first +string/array offset), then measure clean decode rate over all Himegari scripts.""" +import os, re, sys, collections +from pathlib import Path +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import paths +import sys4load + +CPP=paths.KELEBEK_CPP.read_text(encoding="utf-8") +TABLE={}; LABEL={} +for m in re.finditer(r'\{\s*(0x[0-9A-Fa-f]+)\s*,\s*"([^"]*)"\s*,\s*(0x[0-9A-Fa-f]+)\s*\}', CPP): + TABLE[int(m.group(1),16)]=int(m.group(3),16); LABEL[int(m.group(1),16)]=m.group(2) + +files=paths.scripts() + +clean=dirty=0; parsefail=0 +still_unknown=collections.Counter(); examples={} +instr_total=0 +for name,p in sorted(files.items()): + try: scr=sys4load.load(p) + except Exception: parsefail+=1; continue + dw=scr.dwords; cl=scr.code_len + end=cl # data_array_end, starts at F8, shrinks to first string/array off + i=0; ok=True; reason="" + n_instr=0 + while iend: + ok=False; reason=f"overrun 0x{op:x}@{i}"; break + for a in range(argc): + atype=dw[base+2*a]; aval=dw[base+2*a+1] + if atype==2 and 0<=aval shrink code end + end=min(end,aval) + if op==0x64 and a==1 and 0<=aval then argument_count * , where each arg = . + => instruction length in dwords = 1 + 2*argument_count (uniform; type-2/0x64 args seek elsewhere, don't consume inline) + arg type 2 = inline string (value = dword offset into body). types: 0 imm,1 float,3 g-int,9 l-int, etc. +A clean decode consumes exactly code_len dwords with no unknown opcode and no arg overrun. +""" +import os, re, sys, collections +from pathlib import Path +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import paths +import sys4load + +CPP = paths.KELEBEK_CPP.read_text(encoding="utf-8") +# parse {0x1F4, "label", 0x0}, +TABLE = {} +LABEL = {} +for m in re.finditer(r'\{\s*(0x[0-9A-Fa-f]+)\s*,\s*"([^"]*)"\s*,\s*(0x[0-9A-Fa-f]+)\s*\}', CPP): + op = int(m.group(1), 16); lbl = m.group(2); argc = int(m.group(3), 16) + TABLE[op] = argc; LABEL[op] = lbl +print(f"parsed {len(TABLE)} opcode defs from Kelebek1 table (max op 0x{max(TABLE):x})") + +files = paths.scripts() + +clean = dirty = parsefail = 0 +unknown_ops = collections.Counter() +fail_examples = [] +str_ok = str_bad = 0 +opcode_use = collections.Counter() + +for name, p in sorted(files.items()): + try: + scr = sys4load.load(p) + except Exception: + parsefail += 1; continue + dw = scr.dwords; cl = scr.code_len + i = 0; ok = True; reason = "" + while i < cl: + op = dw[i] + if op not in TABLE: + unknown_ops[op] += 1; ok = False; reason = f"unknown op 0x{op:x} @{i}"; break + opcode_use[op] += 1 + argc = TABLE[op] + # check each arg's type; resolve strings + base = i + 1 + if base + 2*argc > cl: + ok = False; reason = f"arg overrun op 0x{op:x} @{i} needs {argc} args"; break + for a in range(argc): + atype = dw[base + 2*a]; aval = dw[base + 2*a + 1] + if atype == 2: # inline string + if 0 <= aval < scr.nbody: + txt, nd = sys4load._decode_string(dw, aval) + if txt is None: str_bad += 1 + else: str_ok += 1 + else: + str_bad += 1 + i = base + 2*argc + if ok and i == cl: + clean += 1 + else: + dirty += 1 + if len(fail_examples) < 15: + fail_examples.append(f" {name}: {reason} (stopped @{i}/{cl})") + +print(f"\n== decode result over {len(files)} scripts ==") +print(f" clean (fully consumed, all opcodes known): {clean}") +print(f" dirty: {dirty}") +print(f" parse-fail (container): {parsefail}") +print(f" string args resolved ok / bad: {str_ok} / {str_bad}") +print(f"\ntop unknown opcodes (op: files affected):") +for op, c in unknown_ops.most_common(25): + print(f" 0x{op:x}: {c}") +print(f"\nsample dirty files:") +print("\n".join(fail_examples)) +print(f"\ntop 25 opcodes actually used in Himegari (op label count):") +for op, c in opcode_use.most_common(25): + print(f" 0x{op:<4x} {LABEL.get(op,'?'):22} {c}") diff --git a/tools/vm0.py b/tools/vm0.py new file mode 100644 index 0000000..b7de699 --- /dev/null +++ b/tools/vm0.py @@ -0,0 +1,425 @@ +#!/usr/bin/env python3 +"""vm0 — headless AGE bytecode interpreter (Phase A0 prototype). + +Executes one script's bytecode to validate the *execution model* before any C#/Godot work. +Reuses tools/sys4load.py for all parsing/decoding. Effectful ops and call-script are STUBBED; +show-text is captured. Named-op semantics come from Kelebek's table; the classified markers are +treated as no-ops (this run TESTS that assumption). + +Purpose (see docs/phase-a-slice-plan.md): + * `--test` run the RECOVER unit test (pointer / 2D-array / loop / control-flow correctness). + * run a script; print captured show-text + an opcode-coverage/stub report. + +Memory model: + * one flat GLOBAL bank G (dict addr->int); globals are raw offsets into one space. + * per-call local frame with sparse typed banks (int/float/string/ptr). + * a `-ptr` variable holds an ADDRESS into G. lookup-array/2d with a ptr dst stores that + address (take-reference); reading a ptr derefs (G[addr]); writing through a ptr writes G[addr]. + This is the model RECOVER forces; the unit test is its litmus. + * jcc(cond, tA, tB): cond truthy -> goto tA else tB; 0xFFFFFFFF = fall through (from RECOVER+SCJUMP). + * call/ret (0x8F/0x05) are intra-script subroutine calls (shared frame); call-script (0x03) is + the inter-script one and is stubbed. +""" +from __future__ import annotations +import sys +import collections +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import paths +import sys4load +from age_opcodes import OPCODES + +NOJUMP = 0xFFFFFFFF + +# operand type tags +T_IMM, T_STR = 0x0, 0x2 +T_GINT, T_GFLOAT, T_GSTR, T_GPTR = 0x3, 0x4, 0x5, 0x6 +T_LINT, T_LFLOAT, T_LSTR, T_LPTR = 0x9, 0xA, 0xB, 0xC + +# opcodes treated as no-ops in v1 (classified markers; see age_opcodes_himegari.py) +# 0x71 = label-definition pseudo-op (count == T1 table size) — structural, no runtime effect. +MARKERS = {0x71, 0x1f4, 0x1f5, 0x1d5, 0x1bc, 0x1bf, 0x21b, 0x1d2, 0x258} + +# loop-guard: halt a run once any single show-text line has been emitted this many times. +# State-gated scenes run with zero initial state can spin a loop that re-emits the same block +# forever; capping re-emission terminates them cleanly and flags them LOOPED (vs. blind step limit). +EMIT_CAP = 2 + + +class Frame: + def __init__(self): + self.i = collections.defaultdict(int) # local-int + self.f = collections.defaultdict(int) # local-float (stored raw) + self.s = collections.defaultdict(str) # local-string + self.p = collections.defaultdict(int) # local-ptr (holds a G address) + + +class VM: + def __init__(self, scr: sys4load.Sys4Script, verbose=False, emit_cap=EMIT_CAP): + self.scr = scr + self.verbose = verbose + self.code = scr.instructions + self.by_off = {ins.offset: idx for idx, ins in enumerate(self.code)} + self.G = collections.defaultdict(int) # global-int bank (flat address space) + self.Gs = collections.defaultdict(str) # global-string bank + self.fr = Frame() + self.callstack = [] # return indices for call/ret + self.text = [] # captured show-text as (str_offset, text) + self.emit_seen = collections.Counter() # per-offset emit count (loop-guard) + self.emit_cap = emit_cap + self.halt_reason = None # 'exit' | 'LOOP:...' | 'STEP-LIMIT' | 'ret-underflow' + self.log = collections.Counter() # stub/unknown opcode hits + self.exec_count = collections.Counter() # opcode coverage + self.steps = 0 + + # ---- operand resolution -------------------------------------------------- + def _string(self, off): + e = self.scr.strings.get(off) + if e is not None: + return e[0] + txt, _ = sys4load._decode_string(self.scr.dwords, off) + return txt if txt is not None else "" + + def read(self, op): + t, v = op + if t == T_IMM: return v + if t == T_STR: return self._string(v) + if t == T_GINT: return self.G[v] + if t == T_GFLOAT: return self.G[v] + if t == T_GSTR: return self.Gs[v] + if t == T_GPTR: return self.G[self.G[v]] + if t == T_LINT: return self.fr.i[v] + if t == T_LFLOAT: return self.fr.f[v] + if t == T_LSTR: return self.fr.s[v] + if t == T_LPTR: return self.G[self.fr.p[v]] # deref ptr + self.log[f"read?t{t:#x}"] += 1 + return v + + def write(self, op, val): + t, v = op + if t == T_GINT or t == T_GFLOAT: self.G[v] = val + elif t == T_GSTR: self.Gs[v] = val + elif t == T_GPTR: self.G[self.G[v]] = val + elif t == T_LINT: self.fr.i[v] = val + elif t == T_LFLOAT: self.fr.f[v] = val + elif t == T_LSTR: self.fr.s[v] = val + elif t == T_LPTR: self.G[self.fr.p[v]] = val # write through + else: self.log[f"write?t{t:#x}"] += 1 + + def base_addr(self, op): + """The base ADDRESS an operand names, for array lookups.""" + t, v = op + if t in (T_IMM, T_GINT, T_GFLOAT, T_GSTR, T_GPTR): return v # global's own offset + if t == T_LINT: return self.fr.i[v] + if t == T_LPTR: return self.fr.p[v] + return v + + def lookup_store(self, dst, addr): + """lookup result: ptr dst gets the reference (address); non-ptr gets the element value.""" + t, v = dst + if t == T_LPTR: self.fr.p[v] = addr + elif t == T_GPTR: self.G[v] = addr + else: self.write(dst, self.G[addr]) + + # ---- execution ----------------------------------------------------------- + def run(self, entry_off=0, max_steps=2_000_000): + pc = self.by_off.get(entry_off, 0) + while 0 <= pc < len(self.code): + if self.steps >= max_steps: + self.log["STEP-LIMIT"] += 1 + self.halt_reason = self.halt_reason or "STEP-LIMIT" + break + self.steps += 1 + ins = self.code[pc] + op = ins.opcode + self.exec_count[op] += 1 + nxt = self.step(ins, pc) + if nxt is None: # halt + break + pc = nxt + else: + self.halt_reason = self.halt_reason or "pc-out-of-range" + return self + + def step(self, ins, pc): + op, a = ins.opcode, ins.args + lbl = OPCODES.get(op, ("?", 0))[0] + + # arithmetic / bitwise: dst = a1 a2 + alu = {"add": lambda x, y: x + y, "sub": lambda x, y: x - y, + "mul": lambda x, y: x * y, "div": lambda x, y: (x // y if y else 0), + "mod": lambda x, y: (x % y if y else 0), "and": lambda x, y: x & y, + "or": lambda x, y: x | y, "sar": lambda x, y: x >> (y & 31), + "shl": lambda x, y: x << (y & 31)} + cmp = {"eq": lambda x, y: int(x == y), "ne": lambda x, y: int(x != y), + "lt": lambda x, y: int(x < y), "lte": lambda x, y: int(x <= y), + "gr": lambda x, y: int(x > y), "gre": lambda x, y: int(x >= y)} + + if lbl in alu: + self.write(a[0], alu[lbl](self.read(a[1]), self.read(a[2]))); return pc + 1 + if lbl in cmp: + self.write(a[0], cmp[lbl](self.read(a[1]), self.read(a[2]))); return pc + 1 + if lbl == "mov": + self.write(a[0], self.read(a[1])); return pc + 1 + if lbl == "set-string": + self.write(a[0], self.read(a[1])); return pc + 1 + if lbl == "lookup-array": # dst = base[idx] + addr = self.base_addr(a[1]) + self.read(a[2]) + self.lookup_store(a[0], addr); return pc + 1 + if lbl == "lookup-array-2d": # dst = base[i*stride + col] + addr = self.base_addr(a[1]) + self.read(a[2]) * self.read(a[3]) + self.read(a[4]) + self.lookup_store(a[0], addr); return pc + 1 + if lbl == "bit-set": + self.write(a[0], self.read(a[0]) | self.read(a[1])); return pc + 1 + if lbl == "bit-reset": + self.write(a[0], self.read(a[0]) & ~self.read(a[1])); return pc + 1 + if lbl == "check-bit": # p1 = (p2 >> p3) & 1 + self.write(a[0], (self.read(a[1]) >> (self.read(a[2]) & 31)) & 1); return pc + 1 + if lbl == "copy-to-global": # best-effort: p1 = p2 (single cell) + self.write(a[0], self.read(a[1])); return pc + 1 + + # control flow + if lbl == "jmp": + return self.by_off.get(a[0][1], pc + 1) + if lbl == "call": # intra-script subroutine + self.callstack.append(pc + 1) + return self.by_off.get(a[0][1], pc + 1) + if lbl == "ret": + if self.callstack: + return self.callstack.pop() + self.halt_reason = "ret-underflow" + return None + if lbl == "jcc": # (cond, tA, tB); 0xFFFFFFFF=fallthrough + tgt = a[1][1] if self.read(a[0]) else a[2][1] + return pc + 1 if tgt == NOJUMP else self.by_off.get(tgt, pc + 1) + if lbl in ("exit", "exit-script"): + self.halt_reason = "exit" + return None + if lbl == "call-script": # STUB (inter-script) + self.log[f"call-script({self.read(a[0]) if a else '?'})"] += 1; return pc + 1 + + # ADV text: capture the string operand as (offset, text); loop-guard on re-emission + if lbl == "show-text": + for t, v in a: + if t == T_STR: + self.emit_seen[v] += 1 + if self.emit_seen[v] > self.emit_cap: + self.halt_reason = f"LOOP:line@{v:#x}×{self.emit_seen[v]}" + return None + self.text.append((v, self._string(v))) + return pc + 1 + if lbl in ("end-text-line", "wait-for-input", "set-font", "comment", + "display-furigana", "dev_ukn"): + return pc + 1 + + if op in MARKERS: # classified no-op markers + return pc + 1 + + # everything else (effectful draw/audio/ui/input, unnamed) -> stub + continue + self.log[f"stub:{lbl if not lbl.startswith('u00') else hex(op)}"] += 1 + return pc + 1 + + +def run_test(): + """RECOVER unit test — pointer/2D-array/loop/control-flow correctness.""" + scr = sys4load.load(paths.DATA1 / "RECOVER.BIN") + vm = VM(scr) + unit = 0 + vm.G[0x152616] = unit + A, B = 0x4e11b, 0x4e085 # block-1 tables (stride 14 / 3) + C, E, F, FL = 0x52383, 0x52f3b, 0x5295f, 0xaacb4 # block-2 tables (stride 30) + flags + # block 1 source: A[unit*14 + 11..13] + for k in range(3): + vm.G[A + unit * 14 + (11 + k)] = 100 + k + # block 2: slot 5 active, slot 6 zero-C (skip), slot 7 flag-off (skip) + vm.G[C + unit * 30 + 5] = 7; vm.G[FL + 5] = 1; vm.G[E + unit * 30 + 5] = 42 + vm.G[C + unit * 30 + 6] = 0; vm.G[FL + 6] = 1; vm.G[E + unit * 30 + 6] = 99 + vm.G[C + unit * 30 + 7] = 3; vm.G[FL + 7] = 0; vm.G[E + unit * 30 + 7] = 88 + vm.run() + + checks = [ + ("block1 B[0]=A[11]", vm.G[B + unit * 3 + 0], 100), + ("block1 B[1]=A[12]", vm.G[B + unit * 3 + 1], 101), + ("block1 B[2]=A[13]", vm.G[B + unit * 3 + 2], 102), + ("block2 s5 C:=E", vm.G[C + unit * 30 + 5], 42), + ("block2 s5 F:=-1", vm.G[F + unit * 30 + 5], -1), + ("block2 s6 C skip", vm.G[C + unit * 30 + 6], 0), # C stayed 0 (guard) + ("block2 s7 C skip", vm.G[C + unit * 30 + 7], 3), # flag off -> untouched + ] + ok = True + for name, got, want in checks: + status = "OK " if got == want else "FAIL" + if got != want: + ok = False + print(f" [{status}] {name}: got {got}, want {want}") + print(f" steps={vm.steps} call-script stubs={sum(v for k,v in vm.log.items() if k.startswith('call-script'))}") + print("RECOVER unit test:", "PASS" if ok else "FAIL") + return 0 if ok else 1 + + +def run_file(path): + scr = sys4load.load(path) + vm = VM(scr).run() + print(f"== {Path(path).name}: {vm.steps} steps, {len(vm.text)} show-text lines captured " + f"(halt: {vm.halt_reason}) ==") + for i, (off, t) in enumerate(vm.text[:20]): + print(f" [{i}] {t}") + if len(vm.text) > 20: + print(f" ... (+{len(vm.text)-20} more)") + stubs = [(k, v) for k, v in vm.log.most_common() if k.startswith("stub:")] + if stubs: + print(" top stubbed effectful/unknown ops:", ", ".join(f"{k[5:]}×{v}" for k, v in stubs[:10])) + ncall = sum(v for k, v in vm.log.items() if k.startswith("call-script")) + print(f" call-script stubs: {ncall} distinct opcodes executed: {len(vm.exec_count)}") + return 0 + + +# ---- dialogue oracle --------------------------------------------------------- +import json +import re + +SCENE_RE = re.compile(r"^S[CP]\d{4}\.BIN$") + + +def load_oracle(path=None): + """file (UPPER) -> ordered list of (str_offset, text): the static show-text lines + extract_phase2 dumped. This is the independent oracle the VM is validated against.""" + path = path or (paths.BUILD / "text" / "dialogue.jsonl") + by = collections.defaultdict(list) + with open(path, encoding="utf-8") as f: + for line in f: + d = json.loads(line) + by[d["file"].upper()].append((int(d["off"], 16), d["text"])) + return by + + +def subsequence_status(emitted_offs, static_offs): + """Classify the emitted show-text offset stream against the static ordered set. + + Emitted may repeat offsets (loop re-emission); we validate the distinct stream in + first-seen order as an in-order subsequence of the static lines. Returns (status, + detail): OK | STRAY (offset never in static) | ORDER (in static but out of order).""" + static_list = list(static_offs) + static_set = set(static_list) + seen, distinct = set(), [] + for o in emitted_offs: + if o not in seen: + seen.add(o) + distinct.append(o) + strays = [o for o in distinct if o not in static_set] + if strays: + return "STRAY", strays + j = 0 + for o in distinct: + while j < len(static_list) and static_list[j] != o: + j += 1 + if j >= len(static_list): + return "ORDER", [o] + j += 1 + return "OK", [] + + +def verdict(status, halt, n_emit, n_static): + """Fold subsequence status + halt reason into one scene verdict.""" + if n_static == 0: + return "NO-DIALOGUE" # not an ADV scene (no static show-text) — skip in scoring + if n_emit == 0: + return "EMPTY" # scene has dialogue but VM emitted none — investigate + if status != "OK": + return status # STRAY / ORDER — a real divergence + return "CLEAN" if halt == "exit" else f"OK/{halt}" # OK subsequence; did it exit cleanly? + + +def run_scene(name, path, oracle): + """Run one scene, return a result dict comparing emitted vs static show-text.""" + scr = sys4load.load(path) + vm = VM(scr).run() + emitted = [off for off, _ in vm.text] + static = oracle.get(name.upper(), []) + static_offs = [o for o, _ in static] + status, detail = subsequence_status(emitted, static_offs) + n_emit = len({o for o in emitted}) + return {"name": name, "vm": vm, "emitted": emitted, "static": static, + "n_emit_distinct": n_emit, "n_static": len(static_offs), + "status": status, "detail": detail, + "verdict": verdict(status, vm.halt_reason, n_emit, len(static_offs))} + + +def run_one_scene(name): + """Detailed single-scene oracle diff (for investigating one script).""" + oracle = load_oracle() + scripts = paths.scripts() + name = name.upper() + if not name.endswith(".BIN"): + name += ".BIN" + if name not in scripts: + print(f"no such script: {name}") + return 1 + r = run_scene(name, scripts[name], oracle) + vm = r["vm"] + print(f"== {name}: verdict {r['verdict']} ==") + print(f" emitted {len(r['emitted'])} lines ({r['n_emit_distinct']} distinct), " + f"static {r['n_static']}, steps {vm.steps}, halt {vm.halt_reason}") + if r["status"] == "STRAY": + stat_set = {o for o, _ in r["static"]} + print(f" STRAY offsets (emitted, not in static): " + f"{', '.join(hex(o) for o in r['detail'])}") + for off, txt in vm.text: + if off in set(r["detail"]): + print(f" @ {off:#x}: {txt!r}") + elif r["status"] == "ORDER": + print(f" first out-of-order offset: {hex(r['detail'][0])}") + return 0 + + +def run_sweep(limit=None): + """Run every SC####/SP#### scene against the dialogue oracle; print a coverage table.""" + oracle = load_oracle() + scripts = paths.scripts() + names = sorted(n for n in scripts if SCENE_RE.match(n)) + if limit: + names = names[:limit] + buckets = collections.Counter() + rows = [] + for name in names: + r = run_scene(name, scripts[name], oracle) + buckets[r["verdict"]] += 1 + rows.append(r) + + # non-clean scenes get listed for follow-up + problem = [r for r in rows if r["verdict"] not in ("CLEAN", "NO-DIALOGUE")] + if problem: + print("non-clean scenes:") + for r in sorted(problem, key=lambda r: r["verdict"]): + d = "" + if r["status"] in ("STRAY", "ORDER"): + d = " " + " ".join(hex(o) for o in r["detail"][:4]) + print(f" {r['name']:<14} {r['verdict']:<16} " + f"emit {r['n_emit_distinct']:>4}/{r['n_static']:<4} " + f"steps {r['vm'].steps:>7} halt {r['vm'].halt_reason}{d}") + print() + scene_total = sum(v for k, v in buckets.items() if k != "NO-DIALOGUE") + valid = sum(v for k, v in buckets.items() + if k == "CLEAN" or k.startswith("OK/")) + print(f"scenes with dialogue: {scene_total} (skipped {buckets['NO-DIALOGUE']} with no static show-text)") + print("verdict breakdown:", dict(sorted(buckets.items()))) + print(f"DIALOGUE-VALID (clean in-order subsequence, no garbage): " + f"{valid}/{scene_total} = {valid/scene_total*100:.1f}%") + return 0 + + +def main(argv=None): + argv = argv if argv is not None else sys.argv[1:] + if not argv or argv[0] == "--test": + return run_test() + if argv[0] == "--sweep": + return run_sweep(limit=int(argv[1]) if len(argv) > 1 else None) + if argv[0] == "--scene": + return run_one_scene(argv[1]) + return run_file(argv[0]) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/vm-map/himegari-opcode-notes.md b/vm-map/himegari-opcode-notes.md new file mode 100644 index 0000000..13420c4 --- /dev/null +++ b/vm-map/himegari-opcode-notes.md @@ -0,0 +1,111 @@ +# Himegari opcode inference notes + +Classification of the top ~20 unnamed opcodes (Phase 3.0, inference-only, 2026-07-06). +Machine-readable form: `tools/age_opcodes_himegari.py` (`INFERRED`). Evidence gathered with +`tools/opcode_context.py` over the 481-script `Output/DATA1` corpus (1,502,986 instructions). + +Baseline: named opcodes = **72.62%** of instruction volume. Classifying the top 20 unnamed +ops covers to **~96.7%** cumulative. Kelebek labels (`u004xxxx`) are engine addresses from a +*later* AGE title — opcode number + argc are validated for Himegari (481/481 clean decode); +the meanings below are inferred and flagged by confidence + confirmation method. + +## Buckets + +### A. Statement/scope scaffolding — zero-arg, no operands, safe VM no-ops (HIGH) +The compiler brackets statements/blocks with operand-less markers. They carry no data and +sit at structural boundaries, so the VM v1 skips them; the Phase-4 dialogue harness confirms. + +- **0x1f4 `stmt-begin`** (60,297; 4.01%) — opens scripts (first instr of ADDEN/ADDEXP/…), and + `stmt-end 0x1f5 → 0x1f4` pairs 25,899×. Successor is the statement's first real op (eq/mov/call). +- **0x1f5 `stmt-end`** (60,297; 4.01%) — equal count to 0x1f4; precedes `exit`/next statement. + `... → 0x1f5 → stmt-begin/lt/show-text`. +- **0x1d5 `cond-block`** (34,238; 2.28%) — **always** preceded by `jcc`; marks the fall-through + (condition-true) block entry. `jcc cond -1 label; 0x1d5; `. +- **0x1bc `block-mark`** (26,865; 1.79%) — `jcc → 0x1bc → mov` (26,413×): another block boundary. +- **0x1bf `call-end`** (8,752; 0.58%) — `call → 0x1bf → stmt-end` (8,600×): end-of-call statement. + +### B. Statement metadata with an id — tentative no-op, confirm via harness (MED/LOW) +Same scaffolding family but carrying an immediate; likely line numbers / statement descriptors. +Treated as no-ops in v1 but flagged `method=harness` because a wrong no-op here could drop state. + +- **0x21b `line-id?`** (27,016; 1.80%) — `mov → 0x21b → stmt-end`. Arg is 0/1/small. Sits + near `savemesskip`/`loadmesskip` dev-comments (MENU/BUNKI) — **verify it isn't a message-control + toggle** before trusting the no-op. +- **0x1d2 `stmt-desc?`** (17,323; 1.15%) — always immediately after `stmt-begin 0x1f4`; 2 imm args + (e.g. `0x2 0x5`) then a `mov`. Reads like a per-statement descriptor. +- **0x258 `decl?`** (7,224; 0.48%) — runs in a chain right after script-entry `0x259`, enumerating + ids (`0x4 0x1`, `0x5 0x1`, `0x6 0x1`, …). Prologue declaration/registration table. + +### C. Structural (HIGH) +- **0x71 `label-def`** (26,445; 1.76%) — its corpus count **exactly equals the T1 table size**, so + this is the instruction T1 (tag 0x71) indexes: a label/anchor definition. Arg is a small id. + Preceded by `end-text-line`/`call`. VM v1 = no-op; **revisit if menu/callback dispatch resolves + entries by this id** (T1 may be a named-entry-point table). + +### D. Computational — exact behaviour via Unicorn/inference (MED/LOW) +- **0x215 `count?`** (5,675; 0.38%) — writes its first operand (a global) which is then tested + `gre …, 0` / `lt` and branched on. Signature (global,global). Looks like a count / search-returns + -index helper. Micro-exec candidate. +- **0x1a2 `resolve-handle?`** (17,585; 1.17%) — takes a `local-ptr` just filled by `lookup-array`, + then `create-texture` follows. Resolves a looked-up resource/handle. Borderline effectful → Frida. + +### E. Effectful — draw / UI / audio / ADV; need Frida to confirm (see shortlist) +All in the 0x420–0x421 graphics family or 0x41D audio family, clustered with named draw/audio ops. +Not semantically no-ops; VM must implement (or knowingly stub) them. + +- **0x7a `text-param?`** (17,324; 1.15%) — `sub` computes a value, then `0x7a `, + then `show-text`. A text speed / auto-wait / window parameter for the following line. (The review + guessed "computational"; context says ADV-text-effectful.) +- **0x202 `draw-blit?`** (9,223) / **0x203 `draw?`** (9,149) — 4–5 coord/size args, preceded by + coordinate arithmetic, chained with `draw-texture`/`u00420950`. Blit/positioned-draw. +- **0x1f7 `ui-elem?`** (7,735) / **0x1fa `ui-clear?`** (6,126) — paired (`0x1f7 → 0x1fa` 5,014×); + create/begin a UI element then show/hide/clear it by id. From MENU: `0x1f7 0x11170 0xc80`, `0x1fa 0x49/0x4a/0x4b`. +- **0x217 / 0x218 / 0x21a `gfx-geom?`** (~5,000 each) — a chain each taking 4 global-ints + (`0x21a → 0x218`, `0x217 → 0x1ff`); rect/coordinate/geometry transforms in the draw library. +- **0x1ff `draw?`** (3,926) — 4 args then `call`; follows `0x217`. +- **0xb6 `snd-ctrl?`** (4,805) — 0x41D family, self-chains, near `play-sound-effect`/`0xb5`; sound + channel / volume / stop control. + +### F. Input / UI hotspot branch — op 0x90 & 0x97 (A0-slice deep-dive, 2026-07-06 s2, HIGH) +Solved while investigating the A0 EMPTY scenes. Kelebek: `{0x90,"u0041BEB0",0x7} // args 5,6,7 = +code locations` (they left it "ukn"). Corpus evidence (`scratchpad/analyze_0x90.py` over all scripts): + +- **op 0x90 = pointer/input hotspot hit-test branch.** Encoding `0x90 x y w h tgt_a tgt_b tgt_c` + (argc 7; targets `0xffffffff` = unused). Branches to one of 3 targets per interaction outcome and + **falls through to pc+1 when nothing matches** — design-confirmed: a 0x90 is 15 dwords, and the + instruction after the last one lands exactly on the fall-through statement (`0xd0+15 = 0xdf`). +- **Two forms, both only in one shared subroutine:** + - *Mode A* (1505 = 5×301): all-immediate, `w=h=20`. The five on-screen ADV buttons — identical in + all 301 scripts at `(x,y) = (684|706|728|750|772, 572)`, each setting one of `G[0x6c9..0x6cd]` to + 1 (target a) / 0 (target b) / 0+run-action (target c). Reads as hover-enter / hover-leave / click. + - *Mode B* (903 = 3×301): local-int operands, `w=h=1`, only target c real — a keyed/degenerate + 2-way input test (fall through, or jump to c). Operands are `0-1000`-based sentinels (key codes?). +- **Distribution is unanimous: every one of the 301 ADV scripts has EXACTLY 8 sites (5 A + 3 B).** + Zero scene-specific use — it is boilerplate ADV chrome (the system button row + input polling), not + gameplay branching. +- **VM handling:** headless (no cursor/input) ⇒ fall through ⇒ **our stub is already correct**; proven + safe by all 279 CLEAN dialogue scenes (which contain these same 8 sites). So 0x90 is NOT a headless + correctness hole. It must be modelled for real in **A2** (Godot input backend) as a live hotspot test; + confirm the exact target→interaction-state mapping via input capture / Frida then. +- **op 0x97** (`u0041C150`, argc 5: `v1 v2 1 1 `, NO code targets) interleaves with 0x90 in + the same subroutine → companion *register-hotspot / set-widget-action* call (trailing imm = action id + 0x0/0x7/0x8). Not control flow; part of the `0x90/0x91/0x92/0x95/0x97` widget cluster. + +**Consequence for A0/A1:** the 12 EMPTY scenes are gated by **state + interactive input-wait loops** +(this chrome polls hotspots until a click), NOT by unmodelled 0x90 semantics. A1 can keep 0x90 as +fall-through with confidence; unlocking the EMPTY family is a Phase-A2/B task (seed state + input). + +## Coverage after classification +See `build/opcode-coverage.md`. Roughly: named 72.6% + no-op scaffolding & label-def (A+C, high +conf) ≈ 14.4% + tentative-no-op metadata (B) ≈ 3.4% → **~90% handleable without live tools**; +the effectful/computational remainder (D+E) ≈ 6–7% is classified but needs Frida/Unicorn for +exact behaviour. + +## Frida / Unicorn shortlist (feeds Phase 3.1/3.2) +- **Frida (effectful):** 0x7a (adv text param), 0x202/0x203/0x1f7/0x1fa/0x217/0x218/0x21a/0x1ff + (draw/UI), 0xb6 (audio), 0x1a2 (resource resolve). Also **Frida-confirm already-named effectful + ops the VM will rely on**: `play-voice 0xc4`, `play-bgm 0xbf`, `play-sound-effect 0xb4`, + `draw-texture 0x1fb`, `draw-string 0x204`, `create-texture 0x1f8`, `set-texture 0x1f9`. +- **Unicorn (computational):** 0x215 (count/search). +- **Harness (no live tools):** 0x21b / 0x1d2 / 0x258 — confirm no-op assumption by diffing the + VM's `show-text` sequence per scene against `build/text/dialogue.jsonl`. diff --git a/vm-map/opcode-leads.json b/vm-map/opcode-leads.json new file mode 100644 index 0000000..11d4bb6 --- /dev/null +++ b/vm-map/opcode-leads.json @@ -0,0 +1,1194 @@ +{ + "note": "Candidate SYS4 opcodes seeded from static byte analysis of 483 scripts. Statement leads = first dword after each [0x8F,0,line#] T3 record. These are HYPOTHESES to confirm against the (packed) AGE.EXE VM dispatch.", + "statement_leads": [ + { + "op": "0x55", + "dec": 85, + "count": 21687, + "pct": 30.743, + "examples": [ + "ADDEXP.BIN", + "ALCHEMY.BIN", + "BTL.BIN", + "BUNKI.BIN", + "CAMP.BIN" + ] + }, + { + "op": "0x8f", + "dec": 143, + "count": 10070, + "pct": 14.275, + "examples": [ + "AIM.BIN", + "ALCHEMY.BIN", + "BTL.BIN", + "BUNKI.BIN", + "CALLBACK_LOAD.BIN" + ] + }, + { + "op": "0x1bf", + "dec": 447, + "count": 8393, + "pct": 11.898, + "examples": [ + "DEBUGADV.BIN", + "DEBUGADV2.BIN", + "GAMECLEAR.BIN", + "SC0000.BIN", + "SC0010.BIN" + ] + }, + { + "op": "0x71", + "dec": 113, + "count": 8168, + "pct": 11.579, + "examples": [ + "DEBUG.BIN", + "DEBUGADV.BIN", + "DEBUGADV2.BIN", + "GAMECLEAR.BIN", + "SC0000.BIN" + ] + }, + { + "op": "0x5", + "dec": 5, + "count": 3777, + "pct": 5.354, + "examples": [ + "ADDEXP.BIN", + "ALCHEMY.BIN", + "BUNKI.BIN", + "CALLBACK_LOAD.BIN", + "CGMODE.BIN" + ] + }, + { + "op": "0x1f5", + "dec": 501, + "count": 3267, + "pct": 4.631, + "examples": [ + "DEBUGADV.BIN", + "DEBUGADV2.BIN", + "SC0000.BIN", + "SC0010.BIN", + "SC0020.BIN" + ] + }, + { + "op": "0x20c", + "dec": 524, + "count": 1934, + "pct": 2.742, + "examples": [ + "AIM.BIN", + "ALCHEMY.BIN", + "BUNKI.BIN", + "CALLBACK_SETTING.BIN", + "CALLBACK_WINDOW.BIN" + ] + }, + { + "op": "0x215", + "dec": 533, + "count": 1639, + "pct": 2.323, + "examples": [ + "DEBUGADV.BIN", + "DEBUGADV2.BIN", + "SC0000.BIN", + "SC0010.BIN", + "SC0020.BIN" + ] + }, + { + "op": "0xa0", + "dec": 160, + "count": 1634, + "pct": 2.316, + "examples": [ + "ALCHEMY.BIN", + "BUNKI.BIN", + "CALCSCOPE.BIN", + "CGMODE.BIN", + "DEBUGADV.BIN" + ] + }, + { + "op": "0x1f4", + "dec": 500, + "count": 1592, + "pct": 2.257, + "examples": [ + "DEBUGADV.BIN", + "DEBUGADV2.BIN", + "SC0000.BIN", + "SC0010.BIN", + "SC0020.BIN" + ] + }, + { + "op": "0xc8", + "dec": 200, + "count": 1553, + "pct": 2.201, + "examples": [ + "CONFIG.BIN", + "DEBUGADV.BIN", + "DEBUGADV2.BIN", + "FIELD.BIN", + "SC0000.BIN" + ] + }, + { + "op": "0x51", + "dec": 81, + "count": 650, + "pct": 0.921, + "examples": [ + "ALCHEMY.BIN", + "BTL.BIN", + "CGMODE.BIN", + "CHMENU.BIN", + "DEBUGADV.BIN" + ] + }, + { + "op": "0x217", + "dec": 535, + "count": 630, + "pct": 0.893, + "examples": [ + "DEBUG.BIN", + "DEBUGADV.BIN", + "DEBUGADV2.BIN", + "SC0000.BIN", + "SC0010.BIN" + ] + }, + { + "op": "0x1fb", + "dec": 507, + "count": 579, + "pct": 0.821, + "examples": [ + "DEBUGADV.BIN", + "DEBUGADV2.BIN", + "SAVE.BIN", + "SC0000.BIN", + "SC0010.BIN" + ] + }, + { + "op": "0x8c", + "dec": 140, + "count": 446, + "pct": 0.632, + "examples": [ + "ALCHEMY.BIN", + "ATSEEK.BIN", + "BTANINIT.BIN", + "BTL.BIN", + "BUNKI.BIN" + ] + }, + { + "op": "0x6c", + "dec": 108, + "count": 434, + "pct": 0.615, + "examples": [ + "DEBUG.BIN", + "DEBUGADV.BIN", + "DEBUGADV2.BIN", + "GAMECLEAR.BIN", + "GAMESTART.BIN" + ] + }, + { + "op": "0x21a", + "dec": 538, + "count": 399, + "pct": 0.566, + "examples": [ + "DEBUGADV.BIN", + "DEBUGADV2.BIN", + "SC0000.BIN", + "SC0010.BIN", + "SC0030.BIN" + ] + }, + { + "op": "0x61", + "dec": 97, + "count": 348, + "pct": 0.493, + "examples": [ + "ALCHEMY.BIN", + "BTL.BIN", + "CHMENU.BIN", + "DEBUGADV.BIN", + "FIELD.BIN" + ] + }, + { + "op": "0x5a", + "dec": 90, + "count": 311, + "pct": 0.441, + "examples": [ + "AIM.BIN", + "ALCHEMY.BIN", + "BTL.BIN", + "CGMODE.BIN", + "CHMENU.BIN" + ] + }, + { + "op": "0x1ad", + "dec": 429, + "count": 297, + "pct": 0.421, + "examples": [ + "DEBUGADV.BIN", + "DEBUGADV2.BIN", + "SC0000.BIN", + "SC0010.BIN", + "SC0020.BIN" + ] + }, + { + "op": "0x199", + "dec": 409, + "count": 297, + "pct": 0.421, + "examples": [ + "DEBUGADV.BIN", + "DEBUGADV2.BIN", + "SC0000.BIN", + "SC0010.BIN", + "SC0020.BIN" + ] + }, + { + "op": "0x7c", + "dec": 124, + "count": 297, + "pct": 0.421, + "examples": [ + "DEBUGADV.BIN", + "DEBUGADV2.BIN", + "SC0000.BIN", + "SC0010.BIN", + "SC0020.BIN" + ] + }, + { + "op": "0x50", + "dec": 80, + "count": 264, + "pct": 0.374, + "examples": [ + "BTL.BIN", + "CGMODE.BIN", + "DEBUGADV.BIN", + "DEBUGADV2.BIN", + "FIELD.BIN" + ] + }, + { + "op": "0xb4", + "dec": 180, + "count": 260, + "pct": 0.369, + "examples": [ + "ADDEXP.BIN", + "CGMODE.BIN", + "CHMENU.BIN", + "CONFIG.BIN", + "DEBUGADV.BIN" + ] + }, + { + "op": "0x22f", + "dec": 559, + "count": 256, + "pct": 0.363, + "examples": [ + "DEBUGADV.BIN", + "SC0000.BIN", + "SC0010.BIN", + "SC0040.BIN", + "SC0050.BIN" + ] + }, + { + "op": "0x218", + "dec": 536, + "count": 255, + "pct": 0.361, + "examples": [ + "DEBUGADV.BIN", + "DEBUGADV2.BIN", + "SC0000.BIN", + "SC0010.BIN", + "SC0030.BIN" + ] + }, + { + "op": "0x2", + "dec": 2, + "count": 153, + "pct": 0.217, + "examples": [ + "DEBUGADV.BIN", + "DEBUGADV2.BIN", + "SC0000.BIN", + "SC0010.BIN", + "SC0020.BIN" + ] + }, + { + "op": "0x75", + "dec": 117, + "count": 139, + "pct": 0.197, + "examples": [ + "CAMP.BIN", + "DEBUGADV.BIN", + "DEBUGADV2.BIN", + "FIELD.BIN", + "FORT.BIN" + ] + }, + { + "op": "0xae", + "dec": 174, + "count": 134, + "pct": 0.19, + "examples": [ + "DEBUGADV.BIN", + "DEBUGADV2.BIN", + "SC0000.BIN", + "SC0010.BIN", + "SC0020.BIN" + ] + }, + { + "op": "0xbf", + "dec": 191, + "count": 115, + "pct": 0.163, + "examples": [ + "DEBUGADV2.BIN", + "FIELD.BIN", + "SC0000.BIN", + "SC0010.BIN", + "SC0020.BIN" + ] + }, + { + "op": "0x5c", + "dec": 92, + "count": 83, + "pct": 0.118, + "examples": [ + "ALCHEMY.BIN", + "CHMENU.BIN", + "EVOLVE.BIN", + "EXILE.BIN", + "FIELD.BIN" + ] + }, + { + "op": "0x202", + "dec": 514, + "count": 58, + "pct": 0.082, + "examples": [ + "DEBUGADV.BIN", + "SC0010.BIN", + "SC0030.BIN", + "SC0060.BIN", + "SC0090.BIN" + ] + }, + { + "op": "0x9", + "dec": 9, + "count": 54, + "pct": 0.077, + "examples": [ + "GAMECLEAR.BIN", + "SC0000.BIN", + "SC0020.BIN", + "SC0030.BIN", + "SC0040.BIN" + ] + }, + { + "op": "0x52", + "dec": 82, + "count": 46, + "pct": 0.065, + "examples": [ + "SC0000.BIN", + "SC0010.BIN", + "SC0020.BIN", + "SC0040.BIN", + "SC0060.BIN" + ] + }, + { + "op": "0x203", + "dec": 515, + "count": 44, + "pct": 0.062, + "examples": [ + "BTL.BIN", + "DEBUGADV.BIN", + "SC0020.BIN", + "SC0030.BIN", + "SC0040.BIN" + ] + }, + { + "op": "0x56", + "dec": 86, + "count": 37, + "pct": 0.052, + "examples": [ + "ALCHEMY.BIN", + "BUNKI.BIN", + "CGMODE.BIN", + "CHMENU.BIN", + "EVOLVE.BIN" + ] + }, + { + "op": "0x3", + "dec": 3, + "count": 35, + "pct": 0.05, + "examples": [ + "ALCHEMY.BIN", + "CHMENU.BIN", + "EVOLVE.BIN", + "FIELD.BIN", + "MENU.BIN" + ] + }, + { + "op": "0x1fe", + "dec": 510, + "count": 25, + "pct": 0.035, + "examples": [ + "DEBUGADV.BIN", + "SC0010.BIN", + "SC0020.BIN", + "SC0070.BIN", + "SC0130.BIN" + ] + }, + { + "op": "0xcd", + "dec": 205, + "count": 24, + "pct": 0.034, + "examples": [ + "AIM.BIN", + "ALCHEMY.BIN", + "CHMENU.BIN", + "EVOLVE.BIN", + "EXILE.BIN" + ] + }, + { + "op": "0xb6", + "dec": 182, + "count": 18, + "pct": 0.026, + "examples": [ + "SC0000.BIN", + "SC0140.BIN", + "SP0044.BIN", + "SP0406.BIN", + "SP0822.BIN" + ] + }, + { + "op": "0x5f", + "dec": 95, + "count": 17, + "pct": 0.024, + "examples": [ + "ALCHEMY.BIN", + "CHMENU.BIN", + "EVOLVE.BIN", + "FIELD.BIN", + "IMPROVE.BIN" + ] + }, + { + "op": "0x93", + "dec": 147, + "count": 17, + "pct": 0.024, + "examples": [ + "DEBUGADV2.BIN", + "SC0000.BIN", + "SC0010.BIN", + "SC0050.BIN", + "SC0080.BIN" + ] + }, + { + "op": "0x59", + "dec": 89, + "count": 17, + "pct": 0.024, + "examples": [ + "DEBUGADV2.BIN", + "SC0000.BIN", + "SC0010.BIN", + "SC0020.BIN", + "SC0040.BIN" + ] + }, + { + "op": "0x101", + "dec": 257, + "count": 16, + "pct": 0.023, + "examples": [ + "DEBUGADV2.BIN", + "FIELD.BIN", + "SHOWGROW.BIN" + ] + }, + { + "op": "0xb5", + "dec": 181, + "count": 11, + "pct": 0.016, + "examples": [ + "ALCHEMY.BIN", + "CHMENU.BIN", + "EXILE.BIN", + "IMPROVE.BIN", + "INFOCH.BIN" + ] + }, + { + "op": "0x21f", + "dec": 543, + "count": 9, + "pct": 0.013, + "examples": [ + "DEBUGADV.BIN", + "SC0000.BIN", + "SC0010.BIN", + "SC0130.BIN", + "SC0710.BIN" + ] + }, + { + "op": "0x64", + "dec": 100, + "count": 5, + "pct": 0.007, + "examples": [ + "CGMODE.BIN", + "HMODE.BIN", + "INFO.BIN", + "INIT2.BIN", + "SELACT.BIN" + ] + }, + { + "op": "0x230", + "dec": 560, + "count": 5, + "pct": 0.007, + "examples": [ + "DEBUGADV.BIN" + ] + }, + { + "op": "0x5b", + "dec": 91, + "count": 5, + "pct": 0.007, + "examples": [ + "FIELD.BIN", + "HISTORY.BIN" + ] + }, + { + "op": "0x12c", + "dec": 300, + "count": 4, + "pct": 0.006, + "examples": [ + "BTL.BIN", + "FIELD.BIN", + "SALLY.BIN" + ] + }, + { + "op": "0x1f8", + "dec": 504, + "count": 4, + "pct": 0.006, + "examples": [ + "CGMODE.BIN", + "HMODE.BIN", + "SAVE.BIN" + ] + }, + { + "op": "0xa1", + "dec": 161, + "count": 4, + "pct": 0.006, + "examples": [ + "FIELD.BIN", + "FORT.BIN", + "INFO.BIN", + "TITLE.BIN" + ] + }, + { + "op": "0x1f7", + "dec": 503, + "count": 4, + "pct": 0.006, + "examples": [ + "SC0000.BIN", + "SC0130.BIN", + "SC0240.BIN" + ] + }, + { + "op": "0x135", + "dec": 309, + "count": 3, + "pct": 0.004, + "examples": [ + "FIELD.BIN" + ] + }, + { + "op": "0x1fa", + "dec": 506, + "count": 3, + "pct": 0.004, + "examples": [ + "HISTORY.BIN", + "MENU.BIN", + "SAVE.BIN" + ] + }, + { + "op": "0x192", + "dec": 402, + "count": 3, + "pct": 0.004, + "examples": [ + "INPUTNAME.BIN", + "SC0320.BIN", + "SC0450.BIN" + ] + }, + { + "op": "0x53", + "dec": 83, + "count": 2, + "pct": 0.003, + "examples": [ + "CGMODE.BIN", + "HMODE.BIN" + ] + }, + { + "op": "0x136", + "dec": 310, + "count": 2, + "pct": 0.003, + "examples": [ + "FIELD.BIN" + ] + }, + { + "op": "0x5e", + "dec": 94, + "count": 1, + "pct": 0.001, + "examples": [ + "BTL.BIN" + ] + }, + { + "op": "0x243", + "dec": 579, + "count": 1, + "pct": 0.001, + "examples": [ + "CALLBACK_LOAD.BIN" + ] + } + ], + "string_consuming_ops": [ + { + "op": "0x0", + "dec": 0, + "count": 30537, + "pct": 80.407, + "examples": [ + "ADDEXP.BIN", + "ADDRANDOMITEM.BIN", + "ADDSKILL.BIN", + "AIM.BIN", + "ALCHEMY.BIN" + ] + }, + { + "op": "0x1a7", + "dec": 423, + "count": 1955, + "pct": 5.148, + "examples": [ + "ADDEN.BIN", + "BTANINIT.BIN", + "BTL.BIN", + "BUNKI.BIN", + "CALLBACK_LOAD.BIN" + ] + }, + { + "op": "0x9", + "dec": 9, + "count": 364, + "pct": 0.958, + "examples": [ + "DRAWOBJ.BIN", + "INPUTNAME.BIN", + "RTN_B001.BIN", + "RTN_M008.BIN", + "RTN_M009.BIN" + ] + }, + { + "op": "0x1a5", + "dec": 421, + "count": 203, + "pct": 0.535, + "examples": [ + "AIM.BIN", + "ALCHEMY.BIN", + "BTL.BIN", + "BUNKI.BIN", + "CAMP.BIN" + ] + }, + { + "op": "0x278", + "dec": 632, + "count": 140, + "pct": 0.369, + "examples": [ + "CAMP.BIN", + "DEBUGADV.BIN", + "DEBUGADV2.BIN", + "FIELD.BIN", + "FORT.BIN" + ] + }, + { + "op": "0x6be", + "dec": 1726, + "count": 134, + "pct": 0.353, + "examples": [ + "DEBUGADV.BIN", + "DEBUGADV2.BIN", + "SC0000.BIN", + "SC0010.BIN", + "SC0020.BIN" + ] + }, + { + "op": "0x7da", + "dec": 2010, + "count": 110, + "pct": 0.29, + "examples": [ + "ALCHEMY.BIN", + "BUNKI.BIN", + "BUNKIMOVE.BIN", + "EVOLVE.BIN", + "FIELD.BIN" + ] + }, + { + "op": "0x27b9", + "dec": 10169, + "count": 100, + "pct": 0.263, + "examples": [ + "FIELD.BIN", + "ROUND.BIN", + "SC0310.BIN", + "SC0320.BIN", + "SC0450.BIN" + ] + }, + { + "op": "0x27bb", + "dec": 10171, + "count": 100, + "pct": 0.263, + "examples": [ + "FIELD.BIN", + "ROUND.BIN", + "SC0310.BIN", + "SC0320.BIN", + "SC0450.BIN" + ] + }, + { + "op": "0x27bc", + "dec": 10172, + "count": 100, + "pct": 0.263, + "examples": [ + "FIELD.BIN", + "ROUND.BIN", + "SC0310.BIN", + "SC0320.BIN", + "SC0450.BIN" + ] + }, + { + "op": "0x27ba", + "dec": 10170, + "count": 99, + "pct": 0.261, + "examples": [ + "ROUND.BIN", + "SC0310.BIN", + "SC0320.BIN", + "SC0450.BIN", + "STINIT.BIN" + ] + }, + { + "op": "0x26b4", + "dec": 9908, + "count": 71, + "pct": 0.187, + "examples": [ + "CCINIT.BIN" + ] + }, + { + "op": "0x27a", + "dec": 634, + "count": 25, + "pct": 0.066, + "examples": [ + "ROUND.BIN", + "SC0310.BIN", + "SC0320.BIN", + "SC0450.BIN" + ] + }, + { + "op": "0x27b", + "dec": 635, + "count": 25, + "pct": 0.066, + "examples": [ + "ROUND.BIN", + "SC0310.BIN", + "SC0320.BIN", + "SC0450.BIN" + ] + }, + { + "op": "0x27c", + "dec": 636, + "count": 25, + "pct": 0.066, + "examples": [ + "ROUND.BIN", + "SC0310.BIN", + "SC0320.BIN", + "SC0450.BIN" + ] + }, + { + "op": "0x27d", + "dec": 637, + "count": 25, + "pct": 0.066, + "examples": [ + "ROUND.BIN", + "SC0310.BIN", + "SC0320.BIN", + "SC0450.BIN" + ] + }, + { + "op": "0x276", + "dec": 630, + "count": 18, + "pct": 0.047, + "examples": [ + "SAVE.BIN", + "SC0000.BIN", + "SC0010.BIN", + "SC0020.BIN", + "SC0040.BIN" + ] + }, + { + "op": "0x277", + "dec": 631, + "count": 16, + "pct": 0.042, + "examples": [ + "SC0000.BIN", + "SC0010.BIN", + "SC0020.BIN", + "SC0040.BIN", + "SC0050.BIN" + ] + }, + { + "op": "0x849", + "dec": 2121, + "count": 15, + "pct": 0.039, + "examples": [ + "ADDSKILL.BIN", + "CHMENU.BIN", + "SAVE.BIN", + "SBUNKI.BIN", + "SBUNKIMOVE.BIN" + ] + }, + { + "op": "0x279", + "dec": 633, + "count": 11, + "pct": 0.029, + "examples": [ + "GAMESTART.BIN", + "INIT2.BIN", + "INPUTNAME.BIN", + "SALLY.BIN" + ] + }, + { + "op": "0x57a", + "dec": 1402, + "count": 7, + "pct": 0.018, + "examples": [ + "SAVE.BIN" + ] + }, + { + "op": "0xd", + "dec": 13, + "count": 6, + "pct": 0.016, + "examples": [ + "ALCHEMY.BIN", + "EVOLVE.BIN", + "INPUTNAME.BIN", + "SELSTAGE.BIN", + "SUMMON.BIN" + ] + }, + { + "op": "0x1", + "dec": 1, + "count": 5, + "pct": 0.013, + "examples": [ + "ADDEXP.BIN", + "FIELD.BIN", + "SELSTAGE.BIN", + "SETCH.BIN", + "STUDY.BIN" + ] + }, + { + "op": "0xc", + "dec": 12, + "count": 5, + "pct": 0.013, + "examples": [ + "CGMODE.BIN", + "DRAWCHP.BIN", + "INPUTNAME.BIN", + "RTN_B001.BIN", + "RTN_M018.BIN" + ] + }, + { + "op": "0x18", + "dec": 24, + "count": 5, + "pct": 0.013, + "examples": [ + "DRAWENP.BIN", + "INPUTNAME.BIN" + ] + }, + { + "op": "0x27", + "dec": 39, + "count": 5, + "pct": 0.013, + "examples": [ + "DRAWENP.BIN", + "INPUTNAME.BIN" + ] + }, + { + "op": "0x36", + "dec": 54, + "count": 5, + "pct": 0.013, + "examples": [ + "DRAWENP.BIN", + "INPUTNAME.BIN" + ] + }, + { + "op": "0x2", + "dec": 2, + "count": 5, + "pct": 0.013, + "examples": [ + "FIELD.BIN", + "INPUTNAME.BIN", + "STUDY.BIN" + ] + }, + { + "op": "0x135", + "dec": 309, + "count": 5, + "pct": 0.013, + "examples": [ + "INPUTNAME.BIN" + ] + }, + { + "op": "0x407", + "dec": 1031, + "count": 5, + "pct": 0.013, + "examples": [ + "SELSTAGE.BIN" + ] + } + ], + "after_0x09_prefix": [ + { + "op": "0x0", + "count": 257496 + }, + { + "op": "0x1", + "count": 29591 + }, + { + "op": "0x2", + "count": 12952 + }, + { + "op": "0x3", + "count": 6647 + }, + { + "op": "0x9", + "count": 4526 + }, + { + "op": "0x4", + "count": 4059 + }, + { + "op": "0x6", + "count": 3769 + }, + { + "op": "0x5", + "count": 3703 + }, + { + "op": "0x8", + "count": 3029 + }, + { + "op": "0x7", + "count": 2649 + }, + { + "op": "0xc", + "count": 2586 + }, + { + "op": "0xa", + "count": 2486 + }, + { + "op": "0xb", + "count": 2379 + }, + { + "op": "0x13e3d", + "count": 2024 + }, + { + "op": "0x55", + "count": 1604 + }, + { + "op": "0xf", + "count": 1516 + }, + { + "op": "0x3f7", + "count": 1157 + }, + { + "op": "0xe", + "count": 1028 + }, + { + "op": "0xd", + "count": 950 + }, + { + "op": "0x31", + "count": 823 + } + ], + "confirmed": { + "0x02": "string-pointer operand tag (dword 0x02 followed by dword offset-into-body)", + "0x71": "T1 label/call-target record tag [0x71,0,val]", + "0x03": "T2 data/var record tag", + "0x8F": "T3 line-index record tag [0x8F,0,line#]" + } +} \ No newline at end of file diff --git a/vm-map/opcodes-himegari.json b/vm-map/opcodes-himegari.json new file mode 100644 index 0000000..7f002c5 --- /dev/null +++ b/vm-map/opcodes-himegari.json @@ -0,0 +1,2023 @@ +{ + "source": "Kelebek1/Eushully-Decompiler age-shared.cpp (validated against 476/476 Himegari scripts, 1.46M instructions, 0 unknown opcodes, 37392/0 string args resolved)", + "instruction_model": "code = seq of instructions; instr = then argc*(); len_dwords = 1 + 2*argc. Inline strings live after code inside [0,F8); stop decoding at the lowest string/array offset referenced (type-2 arg or op 0x64 arg1).", + "arg_types": { + "0x0": "immediate", + "0x1": "float", + "0x2": "string", + "0x3": "global-int", + "0x4": "global-float", + "0x5": "global-string", + "0x6": "global-ptr", + "0x8": "global-string-ptr", + "0x9": "local-int", + "0xa": "local-float", + "0xb": "local-string", + "0xc": "local-ptr", + "0xd": "local-float-ptr", + "0xe": "local-string-ptr" + }, + "header_fields": { + "F0": "local_integer_1", + "F1": "local_floats", + "F2": "local_strings_1", + "F3": "local_integer_2", + "F4": "unknown_data", + "F5": "local_strings_2", + "F6": "sub_header_length(=0x1C)", + "F7": "table_1_length", + "F8": "table_1_offset(=code end)", + "F9": "table_2_length", + "F10": "table_2_offset", + "F11": "table_3_length", + "F12": "table_3_offset" + }, + "opcodes_used_by_himegari": 248, + "named_semantics": 52, + "opcodes": [ + { + "op": "0x1", + "dec": 1, + "label": "u004149C0", + "argc": 0, + "named": false, + "uses": 2 + }, + { + "op": "0x2", + "dec": 2, + "label": "exit", + "argc": 0, + "named": true, + "uses": 5693 + }, + { + "op": "0x3", + "dec": 3, + "label": "call-script", + "argc": 1, + "named": true, + "uses": 2783 + }, + { + "op": "0x5", + "dec": 5, + "label": "ret", + "argc": 0, + "named": true, + "uses": 13846 + }, + { + "op": "0x6", + "dec": 6, + "label": "u00417E80", + "argc": 2, + "named": false, + "uses": 3 + }, + { + "op": "0x8", + "dec": 8, + "label": "u00417FC0", + "argc": 1, + "named": false, + "uses": 64 + }, + { + "op": "0x9", + "dec": 9, + "label": "exit-script", + "argc": 0, + "named": true, + "uses": 75 + }, + { + "op": "0x21", + "dec": 33, + "label": "u00418860", + "argc": 2, + "named": false, + "uses": 3 + }, + { + "op": "0x22", + "dec": 34, + "label": "u00418920", + "argc": 2, + "named": false, + "uses": 8 + }, + { + "op": "0x25", + "dec": 37, + "label": "u00418B40", + "argc": 3, + "named": false, + "uses": 19 + }, + { + "op": "0x50", + "dec": 80, + "label": "add", + "argc": 3, + "named": false, + "uses": 35070 + }, + { + "op": "0x51", + "dec": 81, + "label": "sub", + "argc": 3, + "named": true, + "uses": 50202 + }, + { + "op": "0x52", + "dec": 82, + "label": "mul", + "argc": 3, + "named": true, + "uses": 2961 + }, + { + "op": "0x53", + "dec": 83, + "label": "div", + "argc": 3, + "named": true, + "uses": 2570 + }, + { + "op": "0x54", + "dec": 84, + "label": "mod", + "argc": 3, + "named": true, + "uses": 455 + }, + { + "op": "0x55", + "dec": 85, + "label": "mov", + "argc": 2, + "named": true, + "uses": 268214 + }, + { + "op": "0x56", + "dec": 86, + "label": "and", + "argc": 3, + "named": true, + "uses": 11855 + }, + { + "op": "0x57", + "dec": 87, + "label": "or", + "argc": 3, + "named": true, + "uses": 1652 + }, + { + "op": "0x58", + "dec": 88, + "label": "sar", + "argc": 3, + "named": true, + "uses": 26 + }, + { + "op": "0x59", + "dec": 89, + "label": "shl", + "argc": 3, + "named": true, + "uses": 691 + }, + { + "op": "0x5a", + "dec": 90, + "label": "eq", + "argc": 3, + "named": true, + "uses": 61676 + }, + { + "op": "0x5b", + "dec": 91, + "label": "ne", + "argc": 3, + "named": true, + "uses": 3437 + }, + { + "op": "0x5c", + "dec": 92, + "label": "lt", + "argc": 3, + "named": true, + "uses": 20008 + }, + { + "op": "0x5d", + "dec": 93, + "label": "lte", + "argc": 3, + "named": true, + "uses": 2006 + }, + { + "op": "0x5e", + "dec": 94, + "label": "gr", + "argc": 3, + "named": true, + "uses": 728 + }, + { + "op": "0x5f", + "dec": 95, + "label": "gre", + "argc": 3, + "named": true, + "uses": 5801 + }, + { + "op": "0x60", + "dec": 96, + "label": "u0041A270", + "argc": 2, + "named": false, + "uses": 64 + }, + { + "op": "0x61", + "dec": 97, + "label": "lookup-array", + "argc": 3, + "named": true, + "uses": 140438 + }, + { + "op": "0x63", + "dec": 99, + "label": "u00414A60", + "argc": 2, + "named": false, + "uses": 92 + }, + { + "op": "0x64", + "dec": 100, + "label": "copy-local-array", + "argc": 2, + "named": true, + "uses": 5384 + }, + { + "op": "0x6c", + "dec": 108, + "label": "copy-to-global", + "argc": 2, + "named": true, + "uses": 910 + }, + { + "op": "0x6e", + "dec": 110, + "label": "show-text", + "argc": 2, + "named": true, + "uses": 30918 + }, + { + "op": "0x6f", + "dec": 111, + "label": "end-text-line", + "argc": 1, + "named": true, + "uses": 53619 + }, + { + "op": "0x70", + "dec": 112, + "label": "u0041A750", + "argc": 5, + "named": false, + "uses": 9 + }, + { + "op": "0x71", + "dec": 113, + "label": "u0041A7B0", + "argc": 1, + "named": false, + "uses": 25714 + }, + { + "op": "0x72", + "dec": 114, + "label": "wait-for-input", + "argc": 1, + "named": true, + "uses": 25678 + }, + { + "op": "0x73", + "dec": 115, + "label": "u0041AB30", + "argc": 10, + "named": false, + "uses": 1 + }, + { + "op": "0x75", + "dec": 117, + "label": "u0041AC30", + "argc": 1, + "named": false, + "uses": 221 + }, + { + "op": "0x76", + "dec": 118, + "label": "u0041AC60", + "argc": 1, + "named": false, + "uses": 257 + }, + { + "op": "0x77", + "dec": 119, + "label": "u0041ACB0", + "argc": 1, + "named": false, + "uses": 210 + }, + { + "op": "0x78", + "dec": 120, + "label": "u0041AD00", + "argc": 1, + "named": false, + "uses": 204 + }, + { + "op": "0x79", + "dec": 121, + "label": "u0041AD30", + "argc": 3, + "named": false, + "uses": 9 + }, + { + "op": "0x7a", + "dec": 122, + "label": "u0041AD70", + "argc": 3, + "named": false, + "uses": 16800 + }, + { + "op": "0x7b", + "dec": 123, + "label": "u0041ADB0", + "argc": 2, + "named": false, + "uses": 1889 + }, + { + "op": "0x7c", + "dec": 124, + "label": "u00416A90", + "argc": 0, + "named": false, + "uses": 594 + }, + { + "op": "0x7f", + "dec": 127, + "label": "u00414C60", + "argc": 1, + "named": false, + "uses": 7 + }, + { + "op": "0x80", + "dec": 128, + "label": "u0041AF00", + "argc": 1, + "named": false, + "uses": 20 + }, + { + "op": "0x85", + "dec": 133, + "label": "u00414CF0", + "argc": 0, + "named": false, + "uses": 278 + }, + { + "op": "0x86", + "dec": 134, + "label": "u0041B210", + "argc": 1, + "named": false, + "uses": 25 + }, + { + "op": "0x87", + "dec": 135, + "label": "u00414D10", + "argc": 0, + "named": false, + "uses": 10 + }, + { + "op": "0x88", + "dec": 136, + "label": "u0041B290", + "argc": 1, + "named": false, + "uses": 298 + }, + { + "op": "0x8b", + "dec": 139, + "label": "u0041B3D0", + "argc": 1, + "named": false, + "uses": 178 + }, + { + "op": "0x8c", + "dec": 140, + "label": "jmp", + "argc": 1, + "named": true, + "uses": 37513 + }, + { + "op": "0x8f", + "dec": 143, + "label": "call", + "argc": 1, + "named": true, + "uses": 70677 + }, + { + "op": "0x90", + "dec": 144, + "label": "u0041BEB0", + "argc": 7, + "named": false, + "uses": 2376 + }, + { + "op": "0x93", + "dec": 147, + "label": "u00415040", + "argc": 0, + "named": false, + "uses": 1731 + }, + { + "op": "0x94", + "dec": 148, + "label": "u00415090", + "argc": 0, + "named": false, + "uses": 297 + }, + { + "op": "0x97", + "dec": 151, + "label": "u0041C150", + "argc": 5, + "named": false, + "uses": 891 + }, + { + "op": "0xa0", + "dec": 160, + "label": "jcc", + "argc": 3, + "named": true, + "uses": 114366 + }, + { + "op": "0xa1", + "dec": 161, + "label": "u00427C00", + "argc": 0, + "named": false, + "uses": 12 + }, + { + "op": "0xa2", + "dec": 162, + "label": "u00427FD0", + "argc": 2, + "named": false, + "uses": 80 + }, + { + "op": "0xa3", + "dec": 163, + "label": "u004244D0", + "argc": 2, + "named": false, + "uses": 12 + }, + { + "op": "0xae", + "dec": 174, + "label": "u00415130", + "argc": 0, + "named": false, + "uses": 301 + }, + { + "op": "0xb4", + "dec": 180, + "label": "play-sound-effect", + "argc": 2, + "named": true, + "uses": 1596 + }, + { + "op": "0xb5", + "dec": 181, + "label": "u0041D050", + "argc": 1, + "named": false, + "uses": 1308 + }, + { + "op": "0xb6", + "dec": 182, + "label": "u0041D080", + "argc": 1, + "named": false, + "uses": 4765 + }, + { + "op": "0xb7", + "dec": 183, + "label": "u0041D0E0", + "argc": 1, + "named": false, + "uses": 2 + }, + { + "op": "0xb8", + "dec": 184, + "label": "u00415520", + "argc": 0, + "named": false, + "uses": 2 + }, + { + "op": "0xb9", + "dec": 185, + "label": "u0041D140", + "argc": 1, + "named": false, + "uses": 1 + }, + { + "op": "0xba", + "dec": 186, + "label": "u0041D0B0", + "argc": 1, + "named": false, + "uses": 334 + }, + { + "op": "0xbf", + "dec": 191, + "label": "play-bgm", + "argc": 1, + "named": true, + "uses": 821 + }, + { + "op": "0xc0", + "dec": 192, + "label": "u00415620", + "argc": 1, + "named": false, + "uses": 1 + }, + { + "op": "0xc2", + "dec": 194, + "label": "u0041D2B0", + "argc": 2, + "named": false, + "uses": 529 + }, + { + "op": "0xc4", + "dec": 196, + "label": "play-voice", + "argc": 1, + "named": true, + "uses": 16610 + }, + { + "op": "0xc5", + "dec": 197, + "label": "u0041D4A0", + "argc": 2, + "named": false, + "uses": 9 + }, + { + "op": "0xc6", + "dec": 198, + "label": "u0041D5D0", + "argc": 2, + "named": false, + "uses": 12 + }, + { + "op": "0xc7", + "dec": 199, + "label": "u0041D760", + "argc": 2, + "named": false, + "uses": 1 + }, + { + "op": "0xc8", + "dec": 200, + "label": "sleep", + "argc": 1, + "named": true, + "uses": 1974 + }, + { + "op": "0xcc", + "dec": 204, + "label": "mouse_callback", + "argc": 2, + "named": true, + "uses": 38 + }, + { + "op": "0xcd", + "dec": 205, + "label": "get-input-type", + "argc": 0, + "named": true, + "uses": 37 + }, + { + "op": "0xd0", + "dec": 208, + "label": "u00415830", + "argc": 1, + "named": false, + "uses": 6 + }, + { + "op": "0xd3", + "dec": 211, + "label": "u00425960", + "argc": 0, + "named": false, + "uses": 17 + }, + { + "op": "0xd4", + "dec": 212, + "label": "u004266F0", + "argc": 4, + "named": false, + "uses": 43 + }, + { + "op": "0xd5", + "dec": 213, + "label": "u004262C0", + "argc": 1, + "named": false, + "uses": 17 + }, + { + "op": "0xd9", + "dec": 217, + "label": "u00415880", + "argc": 0, + "named": false, + "uses": 139 + }, + { + "op": "0xfb", + "dec": 251, + "label": "joy_callback", + "argc": 2, + "named": true, + "uses": 410 + }, + { + "op": "0xfe", + "dec": 254, + "label": "u0041E360", + "argc": 1, + "named": false, + "uses": 1 + }, + { + "op": "0xff", + "dec": 255, + "label": "u00415A10", + "argc": 0, + "named": false, + "uses": 39 + }, + { + "op": "0x100", + "dec": 256, + "label": "u00415A60", + "argc": 0, + "named": false, + "uses": 39 + }, + { + "op": "0x101", + "dec": 257, + "label": "u00415BF0", + "argc": 0, + "named": false, + "uses": 1391 + }, + { + "op": "0x107", + "dec": 263, + "label": "u0041E500", + "argc": 2, + "named": false, + "uses": 6 + }, + { + "op": "0x108", + "dec": 264, + "label": "u00415E70", + "argc": 1, + "named": false, + "uses": 37 + }, + { + "op": "0x109", + "dec": 265, + "label": "u00415EC0", + "argc": 2, + "named": false, + "uses": 1258 + }, + { + "op": "0x10a", + "dec": 266, + "label": "u0041E540", + "argc": 2, + "named": false, + "uses": 1192 + }, + { + "op": "0x10b", + "dec": 267, + "label": "u0041E5A0", + "argc": 2, + "named": false, + "uses": 1 + }, + { + "op": "0x10c", + "dec": 268, + "label": "u0041E5E0", + "argc": 2, + "named": false, + "uses": 8 + }, + { + "op": "0x10d", + "dec": 269, + "label": "u00415F10", + "argc": 1, + "named": false, + "uses": 34 + }, + { + "op": "0x12c", + "dec": 300, + "label": "lookup-array-2d", + "argc": 5, + "named": true, + "uses": 4567 + }, + { + "op": "0x12e", + "dec": 302, + "label": "u0041E940", + "argc": 8, + "named": false, + "uses": 33 + }, + { + "op": "0x12f", + "dec": 303, + "label": "u0041ECB0", + "argc": 4, + "named": false, + "uses": 40 + }, + { + "op": "0x130", + "dec": 304, + "label": "u00415F40", + "argc": 1, + "named": false, + "uses": 1 + }, + { + "op": "0x131", + "dec": 305, + "label": "u00415F70", + "argc": 1, + "named": false, + "uses": 619 + }, + { + "op": "0x132", + "dec": 306, + "label": "u0041EF00", + "argc": 1, + "named": false, + "uses": 2 + }, + { + "op": "0x133", + "dec": 307, + "label": "u0041EFF0", + "argc": 2, + "named": false, + "uses": 4 + }, + { + "op": "0x134", + "dec": 308, + "label": "u0041F050", + "argc": 3, + "named": false, + "uses": 2 + }, + { + "op": "0x135", + "dec": 309, + "label": "bit-set", + "argc": 2, + "named": true, + "uses": 218 + }, + { + "op": "0x136", + "dec": 310, + "label": "bit-reset", + "argc": 2, + "named": true, + "uses": 143 + }, + { + "op": "0x137", + "dec": 311, + "label": "u0041F1C0", + "argc": 1, + "named": false, + "uses": 1 + }, + { + "op": "0x13a", + "dec": 314, + "label": "u0041F3A0", + "argc": 6, + "named": false, + "uses": 74 + }, + { + "op": "0x13f", + "dec": 319, + "label": "check-bit", + "argc": 3, + "named": true, + "uses": 55 + }, + { + "op": "0x140", + "dec": 320, + "label": "u0041F9C0", + "argc": 4, + "named": false, + "uses": 135 + }, + { + "op": "0x141", + "dec": 321, + "label": "u0041FAA0", + "argc": 1, + "named": false, + "uses": 3 + }, + { + "op": "0x142", + "dec": 322, + "label": "u0041FB10", + "argc": 1, + "named": false, + "uses": 2 + }, + { + "op": "0x143", + "dec": 323, + "label": "u00415FB0", + "argc": 0, + "named": false, + "uses": 1 + }, + { + "op": "0x144", + "dec": 324, + "label": "u004259D0", + "argc": 2, + "named": false, + "uses": 1 + }, + { + "op": "0x149", + "dec": 329, + "label": "u0041FCE0", + "argc": 1, + "named": false, + "uses": 1 + }, + { + "op": "0x191", + "dec": 401, + "label": "u0041A4A0", + "argc": 2, + "named": false, + "uses": 5 + }, + { + "op": "0x192", + "dec": 402, + "label": "set-string", + "argc": 2, + "named": true, + "uses": 5134 + }, + { + "op": "0x193", + "dec": 403, + "label": "concat", + "argc": 3, + "named": true, + "uses": 58 + }, + { + "op": "0x194", + "dec": 404, + "label": "u00425480", + "argc": 3, + "named": false, + "uses": 15 + }, + { + "op": "0x195", + "dec": 405, + "label": "u00425580", + "argc": 3, + "named": false, + "uses": 17 + }, + { + "op": "0x196", + "dec": 406, + "label": "display-furigana", + "argc": 3, + "named": true, + "uses": 272 + }, + { + "op": "0x197", + "dec": 407, + "label": "u0041B510", + "argc": 1, + "named": false, + "uses": 178 + }, + { + "op": "0x198", + "dec": 408, + "label": "u0041B540", + "argc": 3, + "named": false, + "uses": 446 + }, + { + "op": "0x199", + "dec": 409, + "label": "u00414D50", + "argc": 0, + "named": false, + "uses": 594 + }, + { + "op": "0x19a", + "dec": 410, + "label": "u00414E50", + "argc": 1, + "named": false, + "uses": 317 + }, + { + "op": "0x19b", + "dec": 411, + "label": "u00414E80", + "argc": 0, + "named": false, + "uses": 145 + }, + { + "op": "0x19c", + "dec": 412, + "label": "u00414EC0", + "argc": 0, + "named": false, + "uses": 146 + }, + { + "op": "0x19d", + "dec": 413, + "label": "u0041C680", + "argc": 2, + "named": false, + "uses": 4 + }, + { + "op": "0x19e", + "dec": 414, + "label": "u0041C6E0", + "argc": 2, + "named": false, + "uses": 2 + }, + { + "op": "0x1a0", + "dec": 416, + "label": "u0041C9B0", + "argc": 9, + "named": false, + "uses": 3 + }, + { + "op": "0x1a1", + "dec": 417, + "label": "u0041CB40", + "argc": 2, + "named": false, + "uses": 1 + }, + { + "op": "0x1a2", + "dec": 418, + "label": "u00428010", + "argc": 1, + "named": false, + "uses": 17057 + }, + { + "op": "0x1a3", + "dec": 419, + "label": "string-lookup-set", + "argc": 1, + "named": true, + "uses": 77 + }, + { + "op": "0x1a4", + "dec": 420, + "label": "u0041B580", + "argc": 2, + "named": false, + "uses": 203 + }, + { + "op": "0x1a5", + "dec": 421, + "label": "set-font", + "argc": 1, + "named": true, + "uses": 203 + }, + { + "op": "0x1a6", + "dec": 422, + "label": "halve-strlen", + "argc": 2, + "named": true, + "uses": 17 + }, + { + "op": "0x1a7", + "dec": 423, + "label": "comment", + "argc": 1, + "named": true, + "uses": 1986 + }, + { + "op": "0x1a8", + "dec": 424, + "label": "dev_ukn", + "argc": 0, + "named": false, + "uses": 1937 + }, + { + "op": "0x1a9", + "dec": 425, + "label": "u00428090", + "argc": 1, + "named": false, + "uses": 17 + }, + { + "op": "0x1aa", + "dec": 426, + "label": "u00425920", + "argc": 1, + "named": false, + "uses": 7 + }, + { + "op": "0x1ab", + "dec": 427, + "label": "u0041CCA0", + "argc": 2, + "named": false, + "uses": 1 + }, + { + "op": "0x1ac", + "dec": 428, + "label": "u0041CD80", + "argc": 3, + "named": false, + "uses": 2 + }, + { + "op": "0x1ad", + "dec": 429, + "label": "u004154F0", + "argc": 0, + "named": false, + "uses": 1893 + }, + { + "op": "0x1ae", + "dec": 430, + "label": "u0041CED0", + "argc": 3, + "named": false, + "uses": 2 + }, + { + "op": "0x1af", + "dec": 431, + "label": "u004245C0", + "argc": 3, + "named": false, + "uses": 1 + }, + { + "op": "0x1b0", + "dec": 432, + "label": "u0041A510", + "argc": 3, + "named": false, + "uses": 65 + }, + { + "op": "0x1b2", + "dec": 434, + "label": "u00425790", + "argc": 1, + "named": false, + "uses": 3 + }, + { + "op": "0x1b3", + "dec": 435, + "label": "u004257D0", + "argc": 0, + "named": false, + "uses": 2 + }, + { + "op": "0x1b4", + "dec": 436, + "label": "u004237C0", + "argc": 0, + "named": false, + "uses": 1 + }, + { + "op": "0x1b5", + "dec": 437, + "label": "u0041B5F0", + "argc": 1, + "named": false, + "uses": 11 + }, + { + "op": "0x1b6", + "dec": 438, + "label": "u00414F60", + "argc": 1, + "named": false, + "uses": 614 + }, + { + "op": "0x1b7", + "dec": 439, + "label": "u0041B640", + "argc": 1, + "named": false, + "uses": 297 + }, + { + "op": "0x1b8", + "dec": 440, + "label": "u0041B670", + "argc": 2, + "named": false, + "uses": 6 + }, + { + "op": "0x1b9", + "dec": 441, + "label": "u0041B710", + "argc": 2, + "named": false, + "uses": 6 + }, + { + "op": "0x1ba", + "dec": 442, + "label": "u0041D850", + "argc": 2, + "named": false, + "uses": 15 + }, + { + "op": "0x1bb", + "dec": 443, + "label": "u0041B7B0", + "argc": 1, + "named": false, + "uses": 33 + }, + { + "op": "0x1bc", + "dec": 444, + "label": "u00415670", + "argc": 0, + "named": false, + "uses": 26126 + }, + { + "op": "0x1bd", + "dec": 445, + "label": "u0041D910", + "argc": 1, + "named": false, + "uses": 1 + }, + { + "op": "0x1bf", + "dec": 447, + "label": "u004156C0", + "argc": 0, + "named": false, + "uses": 8541 + }, + { + "op": "0x1c1", + "dec": 449, + "label": "u0041B820", + "argc": 3, + "named": false, + "uses": 9 + }, + { + "op": "0x1c7", + "dec": 455, + "label": "u00414F90", + "argc": 1, + "named": false, + "uses": 299 + }, + { + "op": "0x1c8", + "dec": 456, + "label": "toString", + "argc": 2, + "named": true, + "uses": 5 + }, + { + "op": "0x1ca", + "dec": 458, + "label": "u0041B9B0", + "argc": 1, + "named": false, + "uses": 298 + }, + { + "op": "0x1cb", + "dec": 459, + "label": "u00414FD0", + "argc": 1, + "named": false, + "uses": 614 + }, + { + "op": "0x1cc", + "dec": 460, + "label": "u00415010", + "argc": 1, + "named": false, + "uses": 299 + }, + { + "op": "0x1ce", + "dec": 462, + "label": "u0041B9F0", + "argc": 1, + "named": false, + "uses": 146 + }, + { + "op": "0x1cf", + "dec": 463, + "label": "u0041DA10", + "argc": 1, + "named": false, + "uses": 1204 + }, + { + "op": "0x1d0", + "dec": 464, + "label": "u0041BA80", + "argc": 3, + "named": false, + "uses": 3 + }, + { + "op": "0x1d1", + "dec": 465, + "label": "u0041BAE0", + "argc": 5, + "named": false, + "uses": 1 + }, + { + "op": "0x1d2", + "dec": 466, + "label": "u0041BB40", + "argc": 2, + "named": false, + "uses": 16799 + }, + { + "op": "0x1d3", + "dec": 467, + "label": "u0041BB90", + "argc": 5, + "named": false, + "uses": 3 + }, + { + "op": "0x1d4", + "dec": 468, + "label": "u0041BC00", + "argc": 4, + "named": false, + "uses": 1 + }, + { + "op": "0x1d5", + "dec": 469, + "label": "u00415700", + "argc": 0, + "named": false, + "uses": 33200 + }, + { + "op": "0x1f4", + "dec": 500, + "label": "u004160D0", + "argc": 0, + "named": false, + "uses": 58716 + }, + { + "op": "0x1f5", + "dec": 501, + "label": "u00416120", + "argc": 0, + "named": false, + "uses": 58716 + }, + { + "op": "0x1f6", + "dec": 502, + "label": "u00416170", + "argc": 0, + "named": false, + "uses": 285 + }, + { + "op": "0x1f7", + "dec": 503, + "label": "u00420270", + "argc": 2, + "named": false, + "uses": 7537 + }, + { + "op": "0x1f8", + "dec": 504, + "label": "create-texture", + "argc": 4, + "named": true, + "uses": 16924 + }, + { + "op": "0x1f9", + "dec": 505, + "label": "set-texture", + "argc": 3, + "named": true, + "uses": 748 + }, + { + "op": "0x1fa", + "dec": 506, + "label": "u00420480", + "argc": 1, + "named": false, + "uses": 5948 + }, + { + "op": "0x1fb", + "dec": 507, + "label": "draw-texture", + "argc": 8, + "named": true, + "uses": 25254 + }, + { + "op": "0x1fd", + "dec": 509, + "label": "u00420620", + "argc": 4, + "named": false, + "uses": 816 + }, + { + "op": "0x1fe", + "dec": 510, + "label": "u004206C0", + "argc": 5, + "named": false, + "uses": 181 + }, + { + "op": "0x1ff", + "dec": 511, + "label": "u00420770", + "argc": 4, + "named": false, + "uses": 3752 + }, + { + "op": "0x202", + "dec": 514, + "label": "u00420880", + "argc": 5, + "named": false, + "uses": 8927 + }, + { + "op": "0x203", + "dec": 515, + "label": "u00420950", + "argc": 4, + "named": false, + "uses": 8929 + }, + { + "op": "0x204", + "dec": 516, + "label": "draw-string", + "argc": 4, + "named": true, + "uses": 16925 + }, + { + "op": "0x205", + "dec": 517, + "label": "u00420A60", + "argc": 6, + "named": false, + "uses": 60 + }, + { + "op": "0x207", + "dec": 519, + "label": "u00420B00", + "argc": 8, + "named": false, + "uses": 15 + }, + { + "op": "0x208", + "dec": 520, + "label": "u00420BF0", + "argc": 3, + "named": false, + "uses": 1488 + }, + { + "op": "0x20a", + "dec": 522, + "label": "u00420CE0", + "argc": 1, + "named": false, + "uses": 311 + }, + { + "op": "0x20b", + "dec": 523, + "label": "u00420D50", + "argc": 7, + "named": false, + "uses": 11 + }, + { + "op": "0x20c", + "dec": 524, + "label": "u00416200", + "argc": 0, + "named": false, + "uses": 2342 + }, + { + "op": "0x20d", + "dec": 525, + "label": "u00420E10", + "argc": 1, + "named": false, + "uses": 113 + }, + { + "op": "0x20e", + "dec": 526, + "label": "u00416250", + "argc": 0, + "named": false, + "uses": 337 + }, + { + "op": "0x20f", + "dec": 527, + "label": "u00420E40", + "argc": 3, + "named": false, + "uses": 3 + }, + { + "op": "0x212", + "dec": 530, + "label": "u00421090", + "argc": 2, + "named": false, + "uses": 1 + }, + { + "op": "0x213", + "dec": 531, + "label": "u004210D0", + "argc": 3, + "named": false, + "uses": 9 + }, + { + "op": "0x215", + "dec": 533, + "label": "u00421160", + "argc": 2, + "named": false, + "uses": 5500 + }, + { + "op": "0x216", + "dec": 534, + "label": "u004211A0", + "argc": 2, + "named": false, + "uses": 4 + }, + { + "op": "0x217", + "dec": 535, + "label": "u004211E0", + "argc": 4, + "named": false, + "uses": 5117 + }, + { + "op": "0x218", + "dec": 536, + "label": "u00421270", + "argc": 4, + "named": false, + "uses": 5106 + }, + { + "op": "0x219", + "dec": 537, + "label": "u004212E0", + "argc": 4, + "named": false, + "uses": 3723 + }, + { + "op": "0x21a", + "dec": 538, + "label": "u00421370", + "argc": 4, + "named": false, + "uses": 4699 + }, + { + "op": "0x21b", + "dec": 539, + "label": "u004213E0", + "argc": 1, + "named": false, + "uses": 26273 + }, + { + "op": "0x21c", + "dec": 540, + "label": "u00416270", + "argc": 0, + "named": false, + "uses": 371 + }, + { + "op": "0x21d", + "dec": 541, + "label": "u00421410", + "argc": 2, + "named": false, + "uses": 260 + }, + { + "op": "0x21e", + "dec": 542, + "label": "u00421450", + "argc": 6, + "named": false, + "uses": 984 + }, + { + "op": "0x21f", + "dec": 543, + "label": "u00421510", + "argc": 7, + "named": false, + "uses": 110 + }, + { + "op": "0x220", + "dec": 544, + "label": "u004215D0", + "argc": 6, + "named": false, + "uses": 1320 + }, + { + "op": "0x222", + "dec": 546, + "label": "u004216C0", + "argc": 2, + "named": false, + "uses": 24 + }, + { + "op": "0x223", + "dec": 547, + "label": "u00421700", + "argc": 8, + "named": false, + "uses": 253 + }, + { + "op": "0x224", + "dec": 548, + "label": "u00416290", + "argc": 0, + "named": false, + "uses": 301 + }, + { + "op": "0x228", + "dec": 552, + "label": "u00421940", + "argc": 5, + "named": false, + "uses": 649 + }, + { + "op": "0x229", + "dec": 553, + "label": "u004219E0", + "argc": 5, + "named": false, + "uses": 676 + }, + { + "op": "0x22a", + "dec": 554, + "label": "u00421A90", + "argc": 3, + "named": false, + "uses": 2 + }, + { + "op": "0x22c", + "dec": 556, + "label": "u00421BD0", + "argc": 3, + "named": false, + "uses": 2 + }, + { + "op": "0x22d", + "dec": 557, + "label": "u00421C60", + "argc": 5, + "named": false, + "uses": 1 + }, + { + "op": "0x22f", + "dec": 559, + "label": "u00421DD0", + "argc": 5, + "named": false, + "uses": 594 + }, + { + "op": "0x230", + "dec": 560, + "label": "u00421E70", + "argc": 1, + "named": false, + "uses": 6 + }, + { + "op": "0x231", + "dec": 561, + "label": "u00421EA0", + "argc": 4, + "named": false, + "uses": 90 + }, + { + "op": "0x232", + "dec": 562, + "label": "u00421EF0", + "argc": 4, + "named": false, + "uses": 5 + }, + { + "op": "0x233", + "dec": 563, + "label": "u00421FB0", + "argc": 5, + "named": false, + "uses": 31 + }, + { + "op": "0x234", + "dec": 564, + "label": "u00422060", + "argc": 5, + "named": false, + "uses": 18 + }, + { + "op": "0x236", + "dec": 566, + "label": "u004221A0", + "argc": 4, + "named": false, + "uses": 23 + }, + { + "op": "0x238", + "dec": 568, + "label": "u00422390", + "argc": 1, + "named": false, + "uses": 370 + }, + { + "op": "0x239", + "dec": 569, + "label": "u004223C0", + "argc": 6, + "named": false, + "uses": 233 + }, + { + "op": "0x23a", + "dec": 570, + "label": "u00422420", + "argc": 2, + "named": false, + "uses": 4 + }, + { + "op": "0x23b", + "dec": 571, + "label": "u00422460", + "argc": 7, + "named": false, + "uses": 147 + }, + { + "op": "0x23c", + "dec": 572, + "label": "u004162B0", + "argc": 0, + "named": false, + "uses": 16 + }, + { + "op": "0x23d", + "dec": 573, + "label": "u004162F0", + "argc": 0, + "named": false, + "uses": 145 + }, + { + "op": "0x23f", + "dec": 575, + "label": "u00422930", + "argc": 2, + "named": false, + "uses": 21 + }, + { + "op": "0x241", + "dec": 577, + "label": "u00422B80", + "argc": 5, + "named": false, + "uses": 1 + }, + { + "op": "0x242", + "dec": 578, + "label": "u00422D60", + "argc": 2, + "named": false, + "uses": 298 + }, + { + "op": "0x243", + "dec": 579, + "label": "u00417070", + "argc": 0, + "named": false, + "uses": 308 + }, + { + "op": "0x248", + "dec": 584, + "label": "u00422E80", + "argc": 1, + "named": false, + "uses": 1 + }, + { + "op": "0x249", + "dec": 585, + "label": "u00422EB0", + "argc": 3, + "named": false, + "uses": 26 + }, + { + "op": "0x24d", + "dec": 589, + "label": "u00422E90", + "argc": 12, + "named": false, + "uses": 2 + }, + { + "op": "0x24e", + "dec": 590, + "label": "u00422EA0", + "argc": 1, + "named": false, + "uses": 5 + }, + { + "op": "0x258", + "dec": 600, + "label": "u00422FE0", + "argc": 2, + "named": false, + "uses": 7128 + }, + { + "op": "0x259", + "dec": 601, + "label": "u00416410", + "argc": 0, + "named": false, + "uses": 436 + }, + { + "op": "0x2bd", + "dec": 701, + "label": "u00423100", + "argc": 1, + "named": false, + "uses": 203 + }, + { + "op": "0x2bf", + "dec": 703, + "label": "u00423180", + "argc": 3, + "named": false, + "uses": 75 + }, + { + "op": "0x2c0", + "dec": 704, + "label": "u004231C0", + "argc": 3, + "named": false, + "uses": 1 + }, + { + "op": "0x2c5", + "dec": 709, + "label": "strlen", + "argc": 2, + "named": true, + "uses": 23 + }, + { + "op": "0x2c6", + "dec": 710, + "label": "u0042B5E0", + "argc": 2, + "named": false, + "uses": 1 + }, + { + "op": "0x2c8", + "dec": 712, + "label": "u0042B610", + "argc": 4, + "named": false, + "uses": 1 + } + ] +} \ No newline at end of file diff --git a/vm-map/small-script-listings.md b/vm-map/small-script-listings.md new file mode 100644 index 0000000..21e59d7 --- /dev/null +++ b/vm-map/small-script-listings.md @@ -0,0 +1,140 @@ +# Smallest scripts — full pseudo-listings (opcodes NOT yet decoded) + +### INIT.BIN (64 bytes) +``` +file INIT.BIN +body 1 dwords (4 bytes) +header F0=0x1 F1=1 F2=0x1 F3=0x1 F4=1 F5=0x1 F6=0x1c +code [0x00000 .. 0x00001) 1 dwords +table T1 off 0x00001 count 0 tag 0x71 purity 0/0 +table T2 off 0x00001 count 0 tag 0x03 purity 0/0 +table T3 off 0x00001 count 0 tag 0x8f purity 0/0 +strings 0 inline, 0 references + +; ---- CODE (chunked by T3 line-index; opcodes not yet decoded) ---- + 0x00000: 2 + +; ---- TABLES ---- +T1 (count 0, off 0x00001): +T2 (count 0, off 0x00001): +T3 (count 0, off 0x00001): + +; ---- STRINGS ---- + (no inline strings) +``` + +### ADDILL.BIN (104 bytes) +``` +file ADDILL.BIN +body 11 dwords (44 bytes) +header F0=0x1 F1=1 F2=0x1 F3=0x1 F4=1 F5=0x1 F6=0x1c +code [0x00000 .. 0x00009) 9 dwords +table T1 off 0x00009 count 0 tag 0x71 purity 0/0 +table T2 off 0x00009 count 2 tag 0x03 purity 2/2 +table T3 off 0x0000b count 0 tag 0x8f purity 0/0 +strings 0 inline, 0 references + +; ---- CODE (chunked by T3 line-index; opcodes not yet decoded) ---- + 0x00000: 1f4 3 0 329e 3 0 329d 1f5 2 + +; ---- TABLES ---- +T1 (count 0, off 0x00009): +T2 (count 2, off 0x00009): 1 4 +T3 (count 0, off 0x0000b): + +; ---- STRINGS ---- + (no inline strings) +``` + +### RTN_B002.BIN (160 bytes) +``` +file RTN_B002.BIN +body 25 dwords (100 bytes) +header F0=0x1 F1=1 F2=0x1 F3=0x2 F4=1 F5=0x1 F6=0x1c +code [0x00000 .. 0x00019) 25 dwords +table T1 off 0x00019 count 0 tag 0x71 purity 0/0 +table T2 off 0x00019 count 0 tag 0x03 purity 0/0 +table T3 off 0x00019 count 0 tag 0x8f purity 0/0 +strings 0 inline, 0 references + +; ---- CODE (chunked by T3 line-index; opcodes not yet decoded) ---- + 0x00000: a0 3 cca45 0 ffffffff 0 18 61 c 0 3 52289 3 152616 55 c 0 0 b 55 3 eff77 0 1 2 + +; ---- TABLES ---- +T1 (count 0, off 0x00019): +T2 (count 0, off 0x00019): +T3 (count 0, off 0x00019): + +; ---- STRINGS ---- + (no inline strings) +``` + +### RTN_B003.BIN (160 bytes) +``` +file RTN_B003.BIN +body 25 dwords (100 bytes) +header F0=0x1 F1=1 F2=0x1 F3=0x2 F4=1 F5=0x1 F6=0x1c +code [0x00000 .. 0x00019) 25 dwords +table T1 off 0x00019 count 0 tag 0x71 purity 0/0 +table T2 off 0x00019 count 0 tag 0x03 purity 0/0 +table T3 off 0x00019 count 0 tag 0x8f purity 0/0 +strings 0 inline, 0 references + +; ---- CODE (chunked by T3 line-index; opcodes not yet decoded) ---- + 0x00000: a0 3 cca3a 0 ffffffff 0 18 61 c 0 3 52289 3 152616 55 c 0 0 0 55 3 eff77 0 1 2 + +; ---- TABLES ---- +T1 (count 0, off 0x00019): +T2 (count 0, off 0x00019): +T3 (count 0, off 0x00019): + +; ---- STRINGS ---- + (no inline strings) +``` + +### DEBUGANIME.BIN (164 bytes) +``` +file DEBUGANIME.BIN +body 26 dwords (104 bytes) +header F0=0x1 F1=1 F2=0x1 F3=0x1 F4=1 F5=0x1 F6=0x1c +code [0x00000 .. 0x0001a) 26 dwords +table T1 off 0x0001a count 0 tag 0x71 purity 0/0 +table T2 off 0x0001a count 0 tag 0x03 purity 0/0 +table T3 off 0x0001a count 0 tag 0x8f purity 0/0 +strings 0 inline, 0 references + +; ---- CODE (chunked by T3 line-index; opcodes not yet decoded) ---- + 0x00000: 55 3 15288c 0 8 55 3 15261a 0 1 55 3 15261b 0 0 55 3 15261c 0 1 55 3 15261d 0 32 2 + +; ---- TABLES ---- +T1 (count 0, off 0x0001a): +T2 (count 0, off 0x0001a): +T3 (count 0, off 0x0001a): + +; ---- STRINGS ---- + (no inline strings) +``` + +### CALLBACK_LOST.BIN (176 bytes) +``` +file CALLBACK_LOST.BIN +body 29 dwords (116 bytes) +header F0=0x3 F1=1 F2=0x1 F3=0x1 F4=1 F5=0x1 F6=0x1c +code [0x00000 .. 0x0001d) 29 dwords +table T1 off 0x0001d count 0 tag 0x71 purity 0/0 +table T2 off 0x0001d count 0 tag 0x03 purity 0/0 +table T3 off 0x0001d count 0 tag 0x8f purity 0/0 +strings 1 inline, 1 references + +; ---- CODE (chunked by T3 line-index; opcodes not yet decoded) ---- + 0x00000: 55 9 0 0 3314 1a7 2 str->0x00019 1a8 5a 9 1 3 62428 0 0 a0 9 1 0 ffffffff 0 18 20c 2 ; 'callback_lost' + 0x00019: STR 'callback_lost' + +; ---- TABLES ---- +T1 (count 0, off 0x0001d): +T2 (count 0, off 0x0001d): +T3 (count 0, off 0x0001d): + +; ---- STRINGS ---- + [0x00019 @file 0x0000a0] (4dw) 'callback_lost' +```