From ace8ddf8ebdd7db179a688d267ffcae6bc618c68 Mon Sep 17 00:00:00 2001 From: gamer147 Date: Thu, 9 Jul 2026 08:32:01 -0400 Subject: [PATCH] re: auto-label all opcode dispatch handlers from FUN_00413860 Extract op->real-handler map (handler(op)=ctx[0x26c93+op]) from the registration routine's override stores; ghidra_handler_map.py + build/op-handler-map.json (420 overrides). Cross-check vs opcodes.toml found 0 real drift. One-shot Ghidra pass then labeled the /v2 image: 281 raw FUN_/LAB_ handlers -> op_0xNN_handler, 107 bare VAs -> functions, 31 hand-named preserved, opcode plate comment on every handler. Includes the Task A spec + plan and the two-program (/v2 vs SMM) gotcha. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/engine-re.md | 29 ++ ...07-09-re-tooling-handler-labels-and-iat.md | 254 ++++++++++++++++++ ...e-tooling-handler-labels-and-iat-design.md | 103 +++++++ docs/tools-reference.md | 7 + tools/ghidra_handler_map.py | 139 ++++++++++ tools/test_ghidra_handler_map.py | 61 +++++ 6 files changed, 593 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-09-re-tooling-handler-labels-and-iat.md create mode 100644 docs/superpowers/specs/2026-07-09-re-tooling-handler-labels-and-iat-design.md create mode 100644 tools/ghidra_handler_map.py create mode 100644 tools/test_ghidra_handler_map.py diff --git a/docs/engine-re.md b/docs/engine-re.md index 305bc48..1cf79e0 100644 --- a/docs/engine-re.md +++ b/docs/engine-re.md @@ -58,6 +58,35 @@ which **drift** in ours. This table resolves the *real* handler for any opcode i general fix for VA drift project-wide. To find op `N`'s handler: read `ctx[0x26c93 + N]` from the `FUN_00413860` decompile (or `*(ctx + 0x9b24c + N*4)` at runtime). +### Materialized + applied image-wide (2026-07-09) + +The table is no longer resolved op-by-op by hand — it is **extracted once and applied to the whole +image**. `tools/ghidra_handler_map.py` parses the override stores in `FUN_00413860` (dump at +`build/engine-dump/FUN_00413860.disasm.txt`) → **`build/op-handler-map.json`** (`{op → handler VA}`, +**420 overrides**). Regenerate: `py -3.11 -X utf8 tools/ghidra_handler_map.py +build/engine-dump/FUN_00413860.disasm.txt --check`. The `--check` diffs the derived handlers against the +handler VAs mentioned in `vm-map/opcodes.toml` prose and found **0 real drift** — the only 7 flags are +ops whose toml text records the *worker* VA, not the handler (`0x20c→0x4174a0`, and the `0x21c–0x243` +cluster entries), each already matching the recon tables below. + +A one-shot Ghidra script (via `run_script_inline`; needs `GHIDRA_MCP_ALLOW_SCRIPTS=1`) then labeled the +image from that map: **281 raw `FUN_`/`LAB_` handlers renamed `op_0xNN_handler`, 107 bare handler VAs +turned into functions, 31 hand-named handlers preserved** (source `USER_DEFINED` is never renamed), and +a plate comment `opcode 0xNN dispatch handler; ctx[0x26c93+op] in FUN_00413860` set on every one +(appended to existing decode comments, never clobbering). The one shared handler `0x416650` (ops +`0xaf`/`0x1a8`) is `op_0xaf_0x1a8_handler`. ⇒ every dispatch handler in the image now self-identifies its +opcode; a bare `op_0xNN_handler` is a handler not yet role-RE'd. Enrich with a descriptive name + +decode when you reverse one (the generic name is a floor, not a final). + +> **⚠ Two-program gotcha (cost time 2026-07-09).** The Ghidra project holds **two** imports named +> `range_00400000.bin`: the GOOD one at project path **`/v2/range_00400000.bin`** (`x86:LE:32:default`, +> image base `0x400000`, 4308 functions — all our annotations live here) and a BROKEN early import at +> **`/range_00400000.bin`** (the `x86:LE:32:System Management Mode` mis-import: base `0000:0000`, **0 +> functions**; see the language gotcha in the runbook). After a Ghidra restart the broken one can become +> active. **Always confirm `get_current_program_info` shows base `0x400000` / 4308 functions (or +> `switch_program /v2/range_00400000.bin`) before doing anything** — `run_script_inline` runs against the +> GUI's active program, so a wrong-program script would mutate/measure garbage. + **Other confirmed engine-context offsets** (`ctx`/`esi`): `+0x53d14` = current gfx-object index; `+0x53d88` = per-object cmd-type table (stride `0x78` = 120 bytes); operand-fetch helper = `call 0x41b940` (thiscall, `ecx=ctx`, arg = operand index → returns the operand value); `FUN_00415f30(i)` = diff --git a/docs/superpowers/plans/2026-07-09-re-tooling-handler-labels-and-iat.md b/docs/superpowers/plans/2026-07-09-re-tooling-handler-labels-and-iat.md new file mode 100644 index 0000000..430db40 --- /dev/null +++ b/docs/superpowers/plans/2026-07-09-re-tooling-handler-labels-and-iat.md @@ -0,0 +1,254 @@ +# RE Tooling: Handler Labeling + IAT Reconstruction — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Eliminate two recurring native-RE taxes — Kelebek VA-drift and unnamed imports — by (A) auto-labeling every opcode dispatch handler in the Ghidra image and (B) grafting reconstructed IAT names onto that same image. + +**Architecture:** Two independent workstreams against the existing annotated Ghidra program (`range_00400000.bin`, base `0x400000`), driven over the ghidra-mcp bridge. A is fully static (no running game). B needs a live `AGE.EXE` for `pe-sieve32`. A pure-Python parser does the mechanical work in both; MCP calls apply the results to the image; annotations are always preserved. + +**Tech Stack:** Python 3.11 (`py -3.11 -X utf8`), ghidra-mcp bridge (Ghidra 12.1.2), `bin/pe-sieve32.exe`, `pytest`. + +## Global Constraints + +- Run Python as `py -3.11 -X utf8 tools/.py …` (utf8 mandatory on Windows). +- Tools import `tools/paths.py`; never hard-code paths. +- Engine image: Raw Binary, `x86:LE:32:default`, Image Base `0x400000`; VA→file offset = `VA − 0x400000`. +- Dispatch identity: `handler(op) = ctx[0x26c93 + op] = *(ctx + 0x9b24c + op*4)`; overrides written in `FUN_00413860`; default handler = `FUN_004162b0`. +- **Never hand-edit generated files** (`build/*`); rebuild via the owning tool. +- **Preserve annotations** — never reimport the program in B; never rename an already-named handler in A. +- End every image-mutating task with `save_program` (MCP `mcp__ghidra__save_program`). +- `pe-sieve32.exe` runs from **PowerShell** (Git Bash mangles `/flags`). +- Deferred / out of scope: outer-loop RE (wider dump / live debugger), STL/CRT demangling. + +--- + +## Task A: Auto-label every dispatch handler + +**Files:** +- Create: `tools/ghidra_handler_map.py` (parser + JSON emitter + opcodes.toml cross-check) +- Create: `tools/test_ghidra_handler_map.py` (unit test for the parser) +- Create: `build/op-handler-map.json` (generated — do not hand-edit) +- Create: `scratchpad` fixture `FUN_00413860.disasm.txt` (captured disassembly, input to the parser) +- Modify: `docs/engine-re.md` (dispatch-table section links the generated map) +- Modify: `docs/tools-reference.md` (document `ghidra_handler_map.py`) + +**Interfaces:** +- Produces: `parse_overrides(disasm_lines: list[str]) -> dict[int, int]` mapping `op → handler_va`, excluding the default handler `0x4162b0`. +- Produces: `build/op-handler-map.json` shape `{ "0x1ac": {"handler": "0x427fb0"}, ... }`. +- Consumes (Step A5): `vm-map/opcodes.toml` handler VAs for cross-check. + +- [ ] **Step A1: Capture `FUN_00413860` disassembly as a parser fixture** + +Via MCP against the open program, dump the registration routine's instructions to a fixture file: +Run: `mcp__ghidra__disassemble_function` with `address="0x413860"` (or `get_function_pcode` if disasm is cleaner), save raw text to `scratchpad/FUN_00413860.disasm.txt`. +Expected: a long list of `MOV dword ptr [ + 0x9b...], 0x4?????` stores plus a default-fill loop. + +- [ ] **Step A2: Write the failing parser test** + +```python +# tools/test_ghidra_handler_map.py +from ghidra_handler_map import parse_overrides + +def test_parses_known_override_anchors(): + # Minimal fixture lines in the exact shape captured in A1. + lines = [ + "MOV dword ptr [EAX + 0x9b8fc],0x427fb0", # 0x9b8fc = 0x9b24c + 0x1ac*4 -> op 0x1ac + "MOV dword ptr [EAX + 0x9b688],0x420ec0", # 0x9b688 = 0x9b24c + 0xc8*4 -> op 0xc8 + "MOV dword ptr [EAX + 0x100],0x4162b0", # not in table window -> ignored + ] + m = parse_overrides(lines) + assert m[0x1ac] == 0x427fb0 + assert m[0xc8] == 0x420ec0 + assert 0x4162b0 not in m.values() # default handler excluded + +def test_excludes_default_handler_writes(): + lines = ["MOV dword ptr [EAX + 0x9b24c],0x4162b0"] # op 0x0 default fill + assert parse_overrides(lines) == {} +``` + +- [ ] **Step A3: Run the test to verify it fails** + +Run: `py -3.11 -X utf8 -m pytest tools/test_ghidra_handler_map.py -v` +Expected: FAIL — `ModuleNotFoundError`/`cannot import name 'parse_overrides'`. + +- [ ] **Step A4: Implement the parser** + +```python +# tools/ghidra_handler_map.py +import json, re, sys +import paths # path anchor + +TABLE_BASE = 0x9b24c # byte offset of ctx[0x26c93] +TABLE_SPAN = 0x400 * 4 # 1024 word slots +DEFAULT_HANDLER = 0x4162b0 + +_STORE = re.compile( + r"MOV\s+dword ptr\s+\[\w+\s*\+\s*(0x[0-9a-fA-F]+)\]\s*,\s*(0x[0-9a-fA-F]+)") + +def parse_overrides(lines): + out = {} + for ln in lines: + m = _STORE.search(ln) + if not m: + continue + disp = int(m.group(1), 16) + va = int(m.group(2), 16) + if not (TABLE_BASE <= disp < TABLE_BASE + TABLE_SPAN): + continue + if (disp - TABLE_BASE) % 4 != 0: + continue + op = (disp - TABLE_BASE) // 4 + if va == DEFAULT_HANDLER: + continue + out[op] = va + return out + +def build(disasm_path, out_path): + lines = disasm_path.read_text(encoding="utf-8").splitlines() + m = parse_overrides(lines) + data = {hex(op): {"handler": hex(va)} for op, va in sorted(m.items())} + out_path.write_text(json.dumps(data, indent=2), encoding="utf-8") + return m + +if __name__ == "__main__": + from pathlib import Path + disasm = Path(sys.argv[1]) + out = paths.BUILD / "op-handler-map.json" + m = build(disasm, out) + print(f"{len(m)} overrides -> {out}") +``` + +- [ ] **Step A5: Run the test to verify it passes** + +Run: `py -3.11 -X utf8 -m pytest tools/test_ghidra_handler_map.py -v` +Expected: PASS (both tests). + +- [ ] **Step A6: Generate the real map and verify anchors** + +Run: `py -3.11 -X utf8 tools/ghidra_handler_map.py scratchpad/FUN_00413860.disasm.txt` +Then inspect `build/op-handler-map.json` and assert the four known anchors: +- `0x1ac → 0x427fb0`, `0x1a2 → 0x42d360`, `0x215 → 0x42a0b0`, `0xc8 → 0x420ec0`. +Expected: all four present and exact; total override count is reported and plausible (record it in the doc). If any anchor is missing, the A1 capture missed stores (e.g. non-`MOV`/computed writes) — re-capture with `get_function_pcode` and widen the regex before proceeding. + +- [ ] **Step A7: Cross-check against `opcodes.toml` (report only, no auto-edit)** + +Extend `ghidra_handler_map.py` with a `--check` mode that loads each opcode's recorded handler VA from `vm-map/opcodes.toml` (where present) and prints every op where `toml_handler != derived_handler`. +Run: `py -3.11 -X utf8 tools/ghidra_handler_map.py scratchpad/FUN_00413860.disasm.txt --check` +Expected: a (possibly empty) disagreement list. **Do not edit opcodes.toml here** — capture the list in the task notes for human reconciliation (each disagreement is a latent VA-drift bug). + +- [ ] **Step A8: Apply annotations to the image (preserve good names)** + +For each `(op, va)` in the map, drive MCP: +1. `get_function_by_address(va)` — read the current name. +2. If the name is missing or matches `^(FUN_|LAB_)` → `create_function` (if absent) then `rename_function_by_address(va, f"op_0x{op:x}_handler")`. +3. If the name is already descriptive (e.g. `gfx_op_*`, `sleep_op_*`, `call_script*`) → **do not rename**. +4. Either way `set_plate_comment(va, f"opcode 0x{op:x} dispatch handler; ctx[0x26c93+op] in FUN_00413860")` — appending, not replacing an existing decode comment. +Expected: every raw handler gets an `op_0xNN_handler` name; every previously-named handler keeps its name. + +- [ ] **Step A9: Verify no clobbers, then save** + +Re-read the four anchors and 3 previously-named handlers via `get_function_by_address`; confirm named ones are unchanged and raw ones now carry `op_0x…`. +Run: `mcp__ghidra__save_program`. +Expected: save succeeds; spot-checks pass. + +- [ ] **Step A10: Update docs and commit** + +Edit `docs/engine-re.md` dispatch-table section to reference `build/op-handler-map.json` (the full op→handler map, generated) instead of listing ops by hand; note the total override count and any A7 disagreements. Add `ghidra_handler_map.py` to `docs/tools-reference.md`. + +```bash +git add tools/ghidra_handler_map.py tools/test_ghidra_handler_map.py build/op-handler-map.json docs/engine-re.md docs/tools-reference.md +git commit -m "re: auto-label all opcode dispatch handlers from FUN_00413860" +``` + +--- + +## Task B: Reconstruct the IAT and graft named imports + +**Prerequisite:** a running `AGE.EXE` instance (needed by `pe-sieve32`). + +**Files:** +- Create: `tools/apply_imports.py` (parse pe-sieve import report → `{thunk_addr: "dll!Func"}`) +- Create: `tools/test_apply_imports.py` (unit test for the report parser) +- Create: `build/pe-sieve/` (pe-sieve output dir — reconstructed module + import report) +- Modify: `docs/engine-re.md` (IAT-reconstruction runbook + fallback usage, if taken) +- Modify: `docs/tools-reference.md` (document `apply_imports.py`) + +**Interfaces:** +- Consumes: pe-sieve's import report text. +- Produces: `parse_import_report(text: str) -> dict[int, str]` mapping `thunk_va → "dll!Function"`. + +- [ ] **Step B1: Run pe-sieve against the live game** + +Find the PID, then from **PowerShell**: +```powershell +$p = Get-Process AGE +bin\pe-sieve32.exe /pid $p.Id /imp 3 /dmode 3 /dir build\pe-sieve +``` +Expected: `build/pe-sieve/` contains the reconstructed module and an import report (`*.imports.txt`/tag file). Note the report's exact format for B2. + +- [ ] **Step B2: Write the failing import-report parser test** + +```python +# tools/test_apply_imports.py +from apply_imports import parse_import_report + +def test_parses_thunk_to_named_import(): + # Fixture lines in the EXACT shape observed in B1 (adjust to pe-sieve's real format). + text = "\n".join([ + "0x00456120: kernel32.dll.CreateFileA", + "0x00456124: kernel32.dll.ReadFile", + ]) + m = parse_import_report(text) + assert m[0x456120] == "kernel32.dll!CreateFileA" + assert m[0x456124] == "kernel32.dll!ReadFile" +``` + +- [ ] **Step B3: Run the test to verify it fails** + +Run: `py -3.11 -X utf8 -m pytest tools/test_apply_imports.py -v` +Expected: FAIL — import error. + +- [ ] **Step B4: Implement the report parser** + +Implement `parse_import_report(text)` in `tools/apply_imports.py` to match the real B1 format (the fixture above is a placeholder shape — reconcile the regex to pe-sieve's actual columns). Return `{thunk_va: "dll!Func"}`. +Run: `py -3.11 -X utf8 -m pytest tools/test_apply_imports.py -v` +Expected: PASS. + +- [ ] **Step B5: Graft names onto the existing program** + +For each `(thunk_va, "dll!Func")`, drive MCP against the current annotated program (do **not** reimport): apply the name at the IAT thunk via the appropriate label/reference/external-location API (try `create_label`; if the thunk should be an external function, use the external-location API). Verify one call site reads as a named call: +- Save-path handler (`0x1ac` region) → named `CreateFileA`/`SetFilePointer`. +Expected: at least the save/resolver-chain sites (`FUN_0040e980`/`FUN_0044f390`) show named `CreateFileA`/`SetFilePointer`/`ReadFile`; the `sleep` timer source shows a `timeGetTime`-class name. + +- [ ] **Step B6: Fallback branch (only if B5 is an ordeal)** + +If thunk addresses don't line up or MCP can't set externals cleanly, stop grafting: import `build/pe-sieve/` as a **second, clean Ghidra program** (Format PE), keep it open alongside ours, and use it purely as a name-lookup reference — port import names into our annotated program by hand as each function is touched. Document the two-program workflow in `docs/engine-re.md`'s runbook. (This preserves all annotations either way.) + +- [ ] **Step B7: Apply the Win32 data-type archive** + +Apply Ghidra's bundled Win32 type archive so named imports carry signatures (via the Data Type Manager / MCP `import_data_types` if exposed). Verify `CreateFileA` shows a typed prototype at a call site. +Expected: named imports carry parameter types. + +- [ ] **Step B8: Save, verify annotations intact, update docs, commit** + +Confirm a sample of pre-existing renames/plate comments still present (`get_function_by_address` on `gfx_op_0x215_register_query`, `sleep_op_0xc8`). Run `mcp__ghidra__save_program`. +Edit `docs/engine-re.md` (runbook: IAT reconstruction is done; note graft vs fallback) and `docs/tools-reference.md` (`apply_imports.py`). + +```bash +git add tools/apply_imports.py tools/test_apply_imports.py docs/engine-re.md docs/tools-reference.md +git commit -m "re: reconstruct IAT via pe-sieve and graft named imports" +``` + +--- + +## Self-Review + +**Spec coverage:** +- A (handler labeling): Task A1–A10 — parser, cross-check (A7), preserve-names (A8), anchors (A6), docs (A10). ✓ +- B (IAT graft + fallback + type lib): B1–B8, fallback = B6, type archive = B7. ✓ +- Sequencing (A first, static; B needs game): task order + B prerequisite. ✓ +- Deferred outer-loop RE: Global Constraints. ✓ + +**Placeholder scan:** The B4/B2 note "reconcile regex to pe-sieve's actual format" is intentional — the exact report columns are only knowable after B1 runs; the fixture shows the target shape and B4 adjusts it. Not a hidden TODO. No other placeholders. + +**Type consistency:** `parse_overrides(list[str]) -> dict[int,int]` and `parse_import_report(str) -> dict[int,str]` are each used consistently by their `build`/graft consumers. Handler-name pattern `op_0xNN_handler` used identically in A8/A9. diff --git a/docs/superpowers/specs/2026-07-09-re-tooling-handler-labels-and-iat-design.md b/docs/superpowers/specs/2026-07-09-re-tooling-handler-labels-and-iat-design.md new file mode 100644 index 0000000..c93a74a --- /dev/null +++ b/docs/superpowers/specs/2026-07-09-re-tooling-handler-labels-and-iat-design.md @@ -0,0 +1,103 @@ +# RE tooling: whole-image handler labeling + IAT reconstruction — design + +**Date:** 2026-07-09 +**Status:** approved (shape), plan pending +**Home in the canonical map:** implementation notes land in `docs/engine-re.md` (native-RE) and +`docs/tools-reference.md` (the new scripts); this spec is the one-time design record. + +## Motivation + +Native RE keeps stalling and self-correcting at the *same* structural spots. Reviewing the +`engine-re.md` log, the recurring taxes are: + +1. **Kelebek VA-drift** — the `handler(op) = ctx[0x26c93 + op]` fix is known but paid *by hand, per + op*, and is still the #1 "read the wrong function" error source. +2. **Unnamed imports** — every `CreateFileA`/`ReadFile`/`timeGetTime`/`d3d9::Present`/`std::map` + shows as a raw indirect call; findings get reconstructed by format-string archaeology instead of + read off a label. Deferred when RE was one handler at a time; now we sweep dozens. + +Both are one-time, static, low-risk investments that make *every* future look at the decomp cheaper. +(A third lever — cracking the statically-unreachable outer frame loop via a wider dump or the live +debugger — is explicitly **out of scope here**; it is gated on a careful anti-tamper probe first, +because the engine has integrity checks that already crash-tested a Frida CModule hook.) + +## Workstream A — auto-label every dispatch handler (fully static, no game needed) + +**Source of truth:** `FUN_00413860` (the handler-registration routine). It fills `0x400` slots at +`ctx[0x26c93]` with the default handler `FUN_004162b0`, then overrides specific opcodes with +`ctx[0x26c93 + op] = ` — in asm, an immediate `handler_va` stored to `[base + disp]` +where `disp = 0x9b24c + op*4` (word index `0x26c93 + op`; e.g. `ctx[0x26e3f]=0x427fb0` → op `0x1ac`). + +**Script (run over MCP into the existing annotated program):** +1. Walk `FUN_00413860`'s instructions; match immediate-to-`[base+disp]` stores with + `disp ∈ [0x9b24c, 0x9b24c + 0x400*4)`. Compute `op = (disp − 0x9b24c)/4`, `handler_va = imm`. +2. Skip any entry whose handler is the default `FUN_004162b0` (not a real override). +3. For each real `(op, handler_va)`: + - `create_function` at `handler_va` if none exists. + - **Preserve good names:** if the function already has a non-`FUN_`/`LAB_` name (e.g. + `gfx_op_0x215_register_query`, `sleep_op_0xc8`), do **not** rename — only ensure a plate comment + records the opcode. Rename only raw `FUN_xxxx`/`LAB_xxxx` → `op_0xNN_handler`. + - `set_plate_comment`: `opcode 0xNN dispatch handler; resolved via ctx[0x26c93+op] in FUN_00413860`. +4. Emit `build/op-handler-map.json` (`{ "0x1ac": {"handler": "0x427fb0", "name": "..."}, ... }`). +5. **Cross-check:** diff the derived map against `vm-map/opcodes.toml` handler VAs; print any op where + toml ≠ derived. Those disagreements are latent VA-drift bugs — surfacing them is a deliverable, not + a warning to suppress. (Do not auto-edit `opcodes.toml`; report for human reconciliation.) +6. `save_program`. + +**Acceptance:** +- `build/op-handler-map.json` lists every override op with its real handler; count is sane + (~248 opcodes are *used* by the corpus, but the table may register more — record whatever + `FUN_00413860` actually writes, and note the total). +- Spot-check ≥3 known anchors reproduce prior findings exactly: op `0x1ac`→`0x427fb0`, + op `0x1a2`→`0x42d360`, op `0x215`→`0x42a0b0`, op `0xc8`→`0x420ec0`. +- Every previously hand-named handler still has its name (zero clobbers). +- The cross-check runs clean or produces an explicit, reviewed disagreement list. +- `engine-re.md`'s dispatch-table section links the generated map instead of enumerating ops by hand. + +## Workstream B — reconstruct the IAT + apply the Win32 type archive (needs the game running) + +**Decision (approved): graft onto the existing program — never reimport.** A fresh PE import would +give named imports but discard every rename/plate comment we've accumulated. So we keep the current +program and *apply* the reconstructed import names onto it. + +**Steps:** +1. With a live game instance, run from **PowerShell** (Git Bash mangles `/flags`): + `bin\pe-sieve32.exe /pid /imp 3 /dmode 3 /dir `. Capture the reconstructed module + + pe-sieve's **import report** (`*.imports.txt`/tag files). +2. A small script reads the import report → `{ thunk_addr → "dll!Function" }` and, in the current + Ghidra program, applies each as an external-function reference / label at the IAT thunk (via the + appropriate MCP label / reference / external-location APIs — exact call pinned in the plan). + Annotations stay intact. +3. Apply Ghidra's bundled **Win32 data-type archive** so the now-named imports carry real signatures. +4. `save_program`. + +**Fallback (sanctioned, not a failure): keep both programs separate for cross-referencing.** If the +graft turns into an ordeal (thunk addresses don't line up cleanly, MCP can't set externals on a raw +image, etc.), import pe-sieve's reconstructed PE as a **second, clean Ghidra program** and use it +purely as a *reference* — look up an import by address there, port the name into our annotated program +by hand as we touch each function. We migrate names incrementally until enough is ported that we trust +grafting the rest (or decide the reference workflow is good enough). Either way we never lose +annotations. + +**Scope boundary:** pe-sieve names *imports* (Win32 APIs) only. Statically-linked STL/CRT demangling +(`std::map`, `operator new`) is **not** in scope — that is BSim/FidDb territory, noted as an optional +future follow-up. + +**Acceptance:** +- Known API sites read as named calls: the save-path handler's `%s\SAVE%2.2d.DAT` formatter shows a + named `sprintf`/file-API neighborhood; the resolver chain (`FUN_0040e980`/`FUN_0044f390`) shows + named `CreateFileA`/`SetFilePointer`/`ReadFile`; the `sleep` timer source resolves to a named + `timeGetTime`-class import. +- Existing annotations (renames + plate comments) are all still present. +- `save_program` succeeds; if fallback taken, the second reference program is documented in + `engine-re.md`'s runbook with how to use it. + +## Sequencing & risk + +- **A first** — static, needs no game, de-risks everything downstream (self-navigating decomp), and + its cross-check may pre-empt B confusion. +- **B second** — when a game instance is available. +- Both update `docs/tools-reference.md` (new scripts) and `docs/engine-re.md` (results) per the doc + discipline; both end with `save_program`. +- **Explicitly deferred:** outer-loop RE (wider dump / live debugger) — gated on a separate, careful + anti-tamper probe. diff --git a/docs/tools-reference.md b/docs/tools-reference.md index f03af2a..32541ee 100644 --- a/docs/tools-reference.md +++ b/docs/tools-reference.md @@ -162,6 +162,13 @@ texture ops (no GPU context) — run windowed for real scenes. User args (after *(Static disassembly of `build/engine-dump/range_00400000.bin` uses **capstone** — `py -3.11 -m pip install capstone`; VA `X` → file offset `X−0x400000`.)* +## Native engine RE (Ghidra) + +| Tool | Purpose | Run | Reads → Writes | +|---|---|---|---| +| `ghidra_handler_map.py` | Extract the opcode→real-handler dispatch table (`handler(op)=ctx[0x26c93+op]`) from `FUN_00413860`'s override stores — the general fix for Kelebek VA-drift. `--check` diffs derived handlers vs `opcodes.toml` prose (found 0 real drift). Feeds the one-shot Ghidra annotation pass that names every handler `op_0xNN_handler` (see `docs/engine-re.md` "Materialized + applied image-wide"). | `ghidra_handler_map.py build/engine-dump/FUN_00413860.disasm.txt [--check]` | ⚙ `build/engine-dump/FUN_00413860.disasm.txt` (from ghidra-mcp `disassemble_function(0x413860)`) → ⚙ `build/op-handler-map.json` | +| `test_ghidra_handler_map.py` | Unit tests for the dispatch-table parser (plain runner, no pytest). | `test_ghidra_handler_map.py` | — | + ## Historical / one-off | Tool | Purpose | diff --git a/tools/ghidra_handler_map.py b/tools/ghidra_handler_map.py new file mode 100644 index 0000000..7745603 --- /dev/null +++ b/tools/ghidra_handler_map.py @@ -0,0 +1,139 @@ +"""Derive the opcode -> real dispatch-handler map from the engine's registration routine. + +The AGE interpreter dispatches every opcode through a per-context handler table: + + handler(op) = ctx[0x26c93 + op] = *(ctx + 0x9b24c + op*4) + +`FUN_00413860` builds that table: it STOSD-fills 0x400 slots at ctx+0x9b24c with the +default handler 0x4162b0, then overrides specific opcodes with +`MOV dword ptr [ESI + (0x9b24c + op*4)], `. Kelebek's `u00XXXXXX` opcode +names encode handler VAs from *Kelebek's* build, which DRIFT in ours; this map is the +general fix -- it resolves the real handler for every opcode in *our* image. + +Input: a text dump of FUN_00413860's disassembly (see build/engine-dump/FUN_00413860.disasm.txt, +produced by the ghidra-mcp `disassemble_function(0x413860)` call). +Output: build/op-handler-map.json ({ "0x1ac": {"handler": "0x427fb0"}, ... }). + +Usage: + py -3.11 -X utf8 tools/ghidra_handler_map.py # build the map + py -3.11 -X utf8 tools/ghidra_handler_map.py --check # + cross-check vs opcodes.toml + +See docs/engine-re.md (dispatch table) and docs/tools-reference.md. +""" +import json +import re +import sys +from pathlib import Path + +import paths + +TABLE_BASE = 0x9b24c # byte offset of ctx[0x26c93] (the op-0 slot) +TABLE_SPAN = 0x400 * 4 # 1024 word slots +DEFAULT_HANDLER = 0x4162b0 # STOSD-seeded default handler (not a real override) + +# MOV dword ptr [ + 0x9b...],0x4????? -- an immediate stored to [base+disp]. +_STORE = re.compile( + r"MOV\s+dword ptr\s+\[\w+\s*\+\s*(0x[0-9a-fA-F]+)\]\s*,\s*(0x[0-9a-fA-F]+)") + + +def parse_overrides(lines): + """Return {op: handler_va} for every real override in FUN_00413860's listing. + + Excludes the default-handler fill and any store outside the aligned table window. + """ + out = {} + for ln in lines: + m = _STORE.search(ln) + if not m: + continue + disp = int(m.group(1), 16) + va = int(m.group(2), 16) + if not (TABLE_BASE <= disp < TABLE_BASE + TABLE_SPAN): + continue + if (disp - TABLE_BASE) % 4 != 0: + continue + if va == DEFAULT_HANDLER: + continue + out[(disp - TABLE_BASE) // 4] = va + return out + + +def build(disasm_path, out_path): + lines = Path(disasm_path).read_text(encoding="utf-8").splitlines() + m = parse_overrides(lines) + data = {hex(op): {"handler": hex(va)} for op, va in sorted(m.items())} + Path(out_path).write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8") + return m + + +# ---- cross-check vs opcodes.toml ------------------------------------------------- +# opcodes.toml records handler VAs only in free text (summary/evidence/details/label), +# e.g. "real handler FUN_0042a0b0", "@0x42d360". We extract every module VA mentioned +# per opcode and flag any op whose DERIVED handler is absent from that set (report only). +_VA_HEX = re.compile(r"0x4[0-9a-fA-F]{5}") # module range 0x400000..0x4fffff +_VA_FUN = re.compile(r"FUN_00(4[0-9a-fA-F]{5})") # FUN_004xxxxx + + +def _toml_text_vas(entry): + """All module VAs mentioned in an opcode entry's text fields.""" + chunks = [str(entry.get("label", ""))] + sem = entry.get("semantics", {}) + if isinstance(sem, dict): + for v in sem.values(): + if isinstance(v, str): + chunks.append(v) + text = "\n".join(chunks) + vas = set(int(x, 16) for x in _VA_HEX.findall(text)) + vas |= set(int("0x" + x, 16) for x in _VA_FUN.findall(text)) + return vas + + +def cross_check(derived): + """Compare derived handlers against opcodes.toml prose. Returns (mismatches, checked, uncheckable).""" + import tomllib + toml = tomllib.loads((paths.VM_MAP / "opcodes.toml").read_text(encoding="utf-8")) + mismatches, checked, uncheckable = [], 0, 0 + for entry in toml.get("opcode", []): + op = entry.get("op") + if op is None or op not in derived: + continue + mentioned = _toml_text_vas(entry) + if not mentioned: + uncheckable += 1 + continue + checked += 1 + if derived[op] not in mentioned: + mismatches.append((op, derived[op], sorted(mentioned))) + return mismatches, checked, uncheckable + + +def main(): + if len(sys.argv) < 2: + print(__doc__) + return 2 + disasm = sys.argv[1] + do_check = "--check" in sys.argv[2:] + out = paths.BUILD / "op-handler-map.json" + m = build(disasm, out) + print(f"{len(m)} dispatch-handler overrides -> {out}") + + anchors = {0x1ac: 0x427fb0, 0x1a2: 0x42d360, 0x215: 0x42a0b0, 0xc8: 0x420ec0} + bad = [f"0x{op:x}->{hex(m.get(op))} (want {hex(want)})" + for op, want in anchors.items() if m.get(op) != want] + if bad: + print("ANCHOR CHECK FAILED:", "; ".join(bad)) + return 1 + print("anchor check OK:", ", ".join(f"0x{op:x}->{hex(v)}" for op, v in anchors.items())) + + if do_check: + mismatches, checked, uncheckable = cross_check(m) + print(f"\ncross-check vs opcodes.toml: {checked} ops checked, " + f"{uncheckable} without a recorded handler VA (skipped), {len(mismatches)} disagreements") + for op, va, mentioned in mismatches: + ms = ", ".join(hex(x) for x in mentioned) + print(f" DISAGREE op 0x{op:x}: derived {hex(va)} not among toml VAs [{ms}]") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/test_ghidra_handler_map.py b/tools/test_ghidra_handler_map.py new file mode 100644 index 0000000..2690588 --- /dev/null +++ b/tools/test_ghidra_handler_map.py @@ -0,0 +1,61 @@ +"""Unit tests for the FUN_00413860 dispatch-table parser (tools/ghidra_handler_map.py). + +Run: py -3.11 -X utf8 tools/test_ghidra_handler_map.py (plain runner, no pytest dependency). +""" +import sys +from ghidra_handler_map import parse_overrides + +FAILS = [] + + +def check(cond, msg): + if not cond: + FAILS.append(msg) + print("FAIL:", msg) + else: + print("ok:", msg) + + +def test_parses_known_override_anchors(): + # Lines in the exact shape captured from disassemble_function(0x413860). + # disp = 0x9b24c + op*4 => 0x9b8fc = 0x9b24c + 0x1ac*4 (op 0x1ac), 0x9b56c = op 0xc8. + lines = [ + "0041476a: MOV dword ptr [ESI + 0x9b8fc],0x427fb0", # op 0x1ac + "004148e6: MOV dword ptr [ESI + 0x9b56c],0x420ec0", # op 0xc8 + "00414738: MOV dword ptr [ESI + 0x9b8d4],0x42d360", # op 0x1a2 + "00414d1e: MOV dword ptr [ESI + 0x9baa0],0x42a0b0", # op 0x215 + ] + m = parse_overrides(lines) + check(m.get(0x1ac) == 0x427fb0, "anchor op 0x1ac -> 0x427fb0") + check(m.get(0xc8) == 0x420ec0, "anchor op 0xc8 -> 0x420ec0") + check(m.get(0x1a2) == 0x42d360, "anchor op 0x1a2 -> 0x42d360") + check(m.get(0x215) == 0x42a0b0, "anchor op 0x215 -> 0x42a0b0") + + +def test_excludes_default_handler_and_out_of_window(): + lines = [ + "MOV EAX,0x4162b0", # STOSD seed, not a store -> ignored + "MOV dword ptr [ESI + 0x9b24c],0x4162b0", # op 0x0 default handler -> excluded + "MOV dword ptr [ESI + 0x5f2e8],0x570410", # disp below table window -> ignored + "MOV dword ptr [ESI + 0x992c0],EAX", # register source, no immediate -> ignored + "MOV dword ptr [ESI],0x570fec", # no displacement -> ignored + ] + check(parse_overrides(lines) == {}, "default handler + out-of-window + non-immediate all excluded") + + +def test_ignores_misaligned_displacement(): + # A store into the table window but not on a 4-byte boundary is not a real slot. + lines = ["MOV dword ptr [ESI + 0x9b24e],0x401234"] # 0x9b24e-0x9b24c = 2, not %4 + check(parse_overrides(lines) == {}, "misaligned displacement in window excluded") + + +def main(): + test_parses_known_override_anchors() + test_excludes_default_handler_and_out_of_window() + test_ignores_misaligned_displacement() + print("FAILURES:", len(FAILS)) + return 1 if FAILS else 0 + + +if __name__ == "__main__": + sys.exit(main())