From a61c0c9abd91a74bb33c5cceb31c188ac481c7b6 Mon Sep 17 00:00:00 2001 From: gamer147 Date: Mon, 6 Jul 2026 20:48:22 -0400 Subject: [PATCH] feat(assets): solve asset resolution (SYS4INI per-scene section manifest) resId -> files[section_base(scene) + resId]. SYS4INI's file list is sectioned, one per scene (SCxxxx.BIN + its cross-archive asset manifest); file_number is the index within the section. Unified for set-texture, play-bgm, play-voice. Fully static/general -> no per-scene capture. - tools/parse_sys4ini.py: SYS4INI (S4IC422, LZSS) -> build/asset-index.json - tools/resolve_asset.py: sections + (scene,resId) resolver -> build/asset-sections.json - validated: 97% structural, SC0000 17/17 vs Frida, 586/595 captured loads - opcodes.toml: set-texture/create/draw-texture, play-bgm/voice enriched (frida-grounded) - Frida tooling (capture_load_order all-archive, correlate_scope, ...) + vm0 --settex - docs: asset-resolution-re (step2 SOLVED), global-memory-re (shelved), tools-reference Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/PROJECT-STRUCTURE.md | 5 +- docs/asset-resolution-re.md | 65 +++++-- docs/global-memory-re.md | 128 ++++++++++++++ docs/opcode-reference.md | 45 ++--- docs/tools-reference.md | 87 ++++++++++ tools/correlate_scope.py | 148 ++++++++++++++++ tools/frida/README.md | 35 ++-- tools/frida/capture_load_order.py | 168 ++++++++++++++++++ tools/frida/capture_resid_args.py | 194 +++++++++++++++++++++ tools/frida/find_global_by_sequence.py | 141 ++++++++++++++++ tools/frida/find_globals_base.py | 183 ++++++++++++++++++++ tools/frida/locate_resource_load.py | 178 +++++++++++++++++++ tools/parse_sys4ini.py | 225 +++++++++++++++++++++++++ tools/resolve_asset.py | 116 +++++++++++++ tools/resolve_frida_reads.py | 119 +++++++++++++ tools/vm0.py | 43 ++++- vm-map/opcodes.toml | 78 ++++----- vm-map/resources.json | 16 ++ 18 files changed, 1887 insertions(+), 87 deletions(-) create mode 100644 docs/global-memory-re.md create mode 100644 docs/tools-reference.md create mode 100644 tools/correlate_scope.py create mode 100644 tools/frida/capture_load_order.py create mode 100644 tools/frida/capture_resid_args.py create mode 100644 tools/frida/find_global_by_sequence.py create mode 100644 tools/frida/find_globals_base.py create mode 100644 tools/frida/locate_resource_load.py create mode 100644 tools/parse_sys4ini.py create mode 100644 tools/resolve_asset.py create mode 100644 tools/resolve_frida_reads.py create mode 100644 vm-map/resources.json diff --git a/docs/PROJECT-STRUCTURE.md b/docs/PROJECT-STRUCTURE.md index 8121635..fedab33 100644 --- a/docs/PROJECT-STRUCTURE.md +++ b/docs/PROJECT-STRUCTURE.md @@ -54,7 +54,10 @@ S:\Game Hacking\Eushully\Himegari\ ← workspace root (three siblings) │ └── opcode-leads.json, small-script-listings.md │ ├── docs/ all documentation - │ ├── PROJECT-STRUCTURE.md this file + │ ├── PROJECT-STRUCTURE.md this file (where things live) + │ ├── tools-reference.md every tool: purpose, usage, I/O (operational companion) + │ ├── asset-resolution-re.md resId→file RE (graphics/audio); asset-index steering + │ ├── global-memory-re.md runtime global observation RE (SHELVED; future starting point) │ ├── remake-architecture-and-roadmap.md THE direction doc (phases A–E) │ ├── phase-a-slice-plan.md the current slice (A0/A1/A2) │ ├── vm-mapping-plan.md the phased decode plan diff --git a/docs/asset-resolution-re.md b/docs/asset-resolution-re.md index c18a73b..5122d04 100644 --- a/docs/asset-resolution-re.md +++ b/docs/asset-resolution-re.md @@ -31,8 +31,9 @@ highest-risk area of the port. This doc is the steering state; it feeds the A2b - **The resolution chain is opaque statically.** `CGINIT` (`build/data/CGINIT.json`) is a 925-column *numeric* record table (row-major, sparse) — **not** an id→filename map. **`SYS4INI.BIN` (magic `S4IC422`) is the authoritative asset index** the game + `BinExtractALF` - use (name ↔ archive ↔ offset ↔ size), but filenames are **not stored as plain ASCII** (an - `EV001AA` search misses), so it needs S4IC-format RE to parse. + use (name ↔ archive ↔ offset ↔ size). Filenames aren't plain ASCII because the whole directory + is **LZSS-compressed** (not encrypted). **DONE (2026-07-06):** `tools/parse_sys4ini.py` parses it + → `build/asset-index.json` (13206 entries). See step 1 below. - **Frida file-I/O is noisy.** `ReadFile` hooks on `DATA2.ALF` capture reads during the opening, but the offsets/spans don't line up with extracted AGF sizes → the game likely **memory-maps** the archives (so `ReadFile` offsets are OS paging, not clean per-asset loads) and/or uses async reads. @@ -40,16 +41,46 @@ highest-risk area of the port. This doc is the steering state; it feeds the A2b ## The RE plan (ordered) -1. **Parse `SYS4INI` (S4IC422) → an asset index** `{name, archive, offset, size}`. *Reusable and - bounded* — it names every asset in every DATA*.ALF, gives archive-offset→name (to rescue Frida - offsets), and is the **answer key** for step 2. Deliverable: `tools/parse_sys4ini.py` + - `build/asset-index.json`. (Format reference: asmodean's `exs4alf`, which `BinExtractALF` is based on.) -2. **Crack `resId → filename`.** With SYS4INI as the answer key, either (a) **order-correlate**: run - SC0000 in our engine to get the `set-texture(resId)` sequence, capture the real game's asset-load - order via a *reliable* Frida hook, and align them; or (b) **hook the internal load-by-id - function** directly (find via the opcode dispatch for `0x1f9`) to read `resId → name` at the - source. Likely underlying rule: `resId → CGINIT/table → name`. Deliverable: the mechanism + - `vm-map/resources.json` (or a generated map) seeding at least SC0000's slideshow. +1. **Parse `SYS4INI` (S4IC422) → an asset index** `{name, archive, offset, size}`. **✅ DONE + (2026-07-06).** `tools/parse_sys4ini.py` → `build/asset-index.json`: 5 archives (DATA1–5), + 13206 real entries (2 `@` placeholders skipped). **Format:** `uint32 packed_size @0x134`, then an + LZSS stream at `0x138` running to EOF (GARbro-style: 0x1000 zero-filled ring buffer, init pos + 0xFEE, control bits LSB→MSB, 1=literal / 0=two-byte backref `off=(hi&0xf0)<<4|lo`, `len=3+(hi&0xf)`). + Decompresses to `uint32 arc_count`, `arc_count × char[256]` archive names, `uint32 file_count`, + then `file_count ×` 80-byte records `{char name[64]; u32 arc_id, file_number, offset, size}`. + **Validated:** decompressed length (1058783) equals the stored size dword at `0x12c`; per-archive + counts match the `extracted/` ground truth exactly (DATA2=985, DATA3=39, DATA4=9733, DATA5=210); + all 13206 `offset+size` fit inside their real `.ALF`; 837 name-matched files → 0 size mismatches. + `files[]` preserves directory order (feeds step 2's order-correlation). Re-run: + `py -3.11 -X utf8 tools/parse_sys4ini.py --check`. (Ref: asmodean's `exs4alf` / GARbro Eushully `ArcALF.cs`.) +2. **Resolve `resId → asset file`.** **✅ SOLVED (2026-07-06) — fully static & general; NO runtime capture.** + + **The rule:** SYS4INI's file list is organized into **SECTIONS, one per scene** — each is a + `SCxxxx.BIN` script entry followed by that scene's **asset MANIFEST**: every asset it references, + across *all* archives and types (EV/BG/CS/AE graphics **and** OGG/WAV audio), interleaved in usage + order. `file_number` is the **0-based index within the section**. So: + + > **`resId → files[ section_base(scene) + resId ]`**, where `section_base` = the start of the SYS4INI + > section containing the scene's `SCxxxx.BIN`. + + Unified for `set-texture(resId)`, `play-bgm(id)`, `play-voice(id)` — one manifest. **Tool:** + `tools/resolve_asset.py --build` → `build/asset-sections.json` (359 sections, 136 scenes); + `resolve_asset.py [resId]` resolves. **Validated:** `file_number == position − section_base` + for 12848/13206 files (97%); SC0000 resolves 17/17 across archives vs the Frida capture (`0x25→EV052CA`, + `0x36→BG030A` background, `0x6c→EM* effect`, `play-bgm 5→BGM006`); 586/595 distinct captured loads + (all sections) satisfy `files[base+fn]==name`. This is the derivable rule that generalizes to any + AGE game with the same container — **the "scope" was just which SYS4INI section the scene lives in.** + + *How we got here (condensed):* first confirmed `resId == file_number` via Frida load-order correlation + for SC0000's opening, but `file_number` is not globally unique so a per-scene "scope" was needed. A long + hunt for the selector (thought it was native scene state; even tried reading `G[0x62424]` live — the + VM global memory is structured/packed, see `docs/global-memory-re.md`) missed the real structure until a + **full multi-archive capture** (user domain tip: DATA1 holds BG/CS/CB/CA/CP graphics by name prefix, not + just DATA2 EV CGs) revealed `file_number == SYS4INI position` inside per-scene sections. Superseded tools: + `tools/correlate_scope.py`, `vm0.py --settex` (VM set-texture trace; still useful, but vm0 diverges on + branchy non-opening scenes — use the C# VM to trace those). Runtime note for future work: the game is + **packed** (main VM logic in a per-run heap `r-x` region) and streams archives through a heap block-cache + via `ReadFile` (not mmap); the stable AGF decoder is `AGE.EXE+0x74f1f`. 3. **Wire the backend** (already designed — A2b-background plan Tasks 3–5): `ResourceMap` resolver + Godot `TextureRect` compositing; render only resolved full-screen slots. Mechanical once (1)+(2) land. 4. **Audio** (parallel, same shape): resolve `play-voice`/`play-bgm` `id → OGG` via SYS4INI + a @@ -67,7 +98,9 @@ rendering what the executed bytecode + the map produce (never a hardcoded image) ## Status -A2b-background: **machinery landed** (texture ops engine-driven, tools, findings). The **render is -blocked on asset resolution** (steps 1–2), which is promoted to its own foundational effort. Next: -either start step 1 (`SYS4INI` parser) or bank momentum with the Frida-free **choices** sub-slice -(static-RE opcode hunt) while resolution waits its scheduled turn. +A2b-background: **machinery landed**; **steps 1 & 2 SOLVED (static, general).** Step 1 = +`build/asset-index.json`. Step 2 = **`resId → files[section_base(scene) + resId]`** via SYS4INI +per-scene sections (`tools/resolve_asset.py` + `build/asset-sections.json`) — no runtime capture, works +across all archives/types and for audio too. Remaining for the render (step 3): wire a `ResourceMap` +(scene → section_base; resId → asset via the index) + Godot `TextureRect` compositing (A2b plan Tasks 3–5, +now purely mechanical). Audio (step 4) uses the *same* resolver (`play-bgm/play-voice id → files[base+id]`). diff --git a/docs/global-memory-re.md b/docs/global-memory-re.md new file mode 100644 index 0000000..f1a910b --- /dev/null +++ b/docs/global-memory-re.md @@ -0,0 +1,128 @@ +# Runtime Global Observation — RE state & starting point + +**Goal.** Read the running game's live VM global variables by their bytecode address (e.g. +`G[0x62424]`, `global-string 0x279`). This is the **VM-validation cornerstone**: with it we can +compare our VM's global state against the real game's at any point, verify effectful opcodes that +have no machine oracle (battle math, flag logic, stat updates), and *name* the ~200 still-unclassified +globals by watching them change. The roadmap flags it as the thing that replaces most Frida/Unicorn +guesswork once the VM is validated-correct. + +**Status: SHELVED (2026-07-06), deliberately.** We learned the shape of the problem but did not +achieve a general "read any global by address" capability. This doc records what we tried, what we +proved, the concrete landmarks found, and the *right* way to resume. It is **not** on the critical +path for anything else, and — importantly — it does **not** crack the asset-resolution scope selector +(see "What this does NOT solve"). + +--- + +## The core finding: our flat-address model ≠ physical layout + +Our VM models globals as one flat address space (`G[addr]`), and that is **execution-correct** +(byte-exact against the dialogue-trace oracle). But it is an *abstraction*. Physically the game's +state is **structured and multi-store**: + +- **Multiple stores.** VM int-globals, string-globals, and float-globals are almost certainly + separate arrays (their address magnitudes differ wildly: `global-string 0x279` = 633 vs + `global-int 0x62424` = 403,492). On top of that, values get **copied into native C++ objects**. +- **Structured records, not flat int32.** Entity data (units, party) is stored as records with + **inline fixed-size string slots + mixed int fields + native heap pointers** — not a packed int + array. A `base + addr*4` scan therefore finds nothing (confirmed: no stride in {2,4,8,12,16} + reproduces a known consecutive-address `*INIT` value run). +- **Transient vs. stable globals.** A global is a real variable, but *how it's used* varies: + - **Transient** (e.g. `G[0x62424]`, the CG resId): a scratch/argument register — `mov G[0x62424] + = resId; call load` — set right before a call, overwritten right after. Never holds a stable + value. Non-atomic value scans **race** against it and miss it. + - **Stable** (e.g. `global-string 0x279`, the player name): persists across play. *These* are + findable by value+stability scans; transient ones are not. +- **Multiple resId globals.** Backgrounds vs. foreground portraits/sprites appear to use different + resId globals / churn `G[0x62424]` between visible background changes (observed live). +- **Packed process.** Main VM logic runs from a **per-run heap `r-x` region** (~30 MB, nonstable + base) — so the interpreter can't be hooked at a fixed `AGE.EXE+off`. Archives are streamed via a + heap block-cache through `ReadFile` (not memory-mapped). See `docs/asset-resolution-re.md`. + +**Why the naive scan failed.** The first attempt scanned for a flat `base+addr*4` int32 array using a +signature of `*INIT` constants. It found 0 matches because (a) the layout isn't flat int32, and (b) +`*INIT` entity data lives in structured records. The individual values exist in memory but only +coincidentally adjacent. + +--- + +## What we tried (chronological, with outcomes) + +1. **Flat-int32 signature scan** — `tools/frida/find_globals_base.py` (+ `build/globals-signature.json`, + 15,481 distinctive `(addr,value)` pairs from `*INIT` `mov (global-int A) IMM`; 215-dword contiguous + anchor @`0x631a9`). **Result: 0 hits** at any anchor length, in the opening AND in the first dungeon + (ruling out load-timing). Diagnostic: the individual values occur (coincidentally); no stride in + {2,4,8,12,16} reproduces the consecutive-address sequence ⇒ **not flat int32**. +2. **"Lily" (player name) anchor** — searched for the entered name. Found as ASCII `Lily\0`. **But the + context proved it's the wrong anchor:** every copy is embedded in a **native unit-record** (inline + string slot + stats + heap pointers), i.e. downstream copies, not the VM string-global store. +3. **Differential value scan on `G[0x62424]`** — `tools/frida/find_global_by_sequence.py`. Self-driven + by the ReadFile→file_number(=resId) signal: scan for resId at load 1, keep those that become the + next resId, etc. **Converged to 3 addresses that tracked 35→37→39→43→46 perfectly — but all were + STACK slots** (region `0x18f000`, self-referential + `0x76xxxxxx` return addresses); their values + are garbage between loads. They're the per-call argument copies, not the global. +4. **Stability filter** (re-read survivors ~1.4 s later, during the pause) — proved the point: 2084 + locations held resId 35 *stably*, but **0** of them became 37 ⇒ **`G[0x62424]` is transient**, never + stable at a value, so neither value-scans nor stability-filters can pin it. + +--- + +## Concrete landmarks (for a future run — but note ASLR: these are per-process) + +- **Player name** = `global-string 0x279`; default `"リリィ"` (set in `INPUTNAME.BIN` via + `set-string (global-string 0x279)`, then the input op `0x1aa`/`u00425920` writes the entered name to + `0x279`, then copied to `local-string 0x135`). **Stable global** — the best future anchor. +- **Native unit records** holding the name: ASCII `Lily\0` in 16-byte-ish slots, followed by int stats + (level/…, e.g. `0c 0f 04 0f 13 1f`) and native pointers; arrays with stride ~`0x1c`. Region example + `0x6e9c000 +0x82000` (rw-, no module). These are **native objects**, not the VM store. +- **Transient-arg stack region** where load-arg resId copies appear: ~`0x18f000 +0x11000` (68 KB rw-). +- **Large heap regions** (candidate VM stores): 51.8 MB @ `0x2f64000`, plus ~10/9.6/8/6 MB regions. +- **Unpacked code (packer)**: 30.3 MB **r-x** @ `0x62411000` (main VM logic; nonstable base per run). +- Element size / mapping: **unknown**; it is *not* uniform `base+addr*4` int32. + +## Tools built (kept for resumption) + +All under `tools/frida/` (see `tools/frida/README.md` and `docs/tools-reference.md`): +- `find_globals_base.py` — `--build-sig` builds `build/globals-signature.json`; scan mode does the + flat-int32 signature scan (robust prefix ladder + verify). *Currently finds nothing → layout isn't flat.* +- `find_global_by_sequence.py` — differential resId scan with stability filter. *Finds stack proxies; + transient globals elude it.* +- `capture_load_order.py`, `locate_resource_load.py`, `capture_resid_args.py` — asset-resolution + captures (context: how the loader/decoder chain was found). + +--- + +## What this does NOT solve (avoid the trap we fell into) + +Reading `G[0x62424]` live would **not** reveal the asset-resolution **scope selector** (why resId 37 → +`EV052CA` and not one of the other 8 fn-37 files). The resId is already free from the ReadFile +`file_number`; the scope is *separate native scene state*. So global observation and the scope selector +are independent problems — don't chase globals expecting to crack resolution. + +--- + +## How to resume properly (recommended plan) + +Heuristic value-scans only ever find *specific, stable* globals one at a time; they give no general +`address → memory` mapping. A **guaranteed, general** capability needs the **interpreter's +address-resolution logic**. Recommended order: + +1. **Anchor on a STABLE global**, not a transient one. Best: `global-string 0x279` (the name). Find its + *VM-store* copy (distinguish from native unit copies: the VM store won't be wrapped in heap + pointers). A distinctive name makes the value scan collapse fast. +2. **Find the interpreter's global-access function.** Set a hardware/`MemoryAccessMonitor` watchpoint on + that stable global's physical location, trigger a bytecode read (e.g. open a menu that draws the + name), and **backtrace into the heap interpreter**. That function's address computation *is* the + `bytecode-address → physical` mapping — read it rather than guessing the layout. +3. **Generalize + verify.** Derive the mapping (likely per-store / per-region), build `read_global(addr)`, + and verify against our VM's known state (e.g. read a stable flag whose value our VM predicts). +4. **Then** build a runtime global-watch tool for VM validation and effectful-op naming. + +Alternative bootstraps if watchpoints are awkward: snapshot-diff at **quiescent** points (change a +known stable global via gameplay, diff memory); or a hardware **write** watchpoint on a stable global's +location to catch the interpreter's write path. + +**Bottom line for the next session:** the pieces (signature, tools, landmarks, the name anchor) are in +place. Resume from a *stable* anchor and target the *interpreter*, not heuristic scans of transient +globals — and only when runtime observation is actually the priority (it isn't blocking other work). diff --git a/docs/opcode-reference.md b/docs/opcode-reference.md index 0931a7e..6dd9757 100644 --- a/docs/opcode-reference.md +++ b/docs/opcode-reference.md @@ -17,6 +17,16 @@ - **grounding:** source=inference, confidence=low - **evidence:** confirm via frida +### 0xbf `play-bgm` (play-bgm, argc 1) +- **summary:** Play background music by id; id resolves via the SYS4INI section manifest -> files[section_base(scene)+id] (OGG in DATA3). Same resolution as set-texture. +- **grounding:** source=frida, confidence=high +- **evidence:** Frida capture: `play-bgm 0x5` in SC0000 (section base 0) loaded BGM006.OGG = files[5]. Unified with set-texture resolution rule. + +### 0xc4 `play-voice` (play-voice, argc 1) +- **summary:** Play a voice clip by id; id resolves via the SYS4INI section manifest -> files[section_base(scene)+id] (voice OGG in DATA1/DATA4). Same rule as set-texture/play-bgm. +- **grounding:** source=investigation, confidence=med +- **evidence:** Section-manifest resolution validated across archives incl. DATA4 voice OGGs (586/595 captured loads); per-clip id->OGG not individually Frida-pinned yet. + ## compute ### 0x1a2 `resolve-handle?` (u00428010, argc 1) @@ -36,11 +46,26 @@ - **grounding:** source=inference, confidence=med - **evidence:** confirm via frida +### 0x1f8 `create-texture` (create-texture, argc 4) +- **summary:** Allocate/prepare a texture slot: (slot, width, height, flag). e.g. `create-texture 0xd 0x190 0x1e 0x0` = slot 13, 400x30. +- **grounding:** source=investigation, confidence=med +- **evidence:** SC0000 CG/UI-draw path disasm; slot/w/h roles read off the operands (400x30 text bars, etc.). + +### 0x1f9 `set-texture` (set-texture, argc 3) +- **summary:** Load asset #resId into texture slot: (resId, slot, flag=-1). resId resolves via the SYS4INI per-scene section manifest: files[section_base(scene)+resId] (same rule for play-bgm/play-voice). See docs/asset-resolution-re.md. +- **grounding:** source=frida, confidence=high +- **evidence:** SC0000 Frida-confirmed 17/17 (0x25->EV052CA, 0x2e->EV052DB, 0x36->BG030A background); resolution rule validated on 586/595 captured loads. Traced in CG-load subroutine label_12649 as `set-texture G[0x62424] -1`. + ### 0x1fa `ui-clear?` (u00420480, argc 1) - **summary:** 1 arg (element id); follows 0x1f7 — show/hide/clear UI element by id - **grounding:** source=inference, confidence=med - **evidence:** confirm via frida +### 0x1fb `draw-texture` (draw-texture, argc 8) +- **summary:** Blit a texture slot to screen. Observed 8 args: (handle, slot, srcx, srcy, w, h, dstx, dsty). e.g. `draw-texture 0xcf08 0x3 0 0 0x320 0x258 0 0` = full-screen (800x600) slot 3 at (0,0). +- **grounding:** source=investigation, confidence=med +- **evidence:** SC0000 CG-load subroutine label_12649: `draw-texture (ptr) (slot) 0 0 (w) (h) (dstx) (dsty)`; full-screen slot-3 draws use 0x320x0x258 (800x600). + ### 0x1ff `draw?` (u00420770, argc 4) - **summary:** 4 args (global+imms); follows 0x217, then call - **grounding:** source=inference, confidence=low @@ -403,10 +428,6 @@ op 0x90 (u0041BEB0, argc 7): `0x90 x y w h tgt_a tgt_b tgt_c`. Kelebek left it " - **summary:** — - **grounding:** source=kelebek, confidence=low -### 0xbf `play-bgm` (play-bgm, argc 1) -- **summary:** — -- **grounding:** source=kelebek, confidence=med - ### 0xc0 `u00415620` (u00415620, argc 1) - **summary:** — - **grounding:** source=kelebek, confidence=low @@ -415,10 +436,6 @@ op 0x90 (u0041BEB0, argc 7): `0x90 x y w h tgt_a tgt_b tgt_c`. Kelebek left it " - **summary:** — - **grounding:** source=kelebek, confidence=low -### 0xc4 `play-voice` (play-voice, argc 1) -- **summary:** — -- **grounding:** source=kelebek, confidence=med - ### 0xc5 `u0041D4A0` (u0041D4A0, argc 2) - **summary:** — - **grounding:** source=kelebek, confidence=low @@ -803,18 +820,6 @@ op 0x90 (u0041BEB0, argc 7): `0x90 x y w h tgt_a tgt_b tgt_c`. Kelebek left it " - **summary:** — - **grounding:** source=kelebek, confidence=low -### 0x1f8 `create-texture` (create-texture, argc 4) -- **summary:** — -- **grounding:** source=kelebek, confidence=med - -### 0x1f9 `set-texture` (set-texture, argc 3) -- **summary:** — -- **grounding:** source=kelebek, confidence=med - -### 0x1fb `draw-texture` (draw-texture, argc 8) -- **summary:** — -- **grounding:** source=kelebek, confidence=med - ### 0x1fd `u00420620` (u00420620, argc 4) - **summary:** — - **grounding:** source=kelebek, confidence=low diff --git a/docs/tools-reference.md b/docs/tools-reference.md new file mode 100644 index 0000000..465bc7a --- /dev/null +++ b/docs/tools-reference.md @@ -0,0 +1,87 @@ +# Tools Reference + +Living catalogue of every script in `tools/` — **what it does, how to run it, and what it +reads/writes**. This is the operational companion to `docs/PROJECT-STRUCTURE.md` (which is the +*where-things-live* map); when they overlap, PROJECT-STRUCTURE owns layout, this file owns +usage + I/O. Keep it current: **add a row here whenever you add a tool, and update the row +whenever a tool's inputs/outputs change.** + +## Conventions (apply to every tool) + +- **Run with** `py -3.11 -X utf8 tools/.py …` — the `-X utf8` is required on Windows so + cp932/Shift-JIS source text renders (and generated files stay UTF-8). +- **Paths are never hard-coded.** Every tool imports `tools/paths.py` for `GAME_DIR` / + `EXTRACTED` / `DATA1` / `BUILD` / `VM_MAP` / `BIN`. Relocate the tree by editing only that file. +- **Generated files are never hand-edited** (they're marked ⚙ below). Edit the source, re-run + the generator. +- **`build/` and `extracted/` are disposable** — everything under them regenerates from a tool. + +## Path anchor + +| Tool | Purpose | I/O | +|---|---|---| +| `paths.py` | ★ Single path anchor — derives all workspace dirs from its own location; `paths.scripts()` returns the override-aware `{NAME.BIN → path}` corpus map (loose game-folder patches shadow `extracted/DATA1`). | *Imported, not run.* | + +## Container parse / disassemble + +| Tool | Purpose | Run | Reads → Writes | +|---|---|---|---| +| `sys4load.py` | Loader + opcode-decoding disassembler for SYS4 `.BIN` scripts (the container-format core every other tool builds on). | `sys4load.py ` · `--summary` · `--strings` · `--json` · `sys4load.py --validate` (corpus check) | `.BIN` + `age_opcodes*.py` + `build/global-var-map.json` → stdout listing, or `build/scripts-json/` with `--json` | +| `age_opcodes.py` | 548-entry Kelebek AGE opcode/arg-type table. **PRISTINE upstream data — never edit.** | *Imported.* | — | + +## Opcode reference toolchain — single source of truth = `vm-map/opcodes.toml` + +All opcode knowledge (ABI, semantics, provenance, `depends_on`) is hand-edited **only** in +`vm-map/opcodes.toml`. Everything else is generated from it. + +| Tool | Purpose | Run | Reads → Writes | +|---|---|---|---| +| `opcodes_build.py` | Generator + linter for the opcode reference. | `--build` · `--lint` · `--bootstrap` | `vm-map/opcodes.toml` → ⚙ `tools/age_opcodes_himegari.py`, ⚙ `build/opcodes.json`, ⚙ `docs/opcode-reference.md`, ⚙ `build/opcode-coverage.md` | +| `opcodes_model.py` | In-memory model + loader + linter (dangling-ref / confidence-ceiling / vocabulary / dependents). | *Imported by `opcodes_build.py`.* | `vm-map/opcodes.toml` → — | +| `test_opcodes.py` | Unit tests for the opcode tooling. | `test_opcodes.py` | — | +| `opcode_context.py` | Read-only evidence gatherer for classifying unnamed opcodes (frequency, argc, operand-type signature, neighbours, disasm snippets, Kelebek comment). | `--top 20` · `opcode_context.py 0x1f4 0x71 …` | corpus → stdout | +| `validate_opcode_table.py` | Definitive decode-coverage validator (replicates Kelebek's `data_array_end` code/data split). | `validate_opcode_table.py` | corpus → stdout | +| `validate_opcode_table_naive.py` | Naïve variant of the above (baseline comparison). | `validate_opcode_table_naive.py` | corpus → stdout | +| `age_opcodes_himegari.py` | ⚙ Inferred Himegari opcode semantics — **generated; do not hand-edit.** | *Imported by `sys4load.py`.* | — | + +## Extraction / data corpora + +| Tool | Purpose | Run | Reads → Writes | +|---|---|---|---| +| `extract_phase2.py` | Batch: disassembly + text corpora for every script. | `extract_phase2.py` | corpus → `build/disasm/*.asm`, `build/text/{dialogue.jsonl,strings.jsonl,*.strings.txt}`, `build/manifest.json` | +| `extract_init.py` | Parse a `*INIT` data table (auto-detects name / numeric / footer shape). | `extract_init.py [OUTNAME] [--mode …]` | `
.BIN` → `build/data/.json` | +| `global_map.py` | Build the partial global-variable name map from static evidence. | `global_map.py` | corpus + `build/data/` → `build/global-var-map.{json,md}` | + +## VM + +| Tool | Purpose | Run | Reads → Writes | +|---|---|---|---| +| `vm0.py` | Headless Python bytecode VM (Phase A0 execution-model prototype; reuses `sys4load`). | `--test` (RECOVER unit test) · `--sweep [N]` (oracle coverage) · `--scene NAME` · `--settex NAME` (set-texture resId trace + exec trace) · `` | corpus → stdout; `build/vm0-trace.json`; `build/settex-.json` | +| `correlate_scope.py` | Align the VM's `set-texture(resId)` trace with the game's Frida load order → tag each load's DATA2 package, flag package transitions, dump the significant ops in each transition span (the **scope selector** hunt). | `correlate_scope.py ` | `build/settex-.json` + `build/frida-load-order-result.json` + index → stdout | + +## Asset resolution / graphics + +| Tool | Purpose | Run | Reads → Writes | +|---|---|---|---| +| `parse_sys4ini.py` | Parse `SYS4INI.BIN` (S4IC422, LZSS-compressed) into the authoritative asset index — name ↔ archive ↔ offset ↔ size for all DATA*.ALF (the `resId→file` answer key). | `parse_sys4ini.py [--check]` (`--check` validates vs `extracted/` + `.ALF` sizes) | `姫狩り…/SYS4INI.BIN` → `build/asset-index.json` | +| `resolve_asset.py` | ★ **The static asset resolver.** SYS4INI is sectioned (one per scene: `SCxxxx.BIN` + its cross-archive manifest; `file_number` = index within section). Resolves `resId → files[section_base(scene) + resId]` for graphics AND audio, no capture. | `resolve_asset.py --build` · `resolve_asset.py [resId]` | `build/asset-index.json` → `build/asset-sections.json`; resolves any (scene, resId) | +| `resolve_frida_reads.py` | Rescue noisy Frida archive-read offsets → asset names via the index (per-archive range search; drops 0x20000 paging reads); recovers the per-scene asset load order. | `resolve_frida_reads.py [reads.log] [-o out.json]` | `build/frida-reads.log` + `build/asset-index.json` → `build/frida-asset-loads.json` | +| `convert_agf.py` | Convert named AGF stills to BMP via `AGF2BMP2AGF.exe`. | `convert_agf.py EV001AA.AGF …` | `extracted/DATA2|DATA5/*.AGF` → `build/textures/*.BMP` | + +## Runtime capture (Frida) + +| Tool | Purpose | Run | Reads → Writes | +|---|---|---|---| +| `tools/frida/capture_graphics.py` | Attach Frida to the running game; log archive reads/opens (ground-truth for asset resolution). See `tools/frida/README.md`. | `py -3.11 -u -X utf8 tools/frida/capture_graphics.py [AGE.EXE]` | running game → `build/frida-reads.log`, `build/frida-opens.log` | +| `tools/frida/capture_load_order.py` | **Primary asset-resolution capture:** recover a scene's per-asset load order from exact-start `ReadFile` reads → names via the index; confirms `resId==file_number`. Attach; replay scene; `--analyze`. | `py -3.11 -u -X utf8 tools/frida/capture_load_order.py [pid]` · `--analyze` | running game + index → `build/frida-load-order.jsonl`, `…-result.json` | +| `tools/frida/locate_resource_load.py` | Phase-1 locator: back-traces asset-opens to find the native AGF load chain (`0x16d5d7→0x74f1f`). | `py -3.11 -u -X utf8 tools/frida/locate_resource_load.py [pid]` · `--aggregate` | running game → `build/frida-resource-bt.jsonl` | +| `tools/frida/capture_resid_args.py` | Phase-2 probe: dumps the decoder's args / context / caller frame (established the loader carries only offsets, not names). | `py -3.11 -u -X utf8 tools/frida/capture_resid_args.py [pid]` · `--analyze` | running game → `build/frida-resid-args.jsonl` | +| `tools/frida/find_globals_base.py` | Runtime-global RE (SHELVED — see `docs/global-memory-re.md`): flat-int32 signature scan for the VM global array. Finds nothing → layout isn't flat. | `--build-sig` · `py -3.11 -u -X utf8 tools/frida/find_globals_base.py [pid]` | `*INIT` → `build/globals-signature.json`; scans running game | +| `tools/frida/find_global_by_sequence.py` | Runtime-global RE (SHELVED): differential resId value-scan + stability filter. Finds stack proxies; proved `G[0x62424]` is a transient arg-register. | `py -3.11 -u -X utf8 tools/frida/find_global_by_sequence.py [pid]` | running game + index → stdout | + +## Historical / one-off + +| Tool | Purpose | +|---|---| +| `probe_*.py` (`probe_header`, `probe_leads`, `probe_refs`, `probe_tables`, `probe_tags`, `probe_types`, `probe_xref`) | Container/opcode format-RE probes used to reverse the format originally. Kept for reproducibility; not part of the normal workflow. | +| `pack_check.py` | Checks whether `AGE.EXE` is packed (it is). No longer a blocker — we run our own VM. | diff --git a/tools/correlate_scope.py b/tools/correlate_scope.py new file mode 100644 index 0000000..e0ea0a9 --- /dev/null +++ b/tools/correlate_scope.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +"""Correlate the VM's set-texture(resId) trace with the game's Frida load order to LOCALIZE the +asset-resolution scope selector (docs/asset-resolution-re.md step 2, scope-selector hunt). + +Inputs: + build/settex-.json our VM's ordered set-texture(resId) trace + bytecode offsets + (produce with: py -3.11 -X utf8 tools/vm0.py --settex ) + build/frida-load-order-result.json the game's ordered (name, file_number=resId) loads + (produce with tools/frida/capture_load_order.py --analyze) + build/asset-index.json for segmenting DATA2 into packages + +Method: `resId == file_number`, and each loaded file belongs to exactly one DATA2 "package" +(a monotonic-file_number run). So the game's load order is a readout of the active package over +time. We greedily align each Frida load to the next same-resId set-texture in the VM trace, which +pins it to a bytecode offset. Where the active package CHANGES between consecutive loads, some +instruction in the bytecode span between their offsets flipped the scope -- the scope selector. +This report lists the aligned loads, flags package transitions, and points at the byte spans to +inspect (dump them with the disassembly in build/disasm/.asm). + +Usage: py -3.11 -X utf8 tools/correlate_scope.py (e.g. SC0000) +""" +from __future__ import annotations +import json +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import paths + + +def packages(): + """Segment DATA2 (directory order) into monotonic-file_number runs; return name->pkg index.""" + idx = json.loads((paths.BUILD / "asset-index.json").read_text(encoding="utf-8")) + d2 = [f for f in idx["files"] if f["archive"] == "DATA2.ALF"] + name2pkg, pkg, prev = {}, 0, None + for f in d2: + fn = f["file_number"] + if prev is not None and fn <= prev: + pkg += 1 + name2pkg[f["name"]] = pkg + prev = fn + return name2pkg + + +# boilerplate ops to hide when dumping the between-loads span (leave the structural/effectful ones) +NOISE = {"mov", "add", "sub", "mul", "div", "mod", "and", "or", "sar", "shl", "xor", + "eq", "ne", "lt", "lte", "gr", "gre", "jcc", "jmp", "set-string", + "show-text", "end-text-line", "wait-for-input", "stmt-begin", "stmt-end", + "block-mark", "cond-block", "label-def", "line-id?", "comment", "set-font", + "lookup-array", "lookup-array-2d", "stmt-desc?", "bit-set", "bit-reset", + "check-bit", "text-param?", "gfx-geom?", "count?", "resolve-handle?", + "create-texture", "draw-texture", "draw-string", "draw?", "draw-blit?"} + + +def disasm_map(scene): + """offset(int) -> stripped disasm line, from build/disasm/.asm.""" + path = paths.BUILD / "disasm" / f"{scene}.asm" + out = {} + if not path.exists(): + return out + for ln in path.read_text(encoding="utf-8").splitlines(): + s = ln.strip() + if s[:2] == "0x" and ":" in s: + off = int(s.split(":", 1)[0], 16) + out[off] = s.split(":", 1)[1].strip() + return out + + +def dump_span(trace, ta, tb, dis): + """Print the significant (non-NOISE) ops executed between trace indices ta..tb.""" + seen = 0 + for t in range(ta, min(tb, len(trace))): + off = int(trace[t], 16) + line = dis.get(off, "") + mnem = line.split()[0] if line else "" + if mnem and mnem not in NOISE: + print(f" {trace[t]:>8} {line}") + seen += 1 + if not seen: + print(" (no structural/effectful ops in span — only boilerplate)") + + +def main() -> int: + args = [a for a in sys.argv[1:] if not a.startswith("-")] + if not args: + raise SystemExit(__doc__) + scene = args[0].upper().removesuffix(".BIN") + + settex_path = paths.BUILD / f"settex-{scene}.json" + loads_path = paths.BUILD / "frida-load-order-result.json" + if not settex_path.exists(): + raise SystemExit(f"missing {settex_path.name} — run: tools/vm0.py --settex {scene}") + if not loads_path.exists(): + raise SystemExit(f"missing {loads_path.name} — run tools/frida/capture_load_order.py --analyze") + + sx = json.loads(settex_path.read_text(encoding="utf-8")) + vm, trace = sx["settex"], sx.get("trace", []) # [{i,off,resId,slot,trace_i}] + loads = json.loads(loads_path.read_text(encoding="utf-8"))["load_order"] # [{name,file_number}] + name2pkg = packages() + dis = disasm_map(scene) + + # greedy align: each Frida load -> next same-resId set-texture in VM order + aligned, j, unmatched = [], 0, 0 + for ld in loads: + fn = ld["file_number"] + k = j + while k < len(vm) and vm[k]["resId"] != fn: + k += 1 + if k < len(vm): + aligned.append({"vm": vm[k], "resId": fn, "name": ld["name"], "pkg": name2pkg.get(ld["name"])}) + j = k + 1 + else: + aligned.append({"vm": None, "resId": fn, "name": ld["name"], "pkg": name2pkg.get(ld["name"])}) + unmatched += 1 + + print(f"scene {scene}: {len(vm)} VM set-textures, {len(loads)} Frida loads, " + f"{len(loads)-unmatched} aligned ({unmatched} unmatched)\n") + print(f"{'off':>8} {'resId':>5} {'pkg':>4} name") + prev = None + transitions = [] + for a in aligned: + off = a["vm"]["off"] if a["vm"] else None + flag = "" + if prev is not None and a["pkg"] != prev["pkg"]: + flag = f" <<< PACKAGE {prev['pkg']} -> {a['pkg']}" + transitions.append((prev, a)) + print(f"{str(off):>8} {a['resId']:>5} {str(a['pkg']):>4} {a['name']}{flag}") + prev = a + + print() + if not transitions: + print("No package transitions in this capture — play deeper into the scene to cross one.") + return 0 + print(f"{len(transitions)} package transition(s). Significant ops executed across each " + f"(the scope selector should be here):\n") + for pa, pb in transitions: + print(f" === pkg {pa['pkg']} ({pa['name']}) -> pkg {pb['pkg']} ({pb['name']}) ===") + if pa["vm"] and pb["vm"]: + dump_span(trace, pa["vm"]["trace_i"], pb["vm"]["trace_i"], dis) + else: + print(" (unaligned — cannot pin the span)") + print() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/frida/README.md b/tools/frida/README.md index 54ed3ac..819b678 100644 --- a/tools/frida/README.md +++ b/tools/frida/README.md @@ -21,15 +21,30 @@ Prereq: `py -3.11 -m pip install frida` (core only — `frida-tools` CLI is not ## Tools -- `capture_graphics.py` — hooks `ReadFile` on the graphics archives (`DATA2/DATA5*.ALF`), resolving - each handle→path via `GetFinalPathNameByHandleW` (cached) + the read offset. Log → - `build/frida-reads.log` (`pathoffsetsize`). +- **`capture_load_order.py`** — ★ the working asset-resolution capture. Hooks `ReadFile` on + `DATA2.ALF`; each asset load starts with header reads **at its exact archive offset**, so exact-start + reads give the clean per-asset **load order** (→ names via `build/asset-index.json`). `--analyze` + prints the order + file_number. This **confirmed `resId == SYS4INI file_number`** (the load order + matches our engine's `set-texture(resId)` order 1:1). Attach by pid, replay the scene, Ctrl-C, `--analyze`. +- `capture_graphics.py` — original `ReadFile` logger for `DATA2/DATA5` (`pathoffsetsize` → + `build/frida-reads.log`); pair with `tools/resolve_frida_reads.py`. +- `locate_resource_load.py` — phase-1 locator (back-trace asset-opens → native load chain). +- `capture_resid_args.py` — phase-2 probe (decoder args/context; showed the loader carries only offsets). +- `find_globals_base.py`, `find_global_by_sequence.py` — **runtime global-variable RE (SHELVED)**. Full + write-up, findings, memory landmarks, and the recommended resume plan: **`docs/global-memory-re.md`**. + TL;DR: the VM global memory is structured/multi-store (not flat int32) and `G[0x62424]` is a transient + arg-register — heuristic value scans can't pin it. Resume from a *stable* anchor (the name global + `0x279`) and target the interpreter's address resolution, not scans. -## Known limitations (see docs/asset-resolution-re.md) +## Runtime architecture (learned 2026-07-06 — read before writing new hooks) -- **File-I/O offsets are noisy** — spans don't match extracted AGF sizes; the game likely - **memory-maps** the archives (so `ReadFile` offsets are OS paging noise, not clean per-asset - loads) and/or uses async/`OVERLAPPED` reads. The robust hook is the game's **internal - load-by-id function** (find via the opcode dispatch), not file I/O — a future tool. -- Correlation still needs the **`SYS4INI` (S4IC422) asset index** parsed to turn an archive offset - into a filename. That parser is the first foundational RE step. +- **Attach, don't spawn**, and **use the pid** (`frida.get_local_device().enumerate_processes()` — the + module-level `frida.enumerate_processes()` was removed in frida 17.x). The process is `AGE.EXE`. +- **The game is packed.** Its main VM logic runs from a per-run heap `r-x` region (~30 MB, nonstable + base). So you **cannot** hook the `set-texture`/VM handlers at a fixed `AGE.EXE+off` — only stable + library code (e.g. the AGF decoder `AGE.EXE+0x74f1f`) keeps a fixed offset. +- **Archives are NOT memory-mapped.** No archive-sized region exists; the game streams them through a + small heap **block-cache via `ReadFile`** (128 KB blocks + big reads). The earlier "memory-mapped" + note was wrong — the 128 KB reads are the block cache, not OS paging. +- **The reliable oracle is the `ReadFile` offset stream** → names via the SYS4INI index + (`tools/parse_sys4ini.py` → `build/asset-index.json`). Native-handler hooking is blocked by the packer. diff --git a/tools/frida/capture_load_order.py b/tools/frida/capture_load_order.py new file mode 100644 index 0000000..5bd3342 --- /dev/null +++ b/tools/frida/capture_load_order.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""Recover the game's per-asset LOAD ORDER across ALL archives (docs/asset-resolution-re.md). + +Assets are typed by name prefix and split across archives: DATA1 = CS/CP/CB/CA/BG/... (ADV +sprites, map sprites, battle & icon portraits, backgrounds), DATA2 = EV/EVM event CGs, +DATA5 = movies. Resolution is per-(archive, type): a set-texture(resId, slot) picks a type via +the slot, and resId indexes within that type. So we must watch EVERY DATA*.ALF, not just DATA2. + +Hooks ReadFile on all DATA*.ALF; each asset load begins with header reads at its exact archive +offset, so exact-start reads give the ordered load list -> resolved to (archive, name, prefix, +file_number) via build/asset-index.json. Feed the result to tools/correlate_scope.py to align +with the VM's set-texture(resId) trace and learn the type/scope rule. + +Flow: game running -> attach -> replay a scene -> Ctrl-C -> --analyze. +Output: build/frida-load-order.jsonl, build/frida-load-order-result.json. +""" +import json +import re +import sys +import time +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] +OUT = REPO / "build" / "frida-load-order.jsonl" +INDEX = REPO / "build" / "asset-index.json" + + +def prefix(name): + m = re.match(r"([A-Za-z]+)", name) + return m.group(1) if m else "?" + + +def index_by_archive(): + """{archive: {offset: (name, file_number, prefix)}} and sorted offset lists for containment.""" + idx = json.loads(INDEX.read_text(encoding="utf-8")) + exact, arr = {}, {} + for f in idx["files"]: + a = f["archive"] + exact.setdefault(a, {})[f["offset"]] = (f["name"], f["file_number"], prefix(f["name"])) + arr.setdefault(a, []).append((f["offset"], f["size"], f["name"], f["file_number"])) + for a in arr: + arr[a].sort() + return exact, arr + + +JS = r""" +const k32 = Process.getModuleByName('kernel32.dll'); +const GetFinalPathNameByHandleW = new NativeFunction( + k32.findExportByName('GetFinalPathNameByHandleW'), 'uint32', ['pointer','pointer','uint32','uint32']); +const SetFilePointer = new NativeFunction( + k32.findExportByName('SetFilePointer'), 'uint32', ['pointer','int32','pointer','uint32']); +const NUL = ptr(0); const cache = {}; +function pathOf(h){ const k=h.toString(); let v=cache[k]; if(v!==undefined) return v; let p=null; + try{ const b=Memory.alloc(1040); const n=GetFinalPathNameByHandleW(h,b,519,0); + if(n>0&&n<519) p=b.readUtf16String(); }catch(e){} cache[k]=p; return p; } +Interceptor.attach(k32.findExportByName('ReadFile'), { + onEnter(args){ + const p = pathOf(args[0]); if(!p) return; + const m = /DATA(\d)\.ALF$/i.exec(p); if(!m) return; + const size = args[2].toInt32(); + const ov = args[4]; let off = -1; + try{ off = ov.isNull()? SetFilePointer(args[0],0,NUL,1) : ov.add(8).readU32(); }catch(e){} + if(off < 0) return; + send({kind:'read', archive:'DATA'+m[1]+'.ALF', offset: off, size: size}); + } +}); +send({ready:true}); +""" + + +def analyze(): + if not OUT.exists(): + raise SystemExit(f"no log: {OUT}") + import bisect + exact, arr = index_by_archive() + recs = [json.loads(l) for l in OUT.read_text(encoding="utf-8").splitlines() if l.strip()] + reads = [r for r in recs if r.get("kind") == "read"] + per_arc = {} + for r in reads: + per_arc[r["archive"]] = per_arc.get(r["archive"], 0) + 1 + print(f"{len(reads)} reads across archives: {per_arc}\n") + + def contain(a, off): + v = arr.get(a) + if not v: + return None + i = bisect.bisect_right(v, (off, float("inf"), "", 0)) - 1 + if i < 0: + return None + o, s, n, fn = v[i] + return (n, fn) if off < o + s else None + + # exact-start order (unambiguous), plus containment first-touch (fuller, noisier) + exact_order, contain_order, seen_e, seen_c = [], [], set(), set() + for r in reads: + a, off = r["archive"], r["offset"] + if off in exact.get(a, {}): + name, fn, pfx = exact[a][off] + if name not in seen_e: + seen_e.add(name); exact_order.append({"archive": a, "name": name, "file_number": fn, "prefix": pfx}) + hit = contain(a, off) + if hit and hit[0] not in seen_c: + seen_c.add(hit[0]) + contain_order.append({"archive": a, "name": hit[0], "file_number": hit[1], "prefix": prefix(hit[0])}) + + print(f"=== exact-start load order ({len(exact_order)}) ===") + for e in exact_order: + print(f" {e['archive'][:5]} {e['prefix']:<4} {e['name']:<14} fn={e['file_number']} (resId 0x{e['file_number']:x})") + print(f"\n=== containment first-touch ({len(contain_order)}, includes prefetch noise) ===") + for e in contain_order: + print(f" {e['archive'][:5]} {e['prefix']:<4} {e['name']:<14} fn={e['file_number']}") + + res = {"load_order": exact_order, "containment_order": contain_order, + "reads_per_archive": per_arc} + (REPO / "build" / "frida-load-order-result.json").write_text( + json.dumps(res, ensure_ascii=False, indent=1), encoding="utf-8") + print("\n-> build/frida-load-order-result.json") + return 0 + + +def capture(proc): + import frida + OUT.parent.mkdir(parents=True, exist_ok=True) + log = open(OUT, "w", encoding="utf-8") + exact, _ = index_by_archive() + + def on_message(msg, data): + if msg.get("type") == "error": + print("[frida-error]", msg.get("description")); return + if msg.get("type") != "send": + return + pl = msg["payload"] + if pl.get("ready"): + print("[frida] ReadFile hook live on ALL DATA*.ALF — replay the scene now."); return + log.write(json.dumps(pl, ensure_ascii=False) + "\n"); log.flush() + if pl.get("kind") == "read": + hit = exact.get(pl["archive"], {}).get(pl["offset"]) + if hit: + print(f"LOAD {pl['archive'][:5]} {hit[2]:<4} {hit[0]} fn={hit[1]}") + + target = int(proc) if str(proc).isdigit() else proc + try: + session = frida.attach(target) + except frida.ProcessNotFoundError: + procs = frida.get_local_device().enumerate_processes() + print("AGE-like:", [(p.pid, p.name) for p in procs if "age" in p.name.lower()]) + return 2 + script = session.create_script(JS) + script.on("message", on_message) + script.load() + print(f"[frida] attached to {proc}; logging all-archive reads -> {OUT}") + try: + while True: + time.sleep(0.5) + except KeyboardInterrupt: + print("\n[frida] stopped. Now: py -3.11 -X utf8 tools/frida/capture_load_order.py --analyze") + return 0 + + +def main(): + if "--analyze" in sys.argv: + return analyze() + proc = next((a for a in sys.argv[1:] if not a.startswith("-")), "AGE.EXE") + return capture(proc) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/frida/capture_resid_args.py b/tools/frida/capture_resid_args.py new file mode 100644 index 0000000..1466b71 --- /dev/null +++ b/tools/frida/capture_resid_args.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +"""Phase 2 of the native resId->filename crack (see docs/asset-resolution-re.md step 2a). + +Phase 1 (locate_resource_load.py) found the stable asset-load call chain in AGE.EXE: + AGE.EXE+0x16d5d7 -> AGE.EXE+0x74f1f -> AGE.EXE+0x397b -> ReadFile +This hooks the two upper frames and, at entry, dumps their arguments (raw dwords + any +ASCII a pointer argument targets), tagged with thread id. It also keeps the ReadFile -> +offset -> asset-name resolver. Interleaving the two streams lets us pair each load call +with the asset it produced and find which argument carries the resId (== 37 for EV052CA, +39 EV052DA, 43 EV052DC, 46 EV052DB) or a pointer to the SYS4INI entry (name / file_number). +Once identified, that argument IS the resId->name mapping at the source. Read-only. + +Flow: game running -> this attaches -> replay the opening CGs -> Ctrl-C -> --analyze. +Output: build/frida-resid-args.jsonl (ordered events). +""" +import json +import sys +import time +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] +OUT = REPO / "build" / "frida-resid-args.jsonl" +INDEX = REPO / "build" / "asset-index.json" +HANDLER_OFFSETS = [0x74f1f] # phase 1 chain; 0x16d5d7 is a return addr (not a callable entry) + + +def data2_offsets(): + idx = json.loads(INDEX.read_text(encoding="utf-8")) + return {f["offset"]: (f["name"], f["file_number"]) + for f in idx["files"] if f["archive"] == "DATA2.ALF"} + + +JS_TEMPLATE = r""" +const OFFSETS = new Set(__OFFSETS__); +const HANDLERS = __HANDLERS__; +const PS = Process.pointerSize; +const age = Process.getModuleByName('AGE.EXE'); + +// --- load-handler arg dumps --- +function scanAscii(base, len) { // printable ASCII runs (>=4) as "hexoff:text" + const out = []; + try { + const u = new Uint8Array(base.readByteArray(len)); + let start = -1; + for (let i = 0; i <= u.length; i++) { + const c = i < u.length ? u[i] : 0; + if (c >= 0x20 && c < 0x7f) { if (start < 0) start = i; } + else { if (start >= 0 && i - start >= 4) + out.push(start.toString(16) + ':' + String.fromCharCode.apply(null, u.subarray(start, i))); + start = -1; } + } + } catch (e) {} + return out; +} +HANDLERS.forEach(function(off) { + const addr = age.base.add(off); + Interceptor.attach(addr, { + onEnter(args) { + const sp = this.context.sp, ebp = this.context.ebp; + const raw = [], cargs = []; + for (let i = 1; i <= 8; i++) { // [sp+PS*i] = this fn's arg i + try { raw.push(sp.add(PS * i).readPointer().toString()); } catch (e) { raw.push('0x0'); } + } + for (let i = 2; i <= 9; i++) { // [ebp+PS*i] = CALLER's args (ebp is caller's at entry) + try { cargs.push(ebp.add(PS * i).readPointer().toString()); } catch (e) { cargs.push('0x0'); } + } + // scan the loader context object (arg1) and any pointer arg's target for ASCII (filenames?) + const ctx = scanAscii(ptr(raw[0]), 0x400); + const argStr = {}; + raw.concat(cargs).forEach(function(v, k) { + const s = scanAscii(ptr(v), 0x48); + if (s.length) argStr[k] = s; + }); + send({kind: 'call', off: '0x' + off.toString(16), tid: this.threadId, + ret: '0x' + this.returnAddress.sub(age.base).toString(16), + args: raw, cargs: cargs, ctx: ctx, argStr: argStr}); + } + }); +}); + +// --- ReadFile -> asset-start resolver (ground-truth name per load) --- +const k32 = Process.getModuleByName('kernel32.dll'); +const GetFinalPathNameByHandleW = new NativeFunction( + k32.findExportByName('GetFinalPathNameByHandleW'), 'uint32', ['pointer','pointer','uint32','uint32']); +const SetFilePointer = new NativeFunction( + k32.findExportByName('SetFilePointer'), 'uint32', ['pointer','int32','pointer','uint32']); +const NUL = ptr(0); const cache = {}; +function pathOf(h) { const key = h.toString(); let v = cache[key]; if (v !== undefined) return v; + let p = null; try { const buf = Memory.alloc(1040); + const n = GetFinalPathNameByHandleW(h, buf, 519, 0); + if (n > 0 && n < 519) p = buf.readUtf16String(); } catch (e) {} cache[key] = p; return p; } +Interceptor.attach(k32.findExportByName('ReadFile'), { + onEnter(args) { + const p = pathOf(args[0]); if (!p || !/data2\.alf$/i.test(p)) return; + const size = args[2].toInt32(); if (size > 4096) return; + const ov = args[4]; let off = -1; + try { off = ov.isNull() ? SetFilePointer(args[0], 0, NUL, 1) : ov.add(8).readU32(); } catch (e) {} + if (!OFFSETS.has(off)) return; + send({kind: 'read', tid: this.threadId, offset: off}); + } +}); +send({ready: true}); +""" + + +def analyze(): + if not OUT.exists(): + raise SystemExit(f"no log: {OUT}") + import re + recs = [json.loads(l) for l in OUT.read_text(encoding="utf-8").splitlines() if l.strip()] + calls = [r for r in recs if r["kind"] == "call"] + reads = [r for r in recs if r["kind"] == "read"] + print(f"{len(calls)} decode calls, {len(reads)} reads\n") + + # every ASCII string the hook surfaced (ctx object + pointer-arg targets) + def strings_of(c): + out = list(c.get("ctx", [])) + for v in c.get("argStr", {}).values(): + out.extend(v) + return [s.split(":", 1)[1] for s in out] + + seen = {} + for c in calls: + for s in strings_of(c): + seen[s] = seen.get(s, 0) + 1 + evlike = {s: n for s, n in seen.items() if re.search(r"EVM?\d|\.AGF|AGF", s, re.I)} + print(f"distinct ASCII strings surfaced: {len(seen)}") + print(f"asset-name-like strings ({len(evlike)}):") + for s, n in sorted(evlike.items(), key=lambda kv: -kv[1])[:60]: + print(f" x{n:<4} {s!r}") + if not evlike: + print(" (none — filename not in the context object; try the caller frame / go one level up)") + print("\n top non-EV strings (for orientation):") + for s, n in sorted(seen.items(), key=lambda kv: -kv[1])[:20]: + print(f" x{n:<4} {s!r}") + return 0 + + +def capture(proc): + import frida + offs = data2_offsets() + js = (JS_TEMPLATE + .replace("__OFFSETS__", json.dumps(sorted(offs.keys()))) + .replace("__HANDLERS__", json.dumps(HANDLER_OFFSETS))) + OUT.parent.mkdir(parents=True, exist_ok=True) + log = open(OUT, "w", encoding="utf-8") + + def on_message(msg, data): + if msg.get("type") == "error": + print("[frida-error]", msg.get("description")); return + if msg.get("type") != "send": + return + pl = msg["payload"] + if pl.get("ready"): + print("[frida] resId-arg hooks live — replay the opening CGs now."); return + log.write(json.dumps(pl, ensure_ascii=False) + "\n"); log.flush() + if pl["kind"] == "read": + name, fn = offs.get(pl["offset"], ("?", -1)) + print(f"READ {name} (fn={fn})") + else: + import re + names = [s.split(":", 1)[1] for s in pl.get("ctx", []) + if re.search(r"EVM?\d|\.AGF", s.split(":", 1)[1], re.I)] + if names: + print(f" call ret=AGE.EXE+{pl['ret']} names={names}") + + target = int(proc) if proc.isdigit() else proc + try: + session = frida.attach(target) + except frida.ProcessNotFoundError: + procs = frida.get_local_device().enumerate_processes() + print(f"'{proc}' not found. AGE-like:", [(p.pid, p.name) for p in procs if "age" in p.name.lower()]) + return 2 + script = session.create_script(js) + script.on("message", on_message) + script.load() + print(f"[frida] attached to {proc}; hooks at AGE.EXE+{[hex(h) for h in HANDLER_OFFSETS]}; log -> {OUT}") + try: + while True: + time.sleep(0.5) + except KeyboardInterrupt: + print("\n[frida] stopped. Now: py -3.11 -X utf8 tools/frida/capture_resid_args.py --analyze") + return 0 + + +def main(): + if "--analyze" in sys.argv: + return analyze() + proc = next((a for a in sys.argv[1:] if not a.startswith("-")), "AGE.EXE") + return capture(proc) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/frida/find_global_by_sequence.py b/tools/frida/find_global_by_sequence.py new file mode 100644 index 0000000..2d82b7a --- /dev/null +++ b/tools/frida/find_global_by_sequence.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""Locate the VM int-global G[0x62424] (the CG resId) in memory by a differential value scan -- +the confirmed-anchor bootstrap for runtime global observation (docs/asset-resolution-re.md step 2). + +G[0x62424] is unambiguously a VM global-int (`mov (global-int 0x62424) ...`), and because +resId == SYS4INI file_number, each DATA2 asset-start ReadFile tells us the EXACT value it holds +at that instant (the asset's file_number). So we self-drive a Cheat-Engine-style scan: on the +first CG load, scan the heap for int32 == fn; on each later load, keep only candidates that now +equal the new fn. The monotonic distinct opening sequence (35,37,39,43,46,122,129,...) collapses +the set to G[0x62424] in a few steps -- no native-copy contamination, no manual timing. + +Once found, its address anchors the VM int-global store; from there we derive the address->memory +mapping and read any global live. + +Flow: title screen -> attach -> start new game -> advance the opening SLOWLY (one CG at a time). +""" +import json +import sys +import time +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] +INDEX = REPO / "build" / "asset-index.json" + + +BG_MIN_SIZE = 500_000 # backgrounds are big AGFs (~1MB); portraits/sprites are far smaller + + +def data2_off2fn(): + """{offset: file_number} for BACKGROUND-sized DATA2 assets only (excludes sprite/portrait + loads, which churn G[0x62424]/other resId globals between background changes).""" + idx = json.loads(INDEX.read_text(encoding="utf-8")) + return {f["offset"]: f["file_number"] for f in idx["files"] + if f["archive"] == "DATA2.ALF" and f["size"] >= BG_MIN_SIZE} + + +JS_TEMPLATE = r""" +const OFF2FN = __OFF2FN__; // {offset: file_number == resId} +const k32 = Process.getModuleByName('kernel32.dll'); +const GetFinalPathNameByHandleW = new NativeFunction( + k32.findExportByName('GetFinalPathNameByHandleW'), 'uint32', ['pointer','pointer','uint32','uint32']); +const SetFilePointer = new NativeFunction( + k32.findExportByName('SetFilePointer'), 'uint32', ['pointer','int32','pointer','uint32']); +const NUL = ptr(0); const cache = {}; +function pathOf(h){ const k=h.toString(); let v=cache[k]; if(v!==undefined) return v; let p=null; + try{ const b=Memory.alloc(1040); const n=GetFinalPathNameByHandleW(h,b,519,0); + if(n>0&&n<519) p=b.readUtf16String(); }catch(e){} cache[k]=p; return p; } + +function u32le(v){const b=[v&0xff,(v>>>8)&0xff,(v>>>16)&0xff,(v>>>24)&0xff]; + return b.map(x=>('0'+x.toString(16)).slice(-2)).join(' ');} + +const CAP = 600000; +function scanValue(v){ + const out=[]; const pat=u32le(v); + const ranges=Process.enumerateRanges('rw-'); + for(const r of ranges){ + let m; try{ m=Memory.scanSync(r.base, r.size, pat); }catch(e){ continue; } + for(const x of m){ out.push(x.address); if(out.length>=CAP) return out; } + } + return out; +} + +function isStack(a){ // heuristic: self-referential / return-addr neighbourhood + const r=Process.findRangeByAddress(a); + return r && r.size < 0x200000; // small rw- region = likely a stack +} +let cands=null, lastOff=-1, step=0; +function keepEq(v){ cands=cands.filter(a=>{ try{ return a.readU32()===(v>>>0); }catch(e){ return false; } }); } + +Interceptor.attach(k32.findExportByName('ReadFile'), { + onEnter(args){ + const p=pathOf(args[0]); if(!p || !/data2\.alf$/i.test(p)) return; + const size=args[2].toInt32(); if(size>4096) return; + const ov=args[4]; let off=-1; + try{ off = ov.isNull()? SetFilePointer(args[0],0,NUL,1) : ov.add(8).readU32(); }catch(e){} + if(!(off in OFF2FN)) return; + if(off===lastOff) return; lastOff=off; + const fn=OFF2FN[off]; + if(cands===null){ cands=scanValue(fn); } + else { keepEq(fn); } + step++; + send({step:step, phase:'load', fn:fn, count:cands.length}); + // STABILITY FILTER: 900ms later (during the pause before the next click) the global still + // holds fn, but transient stack copies have been overwritten -> drop them. + setTimeout(function(){ + if(cands===null) return; + keepEq(fn); + send({step:step, phase:'stable', fn:fn, count:cands.length, + addrs: cands.length<=12 ? cands.map(a=>({a:a.toString(), stack:isStack(a)})) : []}); + }, 1400); + } +}); +send({ready:true}); +""" + + +def main(): + import frida + args = [a for a in sys.argv[1:] if not a.startswith("-")] + proc = args[0] if args else "AGE.EXE" + js = JS_TEMPLATE.replace("__OFF2FN__", json.dumps(data2_off2fn())) + + def on_message(msg, data): + if msg.get("type") == "error": + print("[frida-error]", msg.get("description")); return + if msg.get("type") != "send": + return + pl = msg["payload"] + if pl.get("ready"): + print("[frida] scan hook live — advance the opening one CG at a time, pausing ~1.5s each.") + return + ph = pl.get("phase") + print(f"[step {pl['step']} {ph:>6}] resId={pl['fn']} (0x{pl['fn']:x}) candidates={pl['count']}") + if ph == "stable" and pl.get("addrs"): + for e in pl["addrs"]: + print(f" {e['a']} {'(stack)' if e['stack'] else '<== STABLE global candidate'}") + stable = [e for e in pl["addrs"] if not e["stack"]] + if 0 < len(stable) <= 3: + print(" >>> stable non-stack survivors — likely G[0x62424].") + + dev = frida.get_local_device() + target = int(proc) if str(proc).isdigit() else proc + try: + session = frida.attach(target) + except frida.ProcessNotFoundError: + print("AGE-like:", [(p.pid, p.name) for p in dev.enumerate_processes() if "age" in p.name.lower()]) + return 2 + script = session.create_script(js) + script.on("message", on_message) + script.load() + print(f"[frida] attached to {proc}; narrowing G[0x62424] by the resId sequence.") + try: + while True: + time.sleep(0.5) + except KeyboardInterrupt: + pass + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/frida/find_globals_base.py b/tools/frida/find_globals_base.py new file mode 100644 index 0000000..50933e0 --- /dev/null +++ b/tools/frida/find_globals_base.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +"""Locate the game's INT-GLOBAL array in memory by a known-value signature scan, so we can +read VM globals live (e.g. G[0x62424] = the CG resId) -- see docs/asset-resolution-re.md step 2. + +The `*INIT` scripts write thousands of known constants to known global-int addresses at boot: +`mov (global-int ADDR) IMM`. Our VM addresses globals as a flat int array, so at runtime the +game holds `int_globals[ADDR]` at `B + ADDR*4` for some base B. We build a signature of those +(ADDR, value) pairs, find a long CONTIGUOUS run as a rare multi-dword anchor, `Memory.scan` for +it, and verify each candidate B against many scattered pairs. Unique high-match B = the array. + +This unlocks runtime global observation generally (the roadmap's VM-validation cornerstone): +read `resId` (G[0x62424]) directly at each CG load, and watch scope-selector globals. + + build: py -3.11 -X utf8 tools/frida/find_globals_base.py --build-sig + scan: py -3.11 -u -X utf8 tools/frida/find_globals_base.py +""" +import collections +import json +import sys +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] +SIG = REPO / "build" / "globals-signature.json" +sys.path.insert(0, str(REPO / "tools")) + +MOV, T_GINT, T_IMM = 0x55, 3, 0 +INIT_SCRIPTS = ["EBINIT", "ITINIT", "SKINIT", "CGINIT"] + + +def build_signature(): + import paths, sys4load + pairs = collections.defaultdict(collections.Counter) + for name in INIT_SCRIPTS: + p = paths.GAME_DIR / f"{name}.BIN" + if not p.exists(): + p = paths.DATA1 / f"{name}.BIN" + if not p.exists(): + continue + for ins in sys4load.load(p).instructions: + if (ins.opcode == MOV and len(ins.args) >= 2 + and ins.args[0][0] == T_GINT and ins.args[1][0] == T_IMM): + pairs[ins.args[0][1]][ins.args[1][1]] += 1 + # stable, distinctive, single-write addresses + single = {a: next(iter(vc)) for a, vc in pairs.items() + if len(vc) == 1 and vc.most_common(1)[0][1] == 1 + and 8 < next(iter(vc)) < 0x7fffffff} + + # longest contiguous run (addr, addr+1, ...) -> rare multi-dword anchor + addrs = sorted(single) + best = (None, 0) + i = 0 + while i < len(addrs): + j = i + while j + 1 < len(addrs) and addrs[j + 1] == addrs[j] + 1: + j += 1 + if j - i + 1 > best[1]: + best = (addrs[i], j - i + 1) + i = j + 1 + anchor_addr, anchor_len = best + anchor = [(anchor_addr + k, single[anchor_addr + k]) for k in range(anchor_len)] + # scattered verification pairs spread across the address range + spread = addrs[::max(1, len(addrs) // 120)][:120] + verify = [(a, single[a]) for a in spread] + + SIG.parent.mkdir(parents=True, exist_ok=True) + SIG.write_text(json.dumps({"anchor_addr": anchor_addr, "anchor": anchor, "verify": verify}, + ensure_ascii=False), encoding="utf-8") + print(f"signature: {len(single)} distinctive pairs; " + f"anchor run @0x{anchor_addr:x} len {anchor_len} ({anchor_len*4} bytes); " + f"{len(verify)} verify pairs -> {SIG.relative_to(REPO)}") + print("anchor values:", [v for _, v in anchor[:12]]) + return 0 + + +JS_TEMPLATE = r""" +const ANCHOR_ADDR = __ANCHOR_ADDR__; +const ANCHOR_VALS = __ANCHOR_VALS__; // consecutive int32 values at ANCHOR_ADDR.. +const VERIFY = __VERIFY__; // [[addr,val],...] + +// build the anchor byte pattern (little-endian int32 each) +function u32le(v){ const b=[v&0xff,(v>>>8)&0xff,(v>>>16)&0xff,(v>>>24)&0xff]; + return b.map(x=>('0'+x.toString(16)).slice(-2)).join(' '); } +const pattern = ANCHOR_VALS.map(u32le).join(' '); + +function verifyBase(B){ + let ok=0, tot=0; + for(const pv of VERIFY){ + tot++; + try { if(B.add(pv[0]*4).readU32() === (pv[1]>>>0)) ok++; } catch(e){} + } + return {ok:ok, tot:tot}; +} + +const ranges = Process.enumerateRanges('rw-').filter(r=>r.size >= 1024*1024); +let best=null; +// robustness ladder: long anchor is rare but fragile to any changed value; short prefixes +// catch it if a value moved. Each candidate is confirmed by the 120 scattered verify pairs. +const LENS = [ANCHOR_VALS.length, 64, 16, 4].filter((v,i,a)=>v<=ANCHOR_VALS.length && a.indexOf(v)===i); +for(const L of LENS){ + const pat = ANCHOR_VALS.slice(0,L).map(u32le).join(' '); + let hits=0; + ranges.forEach(function(r){ + let matches; try { matches = Memory.scanSync(r.base, r.size, pat); } catch(e){ return; } + if(matches.length > 8000) return; // too common at this length, skip + hits += matches.length; + matches.forEach(function(m){ + const B = m.address.sub(ANCHOR_ADDR*4); + const v = verifyBase(B); + if(v.ok >= 30 && (!best || v.ok>best.ok)) + best = {base: B.toString(), ok:v.ok, tot:v.tot, anchor_at: m.address.toString(), anchor_len:L}; + }); + }); + send({phase:'scan', anchor_len:L, hits:hits, found: !!best}); + if(best) break; +} +if(best){ + // read G[0x62424] (the CG resId) as a sanity value + let resid=null; try{ resid = ptr(best.base).add(0x62424*4).readU32(); }catch(e){} + best.G_62424 = resid; + send({phase:'found', best:best}); +} else { + send({phase:'notfound'}); +} +""" + + +def scan(pid): + import frida + if not SIG.exists(): + raise SystemExit("no signature; run --build-sig first") + sig = json.loads(SIG.read_text(encoding="utf-8")) + js = (JS_TEMPLATE + .replace("__ANCHOR_ADDR__", str(sig["anchor_addr"])) + .replace("__ANCHOR_VALS__", json.dumps([v for _, v in sig["anchor"]])) + .replace("__VERIFY__", json.dumps(sig["verify"]))) + out = {} + + def on_message(msg, data): + if msg.get("type") == "error": + print("[frida-error]", msg.get("description")); return + if msg.get("type") != "send": + return + pl = msg["payload"] + ph = pl.get("phase") + if ph == "scan": + print(f"[scan] anchor prefix {pl['anchor_len']} dwords: {pl['hits']} raw hits" + + (" -> base confirmed" if pl.get("found") else "")) + elif ph == "found": + b = pl["best"] + out["base"] = b["base"] + print(f"[FOUND] int-global base = {b['base']} (verify {b['ok']}/{b['tot']} pairs, " + f"anchor @ {b['anchor_at']})") + print(f" G[0x62424] (CG resId right now) = {b['G_62424']} (0x{b['G_62424']:x})" + if b.get("G_62424") is not None else " G[0x62424] unreadable") + elif ph == "notfound": + print("[scan] anchor pattern not found in any heap range (globals not resident, " + "wrong element size, or array not yet populated)") + + dev = frida.get_local_device() + target = int(pid) if str(pid).isdigit() else pid + try: + session = frida.attach(target) + except frida.ProcessNotFoundError: + print("AGE-like:", [(p.pid, p.name) for p in dev.enumerate_processes() if "age" in p.name.lower()]) + return 2 + script = session.create_script(js) + script.on("message", on_message) + script.load() + if "base" in out: + print(f"\nint-global base found: {out['base']}. Next: hook the loader and read " + f"G[0x62424] there for definitive (resId,name) pairs.") + return 0 + + +def main(): + if "--build-sig" in sys.argv: + return build_signature() + pid = next((a for a in sys.argv[1:] if not a.startswith("-")), "AGE.EXE") + return scan(pid) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/frida/locate_resource_load.py b/tools/frida/locate_resource_load.py new file mode 100644 index 0000000..a0b5df1 --- /dev/null +++ b/tools/frida/locate_resource_load.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +"""Phase 1 of the native resId->filename crack: LOCATE the resource-load / set-texture +handler in the running game by back-tracing every asset-open (see docs/asset-resolution-re.md +step 2, option a). + +Idea: to issue a ReadFile at an asset's exact archive offset the game must have *already* +resolved resId -> (archive, offset) inside its native load-by-id / set-texture (op 0x1f9) +handler. So at each asset-start read, a stack backtrace passes straight through that handler. +We backtrace only on reads whose offset EXACTLY equals a DATA2 asset offset (from +build/asset-index.json), resolve the offset -> asset name, and log module-relative frames. +Frames are AGE.EXE-relative (base subtracted) so they're stable across runs despite ASLR. + +Offline, `aggregate` ranks the AGE.EXE return addresses that recur across the MOST distinct +assets: the load-by-id / set-texture handler is the frame common to every asset-open. That +address (module+offset) becomes the hook target for phase 2 (read the resId argument -> +definitive resId->name table). No game state is modified. + +Flow (see tools/frida/README.md): + 1. Launch the game (via `AGE Patch.exe`) to the title. + 2. py -3.11 -u -X utf8 tools/frida/locate_resource_load.py + 3. Start a new game so the SC0000 opening auto-plays; let a dozen CGs load. + 4. Ctrl-C. Then: py -3.11 -X utf8 tools/frida/locate_resource_load.py --aggregate + +Output: build/frida-resource-bt.jsonl ({name, offset, size, frames:[...]} per asset-open). +""" +import json +import sys +import time +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] # age-reimpl/ +OUT = REPO / "build" / "frida-resource-bt.jsonl" +INDEX = REPO / "build" / "asset-index.json" + + +def load_data2_offsets(): + """{offset -> name} for DATA2.ALF from the asset index (asset-start detector + resolver).""" + idx = json.loads(INDEX.read_text(encoding="utf-8")) + return {f["offset"]: f["name"] for f in idx["files"] if f["archive"] == "DATA2.ALF"} + + +def aggregate(): + """Rank AGE.EXE-relative frames by how many DISTINCT assets they appear under.""" + if not OUT.exists(): + raise SystemExit(f"no capture log: {OUT} (run the capture first)") + from collections import defaultdict + assets_per_frame = defaultdict(set) # frame-string -> set(asset names) + depth_of_frame = defaultdict(list) # frame-string -> [stack depths] + n = 0 + for ln in OUT.read_text(encoding="utf-8").splitlines(): + if not ln.strip(): + continue + rec = json.loads(ln) + n += 1 + for depth, fr in enumerate(rec.get("frames", [])): + if fr.startswith("AGE.EXE+"): # ignore kernel32/ntdll/CRT frames + assets_per_frame[fr].add(rec["name"]) + depth_of_frame[fr].append(depth) + distinct_assets = {rec["name"] for rec in + (json.loads(l) for l in OUT.read_text(encoding="utf-8").splitlines() if l.strip())} + print(f"{n} asset-open events over {len(distinct_assets)} distinct assets") + print("AGE.EXE frames ranked by distinct-asset coverage (handler = covers ~all):") + ranked = sorted(assets_per_frame.items(), key=lambda kv: (-len(kv[1]), fr_depth(depth_of_frame[kv[0]]))) + for fr, assets in ranked[:25]: + d = depth_of_frame[fr] + print(f" {fr:<22} assets={len(assets):<3} avg_depth={sum(d)/len(d):.1f}") + print("\nThe handler is the frame covering the most distinct assets at a shallow, stable depth.") + print("Hook it in phase 2 to read the resId argument.") + return 0 + + +def fr_depth(depths): + return sum(depths) / len(depths) + + +JS_TEMPLATE = r""" +const OFFSETS = new Set(__OFFSETS__); // exact DATA2 asset-start offsets +const k32 = Process.getModuleByName('kernel32.dll'); +const GetFinalPathNameByHandleW = new NativeFunction( + k32.findExportByName('GetFinalPathNameByHandleW'), 'uint32', ['pointer','pointer','uint32','uint32']); +const SetFilePointer = new NativeFunction( + k32.findExportByName('SetFilePointer'), 'uint32', ['pointer','int32','pointer','uint32']); +const NUL = ptr(0); +const cache = {}; +function pathOf(h) { + const key = h.toString(); + let v = cache[key]; if (v !== undefined) return v; + let p = null; + try { const buf = Memory.alloc(1040); + const n = GetFinalPathNameByHandleW(h, buf, 519, 0); + if (n > 0 && n < 519) p = buf.readUtf16String(); } catch (e) {} + cache[key] = p; return p; +} +function frameStr(addr) { + const m = Process.findModuleByAddress(addr); + if (!m) return addr.toString(); + return m.name + '+0x' + addr.sub(m.base).toString(16); +} +const rf = k32.findExportByName('ReadFile'); +Interceptor.attach(rf, { + onEnter(args) { + const p = pathOf(args[0]); + if (!p || !/data2\.alf$/i.test(p)) return; + const size = args[2].toInt32(); + if (size > 4096) return; // header reads only (asset-start burst) + const ov = args[4]; + let off = -1; + try { off = ov.isNull() ? SetFilePointer(args[0], 0, NUL, 1) : ov.add(8).readU32(); } catch (e) {} + if (!OFFSETS.has(off)) return; // only exact asset-start offsets + let frames = []; + try { + frames = Thread.backtrace(this.context, Backtracer.ACCURATE).map(frameStr); + } catch (e) { + try { frames = Thread.backtrace(this.context, Backtracer.FUZZY).map(frameStr); } catch (e2) {} + } + send({offset: off, size: size, frames: frames}); + } +}); +send({ready: true}); +""" + + +def capture(proc): + import frida + offsets = load_data2_offsets() + js = JS_TEMPLATE.replace("__OFFSETS__", json.dumps(sorted(offsets.keys()))) + OUT.parent.mkdir(parents=True, exist_ok=True) + log = open(OUT, "w", encoding="utf-8") + seen = set() + + def on_message(msg, data): + if msg.get("type") == "error": + print("[frida-error]", msg.get("description")); return + if msg.get("type") != "send": + return + pl = msg["payload"] + if pl.get("ready"): + print("[frida] backtrace hook live — start a new game; let the opening load CGs.") + return + name = offsets.get(pl["offset"], "?") + rec = {"name": name, "offset": pl["offset"], "size": pl["size"], "frames": pl["frames"]} + log.write(json.dumps(rec, ensure_ascii=False) + "\n"); log.flush() + if name not in seen: + seen.add(name) + age = [f for f in pl["frames"] if f.startswith("AGE.EXE+")] + print(f"OPEN {name} ({len(pl['frames'])} frames, {len(age)} in AGE.EXE)") + for f in pl["frames"][:8]: + print(" " + f) + + target = int(proc) if proc.isdigit() else proc + try: + session = frida.attach(target) + except frida.ProcessNotFoundError: + procs = frida.get_local_device().enumerate_processes() # frida 17.x: device method + print(f"process '{proc}' not found. AGE-like:", + [(p.pid, p.name) for p in procs if "age" in p.name.lower()]) + return 2 + script = session.create_script(js) + script.on("message", on_message) + script.load() + print(f"[frida] attached to {proc}; {len(offsets)} DATA2 asset offsets loaded; log -> {OUT}") + try: + while True: + time.sleep(0.5) + except KeyboardInterrupt: + print("\n[frida] stopped. Now: py -3.11 -X utf8 tools/frida/locate_resource_load.py --aggregate") + return 0 + + +def main(): + if "--aggregate" in sys.argv: + return aggregate() + proc = next((a for a in sys.argv[1:] if not a.startswith("-")), "AGE.EXE") + return capture(proc) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/parse_sys4ini.py b/tools/parse_sys4ini.py new file mode 100644 index 0000000..b9f4ab0 --- /dev/null +++ b/tools/parse_sys4ini.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3 +"""Parse SYS4INI.BIN (Eushully AGE asset index) -> build/asset-index.json. + +SYS4INI.BIN is the authoritative directory the game (and BinExtractALF, which is +based on asmodean's exs4alf) uses to locate every asset inside the DATA*.ALF +archives. It maps name <-> archive <-> offset <-> size and is the reusable +"answer key" for resId->file resolution (see docs/asset-resolution-re.md): it +both names every asset and gives archive-offset->name (to rescue noisy Frida +file-I/O offsets). + +Container format (little-endian x86), signature "S4IC422 " at offset 0: + + 0x000 char signature[?] "S4IC" family; "S4IC" -> data at 0x134 + ... (title / padding) + 0x134 uint32 packed_size length of the LZSS stream that follows + 0x138 byte[] lzss_stream packed_size bytes, runs to EOF + +The LZSS stream (GARbro-style: 0x1000 ring buffer, zero-filled, init pos 0xFEE, +control bits LSB->MSB, 1=literal, 0=two-byte back-reference: offset=(hi&0xf0)<<4 +| lo, length=3+(hi&0xf)) decompresses to a plain directory: + + uint32 arc_count + { char name[256] } x arc_count archive filenames (DATA1.ALF ..) + uint32 file_count + { char name[64]; uint32 arc_id; one record per asset + uint32 file_number; uint32 offset; + uint32 size } x file_count (80 bytes each) + +Entries whose name is "@" are placeholders and skipped (matches GARbro/exs4alf). + +Usage: py -3.11 -X utf8 tools/parse_sys4ini.py [--check] + --check cross-validate against the extracted/ ground truth and .ALF sizes +""" +from __future__ import annotations +import json +import struct +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import paths + +# S4IC family: signature -> offset of the packed-size dword that precedes the stream. +DATA_OFFSETS = {b"S4AC": 0x114, b"S4IC": 0x134, b"S3IC": 0x134, b"S3IN": 0x12C} + +FRAME_SIZE = 0x1000 +FRAME_INIT_POS = 0xFEE +ARC_NAME_LEN = 256 +FILE_NAME_LEN = 64 +FILE_ENTRY_FMT = "<64s4I" # name[64], arc_id, file_number, offset, size +FILE_ENTRY_LEN = struct.calcsize(FILE_ENTRY_FMT) # 80 + + +def lzss_decompress(src: bytes) -> bytes: + """GARbro-compatible LZSS (0x1000 ring buffer, init pos 0xFEE, threshold 3).""" + frame = bytearray(FRAME_SIZE) # zero-filled + fpos = FRAME_INIT_POS + out = bytearray() + i, n = 0, len(src) + while i < n: + ctl = src[i]; i += 1 + for bit in (1, 2, 4, 8, 0x10, 0x20, 0x40, 0x80): + if ctl & bit: # literal + if i >= n: + return bytes(out) + b = src[i]; i += 1 + out.append(b) + frame[fpos] = b; fpos = (fpos + 1) & 0xFFF + else: # back-reference + if i + 1 >= n: + return bytes(out) + lo, hi = src[i], src[i + 1]; i += 2 + off = ((hi & 0xF0) << 4) | lo + for _ in range(3 + (hi & 0x0F)): + b = frame[off & 0xFFF]; off += 1 + out.append(b) + frame[fpos] = b; fpos = (fpos + 1) & 0xFFF + return bytes(out) + + +def _cstr(buf: bytes) -> str: + """Decode a null-terminated cp932 field (filenames are ASCII in practice).""" + return buf.split(b"\x00", 1)[0].decode("cp932", errors="replace") + + +def parse(path: Path) -> dict: + raw = path.read_bytes() + sig4 = raw[:4] + if sig4 not in DATA_OFFSETS: + raise SystemExit(f"{path.name}: unknown signature {raw[:8]!r}") + magic = _cstr(raw[:8]) + doff = DATA_OFFSETS[sig4] + packed_size = struct.unpack_from(" len(blob): + raise SystemExit(f"{path.name}: directory truncated (need {need}, " + f"have {len(blob)} decompressed bytes)") + + files = [] + for _ in range(file_count): + name_b, arc_id, file_number, offset, size = struct.unpack_from(FILE_ENTRY_FMT, blob, p) + p += FILE_ENTRY_LEN + name = _cstr(name_b) + if name == "@" or not name: + continue + files.append({ + "name": name, + "archive": archives[arc_id] if 0 <= arc_id < arc_count else None, + "arc_id": arc_id, + "file_number": file_number, + "offset": offset, + "size": size, + }) + + return { + "source": path.name, + "magic": magic, + "packed_size": packed_size, + "decompressed_size": len(blob), + "archive_count": arc_count, + "archives": archives, + "file_count": file_count, + "entry_count": len(files), + "files": files, + } + + +def check(index: dict) -> int: + """Cross-validate against extracted/ counts and real .ALF file sizes.""" + problems = 0 + per_arc: dict[str, int] = {} + for f in index["files"]: + per_arc[f["archive"]] = per_arc.get(f["archive"], 0) + 1 + + print("per-archive entry counts (from SYS4INI):") + for a in index["archives"]: + print(f" {a:<16} {per_arc.get(a, 0)}") + + # 1) offset+size must fit inside the real archive on disk. + for a in index["archives"]: + alf = paths.GAME_DIR / a + if not alf.exists(): + print(f" ! archive not on disk: {a}") + continue + asize = alf.stat().st_size + over = [f for f in index["files"] + if f["archive"] == a and f["offset"] + f["size"] > asize] + if over: + problems += len(over) + print(f" ! {a}: {len(over)} entries run past EOF ({asize} bytes); " + f"e.g. {over[0]['name']} @ {over[0]['offset']}+{over[0]['size']}") + else: + print(f" ok {a}: all entries within {asize} bytes") + + # 2) spot-check extracted-folder file sizes against the index (name -> size). + by_name = {f["name"].upper(): f for f in index["files"]} + checked = mismatch = 0 + for d in ("DATA1", "DATA2", "DATA3", "DATA4", "DATA5"): + folder = paths.EXTRACTED / d + if not folder.is_dir(): + continue + for fp in list(folder.glob("*"))[:200]: + if not fp.is_file(): + continue + e = by_name.get(fp.name.upper()) + if e is None: + continue + checked += 1 + if e["size"] != fp.stat().st_size: + mismatch += 1 + if mismatch <= 5: + print(f" ! size mismatch {fp.name}: index {e['size']} " + f"vs extracted {fp.stat().st_size}") + print(f"size spot-check: {checked} matched by name, {mismatch} size mismatches") + problems += mismatch + print("CHECK OK" if problems == 0 else f"CHECK: {problems} problems") + return problems + + +def main() -> int: + do_check = "--check" in sys.argv[1:] + src = paths.GAME_DIR / "SYS4INI.BIN" + if not src.exists(): + raise SystemExit(f"not found: {src}") + index = parse(src) + print(f"{index['source']}: magic={index['magic']!r} " + f"packed={index['packed_size']} -> {index['decompressed_size']} bytes") + print(f"archives ({index['archive_count']}): {index['archives']}") + print(f"files: {index['file_count']} declared, {index['entry_count']} real " + f"(after skipping '@' placeholders)") + for f in index["files"][:5]: + print(f" {f['name']:<16} {f['archive']:<12} " + f"off={f['offset']:>12} size={f['size']:>10} #{f['file_number']}") + + out = paths.BUILD / "asset-index.json" + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(index, ensure_ascii=False, indent=1), encoding="utf-8") + print(f"-> {out.relative_to(paths.REPO)}") + + if do_check: + print("--- validation ---") + return 1 if check(index) else 0 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/resolve_asset.py b/tools/resolve_asset.py new file mode 100644 index 0000000..1dc9956 --- /dev/null +++ b/tools/resolve_asset.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Static, general asset resolver: (scene, resId) -> asset file. Solves asset resolution +(docs/asset-resolution-re.md) with NO runtime capture. + +Mechanism (proven): SYS4INI's file list is organized into SECTIONS, one per scene -- each is a +`SCxxxx.BIN` script entry followed by that scene's asset MANIFEST (all assets it references, across +every archive and type: EV/BG/CS/AE event & sprite graphics, OGG/WAV audio, ...). `file_number` is +the 0-based index within the section. So a bytecode resId resolves as: + + resId -> files[ section_base(scene) + resId ] + +where section_base(scene) is the start of the SYS4INI section containing the scene's script. +This is the same rule for set-texture(resId), play-bgm(id), play-voice(id) -- one unified manifest. +Validated: 97% of files fit `fn == position - section_base`; SC0000's opening resolves 17/17 vs +Frida ground truth; 586/595 captured loads across all sections satisfy `files[base+fn] == name`. + +Usage: + py -3.11 -X utf8 tools/resolve_asset.py --build # emit build/asset-sections.json + py -3.11 -X utf8 tools/resolve_asset.py [resId] # resolve one, or dump the manifest +""" +from __future__ import annotations +import json +import re +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import paths + + +def load_index(): + return json.loads((paths.BUILD / "asset-index.json").read_text(encoding="utf-8"))["files"] + + +def sections(files): + """Split the SYS4INI file list into sections at each file_number reset (fn <= prev). + Returns (section_start_per_position[list], sections[list of (start, end, scene_name|None]).""" + base_of, secs, start, prev = [], [], 0, -1 + for i, f in enumerate(files): + if f["file_number"] <= prev: + secs.append((start, i - 1)) + start = i + base_of.append(start) + prev = f["file_number"] + secs.append((start, len(files) - 1)) + # attach the scene script (SCxxxx.BIN) that owns each section, if any + out = [] + for s, e in secs: + scene = next((files[k]["name"] for k in range(s, e + 1) + if re.match(r"SC\d+\.BIN$", files[k]["name"])), None) + out.append({"start": s, "end": e, "scene": scene}) + return base_of, out + + +def scene_base(files, base_of, scene): + key = scene.upper() + if not key.endswith(".BIN"): + key += ".BIN" + pos = next((i for i, f in enumerate(files) if f["name"].upper() == key), None) + if pos is None: + raise SystemExit(f"scene not in SYS4INI: {scene}") + return base_of[pos] + + +def resolve(files, base, resid): + p = base + resid + return files[p] if 0 <= p < len(files) else None + + +def main() -> int: + files = load_index() + base_of, secs = sections(files) + + if "--build" in sys.argv: + scene_bases = {} + for sec in secs: + if sec["scene"]: + scene_bases[sec["scene"].removesuffix(".BIN")] = sec["start"] + out = paths.BUILD / "asset-sections.json" + out.write_text(json.dumps( + {"note": "SYS4INI sections; resolve resId -> files[section_base + resId]. " + "See docs/asset-resolution-re.md.", + "section_count": len(secs), + "scene_base": scene_bases, + "sections": secs}, ensure_ascii=False, indent=1), encoding="utf-8") + print(f"{len(secs)} sections, {len(scene_bases)} scenes -> {out.relative_to(paths.REPO)}") + return 0 + + args = [a for a in sys.argv[1:] if not a.startswith("-")] + if not args: + raise SystemExit(__doc__) + scene = args[0] + base = scene_base(files, base_of, scene) + sec = next(s for s in secs if s["start"] == base) + print(f"{scene}: section [{sec['start']}..{sec['end']}] base {base} " + f"({sec['end']-sec['start']+1} entries)") + + if len(args) > 1: + resid = int(args[1], 0) + f = resolve(files, base, resid) + print(f" resId {resid} -> {f['archive']} {f['name']} (offset {f['offset']}, size {f['size']})" + if f else f" resId {resid} -> out of range") + return 0 + + # dump the scene's manifest (graphics + audio), skipping the leading script entry + print(" manifest (resId -> asset):") + for p in range(base, sec["end"] + 1): + resid = p - base + f = files[p] + print(f" {resid:>4} {f['archive'][:5]} {f['name']}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/resolve_frida_reads.py b/tools/resolve_frida_reads.py new file mode 100644 index 0000000..d05bed1 --- /dev/null +++ b/tools/resolve_frida_reads.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""Resolve Frida archive-read offsets -> asset names via build/asset-index.json. + +The runtime capture (tools/frida/capture_graphics.py) logs raw ReadFile spans on the +DATA*.ALF archives as `\t\t` lines. On their own those offsets are +opaque and mixed with OS memory-map paging (the doc's "Frida file-I/O is noisy"). With +the SYS4INI asset index as the answer key we can turn each offset back into the *asset* +it belongs to, and thereby recover the real per-scene **asset load order** — the ground +truth for cracking resId->filename (see docs/asset-resolution-re.md, step 2). + +Two read signals per asset (observed): a burst of tiny header reads starting exactly at +the asset's archive offset (delta 0), then one bulk read of the payload. Uniform 0x20000 +(131072-byte) reads are memory-map paging and are dropped. We treat a read whose offset +*exactly* equals an index entry's offset as an unambiguous "asset-start" event; the +ordered, de-duplicated sequence of those is the load order. + +Usage: py -3.11 -X utf8 tools/resolve_frida_reads.py [reads.log] [-o out.json] + default reads.log = build/frida-reads.log ; default out = build/frida-asset-loads.json +""" +from __future__ import annotations +import bisect +import json +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import paths + +PAGING_SIZE = 131072 # 0x20000 uniform memory-map paging reads -> noise + + +def load_index() -> dict: + idx = json.loads((paths.BUILD / "asset-index.json").read_text(encoding="utf-8")) + by_arc: dict[str, list[tuple[int, int, str]]] = {} + for f in idx["files"]: + by_arc.setdefault(f["archive"], []).append((f["offset"], f["size"], f["name"])) + for a in by_arc: + by_arc[a].sort() + return by_arc + + +def resolve(by_arc, arc, off): + """Return (name, entry_offset, size, delta) for the asset whose range holds `off`.""" + arr = by_arc.get(arc) + if not arr: + return None + i = bisect.bisect_right(arr, (off, float("inf"), "")) - 1 + if i < 0: + return None + o, s, n = arr[i] + return (n, o, s, off - o) if off < o + s else None + + +def main() -> int: + args = [a for a in sys.argv[1:] if not a.startswith("-")] + out_flag = next((sys.argv[i + 1] for i, a in enumerate(sys.argv) if a == "-o"), None) + log = Path(args[0]) if args else paths.BUILD / "frida-reads.log" + out = Path(out_flag) if out_flag else paths.BUILD / "frida-asset-loads.json" + if not log.exists(): + raise SystemExit(f"reads log not found: {log}") + + by_arc = load_index() + # exact-offset -> name per archive (asset-start detector) + exact = {a: {o: n for o, _, n in v} for a, v in by_arc.items()} + + per_arc: dict[str, int] = {} + paging = unresolved = total = 0 + starts = [] # ordered (arc, name) asset-start events (deduped consecutively) + contained = set() # every distinct asset any read touched + + for ln in log.read_text(encoding="utf-8").splitlines(): + parts = ln.split("\t") + if len(parts) != 3: + continue + path, off_s, size_s = parts + arc = path.replace("\\", "/").rsplit("/", 1)[-1] + off, size = int(off_s), int(size_s) + total += 1 + per_arc[arc] = per_arc.get(arc, 0) + 1 + if size == PAGING_SIZE: + paging += 1 + continue + hit = resolve(by_arc, arc, off) + if hit is None: + unresolved += 1 + continue + name = hit[0] + contained.add((arc, name)) + if off in exact.get(arc, {}): # exact asset-start + ev = (arc, exact[arc][off]) + if not starts or starts[-1] != ev: + starts.append(ev) + + result = { + "source_log": log.name, + "asset_index": "asset-index.json", + "total_reads": total, + "paging_reads_dropped": paging, + "unresolved_reads": unresolved, + "reads_per_archive": per_arc, + "distinct_assets_touched": len(contained), + "load_order_count": len(starts), + "load_order": [{"archive": a, "name": n} for a, n in starts], + } + out.write_text(json.dumps(result, ensure_ascii=False, indent=1), encoding="utf-8") + + print(f"{log.name}: {total} reads ({paging} paging dropped, {unresolved} unresolved)") + print(f"reads/archive: {per_arc}") + print(f"distinct assets touched: {len(contained)}; " + f"asset-start load order: {len(starts)} events") + for a, n in starts: + print(f" {a:<12} {n}") + print(f"-> {out.relative_to(paths.REPO)}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/vm0.py b/tools/vm0.py index 9546386..cbd0cfd 100644 --- a/tools/vm0.py +++ b/tools/vm0.py @@ -57,9 +57,11 @@ class Frame: class VM: - def __init__(self, scr: sys4load.Sys4Script, verbose=False, emit_cap=EMIT_CAP): + def __init__(self, scr: sys4load.Sys4Script, verbose=False, emit_cap=EMIT_CAP, record_trace=False): self.scr = scr self.verbose = verbose + self.record_trace = record_trace + self.trace = [] # executed code offsets (only if record_trace) self.code = scr.instructions self.by_off = {ins.offset: idx for idx, ins in enumerate(self.code)} self.G = collections.defaultdict(int) # global-int bank (flat address space) @@ -67,6 +69,7 @@ class VM: self.fr = Frame() self.callstack = [] # return indices for call/ret self.text = [] # captured show-text as (str_offset, text) + self.settex = [] # set-texture calls: (code_offset, resId, slot) self.emit_seen = collections.Counter() # per-offset emit count (loop-guard) self.emit_cap = emit_cap self.halt_reason = None # 'exit' | 'LOOP:...' | 'STEP-LIMIT' | 'ret-underflow' @@ -135,6 +138,8 @@ class VM: ins = self.code[pc] op = ins.opcode self.exec_count[op] += 1 + if self.record_trace: + self.trace.append(ins.offset) nxt = self.step(ins, pc) if nxt is None: # halt break @@ -214,6 +219,12 @@ class VM: "display-furigana", "dev_ukn"): return pc + 1 + if lbl == "set-texture": # 0x1f9 (resId, slot, flag) — trace the load + resid = self.read(a[0]) if a else None + slot = self.read(a[1]) if len(a) > 1 else None + self.settex.append((ins.offset, resid, slot, len(self.trace))) # +trace index + return pc + 1 + if op in MARKERS: # classified no-op markers return pc + 1 @@ -426,6 +437,34 @@ def run_trace(out_path): return 0 +def run_settex(name): + """Execute a scene and dump its set-texture(resId) trace in execution order. + + This is the VM side of the asset-resolution scope-selector correlation + (docs/asset-resolution-re.md): each entry is {i, off, resId, slot}, and aligning this + ordered resId sequence with the game's Frida load order pins every load to a bytecode + offset -> localizes where the active CG package/scope switches. + """ + scripts = paths.scripts() + key = name.upper() if name.upper().endswith(".BIN") else name.upper() + ".BIN" + if key not in scripts: + raise SystemExit(f"scene not found: {name}") + vm = VM(sys4load.load(scripts[key]), record_trace=True) + vm.run() + out = paths.BUILD / f"settex-{key.removesuffix('.BIN')}.json" + rows = [{"i": i, "off": f"0x{off:x}", "resId": rid, "slot": slot, "trace_i": ti} + for i, (off, rid, slot, ti) in enumerate(vm.settex)] + out.write_text(json.dumps({"scene": key, "halt": vm.halt_reason, "steps": vm.steps, + "count": len(rows), "settex": rows, + "trace": [f"0x{o:x}" for o in vm.trace]}, + ensure_ascii=False), encoding="utf-8") + print(f"{key}: {len(rows)} set-texture calls (halt={vm.halt_reason}, steps={vm.steps}) " + f"-> {out.relative_to(paths.REPO)}") + for r in rows[:20]: + print(f" #{r['i']:<3} {r['off']:>8} resId={r['resId']} (0x{r['resId']:x}) slot={r['slot']}") + return 0 + + def main(argv=None): argv = argv if argv is not None else sys.argv[1:] if not argv or argv[0] == "--test": @@ -434,6 +473,8 @@ def main(argv=None): return run_sweep(limit=int(argv[1]) if len(argv) > 1 else None) if argv[0] == "--scene": return run_one_scene(argv[1]) + if argv[0] == "--settex": + return run_settex(argv[1]) if argv[0] == "--trace": return run_trace(argv[1]) return run_file(argv[0]) diff --git a/vm-map/opcodes.toml b/vm-map/opcodes.toml index 15c35e2..745bf2d 100644 --- a/vm-map/opcodes.toml +++ b/vm-map/opcodes.toml @@ -1849,17 +1849,17 @@ abi_source = "kelebek+decode-validated" [opcode.semantics] name = "play-bgm" -category = "unknown" -summary = "" +category = "audio" +summary = "Play background music by id; id resolves via the SYS4INI section manifest -> files[section_base(scene)+id] (OGG in DATA3). Same resolution as set-texture." noop_headless = false -source = "kelebek" -confidence = "med" +source = "frida" +confidence = "high" depends_on = [] -evidence = "" +evidence = "Frida capture: `play-bgm 0x5` in SC0000 (section base 0) loaded BGM006.OGG = files[5]. Unified with set-texture resolution rule." [[opcode.semantics.args]] i = 1 -role = "" +role = "bgm id (section-manifest index)" observed_types = ["imm", "g-int", "l-int", "l-ptr"] [[opcode]] @@ -1917,17 +1917,17 @@ abi_source = "kelebek+decode-validated" [opcode.semantics] name = "play-voice" -category = "unknown" -summary = "" +category = "audio" +summary = "Play a voice clip by id; id resolves via the SYS4INI section manifest -> files[section_base(scene)+id] (voice OGG in DATA1/DATA4). Same rule as set-texture/play-bgm." noop_headless = false -source = "kelebek" +source = "investigation" confidence = "med" depends_on = [] -evidence = "" +evidence = "Section-manifest resolution validated across archives incl. DATA4 voice OGGs (586/595 captured loads); per-clip id->OGG not individually Frida-pinned yet." [[opcode.semantics.args]] i = 1 -role = "" +role = "voice id (section-manifest index)" observed_types = ["imm", "l-int", "l-ptr"] [[opcode]] @@ -4512,32 +4512,32 @@ abi_source = "kelebek+decode-validated" [opcode.semantics] name = "create-texture" -category = "unknown" -summary = "" +category = "draw" +summary = "Allocate/prepare a texture slot: (slot, width, height, flag). e.g. `create-texture 0xd 0x190 0x1e 0x0` = slot 13, 400x30." noop_headless = false -source = "kelebek" +source = "investigation" confidence = "med" depends_on = [] -evidence = "" +evidence = "SC0000 CG/UI-draw path disasm; slot/w/h roles read off the operands (400x30 text bars, etc.)." [[opcode.semantics.args]] i = 1 -role = "" +role = "texture slot" observed_types = ["imm", "g-int", "l-int", "l-ptr"] [[opcode.semantics.args]] i = 2 -role = "" +role = "width" observed_types = ["imm", "g-int", "l-int", "l-ptr"] [[opcode.semantics.args]] i = 3 -role = "" +role = "height" observed_types = ["imm", "g-int", "l-int", "l-ptr"] [[opcode.semantics.args]] i = 4 -role = "" +role = "flag" observed_types = ["imm"] [[opcode]] @@ -4548,27 +4548,27 @@ abi_source = "kelebek+decode-validated" [opcode.semantics] name = "set-texture" -category = "unknown" -summary = "" +category = "draw" +summary = "Load asset #resId into texture slot: (resId, slot, flag=-1). resId resolves via the SYS4INI per-scene section manifest: files[section_base(scene)+resId] (same rule for play-bgm/play-voice). See docs/asset-resolution-re.md." noop_headless = false -source = "kelebek" -confidence = "med" +source = "frida" +confidence = "high" depends_on = [] -evidence = "" +evidence = "SC0000 Frida-confirmed 17/17 (0x25->EV052CA, 0x2e->EV052DB, 0x36->BG030A background); resolution rule validated on 586/595 captured loads. Traced in CG-load subroutine label_12649 as `set-texture G[0x62424] -1`." [[opcode.semantics.args]] i = 1 -role = "" +role = "resId (section-manifest index into SYS4INI)" observed_types = ["imm", "g-int", "l-int", "l-ptr"] [[opcode.semantics.args]] i = 2 -role = "" +role = "texture slot" observed_types = ["imm", "g-int", "l-int"] [[opcode.semantics.args]] i = 3 -role = "" +role = "flag (typically -1 / 0xffffffff)" observed_types = ["imm", "l-int"] [[opcode]] @@ -4600,52 +4600,52 @@ abi_source = "kelebek+decode-validated" [opcode.semantics] name = "draw-texture" -category = "unknown" -summary = "" +category = "draw" +summary = "Blit a texture slot to screen. Observed 8 args: (handle, slot, srcx, srcy, w, h, dstx, dsty). e.g. `draw-texture 0xcf08 0x3 0 0 0x320 0x258 0 0` = full-screen (800x600) slot 3 at (0,0)." noop_headless = false -source = "kelebek" +source = "investigation" confidence = "med" depends_on = [] -evidence = "" +evidence = "SC0000 CG-load subroutine label_12649: `draw-texture (ptr) (slot) 0 0 (w) (h) (dstx) (dsty)`; full-screen slot-3 draws use 0x320x0x258 (800x600)." [[opcode.semantics.args]] i = 1 -role = "" +role = "handle/source ref" observed_types = ["imm", "g-int", "l-int", "l-ptr"] [[opcode.semantics.args]] i = 2 -role = "" +role = "texture slot" observed_types = ["imm", "g-int", "l-int", "l-ptr"] [[opcode.semantics.args]] i = 3 -role = "" +role = "src x" observed_types = ["imm", "l-int", "l-ptr"] [[opcode.semantics.args]] i = 4 -role = "" +role = "src y" observed_types = ["imm", "l-int", "l-ptr"] [[opcode.semantics.args]] i = 5 -role = "" +role = "width" observed_types = ["imm", "g-int", "l-int", "l-ptr"] [[opcode.semantics.args]] i = 6 -role = "" +role = "height" observed_types = ["imm", "g-int", "l-int", "l-ptr"] [[opcode.semantics.args]] i = 7 -role = "" +role = "dst x" observed_types = ["imm", "g-int", "l-int", "l-ptr"] [[opcode.semantics.args]] i = 8 -role = "" +role = "dst y" observed_types = ["imm", "g-int", "l-int", "l-ptr"] [[opcode]] diff --git a/vm-map/resources.json b/vm-map/resources.json new file mode 100644 index 0000000..2b587fe --- /dev/null +++ b/vm-map/resources.json @@ -0,0 +1,16 @@ +{ + "_doc": "SUPERSEDED. Asset resolution is now DERIVED statically, not curated per scene.", + "_rule": "resId -> files[section_base(scene) + resId], where SYS4INI is organized into sections (one per SCxxxx.BIN scene + its cross-archive asset manifest) and file_number is the index within the section. Same rule for set-texture / play-bgm / play-voice.", + "_use": "tools/resolve_asset.py --build -> build/asset-sections.json (scene -> section_base). Then resolve any (scene, resId) via build/asset-index.json. See docs/asset-resolution-re.md step 2.", + "_validated": "SC0000 opening 17/17 vs Frida (0x25->EV052CA, 0x36->BG030A, ...); 586/595 captured loads across all sections satisfy files[base+fn]==name.", + "scenes": { + "SC0000": { + "_section_base": 0, + "_note": "base 0 (section 0 = system + SC0000); resId -> files[resId]. Confirmed examples:", + "resId": { + "0x23": "EV049AA.AGF", "0x25": "EV052CA.AGF", "0x27": "EV052DA.AGF", + "0x2b": "EV052DC.AGF", "0x2e": "EV052DB.AGF", "0x36": "BG030A.AGF (background)" + } + } + } +}