diff --git a/docs/PROJECT-STRUCTURE.md b/docs/PROJECT-STRUCTURE.md index 2476401..b854b2d 100644 --- a/docs/PROJECT-STRUCTURE.md +++ b/docs/PROJECT-STRUCTURE.md @@ -16,9 +16,9 @@ S:\Game Hacking\Eushully\Himegari\ ← workspace root (three siblings) │ ├── AGE.EXE, AGERC.DLL, *.dll shipped engine (packed). Stays intact and │ │ runnable in place — Frida launches it if needed. │ ├── DATA1-5.ALF, APPEND01.ALF/.AAI shipped archives (~2.3 GB). -│ ├── *.BIN 52 loose patch-override scripts (v1.03) — -│ │ AUTHORITATIVE over their DATA1 copies. Plus -│ │ non-script indices (SYS4INI=S4IC, SYS4AB=S4AB). +│ ├── *.BIN 49 loose patch-override scripts (v1.03) — +│ │ AUTHORITATIVE over DATA1. Plus two root-only +│ │ engine files (SYS4INI=S4IC, SYS4AB=S4AB). │ └── *.exe (uninstallers), SAS0099.OGG … other shipped files. │ ├── extracted/ ← DERIVED (game-side) — extracted ALF contents, @@ -96,6 +96,8 @@ S:\Game Hacking\Eushully\Himegari\ ← workspace root (three siblings) │ └── manifest.json, opcode-coverage.md (opcode-coverage.md GENERATED from opcodes.toml) │ ├── engine/ DELIVERABLE — the .NET VM core (AgeEngine.sln: Age.Engine / Age.Cli / tests) + │ └── Age.Engine/Sys4/ runtime catalog parser, loose-first bounded ALF asset store, + │ script provider, and temporary resource facade ├── tools/frida/ runtime-capture + engine-dump scripts (see tools/frida/README.md) └── godot/ DELIVERABLE — the Godot/C# ADV front-end (references Age.Engine) ``` diff --git a/docs/asset-resolution-re.md b/docs/asset-resolution-re.md index 0338a6f..d92be9d 100644 --- a/docs/asset-resolution-re.md +++ b/docs/asset-resolution-re.md @@ -166,7 +166,7 @@ The current Phase-A backend deliberately continues through the extracted-file bo accepts both OGG and WAV and Godot loads the WAV bytes into its fixed SC0000 channel pool. This does not change the scoped VFS plan below: ALF/AAI mounting and in-process asset reads remain a separate foundation track. -## Candidate runtime asset-VFS track (scoped 2026-07-10; not started) +## Runtime asset-VFS track (VFS-A complete 2026-07-11) The pre-extracted tree and `build/textures/*.BMP` pipeline were a Phase-A bootstrap, not the desired final runtime. The native-compatible target is a read-only virtual filesystem that preserves AGE's translation/mod @@ -184,12 +184,16 @@ store. ### Proposed layers -1. **Catalog + read-only ALF store.** Parse SYS4INI at runtime while preserving all 13208 raw records +1. **Catalog + read-only ALF store (VFS-A DONE).** `Sys4AssetCatalog` parses SYS4INI at runtime while preserving all 13208 raw records (including the two `@` placeholders), archive names, scene sections, and the existing three lookup modes: universal raw id, scene-local manifest id, and direct name where the opcode family genuinely uses one. An ALF is a payload container at this layer: open the named archive and return a bounded stream/byte range at the indexed offset/size. Before that fallback, probe the configured loose override roots by the record's - exact basename. Keep `build/asset-index.json` as a diagnostic artifact, not a runtime dependency. + exact basename. `Sys4AssetStore` opens a separate read-only file handle per request and constrains archive + seek/read operations to the record range. `Sys4ScriptProvider` now loads both root scenes and nested + `call-script` targets through this seam; `ResourceMap` uses the same live catalog. `build/asset-index.json`, + `build/asset-sections.json`, `build/callscript-names.json`, and `extracted/` are validation/temporary + graphics-audio artifacts, not script-runtime dependencies. 2. **AAI append mount.** Parse the installed `APPEND01.AAI` (`S4AC422`) and its paired `APPEND01.ALF` with the same catalog abstractions. First prove whether Himegari joins append records by a separate pack/tag, by name replacement, or by another table selected by the native high-byte-id path; do not invent mount @@ -202,7 +206,7 @@ store. platform-neutral .NET code; do not carry GARbro's WPF/GameRes dependencies. Kelebek's extractor and the on-disk `BinExtractALF.exe` are validation references; the Kelebek repository exposes no clear license, so its code should not be copied without clarification. -4. **Runtime consumers.** Make script loading accept streams/bytes from the store, make texture surfaces own +4. **Runtime consumers.** Script loading is complete. Next make texture surfaces own decoded RGBA pixels rather than BMP paths, and load OGG/WAV from store bytes. Migrate one consumer at a time; retain extraction/conversion tools as diagnostics until parity is established. @@ -222,6 +226,16 @@ store. the translucent textbox/button chrome appears, root `.BIN` overrides still win, and the standard VM/Godot validation matrix remains green. +VFS-A passes these bounded gates in `Sys4AssetStoreTests`: every catalog field matches the generated +diagnostic index, all 136 scene views match the generated section oracle without cross-section spill, all +13206 archive ranges fit, `raw_index 0x337e` is `SO001.AGF`, representative payloads from every base archive +are byte-identical to `extracted/`, and synthetic removal of a loose override reveals the bounded ALF bytes. +Traversal, past-range seek/read, and concurrent reads are covered. Installed override enumeration corrected +an older inventory error: this tree contains 51 loose root BINs, comprising **49 archive-backed v1.03 script +overrides** (all byte-proven to win and differ from DATA1) plus root-only `SYS4INI.BIN` and `SYS4AB.BIN`. +There are not 52 archive copies available to shadow. APPEND01/AAI, AGF decode, audio consumers, and movie +`0x236` remain unimplemented by design. + ### Deliberate non-goals - Writing/repacking ALF or AAI; loose overrides already provide the native mod/translation workflow. diff --git a/docs/himegari-port-reference.md b/docs/himegari-port-reference.md index 82a4654..577866d 100644 --- a/docs/himegari-port-reference.md +++ b/docs/himegari-port-reference.md @@ -93,10 +93,10 @@ save-format work remain deferred. ### Immediate (no tools needed beyond what's on disk) 1. ~~**Relocate the `Output\` tree**~~ **DONE** — workspace now at `S:\Game Hacking\Eushully\Himegari\姫狩りダンジョンマイスター\`. 2. **Convert remaining AGFs** in DATA2 (985 files) and DATA5 (210 files) with `AGF2BMP2AGF.exe`. *(Deferred — graphics not needed yet.)* The 3-file DATA1 gap is `CHAPTER.AGF`, `LOGO.AGF`, `TEST.AGF`. -3. ~~**Inventory the script files**~~ **DONE** — see [script-inventory.md](script-inventory.md). Key findings: all 481 scripts share magic `SYS4422 `; 52 loose root-dir `.BIN` files are patch overrides that shadow DATA1 copies (use those as authoritative); heavy game logic (damage calc, dungeon loop, battle flow) lives in bytecode, favoring a VM re-implementation in Godot. +3. ~~**Inventory the script files**~~ **DONE** — see [script-inventory.md](script-inventory.md). Key findings: all 481 scripts share magic `SYS4422 `; 49 loose root-dir script `.BIN` files shadow DATA1 copies (plus two root-only engine BINs; use overrides as authoritative); heavy game logic (damage calc, dungeon loop, battle flow) lives in bytecode, favoring a VM re-implementation in Godot. 1. ~~**Relocate the `Output\` tree**~~ **DONE** — workspace now at `S:\Game Hacking\Eushully\Himegari\姫狩りダンジョンマイスター\`. 2. **Convert remaining AGFs** in DATA2 (985 files) and DATA5 (210 files) with `AGF2BMP2AGF.exe`. *(Deferred — graphics not needed yet.)* The 3-file DATA1 gap is `CHAPTER.AGF`, `LOGO.AGF`, `TEST.AGF`. -3. ~~**Inventory the script files**~~ **DONE** — see [script-inventory.md](script-inventory.md). Key findings: all 481 scripts share magic `SYS4422 `; 52 loose root-dir `.BIN` files are patch overrides that shadow DATA1 copies (use those as authoritative); heavy game logic (damage calc, dungeon loop, battle flow) lives in bytecode, favoring a VM re-implementation in Godot. +3. ~~**Inventory the script files**~~ **DONE** — see [script-inventory.md](script-inventory.md). Key findings: all 481 scripts share magic `SYS4422 `; 49 loose root-dir script `.BIN` files shadow DATA1 copies (plus two root-only engine BINs; use overrides as authoritative); heavy game logic (damage calc, dungeon loop, battle flow) lives in bytecode, favoring a VM re-implementation in Godot. ### Header structure — DONE (hex-first, pre-Ghidra) See [sys4-format-notes.md](sys4-format-notes.md). Confirmed across all 481 files: 60-byte header (magic `SYS4422 ` + 13 u32 fields, all offsets in dwords), body split into CODE + 3 typed pointer tables (tags 0x71/0x03/0x8F, 1 dword each, 100% pure) + inline strings. Strings are XOR-0xFF cp932, referenced by a `0x02 ` tagged operand — verified by decoding real dialogue out of `SC0030.BIN`. Remaining unknowns (opcode dispatch, flag fields F0/F2/F3/F5) need the VM. diff --git a/docs/name-resolution.md b/docs/name-resolution.md index 3be9d75..70c6fff 100644 --- a/docs/name-resolution.md +++ b/docs/name-resolution.md @@ -24,10 +24,11 @@ mechanism in `engine-re.md` (“op 0x03 (call-script)…”). Tooling: `parse_sy `build/callscript-names.json` (id→name); `sys4load` renders `call-script 0x1ab =ADDITEM.BIN`; the `build/disasm/*.asm` call graph now reads by name. The one caveat: index the RAW SYS4INI records (*including* the 2 `@` placeholders) — `asset-index.json` carries each entry's `raw_index` (= the id) -for exactly this. **Remaining (functional, not naming):** the C# VM still stubs `call-script` -execution; implementing it (load `.BIN` by id, push frame, run, return) is the follow-up. The original -analysis (kept below for provenance) had concluded this was engine-level and deferred — it was, and -the Ghidra loop is what resolved it. +for exactly this. **Runtime (VFS-A):** `Sys4AssetCatalog` now reads that raw table directly and +`Sys4ScriptProvider` opens the selected record through loose-first/bounded-ALF storage; generated JSON is +only the disassembler annotation and parity oracle. The VM executes the loaded target as a nested frame. +The original analysis (kept below for provenance) had concluded this was engine-level and deferred — it +was, and the Ghidra loop is what resolved it. **What it is (original framing).** `call-script N` (Kelebek opcode 0x03) carries a bare number — `0x329d`, `0x2ade` — the id of an engine entry point. To render `call RECOVER` instead of diff --git a/docs/phase-a-slice-plan.md b/docs/phase-a-slice-plan.md index 6680df1..1084995 100644 --- a/docs/phase-a-slice-plan.md +++ b/docs/phase-a-slice-plan.md @@ -1055,7 +1055,7 @@ separate slices. No implementation commit was made. This is an optional high-leverage detour before movie `0x236` or SFX. It replaces the Phase-A pre-extracted/pre-converted asset bootstrap with the native loose-override/archive-fallback model and removes the runtime dependency on `extracted/` plus `build/textures/`. Canonical format/architecture detail and source -references live in `docs/asset-resolution-re.md` §“Candidate runtime asset-VFS track”; this section defines +references live in `docs/asset-resolution-re.md` §“Runtime asset-VFS track”; this section defines slice boundaries only. Land it as three bounded slices, not one archive/codec rewrite: @@ -1090,6 +1090,21 @@ fixtures and installed-game integration checks rather than committing proprietar modding contract and benefits scripts, UI chrome, SFX, and movies. It is not required to continue opcode coverage immediately, so choosing movie/SFX next remains valid. +### VFS-A — base SYS4 catalog + ALF byte reads DONE (2026-07-11) + +`Sys4AssetCatalog` now runtime-parses `SYS4INI.BIN` into all 13208 raw slots (including two `@` +placeholders), 13206 real/name records, bounded scene sections, and universal raw-id lookup. +`Sys4AssetStore` applies native `loose exact-basename -> ALF offset/size` precedence with traversal rejection, +per-open file handles, and a seek/read boundary. `Sys4ScriptProvider` and all CLI/Godot root-script paths use +store bytes; `ResourceMap` consumes the live catalog instead of generated asset JSON. + +Validation: 116 engine tests; every field against `build/asset-index.json`; all 136 generated scene views; +all 13206 archive ranges; representative bytes across DATA1..5 against `extracted/`; synthetic precedence, +range, traversal, and concurrency tests; CLI SC0000 run; Godot build/selftest. The installed root has 49 +archive-backed script overrides and two root-only BIN engine files, not the historically reported 52 +shadowing scripts; all 49 were byte-proven to win and differ from their archive payload. VFS-B APPEND01/AAI, +VFS-C AGF, audio migration, and movie `0x236` remain separate. + ### Phase A — native SC0000 SFX family (`0xb4`/`0xb5`/`0xb6`/`0xc2`/`0xd9`) DONE (2026-07-11) Native RE and the matching trace resolve the bounded family. `0xb4(resource,channel)` synchronously loads diff --git a/docs/remake-architecture-and-roadmap.md b/docs/remake-architecture-and-roadmap.md index c56d60c..3842e32 100644 --- a/docs/remake-architecture-and-roadmap.md +++ b/docs/remake-architecture-and-roadmap.md @@ -23,8 +23,8 @@ foundation; the runtime + backends + mod system is the bulk of the remaining wor asset rules) selected by a manifest. 3. **Modding is architecture, not an afterthought.** The data model, content loading, and script dispatch are designed so mods can override assets, edit data, patch scripts, and inject - host-language hooks. The engine already hints at this: 52 loose root `.BIN` files natively - shadow their archived copies — a built-in override mechanism we generalize. + host-language hooks. The engine already hints at this: 49 loose root script `.BIN` files natively + shadow their archived copies — a built-in override mechanism VFS-A generalizes. 4. **The original owns the content; we own the engine.** Users provide their AGE install; the runtime imports/loads it. This keeps us on the right side of distribution and mirrors ScummVM. 5. **De-risk with vertical slices.** Prove "run one scene end-to-end" before breadth. Nothing is @@ -231,8 +231,8 @@ is shared across the family), disassembler/assembler, the whole extraction metho **Per-game (inherent content work):** the **global-var map** (globals are game-specific), the **data-table layouts** (each game's `*INIT` differs), assets, and any game-specific effectful behavior. The **call-script registry is no longer a per-game long pole** — it's a raw index into that game's -SYS4INI file table, derived automatically by `parse_sys4ini.py` (`build/callscript-names.json`); the -resolver is generic. So the remaining long pole is really just the **global-var map**. Process: point +SYS4INI file table, parsed directly by the runtime catalog (and exported by `parse_sys4ini.py` as +`build/callscript-names.json` for tooling); the resolver is generic. So the remaining long pole is really just the **global-var map**. Process: point the toolchain at the new game's archives, re-run extraction, rebuild its global map, author a profile. **This is the core payoff of the VM approach:** the *engine* cost amortizes across all AGE games; only content-mapping recurs — far less than re-coding each game's logic bespoke. diff --git a/docs/script-inventory.md b/docs/script-inventory.md index 3ba34e8..5d814f2 100644 --- a/docs/script-inventory.md +++ b/docs/script-inventory.md @@ -7,11 +7,12 @@ during disassembler work). Source: `extracted\DATA1\` (extracted from `DATA1.ALF`). -**Patch overrides:** 52 loose `.BIN` files sit in the game root directory and shadow -their DATA1 counterparts at runtime (sizes differ slightly — e.g. `FIELD.BIN` root -200,536 vs archive 200,224). These are the v1.03 / append-patch versions and should be -treated as **authoritative** over the archive copies. Two engine files exist only in -the root: `SYS4INI.BIN` (272 KB) and `SYS4AB.BIN` (1.08 MB). +**Patch overrides (runtime re-counted 2026-07-11):** 49 loose `.BIN` scripts sit in the game root and +shadow DATA1 counterparts at runtime (sizes differ slightly — e.g. `FIELD.BIN` root 200,536 vs archive +200,224). These v1.03 / append-patch versions are **authoritative**. Two additional engine BINs exist only +in the root: `SYS4INI.BIN` (272 KB) and `SYS4AB.BIN` (1.08 MB), for 51 root BINs total. The earlier count of +52 shadowing scripts conflated this inventory and was not reproducible; VFS-A enumerates and byte-checks all +49 real catalog/name intersections. --- @@ -100,8 +101,8 @@ outside scenes lives. `call-script ` (opcode 0x03) loads another script by a **raw index into the SYS4INI file table** (id = the entry's `raw_index` = its global position in SYS4INI). This is the resolved call-graph registry — there is no separate id→code table; SYS4INI is it. Mechanism: `engine-re.md` (op 0x03 -section); id→name single source: `build/callscript-names.json` (from `parse_sys4ini.py`); `sys4load` -and the regenerated `build/disasm/*.asm` corpus now render targets by name +section); the runtime source is `Sys4AssetCatalog` over SYS4INI, while the mechanically generated +`build/callscript-names.json` feeds `sys4load` diagnostics. The regenerated `build/disasm/*.asm` corpus renders targets by name (`call-script 0x1ab =ADDITEM.BIN`). **297 distinct scripts are called** across the corpus (3002 sites); the hottest are `HISTORY` (backlog), `MENU`, `HIDEWIN`, `BUNKI` (branch), `MES` (message), `ADDITEM`, `ADDEN`, `LOOK`, `RENDERMAP`. Scenes (`SCxxxx.BIN`) load through the *same* id-indexed loader. @@ -126,4 +127,4 @@ derivable on demand from the corpus; materialize a doc only if a consumer needs (176 B) → `MENU.BIN` (3 KB) → `CALCDMG` → a mid-size `SC####`. 3. **The `*INIT` giants are likely data tables**, decodable early even with a partial opcode map — instant win for extracting item/skill/enemy/stage databases. -4. **Use root-directory overrides, not archive copies**, for the 52 patched scripts. +4. **Use root-directory overrides, not archive copies**, for the 49 archive-backed patched scripts. diff --git a/docs/sys4-format-notes.md b/docs/sys4-format-notes.md index 46b3b9d..262bb0b 100644 --- a/docs/sys4-format-notes.md +++ b/docs/sys4-format-notes.md @@ -112,8 +112,8 @@ task. These tags give a head start on labeling the disassembly.* ## Patch-override caveat (re-confirmed) -52 loose `.BIN` in the game root shadow their DATA1 copies at runtime and differ -slightly in size. The disassembler should target the **root** copies where present. +49 loose script `.BIN` files in the game root shadow DATA1 copies at runtime and differ slightly in size; +two additional root-only engine BINs bring the root total to 51. The disassembler should target the **root** copies where present. The header format is identical (same magic/layout) so tooling is copy-agnostic. ## What's solid vs. what needs Ghidra diff --git a/docs/tools-reference.md b/docs/tools-reference.md index 98ebef8..52c693e 100644 --- a/docs/tools-reference.md +++ b/docs/tools-reference.md @@ -72,12 +72,16 @@ The `engine/` .NET solution (`AgeEngine.sln`) is the runtime VM; `godot/` is the Python, but listed here as the things you *run*. Build: `dotnet build engine/AgeEngine.sln`; test: `dotnet test engine/AgeEngine.sln`. Run a CLI command: `dotnet run --project engine/Age.Cli -- `. -**call-script executes** on the product paths: they inject `Sys4ScriptProvider` (id→`.BIN`, via -`build/callscript-names.json`), so `call-script ` loads & runs the target as a nested subroutine +**call-script executes** on the product paths: they inject `Sys4ScriptProvider`, which runtime-parses +`SYS4INI.BIN` and opens `.BIN` bytes through the native loose-first/bounded-ALF store, so `call-script ` loads & runs the target as a nested subroutine frame sharing globals. `trace`/`audio`/`gfx` stay **provider-less** (call-script stubbed) — base-ISA / subsystem oracles. Test scenes are **synthesized** via `Age.Engine/Sys4/ScriptAssembler` (see [[testing-synthesize-dont-disable]]: synthesize test data, never disable a feature to keep a golden green). +The runtime SYS4 front-end is `Sys4AssetCatalog` (raw-id, scene-local, and name views), `IAssetStore` / +`Sys4AssetStore` (exact-basename loose roots, then a bounded ALF range), and `Sys4ScriptProvider` (cached +root/call-script parsing). Generated asset/callscript JSON remains a tooling and test oracle only. + | Command | Purpose | Notes | |---|---|---| | `run ` | Execute a script; print steps, show-text count, **call-script dispatch count**, the first 30 lines (each tagged with its source script), and the distinct source scripts. | `CaptureHost` (headless); **executes call-script**. | @@ -153,7 +157,7 @@ texture ops (no GPU context) — run windowed for real scenes. User args (after | Tool | Purpose | Run | Reads → Writes | |---|---|---|---| | `tools/frida/capture_native_transforms.py` | Capture native `0x21f`/`0x223`/`0x234` worker operands, corrected integer base/anchor coordinates, all one-shot/cyclic retained fields, the one-shot 4×4 matrix, and the final post-cyclic 4×4 matrix. Optional handle filter; read-only. | `py -3.11 -u -X utf8 tools/frida/capture_native_transforms.py [secs] [pid|AGE.EXE] [--handle 0xHANDLE]` | running game → `build/native-transform-trace.jsonl` | -| `parse_sys4ini.py` | Parse `SYS4INI.BIN` (S4IC422, LZSS-compressed) into the authoritative asset index — name ↔ archive ↔ offset ↔ size for all DATA*.ALF (the `resId→file` answer key). Each entry carries `raw_index` (its 0-based position in the SYS4INI record table incl. `@` placeholders) = the engine's universal file id. Also emits the **`call-script → name`** map (id = `raw_index`; see `engine-re.md`). | `parse_sys4ini.py [--check]` (`--check` validates vs `extracted/` + `.ALF` sizes) | `姫狩り…/SYS4INI.BIN` → `build/asset-index.json` + `build/callscript-names.json` | +| `parse_sys4ini.py` | Parse `SYS4INI.BIN` (S4IC422, LZSS-compressed) into the diagnostic JSON asset-index mirror — name ↔ archive ↔ offset ↔ size for all DATA*.ALF. Each real entry carries universal `raw_index`; the runtime parses SYS4INI itself, while these generated files remain tooling/test oracles. Also emits the `call-script → name` annotation map. | `parse_sys4ini.py [--check]` (`--check` validates vs `extracted/` + `.ALF` sizes) | `姫狩り…/SYS4INI.BIN` → `build/asset-index.json` + `build/callscript-names.json` | | `resolve_asset.py` | ★ **The static asset resolver.** SYS4INI is sectioned (one per scene: `SCxxxx.BIN` + its cross-archive manifest; `file_number` = index within section). Resolves `resId → files[section_base(scene) + resId]` for graphics AND audio, no capture. | `resolve_asset.py --build` · `resolve_asset.py [resId]` | `build/asset-index.json` → `build/asset-sections.json`; resolves any (scene, resId) | | `resolve_frida_reads.py` | Rescue noisy Frida archive-read offsets → asset names via the index (per-archive range search; drops 0x20000 paging reads); recovers the per-scene asset load order. | `resolve_frida_reads.py [reads.log] [-o out.json]` | `build/frida-reads.log` + `build/asset-index.json` → `build/frida-asset-loads.json` | | `convert_agf.py` | Convert AGF stills to BMP via `AGF2BMP2AGF.exe` (searches all `extracted/DATA*`). `--scene` batch-converts a scene's whole SYS4INI manifest — feeds the Godot render. | `convert_agf.py EV052CA.AGF …` · `convert_agf.py --scene SC0000` | `extracted/DATA*/*.AGF` → `build/textures/*.BMP` | diff --git a/docs/vm-mapping-plan.md b/docs/vm-mapping-plan.md index 6a9a887..1abca91 100644 --- a/docs/vm-mapping-plan.md +++ b/docs/vm-mapping-plan.md @@ -28,7 +28,7 @@ ## Global constraints - **Python:** `py -3.11 -X utf8 …` always (Shift-JIS output needs utf8 mode on Windows). -- **Authoritative copies:** the 52 loose game-folder `.BIN` shadow their `extracted/DATA1/` copies at runtime — target the game-folder copy where both exist. `sys4load.load()` is copy-agnostic; `paths.scripts()` resolves the override. +- **Authoritative copies:** 49 loose game-folder script `.BIN` files shadow `extracted/DATA1/` copies at runtime; two engine BINs are root-only. Target the game-folder copy where both exist. `sys4load.load()` is copy-agnostic; `paths.scripts()` resolves the override. - **Units:** all script offsets/counts are DWORDS (×4 bytes), relative to body start `0x3C`. - **Instruction rule:** `len_dwords = 1 + 2*argc`; args are `(type,value)`; **stop code decode at the first inline-string/array offset**, not blindly at `F8`. - Record confidence per finding (confirmed-by-bytes / confirmed-by-runtime / hypothesis). diff --git a/engine/Age.Cli/Program.cs b/engine/Age.Cli/Program.cs index 5695ba9..7385f6d 100644 --- a/engine/Age.Cli/Program.cs +++ b/engine/Age.Cli/Program.cs @@ -10,6 +10,7 @@ var table = OpcodeTableJson.Load(Paths.OpcodesJson); // call-script execution: resolves ids -> scripts. Product paths pass this so subroutines run; // `trace` stays provider-less on purpose (the base-ISA offset oracle). var provider = Sys4ScriptProvider.Load(table); +Script ScriptByName(string name) => provider.RequireByName(name); // Diagnostics flags (see the TraceSetup class below): --trace (text flow), --trace-steps (every op), // --trace-ops (only these mnemonics/hex, tagged with their script), --trace-histogram (op + @@ -43,7 +44,7 @@ if (args[0] == "audio") var sceneKey = Path.GetFileNameWithoutExtension(sceneName).ToUpperInvariant(); var res = ResourceMap.Load(); var host = new AudioTraceHost(res, sceneKey); - var vm = new VirtualMachine(Sys4Loader.Load(Paths.Scripts()[sceneName.ToUpperInvariant()], table), table, host); + var vm = new VirtualMachine(ScriptByName(sceneName), table, host); // optional: seed globals, e.g. `audio SC0000.BIN 0xa57=1` to set Lily's form-A flag foreach (var s in args.Skip(2)) { @@ -81,10 +82,10 @@ if (args[0] == "gfx") if (boot) foreach (var b in new[] { "INITCONFIG.BIN", "INIT2.BIN", "INIT.BIN" }) { - var bs = session.RunScene(Sys4Loader.Load(Paths.Scripts()[b], table), table, new CaptureHost(), null, provider); + var bs = session.RunScene(ScriptByName(b), table, new CaptureHost(), null, provider); Console.WriteLine($"[boot] {b}: {bs.Steps} steps (halt: {bs.Halt})"); } - var target = Sys4Loader.Load(Paths.Scripts()[sceneName.ToUpperInvariant()], table); + var target = ScriptByName(sceneName); // With --boot, run the target like the real engine (call-scripts on) so subroutine-driven setup runs. var vm = boot ? new VirtualMachine(target, table, host, new VmOptions(MaxSteps: 20_000_000), provider) : new VirtualMachine(target, table, host); @@ -111,7 +112,6 @@ if (args[0] == "play") // The *INIT boot set — all run clean (halt: exit) and populate the game's data tables into globals. string[] bootScripts = { "SKINIT.BIN", "ITINIT.BIN", "EBINIT.BIN", "CGINIT.BIN", "MPINIT.BIN", "AFINIT.BIN", "CCINIT.BIN", "STINIT.BIN", "STINIT2.BIN" }; - var scripts = Paths.Scripts(); bool boot = args.Contains("--boot"); var userScenes = args.Skip(1).Where(a => a.ToUpperInvariant().EndsWith(".BIN")).ToList(); if (userScenes.Count == 0) { Console.WriteLine("usage: play [--boot] [0xADDR=VAL ...]"); return 1; } @@ -134,7 +134,7 @@ if (args[0] == "play") var playOpts = new VmOptions(HaltAtWaitForInput: !args.Contains("--plow")); // faithful by default foreach (var name in scenes) { - var script = Sys4Loader.Load(scripts[name.ToUpperInvariant()], table); + var script = ScriptByName(name); var r = session.RunScene(script, table, new CaptureHost(), playOpts, provider, trace.Sink); totalLines += r.Emitted.Count; Console.WriteLine($" {name,-14} {r.Emitted.Count,4} lines, {r.Steps,7} steps (halt: {r.Halt})"); @@ -151,8 +151,7 @@ if (args[0] == "sweep") // baseline) and report halt distribution + line counts. Validates the VM + state substrate at scale and // surfaces how booted real data affects the corpus. Headless. var sceneRe = new Regex(@"^S[CP]\d{4}\.BIN$"); - var scripts = Paths.Scripts(); - var names = scripts.Keys.Where(n => sceneRe.IsMatch(n)).OrderBy(n => n, StringComparer.Ordinal).ToList(); + var names = provider.ScriptNames.Where(n => sceneRe.IsMatch(n)).OrderBy(n => n, StringComparer.Ordinal).ToList(); bool boot = args.Contains("--boot"); // Sweep DEFAULTS to plow (walk every page) — it's the dialogue-coverage oracle. --halt-at-wait opts into // the faithful "stop at the first prompt" semantics (VmOptions.HaltAtWaitForInput). @@ -163,7 +162,7 @@ if (args[0] == "sweep") var bootSession = new GameSession(); foreach (var s in new[] { "SKINIT.BIN", "ITINIT.BIN", "EBINIT.BIN", "CGINIT.BIN", "MPINIT.BIN", "AFINIT.BIN", "CCINIT.BIN", "STINIT.BIN", "STINIT2.BIN" }) - bootSession.RunScene(Sys4Loader.Load(scripts[s], table), table, new CaptureHost(), null, provider); + bootSession.RunScene(ScriptByName(s), table, new CaptureHost(), null, provider); baseline = bootSession.ToJson(); Console.WriteLine($"[boot] baseline = {bootSession.Globals.Count} globals; running {names.Count} scenes from it."); } @@ -182,7 +181,7 @@ if (args[0] == "sweep") { var session = Fresh(); if (seeded) foreach (var (k, v) in seeds) session.Seed(k, v); - return session.RunScene(Sys4Loader.Load(scripts[name], table), table, new CaptureHost(), sweepOpts, provider).Emitted.Count; + return session.RunScene(ScriptByName(name), table, new CaptureHost(), sweepOpts, provider).Emitted.Count; } if (seeds.Count > 0) @@ -205,7 +204,7 @@ if (args[0] == "sweep") foreach (var name in names) { var session = Fresh(); - var r = session.RunScene(Sys4Loader.Load(scripts[name], table), table, new CaptureHost(), sweepOpts, provider, trace.Sink); + var r = session.RunScene(ScriptByName(name), table, new CaptureHost(), sweepOpts, provider, trace.Sink); var halt = r.Halt ?? "null"; haltDist[halt] = haltDist.GetValueOrDefault(halt) + 1; totalLines += r.Emitted.Count; @@ -233,8 +232,7 @@ if (args[0] == "trace") var outPath = args[tji + 1]; var sceneName = args.First(a => a.EndsWith(".BIN", StringComparison.OrdinalIgnoreCase)); bool boot = args.Contains("--boot"); - var jscripts = Paths.Scripts(); - var target = Sys4Loader.Load(jscripts[sceneName.ToUpperInvariant()], table); + var target = ScriptByName(sceneName); // --state : start from a captured scene-entry snapshot (Frida global-write log → // capture_global_writes.py) — the real engine's full pre-scene state, superseding the partial // --boot. Otherwise fresh + optional --boot. @@ -252,7 +250,7 @@ if (args[0] == "trace") } if (boot && si < 0) // --state already carries boot state; don't re-run the *INIT prefix foreach (var b in new[] { "INITCONFIG.BIN", "INIT2.BIN", "INIT.BIN" }) - session.RunScene(Sys4Loader.Load(jscripts[b], table), table, new CaptureHost(), null, provider); + session.RunScene(ScriptByName(b), table, new CaptureHost(), null, provider); var sink = new JsonOffsetTraceSink(target.Name); var vm = new VirtualMachine(target, table, new CaptureHost(), new VmOptions(HaltAtWaitForInput: true, MaxSteps: 20_000_000), provider, sink); @@ -269,11 +267,10 @@ if (args[0] == "trace") } var scene = new Regex(@"^S[CP]\d{4}\.BIN$"); - var scripts = Paths.Scripts(); var trace = new SortedDictionary(StringComparer.Ordinal); - foreach (var name in scripts.Keys.Where(n => scene.IsMatch(n)).OrderBy(n => n, StringComparer.Ordinal)) + foreach (var name in provider.ScriptNames.Where(n => scene.IsMatch(n)).OrderBy(n => n, StringComparer.Ordinal)) { - var vm = new VirtualMachine(Sys4Loader.Load(scripts[name], table), table, new CaptureHost()); + var vm = new VirtualMachine(ScriptByName(name), table, new CaptureHost()); vm.Run(); trace[name] = new { offsets = vm.Emitted.Select(e => e.Offset).ToArray(), halt = vm.HaltReason, steps = vm.Steps }; } diff --git a/engine/Age.Engine.Tests/Sys4AssetStoreTests.cs b/engine/Age.Engine.Tests/Sys4AssetStoreTests.cs new file mode 100644 index 0000000..ce57f55 --- /dev/null +++ b/engine/Age.Engine.Tests/Sys4AssetStoreTests.cs @@ -0,0 +1,156 @@ +using System.Text.Json; +using Age.Engine.Sys4; +using Xunit; + +public class Sys4AssetStoreTests +{ + [Fact] + public void RuntimeCatalogMatchesDiagnosticCatalogAndSceneViews() + { + var catalog = Sys4AssetCatalog.Load(Paths.Sys4Ini); + using var index = JsonDocument.Parse(File.ReadAllText(Paths.AssetIndexJson)); + var expected = index.RootElement; + + Assert.Equal(expected.GetProperty("magic").GetString(), catalog.Magic); + Assert.Equal(expected.GetProperty("file_count").GetInt32(), catalog.RawSlots.Count); + Assert.Equal(expected.GetProperty("entry_count").GetInt32(), catalog.Files.Count); + Assert.Equal(13208, catalog.RawSlots.Count); + Assert.Equal(13206, catalog.Files.Count); + Assert.Equal(2, catalog.RawSlots.Count(r => r.IsPlaceholder)); + Assert.Equal(expected.GetProperty("archives").EnumerateArray().Select(a => a.GetString()), catalog.Archives); + + var jsonFiles = expected.GetProperty("files").EnumerateArray().ToArray(); + Assert.Equal(jsonFiles.Length, catalog.Files.Count); + for (int i = 0; i < jsonFiles.Length; i++) + { + var j = jsonFiles[i]; + var actual = catalog.Files[i]; + Assert.Equal(j.GetProperty("raw_index").GetInt32(), actual.RawIndex); + Assert.Equal(j.GetProperty("name").GetString(), actual.Name); + Assert.Equal(j.GetProperty("archive").GetString(), actual.Archive); + Assert.Equal(j.GetProperty("arc_id").GetInt32(), actual.ArchiveId); + Assert.Equal(j.GetProperty("file_number").GetInt32(), actual.FileNumber); + Assert.Equal(j.GetProperty("offset").GetInt64(), actual.Offset); + Assert.Equal(j.GetProperty("size").GetInt64(), actual.Size); + } + + Assert.Equal("SO001.AGF", catalog.ResolveRaw(0x337e)?.Name); + Assert.Equal("BGM005.OGG", catalog.ResolveName("bgm005.ogg")?.Name); + Assert.Null(catalog.ResolveRaw(-1)); + Assert.Null(catalog.ResolveRaw(catalog.RawSlots.Count)); + + using var sections = JsonDocument.Parse(File.ReadAllText(Paths.AssetSectionsJson)); + foreach (var scene in sections.RootElement.GetProperty("scene_base").EnumerateObject()) + { + int start = scene.Value.GetInt32(); + int end = start; + while (end + 1 < catalog.Files.Count + && catalog.Files[end + 1].FileNumber > catalog.Files[end].FileNumber) end++; + for (int i = start; i <= end; i++) + Assert.Same(catalog.Files[i], catalog.ResolveScene(scene.Name, i - start)); + Assert.Null(catalog.ResolveScene(scene.Name, -1)); + Assert.Null(catalog.ResolveScene(scene.Name, end - start + 1)); + } + } + + [Fact] + public void EveryCatalogRangeFitsAndRepresentativePayloadsMatchExtractedData() + { + var catalog = Sys4AssetCatalog.Load(Paths.Sys4Ini); + foreach (var entry in catalog.Files) + { + long archiveLength = new FileInfo(Path.Combine(Paths.GameDir, entry.Archive)).Length; + Assert.InRange(entry.Offset, 0, archiveLength); + Assert.InRange(entry.Size, 0, archiveLength - entry.Offset); + } + + var store = new Sys4AssetStore(catalog, Paths.GameDir); + var samples = catalog.Archives.SelectMany(archive => + { + var entries = catalog.Files.Where(e => e.Archive.Equals(archive, StringComparison.OrdinalIgnoreCase)).ToArray(); + return new[] { entries[0], entries[entries.Length / 2], entries[^1] }; + }).Concat(new[] + { + catalog.ResolveName("MENU.BIN")!, + catalog.ResolveName("SO001.AGF")!, + catalog.ResolveName("BGM005.OGG")!, + }).DistinctBy(e => e.RawIndex); + + foreach (var entry in samples) + { + string folder = Path.GetFileNameWithoutExtension(entry.Archive); + string extracted = Path.Combine(Paths.Extracted, folder, entry.Name); + Assert.True(File.Exists(extracted), $"missing extracted oracle: {folder}/{entry.Name}"); + Assert.Equal(File.ReadAllBytes(extracted), store.ReadAll(entry)); + } + } + + [Fact] + public void AllInstalledLooseScriptOverridesShadowArchiveCopies() + { + var catalog = Sys4AssetCatalog.Load(Paths.Sys4Ini); + var archiveOnly = new Sys4AssetStore(catalog, Paths.GameDir); + var looseFirst = new Sys4AssetStore(catalog, Paths.GameDir, Paths.GameDir); + var rootBins = Directory.EnumerateFiles(Paths.GameDir, "*.BIN") + .Where(path => catalog.ResolveName(Path.GetFileName(path)) is { } entry + && entry.Name.EndsWith(".BIN", StringComparison.OrdinalIgnoreCase)) + .OrderBy(Path.GetFileName, StringComparer.OrdinalIgnoreCase).ToArray(); + + // This installed v1.03 tree currently has 49 archive-backed overrides plus the two root-only + // engine catalogs SYS4AB.BIN/SYS4INI.BIN. Exercise every archive-backed override, not a sample. + Assert.Equal(49, rootBins.Length); + foreach (string path in rootBins) + { + var entry = catalog.ResolveName(Path.GetFileName(path))!; + byte[] loose = File.ReadAllBytes(path); + Assert.Equal(loose, looseFirst.ReadAll(entry)); + Assert.NotEqual(loose, archiveOnly.ReadAll(entry)); + } + } + + [Fact] + public async Task SyntheticStoreIsBoundedLooseFirstThreadSafeAndRejectsTraversal() + { + string temp = Path.Combine(Path.GetTempPath(), "age-vfs-" + Guid.NewGuid().ToString("N")); + string archives = Path.Combine(temp, "archives"), loose = Path.Combine(temp, "loose"); + Directory.CreateDirectory(archives); + Directory.CreateDirectory(loose); + try + { + File.WriteAllBytes(Path.Combine(archives, "DATA1.ALF"), new byte[] { 9, 8, 1, 2, 3, 7 }); + var catalog = Sys4AssetCatalog.Load(Paths.Sys4Ini); + var entry = new AssetEntry("TEST.BIN", "DATA1.ALF", 2, 3); + var store = new Sys4AssetStore(catalog, archives, loose); + + using (var stream = store.Open(entry)) + { + Assert.Equal(3, stream.Length); + Assert.Equal(new byte[] { 1, 2, 3 }, ReadToEnd(stream)); + Assert.Equal(-1, stream.ReadByte()); + Assert.Throws(() => stream.Seek(1, SeekOrigin.End)); + } + + File.WriteAllBytes(Path.Combine(loose, "TEST.BIN"), new byte[] { 4, 5 }); + Assert.Equal(new byte[] { 4, 5 }, store.ReadAll(entry)); + File.Delete(Path.Combine(loose, "TEST.BIN")); + Assert.Equal(new byte[] { 1, 2, 3 }, store.ReadAll(entry)); + + var reads = await Task.WhenAll(Enumerable.Range(0, 8).Select(_ => Task.Run(() => store.ReadAll(entry)))); + Assert.All(reads, bytes => Assert.Equal(new byte[] { 1, 2, 3 }, bytes)); + Assert.Throws(() => store.Open(entry with { Name = "../TEST.BIN" })); + Assert.Throws(() => store.Open(entry with { Archive = "../DATA1.ALF" })); + Assert.Throws(() => store.Open(entry with { Offset = 5, Size = 2 })); + } + finally + { + Directory.Delete(temp, recursive: true); + } + } + + private static byte[] ReadToEnd(Stream stream) + { + using var copy = new MemoryStream(); + stream.CopyTo(copy); + return copy.ToArray(); + } +} diff --git a/engine/Age.Engine.Tests/Sys4ScriptProviderTests.cs b/engine/Age.Engine.Tests/Sys4ScriptProviderTests.cs index b419f76..1b4e53d 100644 --- a/engine/Age.Engine.Tests/Sys4ScriptProviderTests.cs +++ b/engine/Age.Engine.Tests/Sys4ScriptProviderTests.cs @@ -17,4 +17,18 @@ public class Sys4ScriptProviderTests Assert.Same(additem, provider.GetById(0x1ab)); // cached: same instance Assert.Null(provider.GetById(long.MaxValue)); // unknown id } + + [Fact] + public void RootScriptLoadingUsesTheSameAssetStoreAndLoosePrecedence() + { + var table = OpcodeTableJson.Load(Paths.OpcodesJson); + var provider = Sys4ScriptProvider.Load(table); + var patched = provider.RequireByName("FIELD.BIN"); + var directLoose = Sys4Loader.Load(Path.Combine(Paths.GameDir, "FIELD.BIN"), table); + + Assert.Equal(directLoose.Instructions.Count, patched.Instructions.Count); + Assert.Same(patched, provider.RequireByName("field.bin")); + Assert.Null(provider.GetByName("../FIELD.BIN")); + Assert.Equal(481, provider.ScriptNames.Count); + } } diff --git a/engine/Age.Engine/Sys4/Paths.cs b/engine/Age.Engine/Sys4/Paths.cs index fda219b..1c4c357 100644 --- a/engine/Age.Engine/Sys4/Paths.cs +++ b/engine/Age.Engine/Sys4/Paths.cs @@ -12,6 +12,7 @@ public static class Paths public static string AssetIndexJson => Path.Combine(Build, "asset-index.json"); public static string CallscriptNamesJson => Path.Combine(Build, "callscript-names.json"); public static string Textures => Path.Combine(Build, "textures"); + public static string Sys4Ini => Path.Combine(GameDir, "SYS4INI.BIN"); private static string FindRepo() { diff --git a/engine/Age.Engine/Sys4/ResourceMap.cs b/engine/Age.Engine/Sys4/ResourceMap.cs index b638ca5..c2e2430 100644 --- a/engine/Age.Engine/Sys4/ResourceMap.cs +++ b/engine/Age.Engine/Sys4/ResourceMap.cs @@ -1,56 +1,23 @@ -using System.Text.Json; - namespace Age.Engine.Sys4; -/// One SYS4INI asset entry. -public sealed record AssetEntry(string Name, string Archive, long Offset, long Size); - /// -/// Static asset resolver. SYS4INI's file list is sectioned (one per scene: SCxxxx.BIN + its +/// Compatibility facade over the runtime SYS4 catalog. SYS4INI's file list is sectioned (one per scene: SCxxxx.BIN + its /// cross-archive asset manifest); file_number is the index within a section. So a bytecode /// resId resolves as files[section_base(scene) + resId] -- unified for graphics and audio. -/// See docs/asset-resolution-re.md. Built from build/asset-index.json + build/asset-sections.json. +/// See docs/asset-resolution-re.md. Extracted paths remain temporary graphics/audio backends only. /// public sealed class ResourceMap { - private readonly IReadOnlyList _files; - private readonly IReadOnlyDictionary _sceneBase; // "SC0000" -> section base index + private readonly Sys4AssetCatalog _catalog; - public ResourceMap(IReadOnlyList files, IReadOnlyDictionary sceneBase) - { - _files = files; - _sceneBase = sceneBase; - } + public ResourceMap(Sys4AssetCatalog catalog) => _catalog = catalog; - public static ResourceMap Load(string indexPath, string sectionsPath) - { - var files = new List(); - using (var idx = JsonDocument.Parse(File.ReadAllText(indexPath))) - foreach (var f in idx.RootElement.GetProperty("files").EnumerateArray()) - files.Add(new AssetEntry( - f.GetProperty("name").GetString()!, - f.GetProperty("archive").GetString()!, - f.GetProperty("offset").GetInt64(), - f.GetProperty("size").GetInt64())); - - var sceneBase = new Dictionary(StringComparer.OrdinalIgnoreCase); - using (var sec = JsonDocument.Parse(File.ReadAllText(sectionsPath))) - foreach (var p in sec.RootElement.GetProperty("scene_base").EnumerateObject()) - sceneBase[p.Name] = p.Value.GetInt32(); - - return new ResourceMap(files, sceneBase); - } - - public static ResourceMap Load() => Load(Paths.AssetIndexJson, Paths.AssetSectionsJson); + public static ResourceMap Load() => new(Sys4AssetCatalog.Load(Paths.Sys4Ini)); /// Resolve a scene-local resId to its asset, or null if out of range / unknown scene. public AssetEntry? Resolve(string scene, long resId) { - var key = scene.EndsWith(".BIN", StringComparison.OrdinalIgnoreCase) - ? scene[..^4] : scene; - if (!_sceneBase.TryGetValue(key, out var b)) return null; - long p = b + resId; - return p >= 0 && p < _files.Count ? _files[(int)p] : null; + return _catalog.ResolveScene(scene, resId); } /// Pre-converted BMP path for an AGF asset (see tools/convert_agf.py). @@ -70,10 +37,8 @@ public sealed class ResourceMap public string? BgmPathById(long id) { var name = $"BGM{id:D3}.OGG"; - foreach (var f in _files) - if (f.Name.Equals(name, StringComparison.OrdinalIgnoreCase)) - return AudioPath(f); - return null; + var f = _catalog.ResolveName(name); + return f == null ? null : AudioPath(f); } /// Loose extracted OGG/WAV path for an audio asset (extracted/DATA{n}/{name}), or null. diff --git a/engine/Age.Engine/Sys4/Sys4AssetCatalog.cs b/engine/Age.Engine/Sys4/Sys4AssetCatalog.cs new file mode 100644 index 0000000..06a535e --- /dev/null +++ b/engine/Age.Engine/Sys4/Sys4AssetCatalog.cs @@ -0,0 +1,185 @@ +using System.Buffers.Binary; +using System.Text; + +namespace Age.Engine.Sys4; + +/// One raw SYS4INI file record. Placeholder records remain addressable by +/// but are excluded from scene and name views. +public sealed record AssetEntry( + string Name, + string Archive, + long Offset, + long Size, + int RawIndex = -1, + int ArchiveId = -1, + int FileNumber = -1, + bool IsPlaceholder = false); + +/// Runtime parser and lookup views for a base S4IC SYS4INI catalog. +public sealed class Sys4AssetCatalog +{ + private const int PackedSizeOffset = 0x134; + private const int ExpandedSizeOffset = 0x12c; + private const int ArchiveNameSize = 256; + private const int RecordSize = 80; + + private readonly Dictionary _byName; + private readonly Dictionary _sceneRanges; + + public string Magic { get; } + public IReadOnlyList Archives { get; } + public IReadOnlyList RawSlots { get; } + public IReadOnlyList Files { get; } + + private Sys4AssetCatalog(string magic, List archives, List rawSlots) + { + Magic = magic; + Archives = archives; + RawSlots = rawSlots; + Files = rawSlots.Where(r => !r.IsPlaceholder).ToArray(); + _byName = Files.ToDictionary(r => r.Name, StringComparer.OrdinalIgnoreCase); + _sceneRanges = BuildSceneRanges(Files); + } + + public static Sys4AssetCatalog Load(string path) => Parse(File.ReadAllBytes(path), Path.GetFileName(path)); + + public static Sys4AssetCatalog Parse(byte[] data, string name = "SYS4INI.BIN") + { + if (data.Length < PackedSizeOffset + 4 || !data.AsSpan(0, 4).SequenceEqual("S4IC"u8)) + throw new InvalidDataException($"{name}: expected an S4IC catalog"); + + uint expandedSize = BinaryPrimitives.ReadUInt32LittleEndian(data.AsSpan(ExpandedSizeOffset, 4)); + uint packedSize = BinaryPrimitives.ReadUInt32LittleEndian(data.AsSpan(PackedSizeOffset, 4)); + if (packedSize > data.Length - (PackedSizeOffset + 4)) + throw new InvalidDataException($"{name}: packed directory is truncated"); + if (expandedSize == 0 || expandedSize > int.MaxValue) + throw new InvalidDataException($"{name}: invalid expanded size {expandedSize}"); + + var blob = DecompressLzss(data.AsSpan(PackedSizeOffset + 4, checked((int)packedSize)), + checked((int)expandedSize), name); + int p = 0; + uint ReadU32() + { + if (p > blob.Length - 4) throw new InvalidDataException($"{name}: directory is truncated"); + uint value = BinaryPrimitives.ReadUInt32LittleEndian(blob.AsSpan(p, 4)); + p += 4; + return value; + } + + uint archiveCount = ReadU32(); + if (archiveCount is 0 or >= 0x1000 || archiveCount > (blob.Length - p) / ArchiveNameSize) + throw new InvalidDataException($"{name}: invalid archive count {archiveCount}"); + var archives = new List(checked((int)archiveCount)); + for (int i = 0; i < archiveCount; i++, p += ArchiveNameSize) + archives.Add(ReadCString(blob.AsSpan(p, ArchiveNameSize))); + + uint fileCount = ReadU32(); + if (fileCount is 0 or >= 0x400000 || fileCount > (blob.Length - p) / RecordSize) + throw new InvalidDataException($"{name}: invalid file count {fileCount}"); + var slots = new List(checked((int)fileCount)); + for (int i = 0; i < fileCount; i++, p += RecordSize) + { + var row = blob.AsSpan(p, RecordSize); + string fileName = ReadCString(row[..64]); + int archiveId = checked((int)BinaryPrimitives.ReadUInt32LittleEndian(row.Slice(64, 4))); + int fileNumber = checked((int)BinaryPrimitives.ReadUInt32LittleEndian(row.Slice(68, 4))); + long offset = BinaryPrimitives.ReadUInt32LittleEndian(row.Slice(72, 4)); + long size = BinaryPrimitives.ReadUInt32LittleEndian(row.Slice(76, 4)); + string archive = archiveId >= 0 && archiveId < archives.Count ? archives[archiveId] : ""; + bool placeholder = fileName is "" or "@"; + slots.Add(new AssetEntry(fileName, archive, offset, size, i, archiveId, fileNumber, placeholder)); + } + + string magic = ReadCString(data.AsSpan(0, Math.Min(8, data.Length))); + return new Sys4AssetCatalog(magic, archives, slots); + } + + /// Universal raw-id lookup. Placeholder slots are returned, not collapsed. + public AssetEntry? ResolveRaw(long rawId) + => rawId >= 0 && rawId < RawSlots.Count ? RawSlots[(int)rawId] : null; + + /// Case-insensitive exact-name lookup over real records. + public AssetEntry? ResolveName(string name) + => _byName.TryGetValue(Path.GetFileName(name), out var entry) && Path.GetFileName(name) == name + ? entry : null; + + /// Resolve within the owning scene section; ids cannot spill into the next section. + public AssetEntry? ResolveScene(string scene, long localId) + { + string key = Path.GetFileNameWithoutExtension(scene); + if (!_sceneRanges.TryGetValue(key, out var range)) return null; + long pos = range.Start + localId; + return localId >= 0 && pos <= range.End ? Files[(int)pos] : null; + } + + public IReadOnlyList ScriptNames => Files + .Where(f => f.Name.EndsWith(".BIN", StringComparison.OrdinalIgnoreCase)) + .Select(f => f.Name.ToUpperInvariant()).ToArray(); + + private static Dictionary BuildSceneRanges(IReadOnlyList files) + { + var ranges = new Dictionary(StringComparer.OrdinalIgnoreCase); + int start = 0; + for (int i = 1; i <= files.Count; i++) + { + bool end = i == files.Count || files[i].FileNumber <= files[i - 1].FileNumber; + if (!end) continue; + for (int k = start; k < i; k++) + if (files[k].Name.Length == 10 && files[k].Name.StartsWith("SC", StringComparison.OrdinalIgnoreCase) + && files[k].Name.EndsWith(".BIN", StringComparison.OrdinalIgnoreCase) + && files[k].Name.AsSpan(2, 4).ToString().All(char.IsDigit)) + { + ranges[Path.GetFileNameWithoutExtension(files[k].Name)] = (start, i - 1); + break; + } + start = i; + } + return ranges; + } + + private static string ReadCString(ReadOnlySpan bytes) + { + int zero = bytes.IndexOf((byte)0); + if (zero >= 0) bytes = bytes[..zero]; + Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); + return Encoding.GetEncoding(932).GetString(bytes); + } + + private static byte[] DecompressLzss(ReadOnlySpan source, int expectedSize, string name) + { + var frame = new byte[0x1000]; + int framePos = 0xfee, input = 0, output = 0; + var result = new byte[expectedSize]; + while (output < expectedSize) + { + if (input >= source.Length) throw new InvalidDataException($"{name}: LZSS stream ended early"); + int control = source[input++]; + for (int bit = 1; bit <= 0x80 && output < expectedSize; bit <<= 1) + { + if ((control & bit) != 0) + { + if (input >= source.Length) throw new InvalidDataException($"{name}: truncated LZSS literal"); + byte value = source[input++]; + result[output++] = value; + frame[framePos] = value; + framePos = (framePos + 1) & 0xfff; + } + else + { + if (input > source.Length - 2) throw new InvalidDataException($"{name}: truncated LZSS back-reference"); + int lo = source[input++], hi = source[input++]; + int readPos = ((hi & 0xf0) << 4) | lo; + int length = 3 + (hi & 0x0f); + for (int j = 0; j < length && output < expectedSize; j++) + { + byte value = frame[readPos++ & 0xfff]; + result[output++] = value; + frame[framePos] = value; + framePos = (framePos + 1) & 0xfff; + } + } + } + } + return result; + } +} diff --git a/engine/Age.Engine/Sys4/Sys4AssetStore.cs b/engine/Age.Engine/Sys4/Sys4AssetStore.cs new file mode 100644 index 0000000..980517b --- /dev/null +++ b/engine/Age.Engine/Sys4/Sys4AssetStore.cs @@ -0,0 +1,129 @@ +namespace Age.Engine.Sys4; + +/// Read-only byte seam after catalog resolution. +public interface IAssetStore +{ + Stream Open(AssetEntry entry); + byte[] ReadAll(AssetEntry entry); +} + +/// Native base-game precedence: exact-basename loose roots first, indexed ALF range second. +public sealed class Sys4AssetStore : IAssetStore +{ + private readonly string _archiveRoot; + private readonly string[] _looseRoots; + + public Sys4AssetCatalog Catalog { get; } + + public Sys4AssetStore(Sys4AssetCatalog catalog, string archiveRoot, params string[] looseRoots) + { + Catalog = catalog; + _archiveRoot = Path.GetFullPath(archiveRoot); + _looseRoots = looseRoots.Select(Path.GetFullPath).ToArray(); + } + + public Stream Open(AssetEntry entry) + { + ValidateBasename(entry.Name, "asset"); + if (entry.IsPlaceholder) throw new FileNotFoundException("SYS4INI placeholder has no payload", entry.Name); + + foreach (string root in _looseRoots) + { + string candidate = Path.GetFullPath(Path.Combine(root, entry.Name)); + if (!IsDirectChild(root, candidate)) throw new InvalidDataException($"unsafe asset name: {entry.Name}"); + try + { + return new FileStream(candidate, FileMode.Open, FileAccess.Read, FileShare.Read, + 64 * 1024, FileOptions.RandomAccess); + } + catch (FileNotFoundException) { } + catch (DirectoryNotFoundException) { } + } + + ValidateBasename(entry.Archive, "archive"); + string archivePath = Path.GetFullPath(Path.Combine(_archiveRoot, entry.Archive)); + if (!IsDirectChild(_archiveRoot, archivePath)) + throw new InvalidDataException($"unsafe archive name: {entry.Archive}"); + var file = new FileStream(archivePath, FileMode.Open, FileAccess.Read, FileShare.Read, + 64 * 1024, FileOptions.RandomAccess); + try + { + if (entry.Offset < 0 || entry.Size < 0 || entry.Offset > file.Length + || entry.Size > file.Length - entry.Offset) + throw new InvalidDataException($"{entry.Name}: ALF range {entry.Offset}+{entry.Size} exceeds {entry.Archive} ({file.Length})"); + return new BoundedReadStream(file, entry.Offset, entry.Size); + } + catch + { + file.Dispose(); + throw; + } + } + + public byte[] ReadAll(AssetEntry entry) + { + using Stream stream = Open(entry); + if (stream.Length > int.MaxValue) throw new InvalidDataException($"{entry.Name}: payload is too large"); + var bytes = new byte[checked((int)stream.Length)]; + stream.ReadExactly(bytes); + return bytes; + } + + private static void ValidateBasename(string value, string kind) + { + if (string.IsNullOrWhiteSpace(value) || Path.IsPathRooted(value) + || value.Contains('/') || value.Contains('\\') || value is "." or "..") + throw new InvalidDataException($"unsafe {kind} name: {value}"); + } + + private static bool IsDirectChild(string root, string child) + => string.Equals(Path.GetDirectoryName(child)?.TrimEnd(Path.DirectorySeparatorChar), + root.TrimEnd(Path.DirectorySeparatorChar), StringComparison.OrdinalIgnoreCase); + + private sealed class BoundedReadStream : Stream + { + private readonly FileStream _file; + private readonly long _start; + private readonly long _length; + private long _position; + + public BoundedReadStream(FileStream file, long start, long length) + { + _file = file; _start = start; _length = length; + _file.Position = start; + } + + public override bool CanRead => true; + public override bool CanSeek => true; + public override bool CanWrite => false; + public override long Length => _length; + public override long Position { get => _position; set => Seek(value, SeekOrigin.Begin); } + public override void Flush() { } + public override int Read(byte[] buffer, int offset, int count) + => Read(buffer.AsSpan(offset, count)); + public override int Read(Span buffer) + { + int wanted = (int)Math.Min(buffer.Length, _length - _position); + if (wanted <= 0) return 0; + int read = _file.Read(buffer[..wanted]); + _position += read; + return read; + } + public override long Seek(long offset, SeekOrigin origin) + { + long target = origin switch + { + SeekOrigin.Begin => offset, + SeekOrigin.Current => checked(_position + offset), + SeekOrigin.End => checked(_length + offset), + _ => throw new ArgumentOutOfRangeException(nameof(origin)), + }; + if (target < 0 || target > _length) throw new IOException("seek outside asset range"); + _file.Position = _start + target; + return _position = target; + } + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + protected override void Dispose(bool disposing) { if (disposing) _file.Dispose(); base.Dispose(disposing); } + } +} diff --git a/engine/Age.Engine/Sys4/Sys4ScriptProvider.cs b/engine/Age.Engine/Sys4/Sys4ScriptProvider.cs index 9ef9990..32dba5a 100644 --- a/engine/Age.Engine/Sys4/Sys4ScriptProvider.cs +++ b/engine/Age.Engine/Sys4/Sys4ScriptProvider.cs @@ -1,39 +1,54 @@ -using System.Text.Json; using Age.Engine.Hosting; using Age.Engine.Model; + namespace Age.Engine.Sys4; -/// Resolves call-script ids (raw SYS4INI file indices) to loaded scripts, using -/// build/callscript-names.json (id→name) + Paths.Scripts() (name→path). Cached per id. -/// The native resolver prefers a loose override before the archive; Paths.Scripts() already -/// shadows extracted/DATA1 with root overrides, so that behavior is preserved. +/// Loads root and call-script bytecode through the native loose-first asset-store seam. public sealed class Sys4ScriptProvider : IScriptProvider { private readonly OpcodeTable _table; - private readonly IReadOnlyDictionary _idToName; - private readonly Dictionary _byName; // NAME(UPPER) -> path + private readonly IAssetStore _store; private readonly Dictionary _cache = new(); + private readonly Dictionary _nameCache = new(StringComparer.OrdinalIgnoreCase); - public Sys4ScriptProvider(OpcodeTable table, IReadOnlyDictionary idToName, - Dictionary byName) - { _table = table; _idToName = idToName; _byName = byName; } + public Sys4AssetCatalog Catalog { get; } + public IReadOnlyList ScriptNames => Catalog.ScriptNames; + + public Sys4ScriptProvider(OpcodeTable table, Sys4AssetCatalog catalog, IAssetStore store) + { _table = table; Catalog = catalog; _store = store; } public static Sys4ScriptProvider Load(OpcodeTable table) { - var raw = JsonSerializer.Deserialize>( - File.ReadAllText(Paths.CallscriptNamesJson)) ?? new(); - var idToName = raw.ToDictionary(kv => long.Parse(kv.Key), kv => kv.Value); - return new Sys4ScriptProvider(table, idToName, Paths.Scripts()); + var catalog = Sys4AssetCatalog.Load(Paths.Sys4Ini); + return new Sys4ScriptProvider(table, catalog, + new Sys4AssetStore(catalog, Paths.GameDir, Paths.GameDir)); } public Script? GetById(long id) { if (_cache.TryGetValue(id, out var cached)) return cached; - Script? s = null; - if (_idToName.TryGetValue(id, out var name) && - _byName.TryGetValue(name.ToUpperInvariant(), out var path)) - s = Sys4Loader.Load(path, _table); - _cache[id] = s; - return s; + var entry = Catalog.ResolveRaw(id); + Script? script = entry is { IsPlaceholder: false } + && entry.Name.EndsWith(".BIN", StringComparison.OrdinalIgnoreCase) + ? Parse(entry) : null; + _cache[id] = script; + return script; } + + public Script? GetByName(string name) + { + string key = name; + if (_nameCache.TryGetValue(key, out var cached)) return cached; + var entry = Catalog.ResolveName(key); + Script? script = entry != null && entry.Name.EndsWith(".BIN", StringComparison.OrdinalIgnoreCase) + ? Parse(entry) : null; + _nameCache[key] = script; + return script; + } + + public Script RequireByName(string name) + => GetByName(name) ?? throw new FileNotFoundException($"script is not in SYS4INI: {name}", name); + + private Script Parse(AssetEntry entry) + => Sys4Loader.Parse(_store.ReadAll(entry), _table, entry.Name); } diff --git a/godot/Main.cs b/godot/Main.cs index 8c440d0..2c4b7a3 100644 --- a/godot/Main.cs +++ b/godot/Main.cs @@ -150,10 +150,11 @@ public partial class Main : Godot.Control // runs a SYNTHESIZED scene (not a real scene in a crippled mode) so its output is deterministic. Script script; IScriptProvider provider; + Sys4ScriptProvider? scripts = null; if (_selftest) (script, provider) = BuildSelfTestScene(table); - else { script = Sys4Loader.Load(Paths.Scripts()[scene.ToUpperInvariant() + ".BIN"], table); provider = Sys4ScriptProvider.Load(table); } + else { scripts = Sys4ScriptProvider.Load(table); script = scripts.RequireByName(scene + ".BIN"); provider = scripts; } if (_timelineLogPath != null) _timeline = new GodotTimelineLog(_timelineLogPath); - _host = new GodotAdvHost(this, ResourceMap.Load(), scene, _clock, _timeline) { SleepScale = sleepScale, TraceOps = _gfxLogPath != null }; + _host = new GodotAdvHost(this, scripts != null ? new ResourceMap(scripts.Catalog) : ResourceMap.Load(), scene, _clock, _timeline) { SleepScale = sleepScale, TraceOps = _gfxLogPath != null }; _trace = new GodotTraceSink(_timeline); // --trace-histogram: aggregate op/call-site execution counts of the REAL Godot run (headless flow // diverges — wait-for-input is a no-op there — so this is the only way to profile the live path). @@ -169,7 +170,7 @@ public partial class Main : Godot.Control { var session = new GameSession(); foreach (var b in new[] { "INITCONFIG.BIN", "INIT2.BIN", "INIT.BIN" }) - session.RunScene(Sys4Loader.Load(Paths.Scripts()[b], table), table, new CaptureHost(), null, provider); + session.RunScene(scripts!.RequireByName(b), table, new CaptureHost(), null, provider); foreach (var kv in session.Globals) _vm.Globals[kv.Key] = kv.Value; foreach (var kv in session.GlobalStrings) _vm.GlobalStrings[kv.Key] = kv.Value; GD.Print($"[boot] system boot done: {session.Globals.Count} globals seeded");