From cbf98642dc477b6c9967da68a371940853a78978 Mon Sep 17 00:00:00 2001 From: gamer147 Date: Thu, 23 Jul 2026 13:41:35 -0400 Subject: [PATCH] Decode RTINIT routine step banks --- docs/global-reference.md | 41 ++--- docs/name-resolution.md | 36 ++++- docs/phase-b-framework.md | 7 +- docs/script-inventory.md | 9 +- docs/tools-reference.md | 6 +- docs/vm-mapping-plan.md | 5 +- tools/extract_init.py | 253 ++++++++++++++++++++++++++++++- tools/init_table_profile.py | 57 ++++++- tools/test_extract_init.py | 67 +++++++- tools/test_init_table_profile.py | 38 +++++ vm-map/globals.toml | 232 +++++++++++++++++++++++++++- 11 files changed, 702 insertions(+), 49 deletions(-) diff --git a/docs/global-reference.md b/docs/global-reference.md index ec8571c..733e681 100644 --- a/docs/global-reference.md +++ b/docs/global-reference.md @@ -1,7 +1,7 @@ # Global Variable Reference (generated) -5036 globals (212 curated, 4824 auto shape-inferred). Source of truth: `vm-map/globals.toml`. +5037 globals (231 curated, 4806 auto shape-inferred). Source of truth: `vm-map/globals.toml`. ## choice-output @@ -37,7 +37,11 @@ | `0x74e8` | unit_skill_ids | high | investigation | Persistent four-skill row for each playable unit. CALCCC copies positive class_change_skill_awards into the first three slots after a promotion; ADDEXP compares the before/after row to report learned or replaced skills. Columns: 0=skill_slot_1, 1=skill_slot_2, 2=skill_slot_3, 3=skill_slot_4. | | `0x7684` | stage_clear_state | high | investigation | Persistent per-stage completion state indexed by stage id. STAGECLEAR sets the current stage cell to 1, while FORT, SELSTAGE, and FIELD use zero versus one to distinguish an uncleared mission from a cleared replay. | | `0x141a4` | stage_object_runtime_state | high | investigation | Per-stage, per-object-slot runtime state. FIELD initializes mode-1 objects from STINIT's first tagged payload on a fresh stage and updates capturable-object ownership; DRAWOBJ uses it as the vertical sprite-row index. Other consumers interpret it by object type. | +| `0x53517` | entity_movement_routine_set_ids | high | investigation | Per-runtime-entity movement routine set for easy/normal/hard. SETEN copies a stage-specific override or the EBINIT default into this row; MVRTN selects the current difficulty column and stores it in current_routine_set_id. | +| `0x535ad` | entity_battle_routine_set_ids | high | investigation | Per-runtime-entity battle routine set for easy/normal/hard. SETEN copies a stage-specific override or the EBINIT default into this row; BTRTN selects the current difficulty column and stores it in current_routine_set_id. | +| `0x53643` | entity_movement_routine_progress | high | investigation | Per-runtime-entity movement-routine progress counters. FIELD clears all fifty rows, movement providers increment the current step, and MVRTN compares it with movement_routine_minimum_progress_counts before enabling a step. | | `0x5660b` | skill_info_revealed_flags | high | investigation | Persistent skill-information visibility flags. ADDSKILL sets the selected skill, BTL marks every equipped skill when it is observed in combat, and INFOIT suppresses a skill's icon/handler-driven details until this flag is nonzero. This is broader than skill_acquired_flags. | +| `0x56738` | entity_battle_routine_random_rolls | high | investigation | Per-runtime-entity battle-step random rolls. FIELD and BTL fill every active entity's twenty cells with random-modulo-100; BTRTN executes a step when the matching roll is below battle_routine_activation_percents. | | `0x57357` | unit_deployment_cost_adjustments | high | investigation | Persistent per-unit delta added to unit_deployment_cost_base. CALCCC increments it by class_change_deployment_cost_delta, while SETCH, SETEN, ADDEXP, and deployment scripts include it when constructing or updating runtime command cost. | | `0x573bb` | unit_class_change_state | high | investigation | Persistent ten-slot class-change state for each playable unit. CALCCC copies the current unit's row to class_change_state_work before invoking the rule scripts and persists the updated row; CCINIT requires the rule's slot to be zero so each promotion is applied once. Columns: 0=promotion_slot_1, 1=promotion_slot_2, 2=promotion_slot_3, 3=promotion_slot_4, 4=promotion_slot_5, 5=promotion_slot_6, 6=promotion_slot_7, 7=promotion_slot_8, 8=promotion_slot_9, 9=promotion_slot_10. | | `0x66716` | unit_voice_asset_ids | high | investigation | EBINIT per-unit voice bank for 116 voiced characters and variants. FIELD directly selects column 0 before WARPD, column 1 when an acting unit takes chest/treasure contents, and column 3 after occupying, losing, or sealing a stage objective. BTL selects the acting unit's columns 7..9 for ordinary attacks, 10 for an ordinary critical, odd columns 11/13/15 for skill uses, and even columns 12/14/16 for critical skills; the three variants follow its six-step exchange selector. If the target survives positive damage, BTL selects target columns 19/20/21 with 60/30/10 percent weights; column 22 belongs to a target reduced to zero HP, while actor column 23 is selected for a finishing blow. SHOWGROW selects column 24 after level gain. Populated columns 4..6 and 17..18 have no reachable selector in the shipped script corpus and are retained as explicit unused authoring slots; columns 17 and 18 duplicate columns 15 and 16 in all 116 populated rows. Columns: 0=warp, 1=treasure_capture, 3=objective_interaction, 4=unused_slot_4, 5=unused_slot_5, 6=unused_slot_6, 7=normal_attack_1, 8=normal_attack_2, 9=normal_attack_3, 10=critical_normal_attack, 11=skill_use_1, 12=critical_skill_1, 13=skill_use_2, 14=critical_skill_2, 15=skill_use_3, 16=critical_skill_3, 17=unused_slot_17, 18=unused_slot_18, 19=damage_reaction_1, 20=damage_reaction_2, 21=damage_reaction_3, 22=defeated, 23=finishing_blow, 24=level_up. | @@ -145,6 +149,20 @@ | `0xe78e3` | stage_enemy_battle_routine_set_ids | high | investigation | Optional three-column battle-AI routine override per stage enemy. SETEN copies a populated row into the runtime unit or falls back to the unit definition, and BTRTN selects the difficulty_index column as its battle routine table row. Columns: 0=difficulty_0, 1=difficulty_1, 2=difficulty_2. | | `0xe793d` | stage_enemy_required_story_flags | high | investigation | Seven positive prerequisites per stage enemy. FIELD subtracts one from every populated id and suppresses the spawn unless the corresponding story_event_flags cell equals 1. Columns: 0=required_flag_1, 1=required_flag_2, 2=required_flag_3, 3=required_flag_4, 4=required_flag_5, 5=required_flag_6, 6=required_flag_7. | | `0xe7a0f` | stage_enemy_forbidden_story_flags | high | investigation | Five negative prerequisites per stage enemy. FIELD subtracts one from every populated id and suppresses the spawn when the corresponding story_event_flags cell equals 1. Columns: 0=forbidden_flag_1, 1=forbidden_flag_2, 2=forbidden_flag_3, 3=forbidden_flag_4, 4=forbidden_flag_5. | +| `0xeff78` | movement_routine_provider_selectors | high | investigation | RTINIT movement bank 0. MVRTN indexes it by current_routine_set_id and routine_step_index, resolves the selector through its RTN_M001..018/051..053/061 provider table, and call-scripts the selected movement routine. | +| `0xf4d98` | movement_routine_activation_percents | high | investigation | RTINIT movement bank 1. After all other step gates pass, MVRTN executes the provider only when random-modulo-100 is below this value. | +| `0xf9bb8` | movement_routine_parameter_1 | high | investigation | RTINIT movement bank 2. RTN_M004..015 providers consume the current row/step value; its exact meaning is tagged by movement_routine_provider_selectors and remains provider-specific. | +| `0xfe9d8` | movement_routine_parameter_2 | high | investigation | RTINIT movement bank 3. RTN_M005/007/010/011/012 consume it as a provider-specific parameter. | +| `0x1037f8` | movement_routine_parameter_3 | high | investigation | RTINIT movement bank 4. RTN_M011 consumes it as a provider-specific parameter. | +| `0x108618` | movement_routine_parameter_4 | high | investigation | RTINIT movement bank 5. RTN_M011 consumes it as a provider-specific parameter. | +| `0x112258` | movement_routine_minimum_progress_counts | high | investigation | RTINIT movement bank 7. MVRTN requires the current entity's matching movement-step progress counter to reach this value; movement providers increment those counters as their steps execute. | +| `0x117078` | movement_routine_required_story_flag_ids | high | investigation | RTINIT movement bank 8. MVRTN subtracts one and rejects the step when the referenced story flag is not set. | +| `0x11be98` | movement_routine_forbidden_story_flag_ids | high | investigation | RTINIT movement bank 9. MVRTN subtracts one and rejects the step when the referenced story flag is set. | +| `0x120cb8` | battle_routine_provider_selectors | high | investigation | RTINIT battle bank 10. BTRTN indexes it by current_routine_set_id and routine_step_index, resolves the selector through RTN_B001..004, and call-scripts the selected battle routine. | +| `0x125ad8` | battle_routine_activation_percents | high | investigation | RTINIT battle bank 11. BTRTN executes a candidate step only when the current entity's matching random-modulo-100 battle-step roll is below this value. | +| `0x12a8f8` | battle_routine_parameter_1 | high | investigation | RTINIT battle bank 12. RTN_B004 uses it to choose an entry from the prepared battle-action candidate table. | +| `0x147db8` | battle_routine_required_story_flag_ids | high | investigation | RTINIT battle bank 18. BTRTN subtracts one and rejects the step when the referenced story flag is not set. | +| `0x14cbd8` | battle_routine_forbidden_story_flag_ids | high | investigation | RTINIT battle bank 19. BTRTN subtracts one and rejects the step when the referenced story flag is set. | | `0x152877` | battle_entity_indices | high | investigation | The two runtime entity rows participating in BTL. battle_actor_side_index and battle_target_side_index select these cells before BTL and CALCDMG access HP, unit definitions, skills, animation state, and voice banks. Columns: 0=side_0, 1=side_1. | | `0x152879` | battle_selected_skill_ids | high | investigation | Selected skill for each battle side. BTL displays a positive entry through the skill-name table and selects skill-use voice columns; CALCDMG uses the same id for skill parameters. A zero entry follows the equipped/default ordinary-attack path. Columns: 0=side_0, 1=side_1. | | `0x5` | — | low | auto-shape | array | @@ -372,7 +390,8 @@ | `0x66715` | current_unit_id | high | investigation | Shared current-unit selector used by character growth and setup scripts. CCINIT keys every class-change rule on this value; CALCCC, ADDEXP, SETEN, SALLY, and related scripts use it to index unit definitions and persistent per-unit state. | | `0x8c877` | current_item_id | high | investigation | Shared item-id argument/selection slot. Item menus and gameplay scripts write a chosen item id, use it to index ITINIT arrays, and dispatch through item_handler_script_id; ITMES compares it against all 287 item ids to select the matching player-facing title and description. | | `0xa6e59` | current_skill_id | high | investigation | Shared skill-id argument/selection slot. Skill menus and combat scripts write the chosen skill id and use it to index SKINIT arrays; SKMES compares it against all 131 skill ids to select the matching player-facing title and description. | -| `0xeff75` | current_entity_index_hi | med | inference | High-purity current-entity row index (purity 0.95 in the auto shape map); dominant 2D-table row selector. | +| `0xeff75` | current_routine_set_id | high | investigation | Shared RTINIT row selector. MVRTN loads the current entity's difficulty-selected movement routine set; BTRTN loads its battle routine set. Both then iterate routine_step_index across the selected twenty-slot row. | +| `0xeff76` | routine_step_index | high | investigation | Shared RTINIT step selector. MVRTN and BTRTN iterate it from zero through nineteen and use it as the column index in every routine bank and matching per-entity runtime row. | | `0x152616` | current_entity_index | med | investigation | Primary current-entity row index (RECOVER-confirmed; purity 0.51, 363 row-index uses). | | `0x152617` | current_stage_enemy_spawn_slot | high | investigation | Current STINIT enemy-template slot. FIELD selects slots 1..29 while materializing stage units, SETEN records the slot on the runtime entity and reads every parallel enemy buffer through it, and ADDEN uses slot 0 for its special generated unit. | | `0x15261a` | battle_actor_side_index | high | investigation | Index of the acting side in BTL's two-entry battle arrays. CALCDMG derives battle_target_side_index as 1 minus this value; BTL uses the actor's unit-definition row for attack, skill, critical, and finishing-blow voices. | @@ -3686,16 +3705,12 @@ | `0x52383` | — | med | auto-shape | record-table[stride 30] | | `0x5295f` | — | med | auto-shape | record-table[stride 30] | | `0x52f3b` | — | med | auto-shape | record-table[stride 30] | -| `0x53517` | — | med | auto-shape | record-table[stride 3] | -| `0x535ad` | — | med | auto-shape | record-table[stride 3] | -| `0x53643` | — | med | auto-shape | record-table[stride 20] | | `0x53a2b` | — | med | auto-shape | record-table[stride 20] | | `0x53e13` | — | med | auto-shape | record-table[stride 2] | | `0x53e77` | — | med | auto-shape | record-table[stride 2] | | `0x53edd` | — | low | auto-shape | index/counter? | | `0x53ef5` | — | low | auto-shape | index/counter? | | `0x53ef7` | — | med | auto-shape | record-table[stride 8] | -| `0x56738` | — | med | auto-shape | record-table[stride 20] | | `0x57356` | — | med | auto-shape | current-entity-index? | | `0x62436` | — | low | auto-shape | index/counter? | | `0x62450` | — | low | auto-shape | index/counter? | @@ -5035,20 +5050,6 @@ | `0xea1b5` | — | med | auto-shape | record-table[stride 7] | | `0xee035` | — | med | auto-shape | record-table[stride 3] | | `0xeefd5` | — | med | auto-shape | record-table[stride 3] | -| `0xeff78` | — | med | auto-shape | record-table[stride 20] | -| `0xf4d98` | — | med | auto-shape | record-table[stride 20] | -| `0xf9bb8` | — | med | auto-shape | record-table[stride 20] | -| `0xfe9d8` | — | med | auto-shape | record-table[stride 20] | -| `0x1037f8` | — | med | auto-shape | record-table[stride 20] | -| `0x108618` | — | med | auto-shape | record-table[stride 20] | -| `0x112258` | — | med | auto-shape | record-table[stride 20] | -| `0x117078` | — | med | auto-shape | record-table[stride 20] | -| `0x11be98` | — | med | auto-shape | record-table[stride 20] | -| `0x120cb8` | — | med | auto-shape | record-table[stride 20] | -| `0x125ad8` | — | med | auto-shape | record-table[stride 20] | -| `0x12a8f8` | — | med | auto-shape | record-table[stride 20] | -| `0x147db8` | — | med | auto-shape | record-table[stride 20] | -| `0x14cbd8` | — | med | auto-shape | record-table[stride 20] | | `0x1519f8` | — | med | auto-shape | current-entity-index? | | `0x151a5d` | — | med | auto-shape | record-table[stride 3] | | `0x151b89` | — | med | auto-shape | record-table[stride 3] | diff --git a/docs/name-resolution.md b/docs/name-resolution.md index 6503716..a46ad30 100644 --- a/docs/name-resolution.md +++ b/docs/name-resolution.md @@ -418,6 +418,32 @@ are retained as legacy/stale authoring metadata. `extract_init.py` detects this resolved script name, authored chapter, SCJUMP chapters, overwrite history, and raw column keys. `field_semantics`/`semantic_fields` resolves those raw keys through `vm-map/globals.toml`. +### RTINIT movement/battle routine banks (2026-07-23) + +RTINIT is a banked sparse program registry, not the generic numeric extractor's former 14-record result. +Its 3,336 static writes address twenty parallel banks separated by exactly 20,000 cells. Direct +`lookup-array-2d` consumers establish that every bank is `int[1000][20]`: `current_routine_set_id` +selects a one-based row and `routine_step_index` iterates slots 0 through 19. Rows 1..176 are populated +except 150..153, yielding 172 routine sets and 3,307 final cells. Twenty-nine cells are written twice; +eleven of those overwrites change the value, all retained in source order. + +Banks 0..9 form the movement family. MVRTN reads bank 0 as a provider selector, resolves it through +RTN_M001..018/051..053/061, applies bank 1 as a random-modulo-100 activation percentage, gates the step +on a per-entity progress count plus required/forbidden story flags in banks 7..9, and call-scripts the +provider. Banks 2..5 are provider-tagged parameters and bank 6 is reserved/empty. This produces 1,043 +final movement steps. + +Banks 10..19 form the battle family. BTRTN dispatches selectors 1..4 to RTN_B001..004, compares each +entity's pre-rolled 0..99 step value against bank 11's activation percentage, and applies the same +required/forbidden story-flag convention in banks 18/19. Bank 12 is an RTN_B004 parameter; banks 13..17 +are reserved/empty. Only fourteen shipped battle steps are populated. + +`extract_init.py` detects this seventh shape as `banked` and writes `build/data/RTINIT.json`. Each record +keeps its raw `base/20/slot` fields and complete offset-tagged assignment history while adding joined +`movement_steps`/`battle_steps` with provider script names. The bank layout explicitly includes all six +empty reserved banks. Structural and consumer-proven meanings live in `vm-map/globals.toml`; the generic +parameter names deliberately remain until each RTN_M/RTN_B provider proves its tagged schema. + ### The curated registry — `vm-map/globals.toml` (2026-07-07) The v1 auto map (`build/global-var-map.json`) infers *shapes* but cannot recover branch-flag @@ -462,13 +488,13 @@ are *not* story flags — the miner over-tags them; they are recategorized `unkn The v1 map labels *shapes and tables*; the next increments add *meaning*, cheapest first: -1. **Continue INIT semantics by evidence density.** ITINIT/SKINIT, EBINIT, STINIT, CCINIT, and SCINIT now have +1. **Continue INIT semantics by evidence density.** ITINIT/SKINIT, EBINIT, STINIT, CCINIT, SCINIT, and RTINIT now have machine-readable investigation surfaces and semantic joins; EBINIT's populated schema is fully named, STINIT's joined object/enemy payloads are decoded, and CCINIT's 71 class-change rules expose predicates - and effects. SCINIT closes the progression decision-to-scene join. Take RTINIT next as a separate - sparse multi-table family: its 3,336 writes span several 20,000-cell banks used by movement/battle - routines, so it needs layout segmentation before column naming. Preserve explicit joins and do not - infer meaning from column position alone. + and effects. SCINIT closes the progression decision-to-scene join, and RTINIT's twenty movement/battle + banks are structurally decoded. Next decode provider-tagged RTINIT parameters by their RTN_M consumers, + starting with high-density providers 5 and 11; never assign one universal meaning to a parameter bank + whose meaning varies by provider selector. 2. **Extend message-table joins beyond the completed ITMES/SKMES pair** (`VIMES`, other id dispatchers, …) and fold in other `set-string`/`copy-to-global` writers not covered by the `*INIT` set. 3. **Label 2D record tables by their readers** — cross-reference which scripts read each diff --git a/docs/phase-b-framework.md b/docs/phase-b-framework.md index 7acf443..a981289 100644 --- a/docs/phase-b-framework.md +++ b/docs/phase-b-framework.md @@ -784,8 +784,11 @@ promotion apply/report protocol. The input, working-output, and persistent desti as one class-change ABI in `vm-map/globals.toml`, and generated rules carry both raw address provenance and the shared `semantic_fields` join. SCINIT is now the sixth shape: 2,179 paired assignments produce a 1,209-row SCJUMP decision-to-packed-scene registry plus authored chapter metadata, with all script ids -resolved and the full overwrite history retained. The next data-semantics target is RTINIT, whose 3,336 -writes span several sparse 20,000-cell movement/battle routine banks and first need structural segmentation. +resolved and the full overwrite history retained. RTINIT is now the seventh shape: its twenty parallel +1000-by-20 banks produce 172 routine sets, 1,043 movement steps, and fourteen battle steps with RTN_M/RTN_B +provider joins, activation/progress/story gates, six explicit reserved banks, and complete overwrite +history. The next data-semantics slice is provider-tagged parameter decoding, beginning with high-density +movement providers 5 and 11 rather than assigning one meaning to their shared raw parameter banks. Once the natural spine and first gameplay loop are trustworthy, broaden in independent tracks: diff --git a/docs/script-inventory.md b/docs/script-inventory.md index 6ef980f..787f1b7 100644 --- a/docs/script-inventory.md +++ b/docs/script-inventory.md @@ -47,7 +47,7 @@ Large, table-like scripts — prime candidates for data extraction: - `EBINIT` (338 KB) — enemy battle data - `MPINIT` (330 KB) — maps - `SCINIT` (88 KB), `CGINIT` (79 KB — CG gallery), `ITINIT` (70 KB — items), - `RTINIT` (67 KB), `CCINIT` (41 KB — class-change rule program), `SKINIT` (37 KB — skills), `CDINIT` (31 KB), + `RTINIT` (67 KB — movement/battle routine-step banks), `CCINIT` (41 KB — class-change rule program), `SKINIT` (37 KB — skills), `CDINIT` (31 KB), `BTANINIT` (105 KB — battle animations) - Smaller: `AFINIT`, `ALINIT`, `CIINIT`, `CNINIT`, `CTINIT`, `CVINIT`, `ILINIT`, `LAINIT`, `MAINIT`, `OBINIT` (46 object definitions), `SPINIT`, `TRINIT`, `VIINIT` @@ -95,6 +95,13 @@ All 135 packed resource ids resolve to numbered SC scripts. The generated view p decision ids and joins every one of SCJUMP's 847 live decision ids; 844 final chapter tags agree, while three legacy/stale mismatches remain explicit. +RTINIT is a seventh banked shape. Its 3,336 writes populate twenty parallel `1000 × 20` banks: ten +movement banks consumed by MVRTN/RTN_M providers and ten battle banks consumed by BTRTN/RTN_B providers. +The generated 172 routine-set records assemble 1,043 movement steps and fourteen battle steps, join +provider selectors to script names, retain all 29 overwritten cells, and keep the six reserved empty +banks visible. Provider-specific parameter banks remain structurally named until their tagged consumers +prove more specific meanings. + ### Message/string tables (`*MES`) `ITMES` (64 KB — item text), `VIMES` (43 KB), `EIMES` (37 KB), `SKMES` (31 KB — skill text), `CIMES` (15 KB), `MAMES`, `INFOMES`, `MES` — where most translatable text diff --git a/docs/tools-reference.md b/docs/tools-reference.md index 348cb5c..5011e1a 100644 --- a/docs/tools-reference.md +++ b/docs/tools-reference.md @@ -55,9 +55,9 @@ All opcode knowledge (ABI, semantics, provenance, `depends_on`) is hand-edited * |---|---|---|---| | `extract_phase2.py` | Batch: disassembly + text corpora for every script. | `extract_phase2.py` | corpus → `build/disasm/*.asm`, `build/text/{dialogue.jsonl,strings.jsonl,*.strings.txt}`, `build/manifest.json` | | `extract_message_table.py` | Discover a repeated global-id dispatch chain such as ITMES/SKMES, reconstruct player-facing title/description lines (including furigana surface text and readings), and emit an ID-keyed message table with bytecode provenance. | `extract_message_table.py [OUTNAME]` | `.BIN` → `build/data/.json` | -| `extract_init.py` | Parse a `*INIT` data source (auto-detects name / numeric / footer / mixed / rules / dispatch shape). Name tables infer their reserved record span, preserve sparse one-based runtime ids, distinguish lookup bases from first written cells, statically evaluate direct and negative-value writes, and separate parallel `fields` from linked row-major `record_fields`; when flat global ranges overlap, a parallel base established by other records wins over a coincidental row-table cell calculation. Mixed tables recover selector-dispatched records, condition strings, scalars, preallocated buffer cells, consumer-confirmed strides, and length-prefixed footer arrays; STINIT additionally assembles confirmed buffers into `object_placements` and `enemy_spawns`, joins object type names and available descriptions from OBINIT, and exposes consumer-proven tagged payloads (teleport, treasure, card, initial faction, non-triggering hazard/barrier faction, and OBINIT-gated initial object state), preserves engine-dead type-27 writes under `ignored_payload_fields`, and exposes reinforcement schedules and first-clear-only enemy gates. Rules mode decodes CCINIT's source-ordered class-change predicates and output accumulators, joining EBINIT unit names, SKINIT skill names, titles, named stat bonuses, cost deltas, and applied-state slots. Dispatch mode decodes SCINIT's paired decision-to-packed-scene and authored-chapter arrays, preserving source-order overwrites, resolving SYS4INI script names, and cross-checking chapter tags against decoded SCJUMP paths. ITINIT and SKINIT join ITMES/SKMES messages. Top-level `field_semantics` maps raw keys to canonical global/column names, while each record's `semantic_fields` is the generated name-keyed join; complete footer copies map to a row and expose its values without the raw provenance wrapper. Raw keys and unresolved tagged payload cells remain provenance. OUTNAME accepts a stem or one `.json` suffix and rejects paths. Refreshes the generated data index. | `extract_init.py [OUTNAME] [--mode …]` | `
.BIN` plus EBINIT/SKINIT for CCINIT, OBINIT for STINIT, `build/callscript-names.json` and `build/scjump-decisions.json` for SCINIT, matching `.BIN` when supported, and `build/globals.json` → `build/data/.json`, `build/data/README.md` | -| `init_table_profile.py` | Build the static investigation surface for an extracted name/numeric/mixed/rules/dispatch table: per-scalar/string/array-cell/footer-array population and value distributions, representative records, and direct opcode/script consumers. Name tables also report message coverage; rule tables summarize covered units, titled and level-independent rules, threshold/slot distributions, and skill awards; dispatch tables summarize assignments, overwrites, script-name resolution, and SCJUMP chapter agreement. `--message-query REGEX` shows every matching name/message beside all populated fields for semantic correlation. Findings are evidence only; confirmed meanings go in `vm-map/globals.toml`. | `init_table_profile.py
[--build] [--limit N] [--message-query REGEX]` | `build/data/
.json` + corpus → stdout; with `--build`, `build/data/
-field-profile.{json,md}` | -| `test_extract_init.py`, `test_init_table_profile.py` | Regression checks for sparse one-based, mixed selector-dispatched, and CCINIT rule extraction; MES reconstruction/joins; footer-array accounting and semantic projection; EBINIT/SKINIT/OBINIT definition joins; STINIT object/enemy joins and tagged-payload behavior; and field/message/rule profiling. | run each directly | — | +| `extract_init.py` | Parse a `*INIT` data source (auto-detects name / numeric / footer / mixed / rules / dispatch / banked shape). Name tables infer their reserved record span, preserve sparse one-based runtime ids, distinguish lookup bases from first written cells, statically evaluate direct and negative-value writes, and separate parallel `fields` from linked row-major `record_fields`; when flat global ranges overlap, a parallel base established by other records wins over a coincidental row-table cell calculation. Mixed tables recover selector-dispatched records, condition strings, scalars, preallocated buffer cells, consumer-confirmed strides, and length-prefixed footer arrays; STINIT additionally assembles confirmed buffers into `object_placements` and `enemy_spawns`, joins object type names and available descriptions from OBINIT, and exposes consumer-proven tagged payloads (teleport, treasure, card, initial faction, non-triggering hazard/barrier faction, and OBINIT-gated initial object state), preserves engine-dead type-27 writes under `ignored_payload_fields`, and exposes reinforcement schedules and first-clear-only enemy gates. Rules mode decodes CCINIT's source-ordered class-change predicates and output accumulators, joining EBINIT unit names, SKINIT skill names, titles, named stat bonuses, cost deltas, and applied-state slots. Dispatch mode decodes SCINIT's paired decision-to-packed-scene and authored-chapter arrays, preserving source-order overwrites, resolving SYS4INI script names, and cross-checking chapter tags against decoded SCJUMP paths. Banked mode decodes RTINIT's twenty parallel 1000-by-20 movement/battle routine banks, retains all assignments and overwrites, assembles final per-slot steps, and resolves provider selectors to RTN_M/RTN_B scripts while keeping tagged provider parameters generic. ITINIT and SKINIT join ITMES/SKMES messages. Top-level `field_semantics` maps raw keys to canonical global/column names, while each record's `semantic_fields` is the generated name-keyed join; complete footer copies map to a row and expose its values without the raw provenance wrapper. Raw keys and unresolved tagged payload cells remain provenance. OUTNAME accepts a stem or one `.json` suffix and rejects paths. Refreshes the generated data index. | `extract_init.py
[OUTNAME] [--mode …]` | `
.BIN` plus EBINIT/SKINIT for CCINIT, OBINIT for STINIT, `build/callscript-names.json` for RTINIT/SCINIT, `build/scjump-decisions.json` for SCINIT, matching `.BIN` when supported, and `build/globals.json` → `build/data/.json`, `build/data/README.md` | +| `init_table_profile.py` | Build the static investigation surface for an extracted name/numeric/mixed/rules/dispatch/banked table: per-scalar/string/array-cell/footer-array population and value distributions, representative records, and direct opcode/script consumers. Name tables also report message coverage; rule tables summarize covered units, titled and level-independent rules, threshold/slot distributions, and skill awards; dispatch tables summarize assignments, overwrites, script-name resolution, and SCJUMP chapter agreement; banked tables summarize populated/reserved banks, movement/battle steps and providers, and overwrite conflicts. `--message-query REGEX` shows every matching name/message beside all populated fields for semantic correlation. Findings are evidence only; confirmed meanings go in `vm-map/globals.toml`. | `init_table_profile.py
[--build] [--limit N] [--message-query REGEX]` | `build/data/
.json` + corpus → stdout; with `--build`, `build/data/
-field-profile.{json,md}` | +| `test_extract_init.py`, `test_init_table_profile.py` | Regression checks for sparse one-based, mixed selector-dispatched, CCINIT rule, SCINIT dispatch, and RTINIT banked extraction; MES reconstruction/joins; footer-array accounting and semantic projection; EBINIT/SKINIT/OBINIT definition joins; STINIT object/enemy joins and tagged-payload behavior; overwrite/provider joins; and field/message/rule/dispatch/banked profiling. | run each directly | — | | `global_map.py` | Build the partial global-variable name map from static evidence. | `global_map.py` | corpus + `build/data/` → `build/global-var-map.{json,md}` | ## VM diff --git a/docs/vm-mapping-plan.md b/docs/vm-mapping-plan.md index 47edc25..096abb1 100644 --- a/docs/vm-mapping-plan.md +++ b/docs/vm-mapping-plan.md @@ -53,12 +53,13 @@ - [x] **2.0 — Project structure.** Established `docs/`, `build/{disasm,text,data,scripts-json}/`, `godot/`; game install stays read-only in place. Also relaxed the loader magic check to the `SYS4` family (`SYS4424` patch scripts now parse — was silently skipping 5 scripts). - [x] **2.1 — Text corpora.** `tools/extract_phase2.py` → 481/481 scripts: full disassembly (`build/disasm/*.asm`), per-script strings, `build/text/dialogue.jsonl` (**30,057 show-text lines** — the translation corpus), `build/text/strings.jsonl` (38,449 strings tagged by source opcode), `build/manifest.json`. -- [x] **2.2 — `*INIT` data and rule sources → JSON.** `tools/extract_init.py` auto-detects shape (`name`/`numeric`/`footer`/`mixed`/`rules`/`dispatch`) → **SKINIT (131 skills), ITINIT (287 items), EBINIT (277 units), OBINIT (46 object definitions)** [name: sparse one-based name/description/fields], **CGINIT (379 CG entries)** [numeric: index-keyed columns], **MPINIT (1472 map records)** [footer: length-prefixed arrays], **STINIT (74 stages)** [mixed: selector-dispatched strings/scalars/buffer cells/footer arrays], **CCINIT (71 class-change rules over 33 units)** [rules: unit/level/state predicates plus title/cost/stat/skill effects], and **SCINIT (1,209 final decision rows)** [dispatch: scene resource plus authored chapter metadata and overwrite history]. Validated; see `build/data/README.md`. Raw addresses remain bytecode provenance; confirmed semantics come from `vm-map/globals.toml`. +- [x] **2.2 — `*INIT` data and rule sources → JSON.** `tools/extract_init.py` auto-detects shape (`name`/`numeric`/`footer`/`mixed`/`rules`/`dispatch`/`banked`) → **SKINIT (131 skills), ITINIT (287 items), EBINIT (277 units), OBINIT (46 object definitions)** [name: sparse one-based name/description/fields], **CGINIT (379 CG entries)** [numeric: index-keyed columns], **MPINIT (1472 map records)** [footer: length-prefixed arrays], **STINIT (74 stages)** [mixed: selector-dispatched strings/scalars/buffer cells/footer arrays], **CCINIT (71 class-change rules over 33 units)** [rules: unit/level/state predicates plus title/cost/stat/skill effects], **SCINIT (1,209 final decision rows)** [dispatch: scene resource plus authored chapter metadata and overwrite history], and **RTINIT (172 routine sets)** [banked: twenty movement/battle step banks, provider joins, and overwrite history]. Validated; see `build/data/README.md`. Raw addresses remain bytecode provenance; confirmed semantics come from `vm-map/globals.toml`. - [x] **2.3 — Extract `STINIT`'s 74 sparse stage records.** The mixed mode identifies the dominant `scjump_progress_a` dispatch, recovers 29 preallocated buffer layouts (including six consumer-confirmed row strides), and keeps four condition strings, six scalars, fixed-buffer writes, and all 1,396 footer-array copies separated by stage id. Profiles supply population/value and direct-consumer evidence. Header/map semantics cover conditions, BGM, turn/replay/clear-reward settings, and map texture overrides. The object join assembles 2,312 placements with type/position/gates, OBINIT names and available descriptions, 604 reinforcement schedules, and typed initial-faction, teleport, treasure, card-list, non-triggering-faction, and initial-object-state payloads; FIELD's dedicated special-spawn path proves the final three populated type-27 tagged writes are engine-dead, so they remain visible as ignored provenance rather than unresolved semantics. The 30-cell enemy family contributes 1,378 joined spawns with unit/faction, placement, difficulty/story gates, level scaling, weighted selection, difficulty-specific movement/battle routine sets, and 485 first-clear-only gates. Raw address views remain alongside generated `semantic_fields`. - [x] **2.3a — Extract `CCINIT`'s 71 class-change rules.** Rules mode preserves source-order guards and detects the unit id, minimum level, clear applied-state slot, selected title/level, deployment-cost delta, named fourteen-stat bonuses, awarded SKINIT skills, and state slot set by each rule. EBINIT/SKINIT definition joins and the common global semantic projection coexist with raw addresses. CALCCC/ADDEXP establish the apply/report ABI; EVOLVE explains the two level-independent empty-title Lily rules. The generated profile covers 33 units, 69 titled rules, 30 skill awards, three used promotion slots, and 19 populated output fields. - [x] **2.3b — Extract `SCINIT`'s scene-dispatch registry.** Dispatch mode recognizes 2,179 alternating writes to two 10,000-cell arrays, preserves all 710 overwritten decision ids, and emits 1,209 final rows. The primary column maps decisions to 135 packed SYS4INI ids, all resolved to numbered SC scripts; the parallel column is authored chapter metadata. All 847 live SCJUMP decisions join to rows and 844 final chapter tags agree with the independently decoded paths; three legacy/stale mismatches remain explicit. +- [x] **2.3c — Extract `RTINIT`'s routine-step banks.** Banked mode recognizes twenty parallel 1000-by-20 tables and emits 172 sparse routine-set rows with all 3,336 source assignments and 3,307 final cells. Movement banks join 1,043 steps to 19 used RTN_M providers (22 dispatchable); battle banks join fourteen steps to all four RTN_B providers. Activation percentages, progress gates, and required/forbidden story flags are consumer-proven; six empty banks remain explicit and provider-tagged parameters stay generic. - [x] **2.4 — Partial global-var map BUILT + wired into the disassembler.** `tools/global_map.py` → `build/global-var-map.{json,md}` (16,354/49,435 globals labelled: string tables, `*INIT` field arrays, 122 record tables w/ strides, current-entity index pointers). `sys4load` renders the labels inline (`=rec[s30]`, `=current-entity-index?`). See `docs/name-resolution.md`. -- [ ] **2.5 — Grow the global-var map (future, incremental).** Static first: segment RTINIT's sparse multi-table banks, fold in remaining `*MES` writers, and label 2D record tables by their reader scripts. Then Frida to name *which stat* each unresolved field is. Full detail: `docs/name-resolution.md` → "Future step — growing the map". Packed `call-script` ids and SCJUMP decision-to-scene dispatch are resolved; see `docs/name-resolution.md` #1 and `docs/scjump-progression.md`. +- [ ] **2.5 — Grow the global-var map (future, incremental).** Static first: decode RTINIT's provider-tagged parameters through their RTN_M/RTN_B readers, fold in remaining `*MES` writers, and label other 2D record tables by their reader scripts. Then Frida to name *which stat* each unresolved field is. Full detail: `docs/name-resolution.md` → "Future step — growing the map". Packed `call-script` ids and SCJUMP decision-to-scene dispatch are resolved; see `docs/name-resolution.md` #1 and `docs/scjump-progression.md`. ## Phase 3 — Name the unnamed opcodes *(top ~20 BEFORE Phase 4; the rest on demand)* diff --git a/tools/extract_init.py b/tools/extract_init.py index 71494db..d20ee1e 100644 --- a/tools/extract_init.py +++ b/tools/extract_init.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Extract a *INIT data table to JSON. Auto-detects the table's shape. -*INIT scripts populate global arrays and work buffers with static game data. Six shapes seen: +*INIT scripts populate global arrays and work buffers with static game data. Seven shapes seen: name — records keyed by a name string. Each record: set-string(name), static field writes, set-string(desc). Arrays indexed by record id in lockstep (+1/record). @@ -16,14 +16,17 @@ buffers. (CCINIT class changes) dispatch—paired parallel arrays map a sparse decision id to a packed script resource id and authored chapter metadata. (SCINIT scene dispatch) + banked —twenty parallel 1000-by-20 banks define sparse movement and battle routine + step records, including provider joins and source overwrites. (RTINIT routines) Records are {id, name?, desc?, fields:{"0x": value}} or, for footer tables, {id, global_addr, footer_off, values:[...]}. Column addresses are raw engine globals; confirmed names come from the generated engine global registry while raw keys remain provenance. -Usage: py -3.11 -X utf8 tools/extract_init.py
[OUTNAME] [--mode name|numeric|footer|mixed|rules|dispatch] +Usage: py -3.11 -X utf8 tools/extract_init.py
[OUTNAME] [--mode name|numeric|footer|mixed|rules|dispatch|banked] """ from __future__ import annotations +import collections import json import sys from functools import cache @@ -55,6 +58,38 @@ CLASS_CHANGE_STATS_OUT = 0xAB8E9 CLASS_CHANGE_SKILLS_OUT = 0xAB8F7 CLASS_CHANGE_FLAGS_OUT = 0xAB8FB +ROUTINE_BANK_ROOT = 0xEFF78 +ROUTINE_BANK_SPAN = 20000 +ROUTINE_BANK_COUNT = 20 +ROUTINE_RECORD_STRIDE = 20 +ROUTINE_RECORD_SPAN = 1000 +ROUTINE_SET_ID = 0xEFF75 +ROUTINE_STEP_INDEX = 0xEFF76 +ROUTINE_EXECUTION_STATE = 0xEFF77 + +ROUTINE_BANK_ROLES = ( + "movement_provider_selector", + "movement_activation_percent", + "movement_parameter_1", + "movement_parameter_2", + "movement_parameter_3", + "movement_parameter_4", + "movement_reserved", + "movement_minimum_progress_count", + "movement_required_story_flag_id", + "movement_forbidden_story_flag_id", + "battle_provider_selector", + "battle_activation_percent", + "battle_parameter_1", + "battle_reserved_1", + "battle_reserved_2", + "battle_reserved_3", + "battle_reserved_4", + "battle_reserved_5", + "battle_required_story_flag_id", + "battle_forbidden_story_flag_id", +) + UNIT_STAT_COLUMNS = ( "accuracy", "evasion", "physical_attack", "physical_defense", "magic_attack", "magic_defense", "speed", "luck", "critical_chance", @@ -188,6 +223,32 @@ def _paired_parallel_writes(scr) -> tuple[list[tuple], int] | None: return writes, span +def _routine_bank_writes(scr) -> list[tuple] | None: + """Recognize RTINIT's twenty reserved 1000-by-20 routine-step banks.""" + writes = [] + for ins in scr.instructions: + write = _static_global_write(ins) + if write is not None and isinstance(write[1], int): + destination, value = write + relative = destination - ROUTINE_BANK_ROOT + if not (0 <= relative < ROUTINE_BANK_COUNT * ROUTINE_BANK_SPAN): + return None + bank_index, cell = divmod(relative, ROUTINE_BANK_SPAN) + record_id, slot = divmod(cell, ROUTINE_RECORD_STRIDE) + if not ( + 0 <= bank_index < ROUTINE_BANK_COUNT + and 0 <= record_id < ROUTINE_RECORD_SPAN + and 0 <= slot < ROUTINE_RECORD_STRIDE + ): + return None + writes.append(( + ins.offset, destination, value, bank_index, record_id, slot + )) + elif sys4load.display_label(ins.opcode) != "exit": + return None + return writes if len(writes) >= 1000 else None + + def detect_mode(scr): ops = [ins.opcode for ins in scr.instructions] has_str = any(ins.opcode == SET_STRING and ins.args and ins.args[0][0] == T_GLOBAL_STRING @@ -206,6 +267,8 @@ def detect_mode(scr): return "name" if _paired_parallel_writes(scr): return "dispatch" + if _routine_bank_writes(scr): + return "banked" n_footer = ops.count(COPY_LOCAL_ARRAY) n_int = ops.count(MOV) + ops.count(COPY_TO_GLOBAL) return "footer" if n_footer >= max(4, n_int) else "numeric" @@ -823,6 +886,180 @@ def extract_dispatch(scr): } +def _movement_provider_names(names: dict[int, str]) -> dict[int, str]: + providers = { + selector: names.get(0x32FB + selector, "") + for selector in range(1, 19) + } + providers.update({ + 51: names.get(0x330E, ""), + 52: names.get(0x330F, ""), + 53: names.get(0x3310, ""), + 61: names.get(0x3311, ""), + }) + return providers + + +def extract_banked(scr): + """Extract RTINIT's sparse routine sets across twenty parallel step banks.""" + writes = _routine_bank_writes(scr) + if writes is None: + return [], {} + + names = callscript_names() + movement_providers = _movement_provider_names(names) + battle_providers = { + selector: names.get(0x32F6 + selector, "") + for selector in range(1, 5) + } + records_by_id: dict[int, dict] = {} + cell_assignments: dict[tuple[int, int, int], list[int]] = collections.defaultdict(list) + bank_cells: dict[int, set[tuple[int, int]]] = collections.defaultdict(set) + + for offset, destination, value, bank_index, record_id, slot in writes: + bank_base = ROUTINE_BANK_ROOT + bank_index * ROUTINE_BANK_SPAN + key = f"0x{bank_base:x}/{ROUTINE_RECORD_STRIDE}/{slot}" + assignment = { + "offset": f"0x{offset:x}", + "bank_index": bank_index, + "bank_base": f"0x{bank_base:x}", + "role": ROUTINE_BANK_ROLES[bank_index], + "slot": slot, + "value": value, + } + record = records_by_id.setdefault(record_id, { + "id": record_id, + "assignments": [], + "record_fields": {}, + }) + record["assignments"].append(assignment) + record["record_fields"][key] = value + cell_assignments[(bank_index, record_id, slot)].append(value) + bank_cells[bank_index].add((record_id, slot)) + + for record in records_by_id.values(): + final_by_bank_slot = {} + for assignment in record["assignments"]: + final_by_bank_slot[ + (assignment["bank_index"], assignment["slot"]) + ] = assignment["value"] + + movement_steps = [] + battle_steps = [] + for slot in range(ROUTINE_RECORD_STRIDE): + movement = { + ROUTINE_BANK_ROLES[bank]: final_by_bank_slot[(bank, slot)] + for bank in range(10) + if (bank, slot) in final_by_bank_slot + } + if movement: + selector = movement.get("movement_provider_selector") + movement_steps.append({ + "slot": slot, + **movement, + **( + {"provider_script": movement_providers.get(selector, "")} + if selector is not None else {} + ), + }) + + battle = { + ROUTINE_BANK_ROLES[bank]: final_by_bank_slot[(bank, slot)] + for bank in range(10, 20) + if (bank, slot) in final_by_bank_slot + } + if battle: + selector = battle.get("battle_provider_selector") + battle_steps.append({ + "slot": slot, + **battle, + **( + {"provider_script": battle_providers.get(selector, "")} + if selector is not None else {} + ), + }) + if movement_steps: + record["movement_steps"] = movement_steps + if battle_steps: + record["battle_steps"] = battle_steps + + records = [records_by_id[key] for key in sorted(records_by_id)] + record_ids = set(records_by_id) + used_movement_providers = sorted({ + step["movement_provider_selector"] + for record in records + for step in record.get("movement_steps", []) + }) + used_battle_providers = sorted({ + step["battle_provider_selector"] + for record in records + for step in record.get("battle_steps", []) + }) + bank_layouts = {} + for bank_index, role in enumerate(ROUTINE_BANK_ROLES): + base = ROUTINE_BANK_ROOT + bank_index * ROUTINE_BANK_SPAN + cells = bank_cells.get(bank_index, set()) + bank_layouts[f"0x{base:x}"] = { + "bank_index": bank_index, + "family": "movement" if bank_index < 10 else "battle", + "role": role, + "reserved_empty": not cells, + "populated_cell_count": len(cells), + "populated_record_count": len({record_id for record_id, _ in cells}), + "populated_slots": sorted({slot for _, slot in cells}), + } + record_columns = sorted( + { + key + for record in records + for key in record.get("record_fields", {}) + }, + key=lambda key: tuple(int(part, 0) for part in key.split("/")), + ) + return records, { + "schema": "routine-step-banks", + "selector_global": f"0x{ROUTINE_SET_ID:x}", + "step_index_global": f"0x{ROUTINE_STEP_INDEX:x}", + "execution_state_global": f"0x{ROUTINE_EXECUTION_STATE:x}", + "bank_root_base": f"0x{ROUTINE_BANK_ROOT:x}", + "bank_span": ROUTINE_BANK_SPAN, + "bank_count": ROUTINE_BANK_COUNT, + "record_stride": ROUTINE_RECORD_STRIDE, + "reserved_record_span": ROUTINE_RECORD_SPAN, + "first_record_id": min(record_ids), + "last_record_id": max(record_ids), + "missing_record_ids": sorted( + set(range(min(record_ids), max(record_ids) + 1)) - record_ids + ), + "assignment_count": len(writes), + "populated_cell_count": len(cell_assignments), + "overwritten_cell_count": sum( + len(values) > 1 for values in cell_assignments.values() + ), + "conflicting_overwrite_count": sum( + len(set(values)) > 1 for values in cell_assignments.values() + ), + "movement_step_count": sum( + len(record.get("movement_steps", [])) for record in records + ), + "battle_step_count": sum( + len(record.get("battle_steps", [])) for record in records + ), + "movement_provider_scripts": { + str(selector): name + for selector, name in sorted(movement_providers.items()) + }, + "battle_provider_scripts": { + str(selector): name + for selector, name in sorted(battle_providers.items()) + }, + "used_movement_provider_selectors": used_movement_providers, + "used_battle_provider_selectors": used_battle_providers, + "bank_layouts": bank_layouts, + "record_field_columns": record_columns, + } + + def extract_footer(scr): records = [] for i, ins in enumerate(scr.instructions): @@ -1172,6 +1409,11 @@ def write_data_index(data_dir: Path) -> None: "while exposing the final sparse decision-id registry. Packed resource ids join to", "SYS4INI script names, authored chapter tags correlate with SCJUMP's decoded decision", "sites, and legacy/stale chapter mismatches remain explicit.", + "", + "Banked-mode tables preserve RTINIT's twenty parallel 1000-by-20 routine banks,", + "source-ordered overwrites, and final row/slot values. Joined movement and battle", + "steps resolve provider selectors to RTN_M/RTN_B scripts while provider-specific", + "parameter banks retain structural names until their individual consumers prove more.", "Use `tools/init_table_profile.py
--build` to generate value/population and", "direct-consumer evidence.", "", @@ -1212,6 +1454,7 @@ def main() -> int: "mixed": extract_mixed, "rules": extract_class_change_rules, "dispatch": extract_dispatch, + "banked": extract_banked, }[mode] recs, meta = extractor(scr) if mode == "name" and name in MESSAGE_TABLES: @@ -1236,7 +1479,11 @@ def main() -> int: outpath.write_text(json.dumps(out, ensure_ascii=False, indent=2), encoding="utf-8") write_data_index(outpath.parent) print(f"{name}: mode={mode}, {len(recs)} records" - + (f", {len(cols)} field-columns" if mode != 'footer' else "") + + ( + f", {len(meta.get('record_field_columns', []))} record-columns" + if mode == "banked" + else f", {len(cols)} field-columns" if mode != "footer" else "" + ) + f" -> build/data/{outname}.json") for r in recs[:4]: if mode == "footer": diff --git a/tools/init_table_profile.py b/tools/init_table_profile.py index 8b327f2..e4c3bef 100644 --- a/tools/init_table_profile.py +++ b/tools/init_table_profile.py @@ -37,7 +37,9 @@ def load_table(name: str) -> dict: if not path.exists(): raise SystemExit(f"missing extracted table: {path}") data = json.loads(path.read_text(encoding="utf8")) - if data.get("mode") not in {"name", "numeric", "mixed", "rules", "dispatch"}: + if data.get("mode") not in { + "name", "numeric", "mixed", "rules", "dispatch", "banked" + }: raise SystemExit(f"{name}: unsupported field-profiling mode {data.get('mode')!r}") return data @@ -205,6 +207,41 @@ def profile_dispatch(data: dict) -> dict: } +def profile_banked(data: dict) -> dict: + """Summarize RTINIT's routine banks, steps, and overwrite history.""" + if data.get("mode") != "banked": + return {} + layouts = data.get("bank_layouts", {}) + return { + "assignment_count": data.get("assignment_count", 0), + "populated_cell_count": data.get("populated_cell_count", 0), + "overwritten_cell_count": data.get("overwritten_cell_count", 0), + "conflicting_overwrite_count": data.get("conflicting_overwrite_count", 0), + "populated_bank_count": sum( + not layout.get("reserved_empty", False) + for layout in layouts.values() + ), + "reserved_bank_count": sum( + layout.get("reserved_empty", False) + for layout in layouts.values() + ), + "movement_step_count": data.get("movement_step_count", 0), + "battle_step_count": data.get("battle_step_count", 0), + "movement_provider_count": len( + data.get("used_movement_provider_selectors", []) + ), + "battle_provider_count": len( + data.get("used_battle_provider_selectors", []) + ), + "available_movement_provider_count": len( + data.get("movement_provider_scripts", {}) + ), + "available_battle_provider_count": len( + data.get("battle_provider_scripts", {}) + ), + } + + def profile_messages(data: dict) -> dict: """Summarize the joined player-facing message evidence.""" records = data["records"] @@ -342,6 +379,23 @@ def render_markdown(data: dict, rows: list[dict], limit: int) -> str: f"- explicit chapter mismatches: " f"{dispatch_profile['scjump_chapter_mismatch_count']}", ]) + elif banked_profile := profile_banked(data): + lines.extend([ + f"- source assignments: {banked_profile['assignment_count']}", + f"- final populated cells: {banked_profile['populated_cell_count']}", + f"- overwritten cells: {banked_profile['overwritten_cell_count']} " + f"({banked_profile['conflicting_overwrite_count']} change value)", + f"- banks: {banked_profile['populated_bank_count']} populated, " + f"{banked_profile['reserved_bank_count']} reserved/empty", + f"- joined movement steps/used providers: " + f"{banked_profile['movement_step_count']}/" + f"{banked_profile['movement_provider_count']} " + f"({banked_profile['available_movement_provider_count']} dispatchable)", + f"- joined battle steps/used providers: " + f"{banked_profile['battle_step_count']}/" + f"{banked_profile['battle_provider_count']} " + f"({banked_profile['available_battle_provider_count']} dispatchable)", + ]) else: lines.extend([ f"- player-facing messages: {message_profile['population']}/{data['record_count']} " @@ -406,6 +460,7 @@ def main() -> int: "message_profile": messages, "rule_profile": profile_rules(data), "dispatch_profile": profile_dispatch(data), + "banked_profile": profile_banked(data), "columns": sorted(rows, key=lambda row: ( int(row["base"], 16), row["stride"] or 0, row["column"] or 0 )), diff --git a/tools/test_extract_init.py b/tools/test_extract_init.py index 9f50d59..98f5b81 100644 --- a/tools/test_extract_init.py +++ b/tools/test_extract_init.py @@ -313,12 +313,6 @@ def test_real_scene_dispatch() -> None: script = sys4load.load(extract_init.resolve("SCINIT")) check(extract_init.detect_mode(script) == "dispatch", "SCINIT auto-detects as paired scene dispatch arrays") - check( - extract_init.detect_mode( - sys4load.load(extract_init.resolve("RTINIT")) - ) == "numeric", - "RTINIT's multi-table writes do not false-positive as paired dispatch", - ) records, meta = extract_init.extract_dispatch(script) by_id = {record["id"]: record for record in records} check(len(records) == 1209 and meta["assignment_count"] == 2179, @@ -349,6 +343,66 @@ def test_real_scene_dispatch() -> None: "SCINIT's paired columns join to canonical semantic names") +def test_real_routine_banks() -> None: + script = sys4load.load(extract_init.resolve("RTINIT")) + check(extract_init.detect_mode(script) == "banked", + "RTINIT auto-detects as parallel routine-step banks") + records, meta = extract_init.extract_banked(script) + by_id = {record["id"]: record for record in records} + check(len(records) == 172 + and meta["first_record_id"] == 1 + and meta["last_record_id"] == 176 + and meta["missing_record_ids"] == [150, 151, 152, 153], + "RTINIT preserves its sparse one-based routine-set ids") + check(meta["bank_root_base"] == "0xeff78" + and meta["bank_span"] == 20000 + and meta["bank_count"] == 20 + and meta["record_stride"] == 20 + and meta["reserved_record_span"] == 1000, + "RTINIT exposes twenty parallel 1000-by-20 banks") + check(meta["assignment_count"] == 3336 + and meta["populated_cell_count"] == 3307 + and meta["overwritten_cell_count"] == 29 + and meta["conflicting_overwrite_count"] == 11, + "RTINIT preserves source assignments and final overwrite accounting") + check(meta["movement_step_count"] == 1043 + and meta["battle_step_count"] == 14 + and len(meta["used_movement_provider_selectors"]) == 19 + and len(meta["used_battle_provider_selectors"]) == 4 + and len(meta["record_field_columns"]) == 117, + "RTINIT assembles every populated movement and battle step") + check([ + layout["bank_index"] + for layout in meta["bank_layouts"].values() + if layout["reserved_empty"] + ] == [6, 13, 14, 15, 16, 17], + "RTINIT keeps all six reserved empty banks in its structural layout") + movement = by_id[1]["movement_steps"][0] + battle = by_id[1]["battle_steps"][0] + check(movement["movement_provider_selector"] == 1 + and movement["movement_activation_percent"] == 100 + and movement["provider_script"] == "RTN_M001.BIN", + "RTINIT joins movement selectors and activation percentages") + check(battle["battle_provider_selector"] == 1 + and battle["battle_activation_percent"] == 100 + and battle["provider_script"] == "RTN_B001.BIN", + "RTINIT joins battle selectors and activation percentages") + check(by_id[2]["battle_steps"][0]["battle_parameter_1"] == 219 + and by_id[2]["battle_steps"][0]["provider_script"] == "RTN_B004.BIN", + "RTINIT retains provider-specific battle parameters") + check(by_id[173]["movement_steps"][0]["movement_parameter_1"] == 2 + and by_id[173]["movement_steps"][0]["movement_parameter_2"] == 158, + "RTINIT final rows reflect source-ordered conflicting overwrites") + semantics = extract_init.field_semantics(records) + check( + semantics["0xeff78/20/0"] + == "movement_routine_provider_selectors.column_0" + and semantics["0x125ad8/20/0"] + == "battle_routine_activation_percents.column_0", + "RTINIT raw banks join to canonical structural field names", + ) + + def test_real_message_tables() -> None: scripts = paths.scripts() expected = { @@ -524,6 +578,7 @@ if __name__ == "__main__": test_real_mixed_table() test_real_class_change_rules() test_real_scene_dispatch() + test_real_routine_banks() test_real_message_tables() test_message_join() test_field_semantics() diff --git a/tools/test_init_table_profile.py b/tools/test_init_table_profile.py index fc28257..a2bfa4f 100644 --- a/tools/test_init_table_profile.py +++ b/tools/test_init_table_profile.py @@ -120,6 +120,44 @@ def main() -> int: assert dispatch_summary["scjump_chapter_match_count"] == 1 assert dispatch_summary["scjump_chapter_mismatch_count"] == 1 + banked_fixture = { + "table": "BANKED", + "mode": "banked", + "assignment_count": 6, + "populated_cell_count": 5, + "overwritten_cell_count": 1, + "conflicting_overwrite_count": 1, + "movement_step_count": 2, + "battle_step_count": 1, + "movement_provider_scripts": {"1": "RTN_M001.BIN"}, + "battle_provider_scripts": {"1": "RTN_B001.BIN"}, + "used_movement_provider_selectors": [1], + "used_battle_provider_selectors": [1], + "bank_layouts": { + "0x100": {"reserved_empty": False}, + "0x200": {"reserved_empty": True}, + }, + "records": [ + { + "id": 1, + "record_fields": { + "0x100/20/0": 1, + "0x100/20/1": 2, + }, + }, + ], + } + banked_rows = { + row["key"]: row for row in profile.profile_columns(banked_fixture) + } + assert banked_rows["0x100/20/0"]["kind"] == "record-column" + banked_summary = profile.profile_banked(banked_fixture) + assert banked_summary["assignment_count"] == 6 + assert banked_summary["populated_bank_count"] == 1 + assert banked_summary["reserved_bank_count"] == 1 + assert banked_summary["movement_step_count"] == 2 + assert banked_summary["battle_provider_count"] == 1 + messages = profile.profile_messages(fixture) assert messages["population"] == 1 assert messages["coverage"] == 1 / 3 diff --git a/vm-map/globals.toml b/vm-map/globals.toml index 5c7e340..b23a7e5 100644 --- a/vm-map/globals.toml +++ b/vm-map/globals.toml @@ -822,6 +822,160 @@ source = "investigation" confidence = "high" depends_on = ["0x87a57", "0x3234", "0x62ccf"] +[[global]] +address = "0xeff78" +name = "movement_routine_provider_selectors" +category = "data-table" +type = "int[1000][20]" +value_domain = "RTN_M provider selector: 1..18, 51..53, or 61; 0 terminates a row" +usage = "RTINIT movement bank 0. MVRTN indexes it by current_routine_set_id and routine_step_index, resolves the selector through its RTN_M001..018/051..053/061 provider table, and call-scripts the selected movement routine." +source = "investigation" +confidence = "high" +depends_on = ["0xeff75", "0xeff76"] + +[[global]] +address = "0xf4d98" +name = "movement_routine_activation_percents" +category = "data-table" +type = "int[1000][20]" +value_domain = "10..100 percent" +usage = "RTINIT movement bank 1. After all other step gates pass, MVRTN executes the provider only when random-modulo-100 is below this value." +source = "investigation" +confidence = "high" +depends_on = ["0xeff78"] + +[[global]] +address = "0xf9bb8" +name = "movement_routine_parameter_1" +category = "data-table" +type = "int[1000][20]" +value_domain = "provider-specific integer" +usage = "RTINIT movement bank 2. RTN_M004..015 providers consume the current row/step value; its exact meaning is tagged by movement_routine_provider_selectors and remains provider-specific." +source = "investigation" +confidence = "high" +depends_on = ["0xeff78"] + +[[global]] +address = "0xfe9d8" +name = "movement_routine_parameter_2" +category = "data-table" +type = "int[1000][20]" +value_domain = "provider-specific integer" +usage = "RTINIT movement bank 3. RTN_M005/007/010/011/012 consume it as a provider-specific parameter." +source = "investigation" +confidence = "high" +depends_on = ["0xeff78"] + +[[global]] +address = "0x1037f8" +name = "movement_routine_parameter_3" +category = "data-table" +type = "int[1000][20]" +value_domain = "provider-specific integer" +usage = "RTINIT movement bank 4. RTN_M011 consumes it as a provider-specific parameter." +source = "investigation" +confidence = "high" +depends_on = ["0xeff78"] + +[[global]] +address = "0x108618" +name = "movement_routine_parameter_4" +category = "data-table" +type = "int[1000][20]" +value_domain = "{1,2}" +usage = "RTINIT movement bank 5. RTN_M011 consumes it as a provider-specific parameter." +source = "investigation" +confidence = "high" +depends_on = ["0xeff78"] + +[[global]] +address = "0x112258" +name = "movement_routine_minimum_progress_counts" +category = "data-table" +type = "int[1000][20]" +value_domain = "positive step-progress threshold; shipped RTINIT writes 1" +usage = "RTINIT movement bank 7. MVRTN requires the current entity's matching movement-step progress counter to reach this value; movement providers increment those counters as their steps execute." +source = "investigation" +confidence = "high" +depends_on = ["0xeff78", "0x53643"] + +[[global]] +address = "0x117078" +name = "movement_routine_required_story_flag_ids" +category = "data-table" +type = "int[1000][20]" +value_domain = "one-based story_event_flags id" +usage = "RTINIT movement bank 8. MVRTN subtracts one and rejects the step when the referenced story flag is not set." +source = "investigation" +confidence = "high" +depends_on = ["0xeff78", "0x6d3"] + +[[global]] +address = "0x11be98" +name = "movement_routine_forbidden_story_flag_ids" +category = "data-table" +type = "int[1000][20]" +value_domain = "one-based story_event_flags id" +usage = "RTINIT movement bank 9. MVRTN subtracts one and rejects the step when the referenced story flag is set." +source = "investigation" +confidence = "high" +depends_on = ["0xeff78", "0x6d3"] + +[[global]] +address = "0x120cb8" +name = "battle_routine_provider_selectors" +category = "data-table" +type = "int[1000][20]" +value_domain = "RTN_B provider selector 1..4; 0 terminates a row" +usage = "RTINIT battle bank 10. BTRTN indexes it by current_routine_set_id and routine_step_index, resolves the selector through RTN_B001..004, and call-scripts the selected battle routine." +source = "investigation" +confidence = "high" +depends_on = ["0xeff75", "0xeff76"] + +[[global]] +address = "0x125ad8" +name = "battle_routine_activation_percents" +category = "data-table" +type = "int[1000][20]" +value_domain = "70 or 100 percent in shipped RTINIT" +usage = "RTINIT battle bank 11. BTRTN executes a candidate step only when the current entity's matching random-modulo-100 battle-step roll is below this value." +source = "investigation" +confidence = "high" +depends_on = ["0x120cb8", "0x56738"] + +[[global]] +address = "0x12a8f8" +name = "battle_routine_parameter_1" +category = "data-table" +type = "int[1000][20]" +value_domain = "provider-specific integer" +usage = "RTINIT battle bank 12. RTN_B004 uses it to choose an entry from the prepared battle-action candidate table." +source = "investigation" +confidence = "high" +depends_on = ["0x120cb8"] + +[[global]] +address = "0x147db8" +name = "battle_routine_required_story_flag_ids" +category = "data-table" +type = "int[1000][20]" +value_domain = "one-based story_event_flags id" +usage = "RTINIT battle bank 18. BTRTN subtracts one and rejects the step when the referenced story flag is not set." +source = "investigation" +confidence = "high" +depends_on = ["0x120cb8", "0x6d3"] + +[[global]] +address = "0x14cbd8" +name = "battle_routine_forbidden_story_flag_ids" +category = "data-table" +type = "int[1000][20]" +value_domain = "one-based story_event_flags id" +usage = "RTINIT battle bank 19. BTRTN subtracts one and rejects the step when the referenced story flag is set." +source = "investigation" +confidence = "high" +depends_on = ["0x120cb8", "0x6d3"] + [[global]] address = "0x3234" name = "chapter_mode" @@ -967,14 +1121,80 @@ depends_on = ["0x3234", "0x87a57"] [[global]] address = "0xeff75" -name = "current_entity_index_hi" +name = "current_routine_set_id" category = "index-pointer" type = "int" -value_domain = "row index" -usage = "High-purity current-entity row index (purity 0.95 in the auto shape map); dominant 2D-table row selector." -source = "inference" -confidence = "med" -depends_on = [] +value_domain = "RTINIT row id 1..176" +usage = "Shared RTINIT row selector. MVRTN loads the current entity's difficulty-selected movement routine set; BTRTN loads its battle routine set. Both then iterate routine_step_index across the selected twenty-slot row." +source = "investigation" +confidence = "high" +depends_on = ["0x53517", "0x535ad", "0xeff76"] + +[[global]] +address = "0xeff76" +name = "routine_step_index" +category = "index-pointer" +type = "int" +value_domain = "0..19" +usage = "Shared RTINIT step selector. MVRTN and BTRTN iterate it from zero through nineteen and use it as the column index in every routine bank and matching per-entity runtime row." +source = "investigation" +confidence = "high" +depends_on = ["0xeff75"] + +[[global]] +address = "0xeff77" +name = "routine_execution_state" +category = "choice-output" +type = "int" +value_domain = "0..3" +usage = "Shared movement/battle routine result state. MVRTN and BTRTN clear it before scanning steps; RTN_M/RTN_B providers set a nonzero result to stop the scan, and the dispatcher supplies state 1 when no provider produced another result. Exact distinctions between provider-written states 1..3 remain unresolved." +source = "investigation" +confidence = "high" +depends_on = ["0xeff75", "0xeff76"] + +[[global]] +address = "0x53517" +name = "entity_movement_routine_set_ids" +category = "data-table" +type = "int[50][3]" +value_domain = "RTINIT row id by difficulty" +usage = "Per-runtime-entity movement routine set for easy/normal/hard. SETEN copies a stage-specific override or the EBINIT default into this row; MVRTN selects the current difficulty column and stores it in current_routine_set_id." +source = "investigation" +confidence = "high" +depends_on = ["0xe7889", "0xeff75"] + +[[global]] +address = "0x535ad" +name = "entity_battle_routine_set_ids" +category = "data-table" +type = "int[50][3]" +value_domain = "RTINIT row id by difficulty" +usage = "Per-runtime-entity battle routine set for easy/normal/hard. SETEN copies a stage-specific override or the EBINIT default into this row; BTRTN selects the current difficulty column and stores it in current_routine_set_id." +source = "investigation" +confidence = "high" +depends_on = ["0xe78e3", "0xeff75"] + +[[global]] +address = "0x53643" +name = "entity_movement_routine_progress" +category = "data-table" +type = "int[50][20]" +value_domain = "nonnegative per-step counter" +usage = "Per-runtime-entity movement-routine progress counters. FIELD clears all fifty rows, movement providers increment the current step, and MVRTN compares it with movement_routine_minimum_progress_counts before enabling a step." +source = "investigation" +confidence = "high" +depends_on = ["0x112258", "0xeff76"] + +[[global]] +address = "0x56738" +name = "entity_battle_routine_random_rolls" +category = "data-table" +type = "int[50][20]" +value_domain = "0..99" +usage = "Per-runtime-entity battle-step random rolls. FIELD and BTL fill every active entity's twenty cells with random-modulo-100; BTRTN executes a step when the matching roll is below battle_routine_activation_percents." +source = "investigation" +confidence = "high" +depends_on = ["0x125ad8", "0xeff76"] [[global]] address = "0x152616"