Decode RTINIT routine step banks

tools/extract_init.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 """Extract a *INIT data table to JSON. Auto-detects the table's shape.
 
-*INIT scripts populate global arrays and work buffers with static game data. Six shapes seen:
+*INIT scripts populate global arrays and work buffers with static game data. Seven shapes seen:
 
   name   — records keyed by a name string. Each record: set-string(name), static field writes,
            set-string(desc). Arrays indexed by record id in lockstep (+1/record).
@@ -16,14 +16,17 @@
            buffers. (CCINIT class changes)
   dispatch—paired parallel arrays map a sparse decision id to a packed script resource id
            and authored chapter metadata. (SCINIT scene dispatch)
+  banked —twenty parallel 1000-by-20 banks define sparse movement and battle routine
+           step records, including provider joins and source overwrites. (RTINIT routines)
 
 Records are {id, name?, desc?, fields:{"0x<col_base>": value}} or, for footer tables,
 {id, global_addr, footer_off, values:[...]}. Column addresses are raw engine globals;
 confirmed names come from the generated engine global registry while raw keys remain provenance.
 
-Usage: py -3.11 -X utf8 tools/extract_init.py <TABLE> [OUTNAME] [--mode name|numeric|footer|mixed|rules|dispatch]
+Usage: py -3.11 -X utf8 tools/extract_init.py <TABLE> [OUTNAME] [--mode name|numeric|footer|mixed|rules|dispatch|banked]
 """
 from __future__ import annotations
+import collections
 import json
 import sys
 from functools import cache
@@ -55,6 +58,38 @@ CLASS_CHANGE_STATS_OUT = 0xAB8E9
 CLASS_CHANGE_SKILLS_OUT = 0xAB8F7
 CLASS_CHANGE_FLAGS_OUT = 0xAB8FB
 
+ROUTINE_BANK_ROOT = 0xEFF78
+ROUTINE_BANK_SPAN = 20000
+ROUTINE_BANK_COUNT = 20
+ROUTINE_RECORD_STRIDE = 20
+ROUTINE_RECORD_SPAN = 1000
+ROUTINE_SET_ID = 0xEFF75
+ROUTINE_STEP_INDEX = 0xEFF76
+ROUTINE_EXECUTION_STATE = 0xEFF77
+
+ROUTINE_BANK_ROLES = (
+    "movement_provider_selector",
+    "movement_activation_percent",
+    "movement_parameter_1",
+    "movement_parameter_2",
+    "movement_parameter_3",
+    "movement_parameter_4",
+    "movement_reserved",
+    "movement_minimum_progress_count",
+    "movement_required_story_flag_id",
+    "movement_forbidden_story_flag_id",
+    "battle_provider_selector",
+    "battle_activation_percent",
+    "battle_parameter_1",
+    "battle_reserved_1",
+    "battle_reserved_2",
+    "battle_reserved_3",
+    "battle_reserved_4",
+    "battle_reserved_5",
+    "battle_required_story_flag_id",
+    "battle_forbidden_story_flag_id",
+)
+
 UNIT_STAT_COLUMNS = (
     "accuracy", "evasion", "physical_attack", "physical_defense",
     "magic_attack", "magic_defense", "speed", "luck", "critical_chance",
@@ -188,6 +223,32 @@ def _paired_parallel_writes(scr) -> tuple[list[tuple], int] | None:
     return writes, span
 
 
+def _routine_bank_writes(scr) -> list[tuple] | None:
+    """Recognize RTINIT's twenty reserved 1000-by-20 routine-step banks."""
+    writes = []
+    for ins in scr.instructions:
+        write = _static_global_write(ins)
+        if write is not None and isinstance(write[1], int):
+            destination, value = write
+            relative = destination - ROUTINE_BANK_ROOT
+            if not (0 <= relative < ROUTINE_BANK_COUNT * ROUTINE_BANK_SPAN):
+                return None
+            bank_index, cell = divmod(relative, ROUTINE_BANK_SPAN)
+            record_id, slot = divmod(cell, ROUTINE_RECORD_STRIDE)
+            if not (
+                0 <= bank_index < ROUTINE_BANK_COUNT
+                and 0 <= record_id < ROUTINE_RECORD_SPAN
+                and 0 <= slot < ROUTINE_RECORD_STRIDE
+            ):
+                return None
+            writes.append((
+                ins.offset, destination, value, bank_index, record_id, slot
+            ))
+        elif sys4load.display_label(ins.opcode) != "exit":
+            return None
+    return writes if len(writes) >= 1000 else None
+
+
 def detect_mode(scr):
     ops = [ins.opcode for ins in scr.instructions]
     has_str = any(ins.opcode == SET_STRING and ins.args and ins.args[0][0] == T_GLOBAL_STRING
@@ -206,6 +267,8 @@ def detect_mode(scr):
         return "name"
     if _paired_parallel_writes(scr):
         return "dispatch"
+    if _routine_bank_writes(scr):
+        return "banked"
     n_footer = ops.count(COPY_LOCAL_ARRAY)
     n_int = ops.count(MOV) + ops.count(COPY_TO_GLOBAL)
     return "footer" if n_footer >= max(4, n_int) else "numeric"
@@ -823,6 +886,180 @@ def extract_dispatch(scr):
     }
 
 
+def _movement_provider_names(names: dict[int, str]) -> dict[int, str]:
+    providers = {
+        selector: names.get(0x32FB + selector, "")
+        for selector in range(1, 19)
+    }
+    providers.update({
+        51: names.get(0x330E, ""),
+        52: names.get(0x330F, ""),
+        53: names.get(0x3310, ""),
+        61: names.get(0x3311, ""),
+    })
+    return providers
+
+
+def extract_banked(scr):
+    """Extract RTINIT's sparse routine sets across twenty parallel step banks."""
+    writes = _routine_bank_writes(scr)
+    if writes is None:
+        return [], {}
+
+    names = callscript_names()
+    movement_providers = _movement_provider_names(names)
+    battle_providers = {
+        selector: names.get(0x32F6 + selector, "")
+        for selector in range(1, 5)
+    }
+    records_by_id: dict[int, dict] = {}
+    cell_assignments: dict[tuple[int, int, int], list[int]] = collections.defaultdict(list)
+    bank_cells: dict[int, set[tuple[int, int]]] = collections.defaultdict(set)
+
+    for offset, destination, value, bank_index, record_id, slot in writes:
+        bank_base = ROUTINE_BANK_ROOT + bank_index * ROUTINE_BANK_SPAN
+        key = f"0x{bank_base:x}/{ROUTINE_RECORD_STRIDE}/{slot}"
+        assignment = {
+            "offset": f"0x{offset:x}",
+            "bank_index": bank_index,
+            "bank_base": f"0x{bank_base:x}",
+            "role": ROUTINE_BANK_ROLES[bank_index],
+            "slot": slot,
+            "value": value,
+        }
+        record = records_by_id.setdefault(record_id, {
+            "id": record_id,
+            "assignments": [],
+            "record_fields": {},
+        })
+        record["assignments"].append(assignment)
+        record["record_fields"][key] = value
+        cell_assignments[(bank_index, record_id, slot)].append(value)
+        bank_cells[bank_index].add((record_id, slot))
+
+    for record in records_by_id.values():
+        final_by_bank_slot = {}
+        for assignment in record["assignments"]:
+            final_by_bank_slot[
+                (assignment["bank_index"], assignment["slot"])
+            ] = assignment["value"]
+
+        movement_steps = []
+        battle_steps = []
+        for slot in range(ROUTINE_RECORD_STRIDE):
+            movement = {
+                ROUTINE_BANK_ROLES[bank]: final_by_bank_slot[(bank, slot)]
+                for bank in range(10)
+                if (bank, slot) in final_by_bank_slot
+            }
+            if movement:
+                selector = movement.get("movement_provider_selector")
+                movement_steps.append({
+                    "slot": slot,
+                    **movement,
+                    **(
+                        {"provider_script": movement_providers.get(selector, "")}
+                        if selector is not None else {}
+                    ),
+                })
+
+            battle = {
+                ROUTINE_BANK_ROLES[bank]: final_by_bank_slot[(bank, slot)]
+                for bank in range(10, 20)
+                if (bank, slot) in final_by_bank_slot
+            }
+            if battle:
+                selector = battle.get("battle_provider_selector")
+                battle_steps.append({
+                    "slot": slot,
+                    **battle,
+                    **(
+                        {"provider_script": battle_providers.get(selector, "")}
+                        if selector is not None else {}
+                    ),
+                })
+        if movement_steps:
+            record["movement_steps"] = movement_steps
+        if battle_steps:
+            record["battle_steps"] = battle_steps
+
+    records = [records_by_id[key] for key in sorted(records_by_id)]
+    record_ids = set(records_by_id)
+    used_movement_providers = sorted({
+        step["movement_provider_selector"]
+        for record in records
+        for step in record.get("movement_steps", [])
+    })
+    used_battle_providers = sorted({
+        step["battle_provider_selector"]
+        for record in records
+        for step in record.get("battle_steps", [])
+    })
+    bank_layouts = {}
+    for bank_index, role in enumerate(ROUTINE_BANK_ROLES):
+        base = ROUTINE_BANK_ROOT + bank_index * ROUTINE_BANK_SPAN
+        cells = bank_cells.get(bank_index, set())
+        bank_layouts[f"0x{base:x}"] = {
+            "bank_index": bank_index,
+            "family": "movement" if bank_index < 10 else "battle",
+            "role": role,
+            "reserved_empty": not cells,
+            "populated_cell_count": len(cells),
+            "populated_record_count": len({record_id for record_id, _ in cells}),
+            "populated_slots": sorted({slot for _, slot in cells}),
+        }
+    record_columns = sorted(
+        {
+            key
+            for record in records
+            for key in record.get("record_fields", {})
+        },
+        key=lambda key: tuple(int(part, 0) for part in key.split("/")),
+    )
+    return records, {
+        "schema": "routine-step-banks",
+        "selector_global": f"0x{ROUTINE_SET_ID:x}",
+        "step_index_global": f"0x{ROUTINE_STEP_INDEX:x}",
+        "execution_state_global": f"0x{ROUTINE_EXECUTION_STATE:x}",
+        "bank_root_base": f"0x{ROUTINE_BANK_ROOT:x}",
+        "bank_span": ROUTINE_BANK_SPAN,
+        "bank_count": ROUTINE_BANK_COUNT,
+        "record_stride": ROUTINE_RECORD_STRIDE,
+        "reserved_record_span": ROUTINE_RECORD_SPAN,
+        "first_record_id": min(record_ids),
+        "last_record_id": max(record_ids),
+        "missing_record_ids": sorted(
+            set(range(min(record_ids), max(record_ids) + 1)) - record_ids
+        ),
+        "assignment_count": len(writes),
+        "populated_cell_count": len(cell_assignments),
+        "overwritten_cell_count": sum(
+            len(values) > 1 for values in cell_assignments.values()
+        ),
+        "conflicting_overwrite_count": sum(
+            len(set(values)) > 1 for values in cell_assignments.values()
+        ),
+        "movement_step_count": sum(
+            len(record.get("movement_steps", [])) for record in records
+        ),
+        "battle_step_count": sum(
+            len(record.get("battle_steps", [])) for record in records
+        ),
+        "movement_provider_scripts": {
+            str(selector): name
+            for selector, name in sorted(movement_providers.items())
+        },
+        "battle_provider_scripts": {
+            str(selector): name
+            for selector, name in sorted(battle_providers.items())
+        },
+        "used_movement_provider_selectors": used_movement_providers,
+        "used_battle_provider_selectors": used_battle_providers,
+        "bank_layouts": bank_layouts,
+        "record_field_columns": record_columns,
+    }
+
+
 def extract_footer(scr):
     records = []
     for i, ins in enumerate(scr.instructions):
@@ -1172,6 +1409,11 @@ def write_data_index(data_dir: Path) -> None:
         "while exposing the final sparse decision-id registry. Packed resource ids join to",
         "SYS4INI script names, authored chapter tags correlate with SCJUMP's decoded decision",
         "sites, and legacy/stale chapter mismatches remain explicit.",
+        "",
+        "Banked-mode tables preserve RTINIT's twenty parallel 1000-by-20 routine banks,",
+        "source-ordered overwrites, and final row/slot values. Joined movement and battle",
+        "steps resolve provider selectors to RTN_M/RTN_B scripts while provider-specific",
+        "parameter banks retain structural names until their individual consumers prove more.",
         "Use `tools/init_table_profile.py <TABLE> --build` to generate value/population and",
         "direct-consumer evidence.",
         "",
@@ -1212,6 +1454,7 @@ def main() -> int:
         "mixed": extract_mixed,
         "rules": extract_class_change_rules,
         "dispatch": extract_dispatch,
+        "banked": extract_banked,
     }[mode]
     recs, meta = extractor(scr)
     if mode == "name" and name in MESSAGE_TABLES:
@@ -1236,7 +1479,11 @@ def main() -> int:
     outpath.write_text(json.dumps(out, ensure_ascii=False, indent=2), encoding="utf-8")
     write_data_index(outpath.parent)
     print(f"{name}: mode={mode}, {len(recs)} records"
-          + (f", {len(cols)} field-columns" if mode != 'footer' else "")
+          + (
+              f", {len(meta.get('record_field_columns', []))} record-columns"
+              if mode == "banked"
+              else f", {len(cols)} field-columns" if mode != "footer" else ""
+          )
           + f" -> build/data/{outname}.json")
     for r in recs[:4]:
         if mode == "footer":

tools/init_table_profile.py

@@ -37,7 +37,9 @@ def load_table(name: str) -> dict:
     if not path.exists():
         raise SystemExit(f"missing extracted table: {path}")
     data = json.loads(path.read_text(encoding="utf8"))
-    if data.get("mode") not in {"name", "numeric", "mixed", "rules", "dispatch"}:
+    if data.get("mode") not in {
+        "name", "numeric", "mixed", "rules", "dispatch", "banked"
+    }:
         raise SystemExit(f"{name}: unsupported field-profiling mode {data.get('mode')!r}")
     return data
 
@@ -205,6 +207,41 @@ def profile_dispatch(data: dict) -> dict:
     }
 
 
+def profile_banked(data: dict) -> dict:
+    """Summarize RTINIT's routine banks, steps, and overwrite history."""
+    if data.get("mode") != "banked":
+        return {}
+    layouts = data.get("bank_layouts", {})
+    return {
+        "assignment_count": data.get("assignment_count", 0),
+        "populated_cell_count": data.get("populated_cell_count", 0),
+        "overwritten_cell_count": data.get("overwritten_cell_count", 0),
+        "conflicting_overwrite_count": data.get("conflicting_overwrite_count", 0),
+        "populated_bank_count": sum(
+            not layout.get("reserved_empty", False)
+            for layout in layouts.values()
+        ),
+        "reserved_bank_count": sum(
+            layout.get("reserved_empty", False)
+            for layout in layouts.values()
+        ),
+        "movement_step_count": data.get("movement_step_count", 0),
+        "battle_step_count": data.get("battle_step_count", 0),
+        "movement_provider_count": len(
+            data.get("used_movement_provider_selectors", [])
+        ),
+        "battle_provider_count": len(
+            data.get("used_battle_provider_selectors", [])
+        ),
+        "available_movement_provider_count": len(
+            data.get("movement_provider_scripts", {})
+        ),
+        "available_battle_provider_count": len(
+            data.get("battle_provider_scripts", {})
+        ),
+    }
+
+
 def profile_messages(data: dict) -> dict:
     """Summarize the joined player-facing message evidence."""
     records = data["records"]
@@ -342,6 +379,23 @@ def render_markdown(data: dict, rows: list[dict], limit: int) -> str:
             f"- explicit chapter mismatches: "
             f"{dispatch_profile['scjump_chapter_mismatch_count']}",
         ])
+    elif banked_profile := profile_banked(data):
+        lines.extend([
+            f"- source assignments: {banked_profile['assignment_count']}",
+            f"- final populated cells: {banked_profile['populated_cell_count']}",
+            f"- overwritten cells: {banked_profile['overwritten_cell_count']} "
+            f"({banked_profile['conflicting_overwrite_count']} change value)",
+            f"- banks: {banked_profile['populated_bank_count']} populated, "
+            f"{banked_profile['reserved_bank_count']} reserved/empty",
+            f"- joined movement steps/used providers: "
+            f"{banked_profile['movement_step_count']}/"
+            f"{banked_profile['movement_provider_count']} "
+            f"({banked_profile['available_movement_provider_count']} dispatchable)",
+            f"- joined battle steps/used providers: "
+            f"{banked_profile['battle_step_count']}/"
+            f"{banked_profile['battle_provider_count']} "
+            f"({banked_profile['available_battle_provider_count']} dispatchable)",
+        ])
     else:
         lines.extend([
             f"- player-facing messages: {message_profile['population']}/{data['record_count']} "
@@ -406,6 +460,7 @@ def main() -> int:
         "message_profile": messages,
         "rule_profile": profile_rules(data),
         "dispatch_profile": profile_dispatch(data),
+        "banked_profile": profile_banked(data),
         "columns": sorted(rows, key=lambda row: (
             int(row["base"], 16), row["stride"] or 0, row["column"] or 0
         )),

tools/test_extract_init.py

@@ -313,12 +313,6 @@ def test_real_scene_dispatch() -> None:
     script = sys4load.load(extract_init.resolve("SCINIT"))
     check(extract_init.detect_mode(script) == "dispatch",
           "SCINIT auto-detects as paired scene dispatch arrays")
-    check(
-        extract_init.detect_mode(
-            sys4load.load(extract_init.resolve("RTINIT"))
-        ) == "numeric",
-        "RTINIT's multi-table writes do not false-positive as paired dispatch",
-    )
     records, meta = extract_init.extract_dispatch(script)
     by_id = {record["id"]: record for record in records}
     check(len(records) == 1209 and meta["assignment_count"] == 2179,
@@ -349,6 +343,66 @@ def test_real_scene_dispatch() -> None:
           "SCINIT's paired columns join to canonical semantic names")
 
 
+def test_real_routine_banks() -> None:
+    script = sys4load.load(extract_init.resolve("RTINIT"))
+    check(extract_init.detect_mode(script) == "banked",
+          "RTINIT auto-detects as parallel routine-step banks")
+    records, meta = extract_init.extract_banked(script)
+    by_id = {record["id"]: record for record in records}
+    check(len(records) == 172
+          and meta["first_record_id"] == 1
+          and meta["last_record_id"] == 176
+          and meta["missing_record_ids"] == [150, 151, 152, 153],
+          "RTINIT preserves its sparse one-based routine-set ids")
+    check(meta["bank_root_base"] == "0xeff78"
+          and meta["bank_span"] == 20000
+          and meta["bank_count"] == 20
+          and meta["record_stride"] == 20
+          and meta["reserved_record_span"] == 1000,
+          "RTINIT exposes twenty parallel 1000-by-20 banks")
+    check(meta["assignment_count"] == 3336
+          and meta["populated_cell_count"] == 3307
+          and meta["overwritten_cell_count"] == 29
+          and meta["conflicting_overwrite_count"] == 11,
+          "RTINIT preserves source assignments and final overwrite accounting")
+    check(meta["movement_step_count"] == 1043
+          and meta["battle_step_count"] == 14
+          and len(meta["used_movement_provider_selectors"]) == 19
+          and len(meta["used_battle_provider_selectors"]) == 4
+          and len(meta["record_field_columns"]) == 117,
+          "RTINIT assembles every populated movement and battle step")
+    check([
+        layout["bank_index"]
+        for layout in meta["bank_layouts"].values()
+        if layout["reserved_empty"]
+    ] == [6, 13, 14, 15, 16, 17],
+          "RTINIT keeps all six reserved empty banks in its structural layout")
+    movement = by_id[1]["movement_steps"][0]
+    battle = by_id[1]["battle_steps"][0]
+    check(movement["movement_provider_selector"] == 1
+          and movement["movement_activation_percent"] == 100
+          and movement["provider_script"] == "RTN_M001.BIN",
+          "RTINIT joins movement selectors and activation percentages")
+    check(battle["battle_provider_selector"] == 1
+          and battle["battle_activation_percent"] == 100
+          and battle["provider_script"] == "RTN_B001.BIN",
+          "RTINIT joins battle selectors and activation percentages")
+    check(by_id[2]["battle_steps"][0]["battle_parameter_1"] == 219
+          and by_id[2]["battle_steps"][0]["provider_script"] == "RTN_B004.BIN",
+          "RTINIT retains provider-specific battle parameters")
+    check(by_id[173]["movement_steps"][0]["movement_parameter_1"] == 2
+          and by_id[173]["movement_steps"][0]["movement_parameter_2"] == 158,
+          "RTINIT final rows reflect source-ordered conflicting overwrites")
+    semantics = extract_init.field_semantics(records)
+    check(
+        semantics["0xeff78/20/0"]
+        == "movement_routine_provider_selectors.column_0"
+        and semantics["0x125ad8/20/0"]
+        == "battle_routine_activation_percents.column_0",
+        "RTINIT raw banks join to canonical structural field names",
+    )
+
+
 def test_real_message_tables() -> None:
     scripts = paths.scripts()
     expected = {
@@ -524,6 +578,7 @@ if __name__ == "__main__":
     test_real_mixed_table()
     test_real_class_change_rules()
     test_real_scene_dispatch()
+    test_real_routine_banks()
     test_real_message_tables()
     test_message_join()
     test_field_semantics()

tools/test_init_table_profile.py

@@ -120,6 +120,44 @@ def main() -> int:
     assert dispatch_summary["scjump_chapter_match_count"] == 1
     assert dispatch_summary["scjump_chapter_mismatch_count"] == 1
 
+    banked_fixture = {
+        "table": "BANKED",
+        "mode": "banked",
+        "assignment_count": 6,
+        "populated_cell_count": 5,
+        "overwritten_cell_count": 1,
+        "conflicting_overwrite_count": 1,
+        "movement_step_count": 2,
+        "battle_step_count": 1,
+        "movement_provider_scripts": {"1": "RTN_M001.BIN"},
+        "battle_provider_scripts": {"1": "RTN_B001.BIN"},
+        "used_movement_provider_selectors": [1],
+        "used_battle_provider_selectors": [1],
+        "bank_layouts": {
+            "0x100": {"reserved_empty": False},
+            "0x200": {"reserved_empty": True},
+        },
+        "records": [
+            {
+                "id": 1,
+                "record_fields": {
+                    "0x100/20/0": 1,
+                    "0x100/20/1": 2,
+                },
+            },
+        ],
+    }
+    banked_rows = {
+        row["key"]: row for row in profile.profile_columns(banked_fixture)
+    }
+    assert banked_rows["0x100/20/0"]["kind"] == "record-column"
+    banked_summary = profile.profile_banked(banked_fixture)
+    assert banked_summary["assignment_count"] == 6
+    assert banked_summary["populated_bank_count"] == 1
+    assert banked_summary["reserved_bank_count"] == 1
+    assert banked_summary["movement_step_count"] == 2
+    assert banked_summary["battle_provider_count"] == 1
+
     messages = profile.profile_messages(fixture)
     assert messages["population"] == 1
     assert messages["coverage"] == 1 / 3