Correct INIT extraction and profile item fields

tools/extract_init.py

@@ -71,26 +71,58 @@ def detect_mode(scr):
     return "footer" if n_footer >= max(4, n_int) else "numeric"
 
 
+def _infer_record_span(string_addrs):
+    """Infer the reserved width of one parallel string-array column.
+
+    The shipped INIT tables reserve a fixed number of ids per column (300 for
+    SKINIT and 1000 for ITINIT/EBINIT).  A populated record commonly writes its
+    name and then its description, so that column stride is the dominant large
+    positive delta between consecutive string destinations.
+    """
+    counts = {}
+    for left, right in zip(string_addrs, string_addrs[1:]):
+        delta = right - left
+        if delta >= 32:
+            counts[delta] = counts.get(delta, 0) + 1
+    if not counts:
+        raise ValueError("cannot infer name-table record span")
+    return max(counts, key=lambda delta: (counts[delta], delta))
+
+
 def extract_name(scr):
-    name_base = None
-    for ins in scr.instructions:
-        if ins.opcode == SET_STRING and ins.args and ins.args[0][0] == T_GLOBAL_STRING:
-            name_base = ins.args[0][1]; break
-    records, cur, prev_gstr, desc_slot, desc_bases = [], None, None, 0, {}
+    string_addrs = [
+        ins.args[0][1]
+        for ins in scr.instructions
+        if ins.opcode == SET_STRING and ins.args and ins.args[0][0] == T_GLOBAL_STRING
+    ]
+    if not string_addrs:
+        return [], {}
+    name_write_base = string_addrs[0]
+    # AGE's shipped entity ids are one-based.  Array lookups use the cell just
+    # before the first populated destination as their base, then add the id.
+    first_record_id = 1
+    name_base = name_write_base - first_record_id
+    record_span = _infer_record_span(string_addrs)
+    records, cur, desc_slot, desc_bases = [], None, 0, {}
     for ins in scr.instructions:
         if ins.opcode == SET_STRING and ins.args and ins.args[0][0] == T_GLOBAL_STRING:
             addr = ins.args[0][1]
             txt = scr.strings.get(ins.args[1][1], (None,))[0] if len(ins.args) > 1 else None
-            if prev_gstr is None or addr < prev_gstr:
+            # Names occupy column zero.  Do not use an address decrease as the
+            # boundary: ITINIT begins with 101 consecutive name-only records,
+            # which the old heuristic collapsed into item zero.
+            if name_write_base <= addr < name_write_base + record_span:
                 cur = {"id": addr - name_base, "name": txt, "fields": {}}
                 records.append(cur); desc_slot = 0
             elif cur is not None:
                 key = "desc" if desc_slot == 0 else f"desc{desc_slot}"
                 cur[key] = txt; desc_bases.setdefault(key, addr - cur["id"]); desc_slot += 1
-            prev_gstr = addr
         elif ins.opcode == MOV and cur is not None and ins.args and ins.args[0][0] == T_GLOBAL_INT:
             cur["fields"][f"0x{ins.args[0][1] - cur['id']:x}"] = _val(ins.args[1])
     return records, {"name_array_base": f"0x{name_base:x}",
+                     "name_write_base": f"0x{name_write_base:x}",
+                     "first_record_id": first_record_id,
+                     "record_span": record_span,
                      "desc_array_bases": {k: f"0x{v:x}" for k, v in sorted(desc_bases.items())}}
 
 
@@ -149,6 +181,45 @@ def extract_footer(scr):
     return records, {}
 
 
+def write_data_index(data_dir: Path) -> None:
+    """Regenerate the disposable build/data index from current table JSONs."""
+    tables = []
+    for path in sorted(data_dir.glob("*.json")):
+        if path.name.endswith("-field-profile.json"):
+            continue
+        try:
+            data = json.loads(path.read_text(encoding="utf8"))
+        except (OSError, json.JSONDecodeError):
+            continue
+        if "table" in data and "record_count" in data:
+            tables.append((path.name, data))
+    lines = [
+        "<!-- DO NOT EDIT -- generated by tools/extract_init.py -->",
+        "# Parsed INIT data tables",
+        "",
+        "The JSON files in this directory are generated from SYS4 `*INIT` scripts. Raw global-array",
+        "bases remain available in every record; confirmed field meanings live in",
+        "`vm-map/globals.toml` and the generated `docs/global-reference.md`.",
+        "",
+        "| file | mode | records | field bases |",
+        "|---|---|---:|---:|",
+    ]
+    for filename, data in tables:
+        columns = len(data.get("field_columns") or [])
+        lines.append(f"| `{filename}` | {data['mode']} | {data['record_count']} | {columns} |")
+    lines += [
+        "",
+        "Name-mode tables expose one-based runtime `id` values, the lookup `name_array_base`,",
+        "the first populated `name_write_base`, and the reserved `record_span`. Fields are keyed",
+        "by the runtime lookup base used by `lookup-array`, not merely the first written cell.",
+        "",
+        "Use `tools/init_table_profile.py <TABLE> --build` to generate value/population and",
+        "direct-consumer evidence. `STINIT` still requires a bespoke mixed numeric/string parser.",
+        "",
+    ]
+    (data_dir / "README.md").write_text("\n".join(lines), encoding="utf8")
+
+
 def main() -> int:
     argv = [a for a in sys.argv[1:] if not a.startswith("--")]
     mode_arg = next((sys.argv[i + 1] for i, a in enumerate(sys.argv) if a == "--mode"), None)
@@ -169,6 +240,7 @@ def main() -> int:
     outpath = paths.BUILD / "data" / f"{outname}.json"
     outpath.parent.mkdir(parents=True, exist_ok=True)
     outpath.write_text(json.dumps(out, ensure_ascii=False, indent=2), encoding="utf-8")
+    write_data_index(outpath.parent)
     print(f"{name}: mode={mode}, {len(recs)} records"
           + (f", {len(cols)} field-columns" if mode != 'footer' else "")
           + f" -> build/data/{outname}.json")