Correct INIT extraction and profile item fields
This commit is contained in:
gamer147
@@ -71,26 +71,58 @@ def detect_mode(scr):
|
||||
return "footer" if n_footer >= max(4, n_int) else "numeric"
|
||||
|
||||
|
||||
def _infer_record_span(string_addrs):
|
||||
"""Infer the reserved width of one parallel string-array column.
|
||||
|
||||
The shipped INIT tables reserve a fixed number of ids per column (300 for
|
||||
SKINIT and 1000 for ITINIT/EBINIT). A populated record commonly writes its
|
||||
name and then its description, so that column stride is the dominant large
|
||||
positive delta between consecutive string destinations.
|
||||
"""
|
||||
counts = {}
|
||||
for left, right in zip(string_addrs, string_addrs[1:]):
|
||||
delta = right - left
|
||||
if delta >= 32:
|
||||
counts[delta] = counts.get(delta, 0) + 1
|
||||
if not counts:
|
||||
raise ValueError("cannot infer name-table record span")
|
||||
return max(counts, key=lambda delta: (counts[delta], delta))
|
||||
|
||||
|
||||
def extract_name(scr):
|
||||
name_base = None
|
||||
for ins in scr.instructions:
|
||||
if ins.opcode == SET_STRING and ins.args and ins.args[0][0] == T_GLOBAL_STRING:
|
||||
name_base = ins.args[0][1]; break
|
||||
records, cur, prev_gstr, desc_slot, desc_bases = [], None, None, 0, {}
|
||||
string_addrs = [
|
||||
ins.args[0][1]
|
||||
for ins in scr.instructions
|
||||
if ins.opcode == SET_STRING and ins.args and ins.args[0][0] == T_GLOBAL_STRING
|
||||
]
|
||||
if not string_addrs:
|
||||
return [], {}
|
||||
name_write_base = string_addrs[0]
|
||||
# AGE's shipped entity ids are one-based. Array lookups use the cell just
|
||||
# before the first populated destination as their base, then add the id.
|
||||
first_record_id = 1
|
||||
name_base = name_write_base - first_record_id
|
||||
record_span = _infer_record_span(string_addrs)
|
||||
records, cur, desc_slot, desc_bases = [], None, 0, {}
|
||||
for ins in scr.instructions:
|
||||
if ins.opcode == SET_STRING and ins.args and ins.args[0][0] == T_GLOBAL_STRING:
|
||||
addr = ins.args[0][1]
|
||||
txt = scr.strings.get(ins.args[1][1], (None,))[0] if len(ins.args) > 1 else None
|
||||
if prev_gstr is None or addr < prev_gstr:
|
||||
# Names occupy column zero. Do not use an address decrease as the
|
||||
# boundary: ITINIT begins with 101 consecutive name-only records,
|
||||
# which the old heuristic collapsed into item zero.
|
||||
if name_write_base <= addr < name_write_base + record_span:
|
||||
cur = {"id": addr - name_base, "name": txt, "fields": {}}
|
||||
records.append(cur); desc_slot = 0
|
||||
elif cur is not None:
|
||||
key = "desc" if desc_slot == 0 else f"desc{desc_slot}"
|
||||
cur[key] = txt; desc_bases.setdefault(key, addr - cur["id"]); desc_slot += 1
|
||||
prev_gstr = addr
|
||||
elif ins.opcode == MOV and cur is not None and ins.args and ins.args[0][0] == T_GLOBAL_INT:
|
||||
cur["fields"][f"0x{ins.args[0][1] - cur['id']:x}"] = _val(ins.args[1])
|
||||
return records, {"name_array_base": f"0x{name_base:x}",
|
||||
"name_write_base": f"0x{name_write_base:x}",
|
||||
"first_record_id": first_record_id,
|
||||
"record_span": record_span,
|
||||
"desc_array_bases": {k: f"0x{v:x}" for k, v in sorted(desc_bases.items())}}
|
||||
|
||||
|
||||
@@ -149,6 +181,45 @@ def extract_footer(scr):
|
||||
return records, {}
|
||||
|
||||
|
||||
def write_data_index(data_dir: Path) -> None:
|
||||
"""Regenerate the disposable build/data index from current table JSONs."""
|
||||
tables = []
|
||||
for path in sorted(data_dir.glob("*.json")):
|
||||
if path.name.endswith("-field-profile.json"):
|
||||
continue
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
continue
|
||||
if "table" in data and "record_count" in data:
|
||||
tables.append((path.name, data))
|
||||
lines = [
|
||||
"<!-- DO NOT EDIT -- generated by tools/extract_init.py -->",
|
||||
"# Parsed INIT data tables",
|
||||
"",
|
||||
"The JSON files in this directory are generated from SYS4 `*INIT` scripts. Raw global-array",
|
||||
"bases remain available in every record; confirmed field meanings live in",
|
||||
"`vm-map/globals.toml` and the generated `docs/global-reference.md`.",
|
||||
"",
|
||||
"| file | mode | records | field bases |",
|
||||
"|---|---|---:|---:|",
|
||||
]
|
||||
for filename, data in tables:
|
||||
columns = len(data.get("field_columns") or [])
|
||||
lines.append(f"| `{filename}` | {data['mode']} | {data['record_count']} | {columns} |")
|
||||
lines += [
|
||||
"",
|
||||
"Name-mode tables expose one-based runtime `id` values, the lookup `name_array_base`,",
|
||||
"the first populated `name_write_base`, and the reserved `record_span`. Fields are keyed",
|
||||
"by the runtime lookup base used by `lookup-array`, not merely the first written cell.",
|
||||
"",
|
||||
"Use `tools/init_table_profile.py <TABLE> --build` to generate value/population and",
|
||||
"direct-consumer evidence. `STINIT` still requires a bespoke mixed numeric/string parser.",
|
||||
"",
|
||||
]
|
||||
(data_dir / "README.md").write_text("\n".join(lines), encoding="utf8")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
argv = [a for a in sys.argv[1:] if not a.startswith("--")]
|
||||
mode_arg = next((sys.argv[i + 1] for i, a in enumerate(sys.argv) if a == "--mode"), None)
|
||||
@@ -169,6 +240,7 @@ def main() -> int:
|
||||
outpath = paths.BUILD / "data" / f"{outname}.json"
|
||||
outpath.parent.mkdir(parents=True, exist_ok=True)
|
||||
outpath.write_text(json.dumps(out, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
write_data_index(outpath.parent)
|
||||
print(f"{name}: mode={mode}, {len(recs)} records"
|
||||
+ (f", {len(cols)} field-columns" if mode != 'footer' else "")
|
||||
+ f" -> build/data/{outname}.json")
|
||||
|
||||
184
tools/init_table_profile.py
Normal file
184
tools/init_table_profile.py
Normal file
@@ -0,0 +1,184 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Profile extracted INIT columns and find their direct script consumers.
|
||||
|
||||
The extractor tells us which global-array bases are populated with each named
|
||||
record. This tool adds the next layer of evidence: population/value shape and
|
||||
every corpus instruction that refers to the array base directly. The output is
|
||||
an investigation surface, not a semantic source of truth; confirmed field names
|
||||
belong in vm-map/globals.toml.
|
||||
|
||||
Usage:
|
||||
py -3.11 -X utf8 tools/init_table_profile.py ITINIT
|
||||
py -3.11 -X utf8 tools/init_table_profile.py ITINIT --build
|
||||
py -3.11 -X utf8 tools/init_table_profile.py ITINIT --limit 80
|
||||
|
||||
With --build, writes build/data/<TABLE>-field-profile.{json,md}.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import collections
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE))
|
||||
import paths
|
||||
import sys4load
|
||||
|
||||
|
||||
GLOBAL_OPERAND_TYPES = {3, 4, 5, 6, 8}
|
||||
|
||||
|
||||
def load_table(name: str) -> dict:
|
||||
path = paths.BUILD / "data" / f"{name}.json"
|
||||
if not path.exists():
|
||||
raise SystemExit(f"missing extracted table: {path}")
|
||||
data = json.loads(path.read_text(encoding="utf8"))
|
||||
if data.get("mode") not in {"name", "numeric"}:
|
||||
raise SystemExit(f"{name}: field profiling requires name/numeric mode")
|
||||
return data
|
||||
|
||||
|
||||
def value_key(value) -> str:
|
||||
if isinstance(value, dict):
|
||||
return json.dumps(value, ensure_ascii=False, sort_keys=True)
|
||||
return str(value)
|
||||
|
||||
|
||||
def profile_columns(data: dict) -> list[dict]:
|
||||
records = data["records"]
|
||||
values: dict[int, list] = collections.defaultdict(list)
|
||||
examples: dict[int, list[dict]] = collections.defaultdict(list)
|
||||
for record in records:
|
||||
for address, value in record.get("fields", {}).items():
|
||||
base = int(address, 16)
|
||||
values[base].append(value)
|
||||
if len(examples[base]) < 5:
|
||||
examples[base].append({
|
||||
"id": record["id"],
|
||||
"name": record.get("name", ""),
|
||||
"value": value,
|
||||
})
|
||||
|
||||
rows = []
|
||||
for base, vals in values.items():
|
||||
common = collections.Counter(value_key(value) for value in vals).most_common(6)
|
||||
numeric = vals and all(isinstance(value, int) for value in vals)
|
||||
rows.append({
|
||||
"base": f"0x{base:x}",
|
||||
"population": len(vals),
|
||||
"coverage": len(vals) / len(records) if records else 0.0,
|
||||
"distinct_values": len({value_key(value) for value in vals}),
|
||||
"min": min(vals) if numeric else None,
|
||||
"max": max(vals) if numeric else None,
|
||||
"common": [{"value": value, "count": count} for value, count in common],
|
||||
"examples": examples[base],
|
||||
"references": 0,
|
||||
"reader_scripts": [],
|
||||
"reference_ops": [],
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def add_direct_references(rows: list[dict], source_name: str) -> None:
|
||||
by_base = {int(row["base"], 16): row for row in rows}
|
||||
scripts: dict[int, collections.Counter] = {
|
||||
base: collections.Counter() for base in by_base
|
||||
}
|
||||
ops: dict[int, collections.Counter] = {
|
||||
base: collections.Counter() for base in by_base
|
||||
}
|
||||
for name, path in paths.scripts().items():
|
||||
if name.upper() == source_name.upper():
|
||||
continue
|
||||
try:
|
||||
script = sys4load.load(path)
|
||||
except sys4load.Sys4Error:
|
||||
continue
|
||||
for ins in script.instructions:
|
||||
for arg_index, (arg_type, value) in enumerate(ins.args):
|
||||
if arg_type not in GLOBAL_OPERAND_TYPES or value not in by_base:
|
||||
continue
|
||||
scripts[value][name] += 1
|
||||
ops[value][f"{sys4load.display_label(ins.opcode)}:arg{arg_index + 1}"] += 1
|
||||
|
||||
for base, row in by_base.items():
|
||||
row["references"] = sum(scripts[base].values())
|
||||
row["reader_scripts"] = [
|
||||
{"script": script, "count": count}
|
||||
for script, count in scripts[base].most_common()
|
||||
]
|
||||
row["reference_ops"] = [
|
||||
{"operation": operation, "count": count}
|
||||
for operation, count in ops[base].most_common()
|
||||
]
|
||||
|
||||
|
||||
def render_markdown(data: dict, rows: list[dict], limit: int) -> str:
|
||||
ranked = sorted(rows, key=lambda row: (-row["population"], -row["references"], row["base"]))
|
||||
shown = ranked[:limit]
|
||||
lines = [
|
||||
f"# {data['table']} field profile",
|
||||
"",
|
||||
"> Generated by `tools/init_table_profile.py` — do not hand-edit.",
|
||||
"> This is evidence for investigation; confirmed names live in `vm-map/globals.toml`.",
|
||||
"",
|
||||
f"- records: {data['record_count']}",
|
||||
f"- populated global-array bases: {len(rows)}",
|
||||
f"- rows shown: {len(shown)} (ranked by record coverage, then consumer references)",
|
||||
"",
|
||||
"| base | populated | distinct | range | direct refs | readers | common values | examples |",
|
||||
"|---|---:|---:|---|---:|---|---|---|",
|
||||
]
|
||||
for row in shown:
|
||||
value_range = "—" if row["min"] is None else f"{row['min']}..{row['max']}"
|
||||
readers = ", ".join(entry["script"].removesuffix(".BIN")
|
||||
for entry in row["reader_scripts"][:5]) or "—"
|
||||
common = ", ".join(f"{entry['value']}×{entry['count']}" for entry in row["common"][:4])
|
||||
examples = ", ".join(
|
||||
f"{entry['id']}:{entry['name']}={value_key(entry['value'])}"
|
||||
for entry in row["examples"][:3]
|
||||
).replace("|", "\\|")
|
||||
lines.append(
|
||||
f"| `{row['base']}` | {row['population']}/{data['record_count']} "
|
||||
f"({row['coverage']:.0%}) | {row['distinct_values']} | {value_range} | "
|
||||
f"{row['references']} | {readers} | {common} | {examples} |"
|
||||
)
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("table", help="extracted table name, e.g. ITINIT")
|
||||
parser.add_argument("--build", action="store_true", help="write JSON and Markdown profiles")
|
||||
parser.add_argument("--limit", type=int, default=40, help="Markdown/console row limit")
|
||||
args = parser.parse_args()
|
||||
|
||||
name = args.table.upper().removesuffix(".JSON").removesuffix(".BIN")
|
||||
data = load_table(name)
|
||||
rows = profile_columns(data)
|
||||
add_direct_references(rows, data["source"])
|
||||
output = {
|
||||
"table": data["table"],
|
||||
"source": data["source"],
|
||||
"record_count": data["record_count"],
|
||||
"field_base_count": len(rows),
|
||||
"columns": sorted(rows, key=lambda row: int(row["base"], 16)),
|
||||
}
|
||||
markdown = render_markdown(data, rows, args.limit)
|
||||
print(markdown)
|
||||
if args.build:
|
||||
stem = paths.BUILD / "data" / f"{name}-field-profile"
|
||||
stem.with_suffix(".json").write_text(
|
||||
json.dumps(output, ensure_ascii=False, indent=2), encoding="utf8"
|
||||
)
|
||||
stem.with_suffix(".md").write_text(markdown, encoding="utf8")
|
||||
print(f"wrote {stem.relative_to(paths.REPO)}.json/.md")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
54
tools/test_extract_init.py
Normal file
54
tools/test_extract_init.py
Normal file
@@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Regression tests for INIT table extraction.
|
||||
|
||||
Run: py -3.11 -X utf8 tools/test_extract_init.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import extract_init
|
||||
import paths
|
||||
import sys4load
|
||||
|
||||
|
||||
FAILS: list[str] = []
|
||||
|
||||
|
||||
def check(condition: bool, message: str) -> None:
|
||||
print((" ok " if condition else " FAIL ") + message)
|
||||
if not condition:
|
||||
FAILS.append(message)
|
||||
|
||||
|
||||
def test_real_name_tables() -> None:
|
||||
expected = {
|
||||
"SKINIT.BIN": (300, 131),
|
||||
"ITINIT.BIN": (1000, 287),
|
||||
"EBINIT.BIN": (1000, 277),
|
||||
}
|
||||
scripts = paths.scripts()
|
||||
for name, (span, count) in expected.items():
|
||||
records, meta = extract_init.extract_name(sys4load.load(scripts[name]))
|
||||
check(meta["record_span"] == span, f"{name}: record span is {span}")
|
||||
check(len(records) == count, f"{name}: extracts {count} named records")
|
||||
check(len({record["id"] for record in records}) == count,
|
||||
f"{name}: record ids are unique")
|
||||
|
||||
items, _ = extract_init.extract_name(sys4load.load(scripts["ITINIT.BIN"]))
|
||||
by_id = {record["id"]: record for record in items}
|
||||
check(by_id[1]["name"] == "銅の鍵", "ITINIT item 1 is the copper key")
|
||||
check(len(by_id[1]["fields"]) == 5, "ITINIT item 1 owns only its five fields")
|
||||
check("desc" not in by_id[1], "ITINIT item 1 has no fabricated description")
|
||||
check(by_id[101]["desc"] == "ＨＰ３０回復", "ITINIT item 101 keeps its description")
|
||||
check(by_id[1]["fields"]["0x8c879"] == 10,
|
||||
"ITINIT columns use the runtime lookup base")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_real_name_tables()
|
||||
if FAILS:
|
||||
raise SystemExit(f"{len(FAILS)} failed checks")
|
||||
print("all extract_init checks passed")
|
||||
33
tools/test_init_table_profile.py
Normal file
33
tools/test_init_table_profile.py
Normal file
@@ -0,0 +1,33 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Tests for INIT field profiling. Run: py -3.11 -X utf8 tools/test_init_table_profile.py"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import init_table_profile as profile
|
||||
|
||||
|
||||
def main() -> int:
|
||||
fixture = {
|
||||
"records": [
|
||||
{"id": 1, "name": "one", "fields": {"0x10": 2, "0x20": 0}},
|
||||
{"id": 3, "name": "three", "fields": {"0x10": 2}},
|
||||
{"id": 7, "name": "seven", "fields": {"0x10": 5}},
|
||||
]
|
||||
}
|
||||
rows = {row["base"]: row for row in profile.profile_columns(fixture)}
|
||||
assert rows["0x10"]["population"] == 3
|
||||
assert rows["0x10"]["coverage"] == 1.0
|
||||
assert rows["0x10"]["distinct_values"] == 2
|
||||
assert rows["0x10"]["min"] == 2 and rows["0x10"]["max"] == 5
|
||||
assert rows["0x10"]["common"][0] == {"value": "2", "count": 2}
|
||||
assert rows["0x20"]["population"] == 1
|
||||
assert rows["0x20"]["examples"][0]["name"] == "one"
|
||||
print("all init_table_profile checks passed")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user