conco/OpenMaidEngine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Add semantic append INIT inspection

Browse Source
This commit is contained in:
gamer147
2026-07-24 09:45:40 -04:00
parent d449a2fe1e
commit bf9639cafc
9 changed files with 326 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

129
tools/extract_init.py
View File

@@ -56,7 +56,9 @@ Records are {id, name?, desc?, fields:{"0x<col_base>": value}} or, for footer ta
{id, global_addr, footer_off, values:[...]}. Column addresses are raw engine globals;
confirmed names come from the generated engine global registry while raw keys remain provenance.
Usage: py -3.11 -X utf8 tools/extract_init.py <TABLE> [OUTNAME] [--mode name|numeric|footer|mixed|rules|dispatch|banked]
Usage: py -3.11 -X utf8 tools/extract_init.py <TABLE> [OUTNAME]
[--mode name|numeric|footer|mixed|rules|dispatch|banked]
[--packed-id 0xPPxxxxxx]
"""
from __future__ import annotations
import collections
@@ -69,6 +71,7 @@ HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
import paths
import extract_message_table
import parse_sys4ini
import sys4load
SET_STRING = 0x192
@@ -855,6 +858,58 @@ def resolve(name: str) -> Path:
raise SystemExit(f"not found: {name}.BIN")
def load_packed_script(packed_id: int) -> sys4load.Sys4Script:
"""Load one selector-keyed AAI script by its native packed resource id."""
if not 0 <= packed_id <= 0xFFFFFFFF:
raise ValueError(f"packed id outside uint32: {packed_id}")
pack_id = (packed_id >> 24) & 0xFF
raw_index = packed_id & 0xFFFFFF
if not 0 < pack_id < 0x80:
raise ValueError(
f"packed append id must use selector 1..127, got 0x{packed_id:08x}"
)
selected = None
for catalog_path in sorted(paths.GAME_DIR.glob("*.AAI")):
header = catalog_path.read_bytes()[:0x10C]
if len(header) < 0x10C or header[:4] != b"S4AC":
continue
selector = int.from_bytes(header[0x108:0x10C], "little")
if selector == pack_id:
# Match the native later-catalog-wins mount behavior deterministically for tools.
selected = (catalog_path, parse_sys4ini.parse(catalog_path))
if selected is None:
raise FileNotFoundError(f"no mounted AAI catalog for selector {pack_id}")
catalog_path, catalog = selected
entry = next(
(item for item in catalog["files"] if item["raw_index"] == raw_index),
None,
)
if entry is None:
raise FileNotFoundError(
f"{catalog_path.name}: no real record at raw index 0x{raw_index:x}"
)
if not entry["name"].upper().endswith(".BIN"):
raise ValueError(
f"0x{packed_id:08x} is not a SYS4 script: {entry['name']}"
)
archive = paths.GAME_DIR / entry["archive"]
archive_size = archive.stat().st_size
if entry["offset"] + entry["size"] > archive_size:
raise ValueError(
f"{catalog_path.name}: {entry['name']} range exceeds {archive.name}"
)
with archive.open("rb") as stream:
stream.seek(entry["offset"])
payload = stream.read(entry["size"])
if len(payload) != entry["size"]:
raise ValueError(
f"{catalog_path.name}: short read for {entry['name']}"
)
return sys4load.load_bytes(payload, entry["name"])
def normalize_outname(value: str) -> str:
"""Accept a generated-file stem, not a path; tolerate one `.json` suffix."""
if not value or Path(value).name != value or "/" in value or "\\" in value:
@@ -1401,7 +1456,7 @@ def _resolve_parallel_record_overlaps(records):
record["record_fields"] = retained
def extract_name(scr):
def extract_name(scr, layout_hint: dict | None = None):
string_addrs = [
ins.args[0][1]
for ins in scr.instructions
@@ -1409,12 +1464,23 @@ def extract_name(scr):
]
if not string_addrs:
return [], {}
name_write_base = string_addrs[0]
# AGE's shipped entity ids are one-based. Array lookups use the cell just
# before the first populated destination as their base, then add the id.
first_record_id = 1
name_base = name_write_base - first_record_id
record_span = _infer_record_span(string_addrs)
if layout_hint is None:
name_write_base = string_addrs[0]
# AGE's shipped entity ids are one-based. Array lookups use the cell just
# before the first populated destination as their base, then add the id.
first_record_id = 1
name_base = name_write_base - first_record_id
record_span = _infer_record_span(string_addrs)
else:
name_base = int(layout_hint["name_array_base"], 0)
name_write_base = int(layout_hint["name_write_base"], 0)
first_record_id = int(layout_hint["first_record_id"])
record_span = int(layout_hint["record_span"])
if not any(
name_write_base <= address < name_write_base + record_span
for address in string_addrs
):
raise ValueError(f"{scr.path.name}: no name writes inside hinted layout")
records, cur, desc_slot, desc_bases = [], None, 0, {}
for ins in scr.instructions:
if ins.opcode == SET_STRING and ins.args and ins.args[0][0] == T_GLOBAL_STRING:
@@ -1445,12 +1511,15 @@ def extract_name(scr):
{key for record in records for key in record.get("record_fields", {})},
key=lambda key: tuple(int(part, 0) for part in key.split("/")),
)
return records, {"name_array_base": f"0x{name_base:x}",
"name_write_base": f"0x{name_write_base:x}",
"first_record_id": first_record_id,
"record_span": record_span,
"record_field_columns": record_columns,
"desc_array_bases": {k: f"0x{v:x}" for k, v in sorted(desc_bases.items())}}
meta = {"name_array_base": f"0x{name_base:x}",
"name_write_base": f"0x{name_write_base:x}",
"first_record_id": first_record_id,
"record_span": record_span,
"record_field_columns": record_columns,
"desc_array_bases": {k: f"0x{v:x}" for k, v in sorted(desc_bases.items())}}
if layout_hint is not None:
meta["fragment_layout_source"] = layout_hint.get("source", "base table")
return records, meta
def extract_vocabulary(scr):
@@ -6961,6 +7030,7 @@ def write_data_index(data_dir: Path) -> None:
def main() -> int:
argv = []
mode_arg = None
packed_id = None
index = 1
while index < len(sys.argv):
arg = sys.argv[index]
@@ -6970,6 +7040,12 @@ def main() -> int:
mode_arg = sys.argv[index + 1]
index += 2
continue
if arg == "--packed-id":
if index + 1 >= len(sys.argv):
raise SystemExit("--packed-id requires a value")
packed_id = int(sys.argv[index + 1], 0)
index += 2
continue
if arg.startswith("--"):
raise SystemExit(f"unknown option: {arg}")
argv.append(arg)
@@ -6978,10 +7054,20 @@ def main() -> int:
raise SystemExit(__doc__)
name = argv[0].upper().removesuffix(".BIN")
try:
outname = normalize_outname(argv[1]) if len(argv) > 1 else name
default_outname = (
f"APPEND{(packed_id >> 24) & 0xff:02d}-{name}"
if packed_id is not None else name
)
outname = normalize_outname(argv[1]) if len(argv) > 1 else default_outname
except ValueError as error:
raise SystemExit(str(error)) from error
scr = sys4load.load(resolve(name))
try:
scr = (
load_packed_script(packed_id)
if packed_id is not None else sys4load.load(resolve(name))
)
except (FileNotFoundError, ValueError) as error:
raise SystemExit(str(error)) from error
if mode_arg is not None:
mode = mode_arg
@@ -7042,8 +7128,13 @@ def main() -> int:
extractor = extract_h_scene_gallery
elif mode == "footer" and name == "MPINIT":
extractor = extract_map_terrain_atlas
recs, meta = extractor(scr)
if mode == "name" and name in MESSAGE_TABLES:
if packed_id is not None and name == "EBINIT":
_, layout_hint = extract_name(sys4load.load(resolve("EBINIT")))
layout_hint["source"] = "EBINIT.BIN"
recs, meta = extract_name(scr, layout_hint)
else:
recs, meta = extractor(scr)
if packed_id is None and mode == "name" and name in MESSAGE_TABLES:
message_name = MESSAGE_TABLES[name]
meta["message_table"] = join_messages(
recs, sys4load.load(extract_message_table.resolve(message_name))
@@ -7063,6 +7154,8 @@ def main() -> int:
"record_count": len(recs), **meta,
"field_columns": cols if mode != "footer" else None,
"field_semantics": semantics, "records": recs}
if packed_id is not None:
out["packed_id"] = f"0x{packed_id:08x}"
outpath = paths.BUILD / "data" / f"{outname}.json"
outpath.parent.mkdir(parents=True, exist_ok=True)
outpath.write_text(json.dumps(out, ensure_ascii=False, indent=2), encoding="utf-8")

98
tools/init_table_profile.py
View File

@@ -701,6 +701,92 @@ def render_message_matches(data: dict, pattern: str) -> str:
return "\n".join(lines)
def find_record_matches(data: dict, selector: str) -> list[dict]:
"""Select a record by numeric id, exact name, or name regex."""
try:
record_id = int(selector, 0)
except ValueError:
folded = selector.casefold()
exact = [
record
for record in data["records"]
if record.get("name", "").casefold() == folded
]
if exact:
return exact
regex = re.compile(selector, re.IGNORECASE)
return [
record
for record in data["records"]
if regex.search(record.get("name", ""))
]
return [record for record in data["records"] if record.get("id") == record_id]
def _render_record_value(value) -> str:
if isinstance(value, int) and value >= 0x10000:
rendered = f"{value} (`0x{value:08x}`)"
else:
rendered = json.dumps(value, ensure_ascii=False)
return rendered.replace("|", "\\|").replace("\n", "<br>")
def render_record_matches(data: dict, selector: str) -> str:
"""Render a compact semantic view of one or more selected INIT records."""
matches = find_record_matches(data, selector)
escaped_selector = selector.replace("`", "\\`")
lines = [
f"# {data['table']} record",
"",
f"- source: `{data.get('source', '(unspecified)')}`",
]
if data.get("packed_id"):
lines.append(f"- packed resource: `{data['packed_id']}`")
lines.extend([
f"- selector: `{escaped_selector}`",
f"- matches: {len(matches)}",
"",
])
semantics = data.get("field_semantics", {})
for record in matches:
name = record.get("name", "")
heading = f"## {record['id']}" + (f" — {name}" if name else "")
lines.extend([heading, ""])
for key in ("desc", "desc1", "desc2", "desc3"):
if key in record:
lines.append(f"- {key}: {_render_record_value(record[key])}")
for key, value in record.get("message", {}).items():
if key != "furigana":
lines.append(f"- message.{key}: {_render_record_value(value)}")
fields = {}
for collection_name in (
"fields",
"record_fields",
"string_fields",
"array_fields",
"footer_arrays",
):
fields.update(record.get(collection_name, {}))
if fields:
lines.extend([
"",
"| semantic field | value | raw provenance |",
"|---|---:|---|",
])
rows = sorted(
fields.items(),
key=lambda item: (semantics.get(item[0], item[0]), item[0]),
)
for raw_key, value in rows:
semantic_name = semantics.get(raw_key, "unresolved")
lines.append(
f"| `{semantic_name}` | {_render_record_value(value)} | "
f"`{raw_key}` |"
)
lines.append("")
return "\n".join(lines)
def add_direct_references(rows: list[dict], source_name: str) -> None:
by_base: dict[int, list[dict]] = collections.defaultdict(list)
for row in rows:
@@ -1028,6 +1114,11 @@ def main() -> int:
metavar="REGEX",
help="show matching names/player-facing messages beside all populated fields",
)
parser.add_argument(
"--record",
metavar="ID_OR_NAME",
help="show a focused semantic record by numeric id, exact name, or name regex",
)
args = parser.parse_args()
name = args.table.upper().removesuffix(".JSON").removesuffix(".BIN")
@@ -1068,9 +1159,12 @@ def main() -> int:
)),
}
markdown = render_markdown(data, rows, args.limit)
if args.message_query:
if args.record:
print(render_record_matches(data, args.record))
elif args.message_query:
print(render_message_matches(data, args.message_query))
print(markdown)
else:
print(markdown)
if args.build:
stem = paths.BUILD / "data" / f"{name}-field-profile"
stem.with_suffix(".json").write_text(

11
tools/sys4load.py
View File

@@ -219,9 +219,9 @@ def _decode_string(dwords, start, limit=4096):
return text, ndwords
def load(path) -> Sys4Script:
path = Path(path)
data = path.read_bytes()
def load_bytes(data: bytes, name: str | Path = "<memory>.BIN") -> Sys4Script:
"""Parse one SYS4 script payload already read from an archive or other byte source."""
path = Path(name)
if len(data) < HEADER_SIZE:
raise Sys4Error(f"{path.name}: too small ({len(data)} bytes)")
if data[:4] != MAGIC_PREFIX:
@@ -239,6 +239,11 @@ def load(path) -> Sys4Script:
return scr
def load(path) -> Sys4Script:
path = Path(path)
return load_bytes(path.read_bytes(), path)
def decode_code(scr: Sys4Script):
"""Walk the code section into instructions, resolving inline strings.

38
tools/test_extract_init.py
View File

@@ -105,6 +105,43 @@ def test_real_name_tables() -> None:
)
def test_append_ebinit_fragment() -> None:
script = extract_init.load_packed_script(0x01000001)
_, base_layout = extract_init.extract_name(
sys4load.load(extract_init.resolve("EBINIT"))
)
base_layout["source"] = "EBINIT.BIN"
records, meta = extract_init.extract_name(script, base_layout)
by_id = {record["id"]: record for record in records}
semantics = extract_init.field_semantics(records)
extract_init.attach_semantic_fields(records, semantics)
check(
script.path.name == "$1$EBINIT.BIN" and len(script.instructions) == 358,
"packed resource 0x01000001 loads the append EBINIT payload",
)
check(
set(by_id) == {81, 900, 901, 902, 903, 904, 905},
"append EBINIT exposes its seven sparse additive unit rows",
)
check(
meta["fragment_layout_source"] == "EBINIT.BIN"
and meta["name_array_base"] == "0x84a"
and meta["record_span"] == 1000,
"append EBINIT reuses the canonical base-table layout",
)
check(
by_id[81]["semantic_fields"]["unit_starting_level"] == 40
and by_id[81]["semantic_fields"]["unit_base_stats.physical_attack"] == 23
and by_id[81]["semantic_fields"]["unit_stat_growth_rates.speed"] == 63,
"append unit 81 exposes named level, base-stat, and growth fields",
)
check(
by_id[81]["semantic_fields"]["unit_battle_sprite_asset_id"] == 0x0100002D,
"append unit 81 retains its selector-keyed battle sprite resource id",
)
def test_character_profiles() -> None:
scripts = paths.scripts()
records, meta = extract_init.extract_character_profiles(
@@ -2293,6 +2330,7 @@ def test_field_semantics() -> None:
if __name__ == "__main__":
test_real_name_tables()
test_append_ebinit_fragment()
test_character_profiles()
test_magic_actions()
test_static_negative_write()

8
tools/test_init_table_profile.py
View File

@@ -513,6 +513,14 @@ def main() -> int:
rendered = profile.render_message_matches(fixture, "First")
assert "| 1 | one | First |" in rendered
assert "`test_record.zero` (`0x30/3/0`)=9" in rendered
assert [record["id"] for record in profile.find_record_matches(fixture, "0x1")] == [1]
assert [record["id"] for record in profile.find_record_matches(fixture, "three")] == [3]
assert [record["id"] for record in profile.find_record_matches(fixture, "sev.*")] == [7]
focused = profile.render_record_matches(fixture, "1")
assert "## 1 — one" in focused
assert "| `test_parallel` | 2 | `0x10` |" in focused
assert "| `test_record.zero` | 9 | `0x30/3/0` |" in focused
assert "- message.description: \"First\"" in focused
enemy_fixture = {
"table": "ENEMY",