conco/OpenMaidEngine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Join VIMES and EIMES semantics

Browse Source
This commit is contained in:
gamer147
2026-07-23 17:39:08 -04:00
parent 98c7b8aba7
commit e84f61d670
13 changed files with 416 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
tools/extract_init.py
View File

@@ -493,10 +493,17 @@ UNIT_STAT_COLUMNS = (
)
MESSAGE_TABLES = {
"EBINIT": "EIMES",
"ITINIT": "ITMES",
"SKINIT": "SKMES",
"VIINIT": "VIMES",
}
VOCABULARY_NAME_ARRAY_BASE = 0x463B
VOCABULARY_RECORD_TABLE_BASE = 0x15A2A9
VOCABULARY_RECORD_STRIDE = 3
VOCABULARY_RECORD_SPAN = 200
def resolve(name: str) -> Path:
for cand in (paths.GAME_DIR / f"{name}.BIN", paths.DATA1 / f"{name}.BIN"):
@@ -1103,6 +1110,70 @@ def extract_name(scr):
"desc_array_bases": {k: f"0x{v:x}" for k, v in sorted(desc_bases.items())}}
def extract_vocabulary(scr):
"""Extract VIINIT's sparse glossary names and pre-name row-table writes."""
records = []
by_id = {}
for ins in scr.instructions:
if (
ins.opcode != SET_STRING
or len(ins.args) < 2
or ins.args[0][0] != T_GLOBAL_STRING
):
continue
record_id = ins.args[0][1] - VOCABULARY_NAME_ARRAY_BASE
if not (1 <= record_id < VOCABULARY_RECORD_SPAN):
raise ValueError(
f"{scr.path.name}: glossary name outside reserved id span: "
f"0x{ins.args[0][1]:x}"
)
text = scr.strings.get(ins.args[1][1], (None,))[0]
record = {
"id": record_id,
"name": text,
"fields": {},
"record_fields": {},
}
records.append(record)
by_id[record_id] = record
for ins in scr.instructions:
write = _static_global_write(ins)
if write is None:
continue
destination, value = write
relative = destination - VOCABULARY_RECORD_TABLE_BASE
if not (0 <= relative < VOCABULARY_RECORD_SPAN * VOCABULARY_RECORD_STRIDE):
raise ValueError(
f"{scr.path.name}: unexpected integer write 0x{destination:x}"
)
record_id, column = divmod(relative, VOCABULARY_RECORD_STRIDE)
if record_id not in by_id:
raise ValueError(
f"{scr.path.name}: integer write for unnamed glossary id {record_id}"
)
_store_unique(
by_id[record_id]["record_fields"],
(
f"0x{VOCABULARY_RECORD_TABLE_BASE:x}/"
f"{VOCABULARY_RECORD_STRIDE}/{column}"
),
value,
record_id,
)
record_columns = sorted({
key for record in records for key in record["record_fields"]
}, key=lambda key: tuple(int(part, 0) for part in key.split("/")))
return records, {
"name_array_base": f"0x{VOCABULARY_NAME_ARRAY_BASE:x}",
"name_write_base": f"0x{VOCABULARY_NAME_ARRAY_BASE + 1:x}",
"first_record_id": 1,
"record_span": VOCABULARY_RECORD_SPAN,
"record_field_columns": record_columns,
}
@cache
def object_type_definitions() -> dict[int, dict]:
"""Load OBINIT's authoritative display and state-row metadata by object type id."""
@@ -1914,6 +1985,8 @@ def main() -> int:
"dispatch": extract_dispatch,
"banked": extract_banked,
}[mode]
if mode == "name" and name == "VIINIT":
extractor = extract_vocabulary
recs, meta = extractor(scr)
if mode == "name" and name in MESSAGE_TABLES:
message_name = MESSAGE_TABLES[name]

79
tools/extract_message_table.py
View File

@@ -1,21 +1,23 @@
#!/usr/bin/env python3
"""Extract an ID-dispatched SYS4 message table.
ITMES and SKMES are long chains of:
The shipped tables use two equivalent control-flow shapes:
eq <temporary>, <selected id global>, <record id>
jcc ...
show-text ...
...
jcc <next guard>
<message body>
jmp <shared exit>
This tool discovers the selector global from that repeated shape, reconstructs
the displayed lines (including the surface text of furigana spans), and emits a
reusable ID-to-message JSON table.
or a compact guard block whose successful branches target message bodies stored
later in the script. This tool discovers the selector global, follows either
layout, reconstructs the displayed lines (including the surface text of
furigana spans), and emits a reusable ID-to-message JSON table.
Usage:
py -3.11 -X utf8 tools/extract_message_table.py ITMES
py -3.11 -X utf8 tools/extract_message_table.py SKMES [OUTNAME]
py -3.11 -X utf8 tools/extract_message_table.py VIMES
py -3.11 -X utf8 tools/extract_message_table.py EIMES
"""
from __future__ import annotations
@@ -33,6 +35,11 @@ import sys4load
T_IMM = 0
T_INLINE_STRING = 2
T_GLOBAL_INT = 3
BRANCH_SENTINEL = 0xFFFFFFFF
MESSAGE_LAYOUTS = {
"EIMES.BIN": "enemy-commentary",
}
def resolve(name: str) -> Path:
@@ -77,7 +84,9 @@ def _strings(ins, scr) -> list[str]:
]
def _message_body(scr, start: int, stop: int) -> dict | None:
def _message_body(
scr, start: int, stop: int, layout: str = "title-description"
) -> dict | None:
fragments: list[str] = []
lines: list[str] = []
furigana: list[dict] = []
@@ -102,15 +111,34 @@ def _message_body(scr, start: int, stop: int) -> dict | None:
lines.append("".join(fragments))
if not lines:
return None
message = {
"title": lines[0],
"description": "\n".join(lines[1:]),
}
if layout == "enemy-commentary":
message = {
"summary": lines[0],
"strategy": "\n".join(lines[1:]),
}
else:
message = {
"title": lines[0],
"description": "\n".join(lines[1:]),
}
if furigana:
message["furigana"] = furigana
return message
def _branch_target_index(scr, guard_index: int, offsets: dict[int, int]) -> int | None:
"""Resolve the conditional branch immediately following a dispatch guard."""
if guard_index + 1 >= len(scr.instructions):
return None
branch = scr.instructions[guard_index + 1]
if sys4load.display_label(branch.opcode) != "jcc":
return None
for arg_type, value in branch.args[1:]:
if arg_type == T_IMM and value != BRANCH_SENTINEL and value in offsets:
return offsets[value]
return None
def extract_messages(scr, selector: int | None = None) -> tuple[list[dict], dict]:
"""Extract ordered message records and dispatch metadata from a script."""
discovered_selector, comparison_count = discover_selector(scr)
@@ -122,24 +150,49 @@ def extract_messages(scr, selector: int | None = None) -> tuple[list[dict], dict
]
records: list[dict] = []
seen: set[int] = set()
offsets = {
instruction.offset: index
for index, instruction in enumerate(scr.instructions)
}
message_layout = MESSAGE_LAYOUTS.get(scr.path.name.upper(), "title-description")
dispatch_layout_counts = collections.Counter()
for guard_index, (instruction_index, record_id) in enumerate(guards):
stop = guards[guard_index + 1][0] if guard_index + 1 < len(guards) else len(scr.instructions)
message = _message_body(scr, instruction_index + 1, stop)
message_start = instruction_index + 2
message = _message_body(scr, message_start, stop, message_layout)
dispatch_layout = "fallthrough"
if message is None:
branch_target = _branch_target_index(scr, instruction_index, offsets)
if branch_target is not None:
message_start = branch_target
message = _message_body(
scr, message_start, len(scr.instructions), message_layout
)
dispatch_layout = "branch-target"
if message is None:
continue
if record_id in seen:
raise ValueError(f"{scr.path.name}: duplicate message id {record_id}")
seen.add(record_id)
dispatch_layout_counts[dispatch_layout] += 1
records.append({
"id": record_id,
"dispatch_offset": f"0x{scr.instructions[instruction_index].offset:x}",
"message_offset": f"0x{scr.instructions[message_start].offset:x}",
**message,
})
dispatch_layout = (
next(iter(dispatch_layout_counts))
if len(dispatch_layout_counts) == 1
else "mixed"
)
return records, {
"selector_global": f"0x{selector:x}",
"dispatch_guard_count": len(guards),
"message_count": len(records),
"selector_discovery_count": comparison_count,
"dispatch_layout": dispatch_layout,
"message_layout": message_layout,
}

42
tools/init_table_profile.py
View File

@@ -32,6 +32,14 @@ import sys4load
GLOBAL_OPERAND_TYPES = {3, 4, 5, 6, 8}
def message_heading_body(message: dict) -> tuple[str, str]:
"""Return presentation-neutral heading/body text for supported MES layouts."""
return (
message.get("title", message.get("summary", "")),
message.get("description", message.get("strategy", "")),
)
def load_table(name: str) -> dict:
path = paths.BUILD / "data" / f"{name}.json"
if not path.exists():
@@ -76,7 +84,8 @@ def profile_columns(data: dict) -> list[dict]:
**extra,
}
if message := record.get("message"):
example["message_description"] = message.get("description", "")
_, body = message_heading_body(message)
example["message_description"] = body
examples[key].append(example)
for record in records:
@@ -264,24 +273,30 @@ def profile_messages(data: dict) -> dict:
with_furigana = [
record for record in with_message if record["message"].get("furigana")
]
examples = []
for record in with_message[:5]:
heading, body = message_heading_body(record["message"])
examples.append({
"id": record["id"],
"name": record.get("name", ""),
"title": heading,
"description": body,
"message_fields": {
key: record["message"][key]
for key in ("title", "description", "summary", "strategy")
if key in record["message"]
},
})
return {
"population": len(with_message),
"coverage": len(with_message) / len(records) if records else 0.0,
"furigana_records": len(with_furigana),
"examples": [
{
"id": record["id"],
"name": record.get("name", ""),
"title": record["message"]["title"],
"description": record["message"]["description"],
}
for record in with_message[:5]
],
"examples": examples,
}
def find_message_matches(data: dict, pattern: str) -> list[dict]:
"""Return records whose name/title/description matches a regular expression."""
"""Return records whose name or supported message text matches a regex."""
regex = re.compile(pattern, re.IGNORECASE)
return [
record
@@ -290,6 +305,8 @@ def find_message_matches(data: dict, pattern: str) -> list[dict]:
record.get("name", ""),
record.get("message", {}).get("title", ""),
record.get("message", {}).get("description", ""),
record.get("message", {}).get("summary", ""),
record.get("message", {}).get("strategy", ""),
]))
]
@@ -314,7 +331,8 @@ def render_message_matches(data: dict, pattern: str) -> str:
for key, value in sorted(fields.items())
)
name = record.get("name", "").replace("|", "\\|")
description = record.get("message", {}).get("description", "").replace("|", "\\|")
_, body = message_heading_body(record.get("message", {}))
description = body.replace("|", "\\|")
lines.append(
f"| {record['id']} | {name} | {description} | {rendered_fields} |"
)

76
tools/test_extract_init.py
View File

@@ -88,6 +88,22 @@ def test_real_name_tables() -> None:
check(object_by_id[28]["name"] == "カード取得",
"OBINIT object ids provide authoritative STINIT type labels")
vocabulary, vocabulary_meta = extract_init.extract_vocabulary(
sys4load.load(scripts["VIINIT.BIN"])
)
vocabulary_by_id = {record["id"]: record for record in vocabulary}
check(
vocabulary_meta["record_span"] == 200
and len(vocabulary) == 65
and vocabulary[-1]["id"] == 120,
"VIINIT extracts 65 sparse glossary topics from its 200-row table",
)
check(
vocabulary_by_id[1]["name"] == "【迷宮】占有率"
and vocabulary_by_id[1]["record_fields"]["0x15a2a9/3/0"] == 201,
"VIINIT associates pre-name unlock writes with the correct topic",
)
def test_static_negative_write() -> None:
class Instruction:
@@ -561,10 +577,12 @@ def test_real_routine_banks() -> None:
def test_real_message_tables() -> None:
scripts = paths.scripts()
expected = {
"ITMES.BIN": (0x8C877, 287),
"SKMES.BIN": (0xA6E59, 131),
"ITMES.BIN": (0x8C877, 287, "fallthrough"),
"SKMES.BIN": (0xA6E59, 131, "fallthrough"),
"VIMES.BIN": (0x15A2A8, 65, "branch-target"),
"EIMES.BIN": (0x15A759, 192, "branch-target"),
}
for name, (selector, count) in expected.items():
for name, (selector, count, dispatch_layout) in expected.items():
records, meta = extract_message_table.extract_messages(
sys4load.load(scripts[name])
)
@@ -575,6 +593,8 @@ def test_real_message_tables() -> None:
f"{name}: every dispatch guard yields a message")
check(len({record['id'] for record in records}) == len(records),
f"{name}: message ids are unique")
check(meta["dispatch_layout"] == dispatch_layout,
f"{name}: recognizes its {dispatch_layout} dispatch layout")
item_messages, _ = extract_message_table.extract_messages(
sys4load.load(scripts["ITMES.BIN"])
@@ -598,6 +618,27 @@ def test_real_message_tables() -> None:
check(skills[1]["description"] == "　床のない地形を移動可能になる",
"SKMES skill 1 keeps its player-facing behavior")
vocabulary_messages, vocabulary_meta = extract_message_table.extract_messages(
sys4load.load(scripts["VIMES.BIN"])
)
vocabulary = {record["id"]: record for record in vocabulary_messages}
check(vocabulary[1]["title"] == "『【迷宮】占有率』"
and "各勢力の占領度合" in vocabulary[1]["description"],
"VIMES follows branch targets and reconstructs glossary help text")
check(vocabulary_meta["message_layout"] == "title-description",
"VIMES retains the title/description message layout")
enemy_messages, enemy_meta = extract_message_table.extract_messages(
sys4load.load(scripts["EIMES.BIN"])
)
enemies = {record["id"]: record for record in enemy_messages}
check(enemies[101]["summary"] == "高い能力を秘めた隣国の姫騎士"
and enemies[101]["strategy"] == "初遭遇時にはまず勝てない",
"EIMES exposes its two lines as enemy summary and strategy")
check(enemy_meta["message_layout"] == "enemy-commentary"
and "title" not in enemies[101],
"EIMES does not mislabel its first commentary line as a title")
def test_message_join() -> None:
scripts = paths.scripts()
@@ -621,6 +662,35 @@ def test_message_join() -> None:
check(joined["IT"][1]["message"]["description"] == expected["IT"][1],
"INIT/MES join uses the shared runtime id")
vocabulary, _ = extract_init.extract_vocabulary(
sys4load.load(scripts["VIINIT.BIN"])
)
vocabulary_meta = extract_init.join_messages(
vocabulary, sys4load.load(scripts["VIMES.BIN"])
)
check(
vocabulary_meta["joined_count"] == 65
and not vocabulary_meta["init_ids_without_message"]
and not vocabulary_meta["message_ids_without_init"],
"VIINIT and VIMES form a complete 65-topic runtime-id join",
)
units, _ = extract_init.extract_name(sys4load.load(scripts["EBINIT.BIN"]))
enemy_meta = extract_init.join_messages(
units, sys4load.load(scripts["EIMES.BIN"])
)
unit_by_id = {record["id"]: record for record in units}
check(
enemy_meta["joined_count"] == 192
and len(enemy_meta["init_ids_without_message"]) == 85
and not enemy_meta["message_ids_without_init"],
"EIMES joins 192 sparse enemy-commentary rows to EBINIT",
)
check(
unit_by_id[101]["message"]["strategy"] == "初遭遇時にはまず勝てない",
"EBINIT records expose EIMES strategy text by unit id",
)
def test_field_semantics() -> None:
scripts = paths.scripts()

11
tools/test_globals.py
View File

@@ -58,6 +58,17 @@ def test_load_and_lint():
"AI action-element eligibility state is curated")
check(entries[0xcc9fe]["name"] == "healing_action_scope_masks",
"AI healing-action eligibility state is curated")
check(entries[0x15a2a8]["name"] == "current_glossary_topic_id"
and entries[0x15a2a9]["name"]
== "glossary_topic_unlock_seen_decision_ids"
and entries[0x463b]["name"] == "glossary_topic_titles",
"VIINIT/VIMES glossary state is curated")
check(entries[0x15a759]["name"] == "current_enemy_encyclopedia_unit_id"
and entries[0x56b85]["name"]
== "enemy_encyclopedia_revealed_flags",
"EBINIT/EIMES enemy-encyclopedia state is curated")
check(entries[0x5f0ed]["name"] == "scene_decision_seen_flags",
"glossary prerequisite seen-state is curated")
check(entries[0x20543]["name"] == "tile_faction_traversal_masks",
"faction-specific tile traversal masks are curated")
check(entries[0xaba64]["name"] == "stage_object_runtime_flags",

22
tools/test_init_table_profile.py
View File

@@ -178,6 +178,28 @@ def main() -> int:
rendered = profile.render_message_matches(fixture, "First")
assert "| 1 | one | First |" in rendered
assert "`test_record.zero` (`0x30/3/0`)=9" in rendered
enemy_fixture = {
"table": "ENEMY",
"records": [
{
"id": 101,
"name": "boss",
"message": {
"summary": "Powerful knight",
"strategy": "Avoid the first encounter",
},
}
],
}
enemy_messages = profile.profile_messages(enemy_fixture)
assert enemy_messages["examples"][0]["title"] == "Powerful knight"
assert enemy_messages["examples"][0]["description"] == "Avoid the first encounter"
assert enemy_messages["examples"][0]["message_fields"] == {
"summary": "Powerful knight",
"strategy": "Avoid the first encounter",
}
assert profile.find_message_matches(enemy_fixture, "first encounter")
print("all init_table_profile checks passed")
return 0