chore: initialize age-reimpl repo
Reverse-engineering + open reimplementation workspace for Eushully's AGE/SYS4 engine (first target: Himegari). The repo root is age-reimpl/; the original game install and the extracted ALF data are siblings outside the repo and are never tracked. build/ (derived corpora) is gitignored and regenerated by the tools. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
gamer147
612
tools/age_opcodes.py
Normal file
612
tools/age_opcodes.py
Normal file
@@ -0,0 +1,612 @@
|
||||
"""AGE (Eushully SYS4/SYS5) opcode + operand-type tables.
|
||||
|
||||
Transcribed verbatim from Kelebek1/Eushully-Decompiler `age-shared.cpp`
|
||||
(vm-map/kelebek1-age-shared.cpp). Validated against all 476 Himegari SYS4
|
||||
scripts: every instruction decodes with zero unknown opcodes when the code
|
||||
stream is walked as `<opcode:u32> + argc*(<argtype:u32><value:u32>)`,
|
||||
instruction length = 1 + 2*argc dwords.
|
||||
|
||||
Handler labels `u004XXXXX` are AGE.EXE addresses from Kelebek's reference
|
||||
title (a later AGE game); the opcode NUMBER + argument_count are the stable
|
||||
ABI and are correct for Himegari. Bare-hex/`dev_ukn` labels = unnamed semantics.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
# op_code -> (label, argument_count)
|
||||
OPCODES: dict[int, tuple[str, int]] = {
|
||||
0x0001: ('u004149C0', 0), # error
|
||||
0x0002: ('exit', 0),
|
||||
0x0003: ('call-script', 1), # call another script, param = SYSTEM4.bin index
|
||||
0x0004: ('u00417E30', 2),
|
||||
0x0005: ('ret', 0),
|
||||
0x0006: ('u00417E80', 2),
|
||||
0x0007: ('u00417F90', 1),
|
||||
0x0008: ('u00417FC0', 1),
|
||||
0x0009: ('exit-script', 0),
|
||||
0x000a: ('u00424170', 2),
|
||||
0x000b: ('u00418090', 11),
|
||||
0x000c: ('u004149E0', 0),
|
||||
0x000d: ('u004181A0', 4),
|
||||
0x000e: ('u00418200', 12),
|
||||
0x000f: ('u00418300', 1),
|
||||
0x0010: ('u00414A00', 4),
|
||||
0x0011: ('u00418330', 9),
|
||||
0x0012: ('u004183F0', 1),
|
||||
0x0013: ('u00418420', 4),
|
||||
0x0014: ('u00414A20', 0),
|
||||
0x0015: ('u00418490', 5),
|
||||
0x0016: ('u00418520', 2),
|
||||
0x0017: ('u00418560', 2),
|
||||
0x001e: ('u004185B0', 8),
|
||||
0x001f: ('u00418690', 12),
|
||||
0x0020: ('u004187C0', 6),
|
||||
0x0021: ('u00418860', 2),
|
||||
0x0022: ('u00418920', 2),
|
||||
0x0023: ('u004189D0', 2),
|
||||
0x0024: ('u00418A90', 2),
|
||||
0x0025: ('u00418B40', 3),
|
||||
0x0026: ('u00418C00', 4),
|
||||
0x0027: ('u00418CC0', 4),
|
||||
0x0028: ('u00418D90', 4),
|
||||
0x002a: ('u00418E60', 4),
|
||||
0x002b: ('u00418F30', 5),
|
||||
0x002c: ('u00419010', 5),
|
||||
0x002d: ('u004190A0', 12),
|
||||
0x002e: ('u004194B0', 5),
|
||||
0x002f: ('u004195A0', 4),
|
||||
0x0030: ('u00419670', 5),
|
||||
0x0031: ('u00419750', 4),
|
||||
0x0032: ('u004197C0', 10),
|
||||
0x0033: ('u00419900', 6),
|
||||
0x0034: ('u004199C0', 12),
|
||||
0x0035: ('u00419AF0', 11),
|
||||
0x0036: ('u00419C00', 3),
|
||||
0x0037: ('u00419C90', 11),
|
||||
0x0038: ('u00419DA0', 12),
|
||||
0x0050: ('add', 3), # add. param1 = param2 + param3
|
||||
0x0051: ('sub', 3), # sub. param1 = param2 - param3
|
||||
0x0052: ('mul', 3), # mul. param1 = param2 * param3
|
||||
0x0053: ('div', 3), # div. param1 = param2 / param3
|
||||
0x0054: ('mod', 3), # mod. param1 = param2 % param3
|
||||
0x0055: ('mov', 2), # mov. param1 = param2
|
||||
0x0056: ('and', 3), # and. param1 = param2 & param3
|
||||
0x0057: ('or', 3), # or. param1 = param2 | param3
|
||||
0x0058: ('sar', 3), # sar. param1 = param2 >> param3
|
||||
0x0059: ('shl', 3), # shl. param1 = param2 << param3
|
||||
0x005a: ('eq', 3), # eq. param1 = param2 == param3
|
||||
0x005b: ('ne', 3), # ne. param1 = param2 != param3
|
||||
0x005c: ('lt', 3), # lt. param1 = param2 < param3
|
||||
0x005d: ('lte', 3), # lte. param1 = param2 <= param3
|
||||
0x005e: ('gr', 3), # gr. param1 = param2 > param3
|
||||
0x005f: ('gre', 3), # gre. param1 = param2 >= param3
|
||||
0x0060: ('u0041A270', 2),
|
||||
0x0061: ('lookup-array', 3), # lookup. param1 = param2[param3]
|
||||
0x0062: ('u0041A360', 3),
|
||||
0x0063: ('u00414A60', 2),
|
||||
0x0064: ('copy-local-array', 2),
|
||||
0x0065: ('u00414AA0', 2),
|
||||
0x0066: ('u00414AE0', 3),
|
||||
0x0067: ('u00414B20', 3),
|
||||
0x0068: ('u00414B60', 3),
|
||||
0x0069: ('u00414BA0', 3),
|
||||
0x006a: ('u00414BE0', 3),
|
||||
0x006b: ('u00414C20', 3),
|
||||
0x006c: ('copy-to-global', 2), # loop copy local value to global array, param1 = array start, param2 = count
|
||||
0x006d: ('u00416960', 0),
|
||||
0x006e: ('show-text', 2),
|
||||
0x006f: ('end-text-line', 1),
|
||||
0x0070: ('u0041A750', 5),
|
||||
0x0071: ('u0041A7B0', 1),
|
||||
0x0072: ('wait-for-input', 1),
|
||||
0x0073: ('u0041AB30', 10),
|
||||
0x0074: ('u0041AC00', 1),
|
||||
0x0075: ('u0041AC30', 1),
|
||||
0x0076: ('u0041AC60', 1),
|
||||
0x0077: ('u0041ACB0', 1),
|
||||
0x0078: ('u0041AD00', 1),
|
||||
0x0079: ('u0041AD30', 3),
|
||||
0x007a: ('u0041AD70', 3),
|
||||
0x007b: ('u0041ADB0', 2), # ukn, both args point to code locations
|
||||
0x007c: ('u00416A90', 0),
|
||||
0x007d: ('u0041AE00', 2),
|
||||
0x007e: ('u0041AEA0', 1),
|
||||
0x007f: ('u00414C60', 1),
|
||||
0x0080: ('u0041AF00', 1),
|
||||
0x0081: ('u0041AF30', 1),
|
||||
0x0082: ('u0041AF80', 5),
|
||||
0x0083: ('u00414C90', 3),
|
||||
0x0084: ('u0041AFE0', 1),
|
||||
0x0085: ('u00414CF0', 0),
|
||||
0x0086: ('u0041B210', 1),
|
||||
0x0087: ('u00414D10', 0),
|
||||
0x0088: ('u0041B290', 1),
|
||||
0x0089: ('u0041B2E0', 4),
|
||||
0x008a: ('u0041B330', 6),
|
||||
0x008b: ('u0041B3D0', 1),
|
||||
0x008c: ('jmp', 1),
|
||||
0x008d: ('u0041BCE0', 2),
|
||||
0x008e: ('u0041BD60', 1),
|
||||
0x008f: ('call', 1),
|
||||
0x0090: ('u0041BEB0', 7), # ukn, args 5, 6 and 7 point to code locations
|
||||
0x0091: ('u0041BFB0', 1),
|
||||
0x0092: ('u0041C030', 2),
|
||||
0x0093: ('u00415040', 0),
|
||||
0x0094: ('u00415090', 0),
|
||||
0x0095: ('u0041C0C0', 2),
|
||||
0x0096: ('u004150C0', 0),
|
||||
0x0097: ('u0041C150', 5),
|
||||
0x00a0: ('jcc', 3),
|
||||
0x00a1: ('u00427C00', 0),
|
||||
0x00a2: ('u00427FD0', 2),
|
||||
0x00a3: ('u004244D0', 2),
|
||||
0x00aa: ('u0041C270', 2),
|
||||
0x00ab: ('u0041C330', 2),
|
||||
0x00ac: ('u0041C3E0', 9),
|
||||
0x00ad: ('u00415110', 0),
|
||||
0x00ae: ('u00415130', 0),
|
||||
0x00af: ('u00415480', 0),
|
||||
0x00b0: ('u0041C530', 1),
|
||||
0x00b1: ('u0041C560', 1),
|
||||
0x00b2: ('u0041C590', 2),
|
||||
0x00b3: ('u004154B0', 0),
|
||||
0x00b4: ('play-sound-effect', 2), # play a sound effect/ambient. param1 = file index, param2 = play mode?
|
||||
0x00b5: ('u0041D050', 1),
|
||||
0x00b6: ('u0041D080', 1),
|
||||
0x00b7: ('u0041D0E0', 1),
|
||||
0x00b8: ('u00415520', 0),
|
||||
0x00b9: ('u0041D140', 1),
|
||||
0x00ba: ('u0041D0B0', 1),
|
||||
0x00bb: ('u0041D250', 1),
|
||||
0x00bc: ('u0041D280', 1),
|
||||
0x00bd: ('u00415570', 1),
|
||||
0x00be: ('u004155E0', 1),
|
||||
0x00bf: ('play-bgm', 1), # param1 = bgm number
|
||||
0x00c0: ('u00415620', 1),
|
||||
0x00c1: ('u00415650', 0),
|
||||
0x00c2: ('u0041D2B0', 2),
|
||||
0x00c3: ('u0041D390', 1),
|
||||
0x00c4: ('play-voice', 1),
|
||||
0x00c5: ('u0041D4A0', 2),
|
||||
0x00c6: ('u0041D5D0', 2),
|
||||
0x00c7: ('u0041D760', 2),
|
||||
0x00c8: ('sleep', 1), # param1 = sleep time?
|
||||
0x00c9: ('u00415770', 0),
|
||||
0x00ca: ('u004157A0', 0),
|
||||
0x00cb: ('u00415800', 1),
|
||||
0x00cc: ('mouse_callback', 2), # sets mouse/keyboard callback location, param1 = id, param2 = offset (minus header, not multiplied by 4)
|
||||
0x00cd: ('get-input-type', 0), # get input type, mouse, keyboard, pad etc
|
||||
0x00ce: ('u0041E0B0', 3),
|
||||
0x00cf: ('u00416D40', 0),
|
||||
0x00d0: ('u00415830', 1),
|
||||
0x00d1: ('u00415860', 0),
|
||||
0x00d2: ('u0041E110', 1),
|
||||
0x00d3: ('u00425960', 0),
|
||||
0x00d4: ('u004266F0', 4), # seems to setup some kind of looping function calls. param1 = ukn, param2 = loop count, param3 = function location?, param4 = function location?
|
||||
0x00d5: ('u004262C0', 1),
|
||||
0x00d6: ('u004267D0', 6),
|
||||
0x00d7: ('u0041E1A0', 1),
|
||||
0x00d8: ('u0041E150', 2),
|
||||
0x00d9: ('u00415880', 0),
|
||||
0x00da: ('u004158B0', 6),
|
||||
0x00fa: ('u00415940', 0),
|
||||
0x00fb: ('joy_callback', 2), # sets code callback for joystick inputs. ID 0-4= left thumb up/down/left/right, 4 = X on Xbox controller etc. param1 = id, param2 = offset (minus header, not multiplied by 4)
|
||||
0x00fc: ('u004159F0', 0),
|
||||
0x00fd: ('u0041E2D0', 2),
|
||||
0x00fe: ('u0041E360', 1),
|
||||
0x00ff: ('u00415A10', 0),
|
||||
0x0100: ('u00415A60', 0),
|
||||
0x0101: ('u00415BF0', 0), # joystick input?
|
||||
0x0102: ('u0041E3C0', 3),
|
||||
0x0103: ('u0041E4A0', 1),
|
||||
0x0104: ('u00415C50', 0),
|
||||
0x0105: ('u0041E4D0', 1),
|
||||
0x0106: ('u00415E40', 1),
|
||||
0x0107: ('u0041E500', 2),
|
||||
0x0108: ('u00415E70', 1),
|
||||
0x0109: ('u00415EC0', 2),
|
||||
0x010a: ('u0041E540', 2),
|
||||
0x010b: ('u0041E5A0', 2),
|
||||
0x010c: ('u0041E5E0', 2),
|
||||
0x010d: ('u00415F10', 1),
|
||||
0x010e: ('u0041E650', 2),
|
||||
0x010f: ('u0041E690', 1),
|
||||
0x012c: ('lookup-array-2d', 5), # 2d array lookup. param1 = param2[(param3 * param4) + param5]
|
||||
0x012d: ('u0041E720', 7),
|
||||
0x012e: ('u0041E940', 8),
|
||||
0x012f: ('u0041ECB0', 4),
|
||||
0x0130: ('u00415F40', 1),
|
||||
0x0131: ('u00415F70', 1),
|
||||
0x0132: ('u0041EF00', 1),
|
||||
0x0133: ('u0041EFF0', 2),
|
||||
0x0134: ('u0041F050', 3),
|
||||
0x0135: ('bit-set', 2), # bts, param1 = param1 OR param2
|
||||
0x0136: ('bit-reset', 2), # btr, param1 = param1 NOR param2
|
||||
0x0137: ('u0041F1C0', 1),
|
||||
0x0138: ('u0041F2B0', 2),
|
||||
0x0139: ('u0041F310', 3),
|
||||
0x013a: ('u0041F3A0', 6),
|
||||
0x013b: ('u0041F440', 7),
|
||||
0x013c: ('u0041F7E0', 1),
|
||||
0x013d: ('u0041F840', 3),
|
||||
0x013e: ('u0041F8D0', 2),
|
||||
0x013f: ('check-bit', 3), # param1 = param2 & (1 << param3). Neg, sbb, neg to get the result as a bool
|
||||
0x0140: ('u0041F9C0', 4),
|
||||
0x0141: ('u0041FAA0', 1),
|
||||
0x0142: ('u0041FB10', 1),
|
||||
0x0143: ('u00415FB0', 0),
|
||||
0x0144: ('u004259D0', 2),
|
||||
0x0145: ('u00416040', 1),
|
||||
0x0146: ('u0041FB40', 1),
|
||||
0x0147: ('u0041FB80', 6),
|
||||
0x0148: ('u004160A0', 1),
|
||||
0x0149: ('u0041FCE0', 1),
|
||||
0x014a: ('u0041FD10', 7),
|
||||
0x014b: ('u0041FF50', 1),
|
||||
0x014c: ('set-agerc-export', 2), # binds an agerc.dll export name to the given number
|
||||
0x014d: ('call-agerc-export', 6), # call the param1 agerc exported function
|
||||
0x0190: ('u0041C5E0', 2),
|
||||
0x0191: ('u0041A4A0', 2),
|
||||
0x0192: ('set-string', 2), # u004252D0 : set-string. param1 = param2
|
||||
0x0193: ('concat', 3), # u00425370 : concat. param1 = param2.concat(param3)
|
||||
0x0194: ('u00425480', 3),
|
||||
0x0195: ('u00425580', 3),
|
||||
0x0196: ('display-furigana', 3), # u0041B400 : display-furigana. param1 = text, param2 = furigana
|
||||
0x0197: ('u0041B510', 1),
|
||||
0x0198: ('u0041B540', 3),
|
||||
0x0199: ('u00414D50', 0),
|
||||
0x019a: ('u00414E50', 1),
|
||||
0x019b: ('u00414E80', 0),
|
||||
0x019c: ('u00414EC0', 0),
|
||||
0x019d: ('u0041C680', 2),
|
||||
0x019e: ('u0041C6E0', 2),
|
||||
0x019f: ('u0041C860', 2),
|
||||
0x01a0: ('u0041C9B0', 9),
|
||||
0x01a1: ('u0041CB40', 2),
|
||||
0x01a2: ('u00428010', 1),
|
||||
0x01a3: ('string-lookup-set', 1), # check the value given exists in save/current data and set. param1 = strings[param1]
|
||||
0x01a4: ('u0041B580', 2),
|
||||
0x01a5: ('set-font', 1), # set-font
|
||||
0x01a6: ('halve-strlen', 2), # halve-strlen? param1 = param2.length() / 2 (rounded down)
|
||||
0x01a7: ('comment', 1), # Developer debug comment
|
||||
0x01a8: ('dev_ukn', 0), # Developer debug something, no function in-game
|
||||
0x01a9: ('u00428090', 1),
|
||||
0x01aa: ('u00425920', 1),
|
||||
0x01ab: ('u0041CCA0', 2),
|
||||
0x01ac: ('u0041CD80', 3),
|
||||
0x01ad: ('u004154F0', 0),
|
||||
0x01ae: ('u0041CED0', 3),
|
||||
0x01af: ('u004245C0', 3),
|
||||
0x01b0: ('u0041A510', 3),
|
||||
0x01b1: ('u0041B5C0', 1),
|
||||
0x01b2: ('u00425790', 1), # to string table?
|
||||
0x01b3: ('u004257D0', 0),
|
||||
0x01b4: ('u004237C0', 0),
|
||||
0x01b5: ('u0041B5F0', 1),
|
||||
0x01b6: ('u00414F60', 1),
|
||||
0x01b7: ('u0041B640', 1),
|
||||
0x01b8: ('u0041B670', 2),
|
||||
0x01b9: ('u0041B710', 2),
|
||||
0x01ba: ('u0041D850', 2),
|
||||
0x01bb: ('u0041B7B0', 1),
|
||||
0x01bc: ('u00415670', 0),
|
||||
0x01bd: ('u0041D910', 1),
|
||||
0x01be: ('u0041D9D0', 2),
|
||||
0x01bf: ('u004156C0', 0),
|
||||
0x01c0: ('u0041DB70', 1),
|
||||
0x01c1: ('u0041B820', 3),
|
||||
0x01c2: ('u0041B860', 2),
|
||||
0x01c3: ('u0041B8A0', 2),
|
||||
0x01c4: ('u00415720', 1),
|
||||
0x01c5: ('u00425800', 4),
|
||||
0x01c6: ('u0041DD80', 2),
|
||||
0x01c7: ('u00414F90', 1),
|
||||
0x01c8: ('toString', 2), # u00425680 : toString
|
||||
0x01c9: ('u0041B8E0', 3),
|
||||
0x01ca: ('u0041B9B0', 1),
|
||||
0x01cb: ('u00414FD0', 1),
|
||||
0x01cc: ('u00415010', 1),
|
||||
0x01cd: ('u0041A560', 2),
|
||||
0x01ce: ('u0041B9F0', 1),
|
||||
0x01cf: ('u0041DA10', 1),
|
||||
0x01d0: ('u0041BA80', 3),
|
||||
0x01d1: ('u0041BAE0', 5),
|
||||
0x01d2: ('u0041BB40', 2),
|
||||
0x01d3: ('u0041BB90', 5),
|
||||
0x01d4: ('u0041BC00', 4),
|
||||
0x01d5: ('u00415700', 0),
|
||||
0x01d6: ('u0041DA40', 2),
|
||||
0x01d7: ('u0041DA80', 2),
|
||||
0x01d8: ('u0041DAD0', 3),
|
||||
0x01d9: ('u0041DB20', 2),
|
||||
0x01f4: ('u004160D0', 0),
|
||||
0x01f5: ('u00416120', 0),
|
||||
0x01f6: ('u00416170', 0),
|
||||
0x01f7: ('u00420270', 2),
|
||||
0x01f8: ('create-texture', 4), # create a new drawable rect. param1 = id, param2 = sizeX, param3 = sizeY, param4 = ukn
|
||||
0x01f9: ('set-texture', 3), # set a texture to a given ID. param1 = file index, param2 = id, param3 = ?
|
||||
0x01fa: ('u00420480', 1),
|
||||
0x01fb: ('draw-texture', 8), # draw a texture. param1 = UI element id?, param2 = textureID, param3 = texX, param4 = texY, param5 = width, param6 = height, param7 = drawX, param8 = drawY
|
||||
0x01fc: ('u004205F0', 1),
|
||||
0x01fd: ('u00420620', 4),
|
||||
0x01fe: ('u004206C0', 5),
|
||||
0x01ff: ('u00420770', 4),
|
||||
0x0200: ('u00420800', 1),
|
||||
0x0201: ('u00416190', 1),
|
||||
0x0202: ('u00420880', 5),
|
||||
0x0203: ('u00420950', 4),
|
||||
0x0204: ('draw-string', 4), # u00420A10 : place-string. param1 = id? param2 = x, param3 = y, param4 = string
|
||||
0x0205: ('u00420A60', 6),
|
||||
0x0206: ('u004161C0', 7),
|
||||
0x0207: ('u00420B00', 8),
|
||||
0x0208: ('u00420BF0', 3),
|
||||
0x0209: ('u00420C50', 5),
|
||||
0x020a: ('u00420CE0', 1),
|
||||
0x020b: ('u00420D50', 7),
|
||||
0x020c: ('u00416200', 0),
|
||||
0x020d: ('u00420E10', 1),
|
||||
0x020e: ('u00416250', 0),
|
||||
0x020f: ('u00420E40', 3),
|
||||
0x0210: ('u00420FF0', 1),
|
||||
0x0211: ('u00421060', 1),
|
||||
0x0212: ('u00421090', 2),
|
||||
0x0213: ('u004210D0', 3),
|
||||
0x0214: ('u00421120', 2),
|
||||
0x0215: ('u00421160', 2),
|
||||
0x0216: ('u004211A0', 2),
|
||||
0x0217: ('u004211E0', 4),
|
||||
0x0218: ('u00421270', 4),
|
||||
0x0219: ('u004212E0', 4),
|
||||
0x021a: ('u00421370', 4),
|
||||
0x021b: ('u004213E0', 1),
|
||||
0x021c: ('u00416270', 0),
|
||||
0x021d: ('u00421410', 2),
|
||||
0x021e: ('u00421450', 6),
|
||||
0x021f: ('u00421510', 7),
|
||||
0x0220: ('u004215D0', 6),
|
||||
0x0221: ('u00421670', 4),
|
||||
0x0222: ('u004216C0', 2),
|
||||
0x0223: ('u00421700', 8),
|
||||
0x0224: ('u00416290', 0),
|
||||
0x0225: ('u00421780', 2),
|
||||
0x0226: ('u004217D0', 5),
|
||||
0x0227: ('u00421880', 6),
|
||||
0x0228: ('u00421940', 5),
|
||||
0x0229: ('u004219E0', 5),
|
||||
0x022a: ('u00421A90', 3),
|
||||
0x022b: ('u00421B30', 4),
|
||||
0x022c: ('u00421BD0', 3),
|
||||
0x022d: ('u00421C60', 5),
|
||||
0x022e: ('u00421D10', 6),
|
||||
0x022f: ('u00421DD0', 5),
|
||||
0x0230: ('u00421E70', 1),
|
||||
0x0231: ('u00421EA0', 4),
|
||||
0x0232: ('u00421EF0', 4),
|
||||
0x0233: ('u00421FB0', 5),
|
||||
0x0234: ('u00422060', 5),
|
||||
0x0235: ('u00422100', 5),
|
||||
0x0236: ('u004221A0', 4),
|
||||
0x0237: ('u00422350', 2),
|
||||
0x0238: ('u00422390', 1),
|
||||
0x0239: ('u004223C0', 6),
|
||||
0x023a: ('u00422420', 2),
|
||||
0x023b: ('u00422460', 7),
|
||||
0x023c: ('u004162B0', 0),
|
||||
0x023d: ('u004162F0', 0),
|
||||
0x023e: ('u004228C0', 2),
|
||||
0x023f: ('u00422930', 2),
|
||||
0x0240: ('u004229A0', 4),
|
||||
0x0241: ('u00422B80', 5),
|
||||
0x0242: ('u00422D60', 2),
|
||||
0x0243: ('u00417070', 0),
|
||||
0x0244: ('u00416360', 0),
|
||||
0x0245: ('u00422DA0', 2),
|
||||
0x0246: ('u00422E10', 2),
|
||||
0x0247: ('u00416390', 1),
|
||||
0x0248: ('u00422E80', 1),
|
||||
0x0249: ('u00422EB0', 3),
|
||||
0x024a: ('u004163C0', 3),
|
||||
0x024d: ('u00422E90', 12),
|
||||
0x024e: ('u00422EA0', 1),
|
||||
0x024f: ('u00422ED0', 10),
|
||||
0x0250: ('u00422F60', 10),
|
||||
0x0251: ('u00422FF0', 12),
|
||||
0x0252: ('u00423000', 1),
|
||||
0x0253: ('u00423019', 2),
|
||||
0x0254: ('u00423049', 5),
|
||||
0x0256: ('u00423050', 5),
|
||||
0x0257: ('257', 5), # Sankai no Yubiwa
|
||||
0x0258: ('u00422FE0', 2),
|
||||
0x0259: ('u00416410', 0),
|
||||
0x025a: ('u00423120', 1),
|
||||
0x025b: ('25B', 1), # Kami no Rhapsody
|
||||
0x025c: ('u00423122', 8),
|
||||
0x025d: ('u00423123', 3),
|
||||
0x025e: ('u00423124', 5),
|
||||
0x025f: ('u00423125', 4),
|
||||
0x0260: ('u00423126', 4),
|
||||
0x0261: ('u00423127', 1),
|
||||
0x0262: ('262', 1), # Amayui 2
|
||||
0x0263: ('263', 1), # Amayui 2
|
||||
0x0264: ('264', 5), # Hyakusen
|
||||
0x02bc: ('u00423020', 11),
|
||||
0x02bd: ('u00423100', 1),
|
||||
0x02be: ('u00423140', 1),
|
||||
0x02bf: ('u00423180', 3),
|
||||
0x02c0: ('u004231C0', 3),
|
||||
0x02c1: ('u00425BC0', 1),
|
||||
0x02c2: ('u00425CD0', 6),
|
||||
0x02c3: ('u00423200', 2),
|
||||
0x02c4: ('u00416450', 0), # using as a way to test logging, originally u00416450
|
||||
0x02c5: ('strlen', 2), # u0042B5D0 : strlen. param1 = param2.length()
|
||||
0x02c6: ('u0042B5E0', 2),
|
||||
0x02c7: ('u0042B5F0', 4),
|
||||
0x02c8: ('u0042B610', 4),
|
||||
0x02c9: ('2C9', 3), # Sankai no Yubiwa
|
||||
0x02cc: ('2CC', 1), # Sankai no Yubiwa
|
||||
0x02cd: ('2CD', 1), # Sankai no Yubiwa
|
||||
0x02ce: ('u0042B616', 1),
|
||||
0x02cf: ('u0042B617', 1),
|
||||
0x02d0: ('u0042B940', 3),
|
||||
0x02d1: ('u0042B950', 3),
|
||||
0x02d2: ('u0042B960', 3),
|
||||
0x02d3: ('u0042B970', 3),
|
||||
0x02d5: ('u0042B990', 2),
|
||||
0x02d7: ('u0042B9B0', 2),
|
||||
0x02d8: ('set-array-to', 3), # Set a given array to the given value x times. loop: param1[param3] = param2; param3++
|
||||
0x02d9: ('u0042BA30', 2),
|
||||
0x02da: ('u004234E0', 8),
|
||||
0x02db: ('u004235C0', 1),
|
||||
0x02dc: ('u0042BA80', 1),
|
||||
0x02dd: ('u0042D880', 2),
|
||||
0x02de: ('u0042BAC0', 2),
|
||||
0x02df: ('u0042BAC1', 3),
|
||||
0x02e0: ('u0042CE0F', 3),
|
||||
0x02e1: ('u0042CE10', 3),
|
||||
0x02e2: ('u0042CE11', 3),
|
||||
0x02e3: ('u0042CE30', 3),
|
||||
0x02e4: ('u0042CE31', 3),
|
||||
0x02e5: ('u0042CE50', 1),
|
||||
0x02e6: ('u0042CE60', 2),
|
||||
0x02e7: ('u0042CE70', 2),
|
||||
0x02e8: ('u0042CE80', 1),
|
||||
0x02e9: ('u0042CE90', 1),
|
||||
0x02ea: ('u0042CEA0', 1),
|
||||
0x02eb: ('u0042CEB0', 1),
|
||||
0x02ec: ('u0042CEC0', 2),
|
||||
0x02ee: ('u0042CEC2', 1),
|
||||
0x02ef: ('u0042CEC3', 11),
|
||||
0x02f0: ('u0042CEC4', 9),
|
||||
0x02f1: ('u0042CEC5', 7),
|
||||
0x02f2: ('u0042CEC6', 6),
|
||||
0x02f3: ('2F3', 6), # La Dea
|
||||
0x02f4: ('2F4', 3), # La Dea
|
||||
0x02f5: ('2F5', 4), # La Dea
|
||||
0x02f6: ('2F6', 1), # La Dea
|
||||
0x02f7: ('2F7', 1), # La Dea
|
||||
0x02f8: ('2F8', 2), # La Dea
|
||||
0x02f9: ('2F9', 7), # La Dea
|
||||
0x02fa: ('2FA', 1), # La Dea
|
||||
0x02fb: ('2FB', 1), # La Dea
|
||||
0x02fc: ('2FC', 5), # Kami no Rhapsody
|
||||
0x02fd: ('2FD', 6), # Kami no Rhapsody
|
||||
0x02fe: ('2FE', 1), # Sankai no Yubiwa
|
||||
0x02ff: ('2FF', 2), # Sankai no Yubiwa
|
||||
0x0300: ('300', 3), # Sankai no Yubiwa
|
||||
0x0301: ('301', 1), # Sankai no Yubiwa
|
||||
0x0302: ('302', 2), # Sankai no Yubiwa
|
||||
0x0303: ('303', 3), # Sankai no Yubiwa
|
||||
0x0304: ('304', 0), # Sankai no Yubiwa
|
||||
0x0305: ('305', 0), # Sankai no Yubiwa
|
||||
0x0306: ('306', 1), # Sankai no Yubiwa
|
||||
0x0307: ('307', 1), # Sankai no Yubiwa
|
||||
0x0308: ('308', 1), # Amayui Alchemy Meister
|
||||
0x030a: ('30A', 2), # Amayui Alchemy Meister
|
||||
0x030c: ('30C', 1), # Tenmei no Conquista
|
||||
0x0320: ('u0043AA20', 10),
|
||||
0x0321: ('u0043AA30', 3),
|
||||
0x0322: ('u0043AA40', 4),
|
||||
0x0323: ('u0043AA50', 5),
|
||||
0x0324: ('u0043AA60', 0),
|
||||
0x0325: ('u0043AA70', 2),
|
||||
0x0326: ('u0043AA80', 4),
|
||||
0x0327: ('u0043AA90', 1),
|
||||
0x0328: ('u0043AAA0', 3),
|
||||
0x0329: ('u0043AAB0', 2),
|
||||
0x032a: ('32A', 1), # Kami no Rhapsody
|
||||
0x032b: ('u0043AAD0', 0),
|
||||
0x032c: ('u0043AAE0', 6),
|
||||
0x032d: ('u0043AAF0', 2),
|
||||
0x032e: ('u0043AB10', 11),
|
||||
0x032f: ('u0043AB11', 1),
|
||||
0x0330: ('u0043AB12', 2),
|
||||
0x0332: ('u0043AB14', 4),
|
||||
0x0334: ('u0043AB16', 1),
|
||||
0x0335: ('u0043AB17', 4),
|
||||
0x0337: ('u0043AB19', 4),
|
||||
0x033b: ('u0043AB1D', 4),
|
||||
0x033d: ('u0043AB1E', 3),
|
||||
0x033e: ('u0043AB1F', 5),
|
||||
0x033f: ('u0043AB20', 3),
|
||||
0x0340: ('340', 1), # Sankai no Yubiwa
|
||||
0x0341: ('341', 2), # Amayui Alchemy Meister
|
||||
0x0342: ('342', 1), # Amayui Alchemy Meister
|
||||
0x0344: ('344', 2), # Amayui Alchemy Meister
|
||||
0x0345: ('345', 3), # Amayui Alchemy Meister
|
||||
0x0349: ('349', 4), # Amayui Alchemy Meister
|
||||
0x034d: ('34D', 6), # Amayui Alchemy Meister
|
||||
0x034e: ('34E', 4), # Amayui Alchemy Meister
|
||||
0x0352: ('352', 3), # Amayui Alchemy Meister
|
||||
0x0353: ('353', 2), # Fuukan no Gransesta
|
||||
0x0354: ('354', 2), # Fuukan no Gransesta
|
||||
0x0358: ('358', 5), # Amayui 2
|
||||
0x035a: ('35A', 5), # Amayui 2
|
||||
0x035b: ('35B', 2), # Fuukan no Gransesta
|
||||
0x035c: ('35C', 2), # Fuukan no Gransesta
|
||||
0x035d: ('35D', 3), # Fuukan no Gransesta
|
||||
0x035f: ('35F', 3), # Fuukan no Gransesta
|
||||
0x0360: ('360', 3), # Fuukan no Gransesta
|
||||
0x0361: ('361', 2), # Fuukan no Gransesta
|
||||
0x0363: ('363', 3), # Amayui 2
|
||||
0x0364: ('364', 3), # Amayui 2
|
||||
0x0384: ('384', 3), # Tenmei no Conquista
|
||||
0x0386: ('386', 11), # Tenmei no Conquista
|
||||
0x0387: ('387', 8), # Tenmei no Conquista
|
||||
0x0388: ('388', 3), # Tenmei no Conquista
|
||||
0x0389: ('389', 6), # Tenmei no Conquista
|
||||
0x038f: ('38F', 6), # Tenmei no Conquista
|
||||
0x0390: ('390', 7), # Tenmei no Conquista
|
||||
0x0391: ('391', 2), # Amayui 2
|
||||
0x0392: ('392', 1), # Tenmei no Conquista
|
||||
0x0393: ('393', 6), # Amayui 2
|
||||
0x0396: ('396', 5), # Tenmei no Conquista
|
||||
0x0398: ('398', 3), # Amayui 2
|
||||
0x0399: ('399', 7), # Tenmei no Conquista
|
||||
0x039b: ('39B', 5), # Amayui 2
|
||||
}
|
||||
|
||||
# argument type tag -> human label (from disassembler.cpp get_type_label)
|
||||
ARG_TYPES: dict[int, str] = {
|
||||
0x0000: 'imm',
|
||||
0x0001: 'float',
|
||||
0x0002: 'string',
|
||||
0x0003: 'global-int',
|
||||
0x0004: 'global-float',
|
||||
0x0005: 'global-string',
|
||||
0x0006: 'global-ptr',
|
||||
0x0008: 'global-string-ptr',
|
||||
0x0009: 'local-int',
|
||||
0x000a: 'local-float',
|
||||
0x000b: 'local-string',
|
||||
0x000c: 'local-ptr',
|
||||
0x000d: 'local-float-ptr',
|
||||
0x000e: 'local-string-ptr',
|
||||
0x8003: 'type-0x8003',
|
||||
0x8005: 'type-0x8005',
|
||||
0x8009: 'type-0x8009',
|
||||
0x800b: 'type-0x800B',
|
||||
}
|
||||
|
||||
# opcodes whose (some) operands are code offsets (label targets), from age-shared.h
|
||||
CONTROL_FLOW = frozenset({0x8C, 0x8F, 0xA0, 0xCC, 0xFB, 0xD4, 0x90, 0x7B})
|
||||
|
||||
# opcode 0x64 arg #1 references an array stored in the footer (not inline)
|
||||
ARRAY_OPCODE = 0x64
|
||||
|
||||
|
||||
def is_label_argument(op: int, arg_index: int, raw_value: int) -> bool:
|
||||
"""Mirror of age-shared.h is_label_argument: is this operand a code-offset label?"""
|
||||
if raw_value == 0xFFFFFFFF:
|
||||
return False
|
||||
if op in (0x8C, 0x8F):
|
||||
return True
|
||||
if op == 0xA0:
|
||||
return arg_index > 0
|
||||
if op in (0xCC, 0xFB):
|
||||
return arg_index > 0
|
||||
if op == 0xD4:
|
||||
return arg_index >= 2
|
||||
if op == 0x90:
|
||||
return arg_index >= 4
|
||||
if op == 0x7B:
|
||||
return True
|
||||
return False
|
||||
98
tools/age_opcodes_himegari.py
Normal file
98
tools/age_opcodes_himegari.py
Normal file
@@ -0,0 +1,98 @@
|
||||
"""Himegari-specific opcode INFERENCE layer (sits on top of age_opcodes.py).
|
||||
|
||||
`age_opcodes.py` is the verbatim Kelebek table (opcode number + argc, validated 481/481).
|
||||
Many opcodes have only engine-address labels (`u004xxxx`). This module records what the
|
||||
top-frequency unnamed opcodes *mean*, inferred from operand types, disassembly context,
|
||||
and neighbouring named ops (see `vm-map/himegari-opcode-notes.md` for the evidence).
|
||||
|
||||
These are INFERENCES, not ground truth. Fields:
|
||||
name short mnemonic for the disassembler
|
||||
category marker | structural | computational | draw | audio | adv | control
|
||||
noop True = safe for the Godot VM v1 to skip (no state/visible effect expected)
|
||||
False = has an effect; must be implemented (or knowingly stubbed)
|
||||
confidence high | med | low
|
||||
method how the inference was reached / how to confirm:
|
||||
structure | context | harness (confirm via dialogue diff in Phase 4) |
|
||||
frida (needs live capture) | unicorn (micro-exec)
|
||||
note one-line rationale / caveat
|
||||
|
||||
Consumed by `sys4load.render_listing` (nicer disassembly) and, later, the Godot VM.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
INFERRED: dict[int, dict] = {
|
||||
# ---- statement / scope scaffolding: zero-arg, no operands, bracket statements ----
|
||||
0x1f4: dict(name="stmt-begin", category="marker", noop=True, confidence="high",
|
||||
method="structure", note="zero-arg; opens scripts, pairs with stmt-end 0x1f5"),
|
||||
0x1f5: dict(name="stmt-end", category="marker", noop=True, confidence="high",
|
||||
method="structure", note="zero-arg; precedes exit/next-stmt, pairs with 0x1f4"),
|
||||
0x1d5: dict(name="cond-block", category="marker", noop=True, confidence="high",
|
||||
method="context", note="zero-arg; ALWAYS follows jcc — marks conditional body entry"),
|
||||
0x1bc: dict(name="block-mark", category="marker", noop=True, confidence="high",
|
||||
method="context", note="zero-arg; follows jcc/mov, precedes mov/ret — block boundary"),
|
||||
0x1bf: dict(name="call-end", category="marker", noop=True, confidence="med",
|
||||
method="context", note="zero-arg; call->0x1bf->stmt-end — end-of-call-statement marker"),
|
||||
|
||||
# ---- statement metadata carrying an id (tentative no-op, confirm via harness) ----
|
||||
0x21b: dict(name="line-id?", category="marker", noop=True, confidence="med",
|
||||
method="harness", note="1 imm; mov->0x21b->stmt-end; near save/load-messkip — likely line/stmt id, verify not msg-control"),
|
||||
0x1d2: dict(name="stmt-desc?", category="marker", noop=True, confidence="med",
|
||||
method="harness", note="2 imm; immediately after stmt-begin 0x1f4 — statement descriptor?"),
|
||||
0x258: dict(name="decl?", category="marker", noop=True, confidence="low",
|
||||
method="harness", note="2 imm; runs in a chain right after script-entry 0x259, enumerating ids — prologue declaration/registration?"),
|
||||
|
||||
# ---- structural ----
|
||||
0x71: dict(name="label-def", category="structural", noop=True, confidence="high",
|
||||
method="structure", note="1 imm; count == T1 table size -> the label/anchor T1 indexes. v1 no-op; revisit if menu/callback dispatch looks up by id"),
|
||||
|
||||
# ---- input / UI hotspot (u0041Bxxx/Cxxx widget module; see himegari-opcode-notes.md) ----
|
||||
0x90: dict(name="hotspot-branch", category="input", noop=False, confidence="high",
|
||||
method="structure", note="argc7: x y w h + 3 code targets (0xffffffff=unused). "
|
||||
"Cursor/input hotspot hit-test; branches per interaction (inferred hover-enter->flag=1 / "
|
||||
"hover-leave->flag=0 / click->run-action), FALLS THROUGH to pc+1 on no-match "
|
||||
"(design-confirmed: enc.len 15 lands on the next stmt). HEADLESS: fall through = correct "
|
||||
"no-input behaviour (proven safe by 279 CLEAN scenes). Occurs ONLY in a shared ADV-chrome "
|
||||
"subroutine, identical in all 301 ADV scripts (8/script = 5 imm-rect buttons @y=572 "
|
||||
"x=684..772 20x20 toggling G[0x6c9..0x6cd] + 3 local-operand keyed forms). Model live in A2; "
|
||||
"confirm target->state mapping via input capture/Frida."),
|
||||
0x97: dict(name="hotspot-reg?", category="input", noop=False, confidence="med",
|
||||
method="frida", note="argc5: v1 v2 1 1 <action-id imm>; NO code targets. Interleaves with "
|
||||
"0x90 in the ADV-chrome subroutine -> companion register-hotspot / set-widget-action "
|
||||
"(trailing imm = action id 0x0/0x7/0x8). u0041C150, same widget cluster as 0x90/0x91/0x92/0x95."),
|
||||
|
||||
# ---- computational (exact behaviour via Unicorn micro-exec) ----
|
||||
0x215: dict(name="count?", category="computational", noop=False, confidence="med",
|
||||
method="unicorn", note="2 args -> writes global then result tested >0 (gre/lt) — count/search-returns-index helper"),
|
||||
0x1a2: dict(name="resolve-handle?", category="computational", noop=False, confidence="low",
|
||||
method="frida", note="1 local-ptr from lookup-array, then create-texture — resolves a looked-up resource/handle"),
|
||||
|
||||
# ---- ADV / text-display (effectful; confirm via Frida) ----
|
||||
0x7a: dict(name="text-param?", category="adv", noop=False, confidence="med",
|
||||
method="frida", note="3 args (imm/computed/imm); sub computes a value then 0x7a then show-text — text speed/wait/window param"),
|
||||
|
||||
# ---- draw / UI (0x420-0x421 graphics family; effectful; Frida) ----
|
||||
0x202: dict(name="draw-blit?", category="draw", noop=False, confidence="med",
|
||||
method="frida", note="5 args (coords/sizes); preceded by coord arithmetic, near draw ops"),
|
||||
0x203: dict(name="draw?", category="draw", noop=False, confidence="med",
|
||||
method="frida", note="4 args; chains with 0x202/draw-texture"),
|
||||
0x1f7: dict(name="ui-elem?", category="draw", noop=False, confidence="med",
|
||||
method="frida", note="2 args; 0x420 family, pairs with 0x1fa — create/begin a UI element"),
|
||||
0x1fa: dict(name="ui-clear?", category="draw", noop=False, confidence="med",
|
||||
method="frida", note="1 arg (element id); follows 0x1f7 — show/hide/clear UI element by id"),
|
||||
0x217: dict(name="gfx-geom?", category="draw", noop=False, confidence="low",
|
||||
method="frida", note="4 global-ints; part of a 0x217/0x218/0x21a geometry chain"),
|
||||
0x218: dict(name="gfx-geom?", category="draw", noop=False, confidence="low",
|
||||
method="frida", note="4 global-ints; chains with 0x21a/0x217"),
|
||||
0x21a: dict(name="gfx-geom?", category="draw", noop=False, confidence="low",
|
||||
method="frida", note="4 global-ints; chains with 0x218/0x217"),
|
||||
0x1ff: dict(name="draw?", category="draw", noop=False, confidence="low",
|
||||
method="frida", note="4 args (global+imms); follows 0x217, then call"),
|
||||
|
||||
# ---- audio (0x41D family, near play-sound-effect 0xb4) ----
|
||||
0xb6: dict(name="snd-ctrl?", category="audio", noop=False, confidence="low",
|
||||
method="frida", note="1 imm; self-chains, 0x41D family near play-sound-effect/0xb5 — sound channel/volume/stop control"),
|
||||
}
|
||||
|
||||
# Names ending in "?" are low/medium-confidence guesses; the disassembler shows them
|
||||
# so listings read better than `u004xxxx`, but the VM must not treat non-noop ops as
|
||||
# no-ops without Frida/Unicorn/harness confirmation (see `method`).
|
||||
186
tools/extract_init.py
Normal file
186
tools/extract_init.py
Normal file
@@ -0,0 +1,186 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract a *INIT data table to JSON. Auto-detects the table's shape.
|
||||
|
||||
*INIT scripts populate parallel global arrays with static game data. Three shapes seen:
|
||||
|
||||
name — records keyed by a name string. Each record: set-string(name), mov(fields..),
|
||||
set-string(desc). Arrays indexed by record id in lockstep (+1/record).
|
||||
(SKINIT skills, ITINIT items, EBINIT units)
|
||||
numeric— column table with NO names: mov/copy-to-global into parallel int arrays, keyed
|
||||
by an incrementing index column. (CGINIT gallery)
|
||||
footer — copy-local-array (op 0x64) bulk-loads length-prefixed arrays from the file
|
||||
footer into per-record global arrays. The data lives in the footer. (MPINIT maps)
|
||||
|
||||
Records are {id, name?, desc?, fields:{"0x<col_base>": value}} or, for footer tables,
|
||||
{id, global_addr, footer_off, values:[...]}. Column addresses are raw engine globals;
|
||||
naming them (attack, cost, …) needs the engine global-var map — later work.
|
||||
|
||||
Usage: py -3.11 -X utf8 tools/extract_init.py <TABLE> [OUTNAME] [--mode name|numeric|footer]
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE))
|
||||
import paths
|
||||
import sys4load
|
||||
|
||||
SET_STRING = 0x192
|
||||
MOV = 0x55
|
||||
COPY_TO_GLOBAL = 0x6C
|
||||
COPY_LOCAL_ARRAY = 0x64
|
||||
T_GLOBAL_INT = 3
|
||||
T_GLOBAL_STRING = 5
|
||||
T_IMM = 0
|
||||
|
||||
|
||||
def resolve(name: str) -> Path:
|
||||
for cand in (paths.GAME_DIR / f"{name}.BIN", paths.DATA1 / f"{name}.BIN"):
|
||||
if cand.exists():
|
||||
return cand
|
||||
raise SystemExit(f"not found: {name}.BIN")
|
||||
|
||||
|
||||
def _val(arg):
|
||||
"""Render an operand as an int (immediate) or a {type,value} ref."""
|
||||
t, v = arg
|
||||
return v if t == T_IMM else {"type": f"0x{t:x}", "value": f"0x{v:x}"}
|
||||
|
||||
|
||||
def read_footer_array(scr, off):
|
||||
"""Read a length-prefixed Data_Array at dword `off`: [length][v0..v_{length-1}]."""
|
||||
dw = scr.dwords
|
||||
if not (0 <= off < scr.nbody):
|
||||
return None
|
||||
length = dw[off]
|
||||
if length > scr.nbody or off + 1 + length > scr.nbody:
|
||||
return None
|
||||
return list(dw[off + 1: off + 1 + length])
|
||||
|
||||
|
||||
def detect_mode(scr):
|
||||
ops = [ins.opcode for ins in scr.instructions]
|
||||
has_str = any(ins.opcode == SET_STRING and ins.args and ins.args[0][0] == T_GLOBAL_STRING
|
||||
for ins in scr.instructions)
|
||||
if has_str:
|
||||
return "name"
|
||||
n_footer = ops.count(COPY_LOCAL_ARRAY)
|
||||
n_int = ops.count(MOV) + ops.count(COPY_TO_GLOBAL)
|
||||
return "footer" if n_footer >= max(4, n_int) else "numeric"
|
||||
|
||||
|
||||
def extract_name(scr):
|
||||
name_base = None
|
||||
for ins in scr.instructions:
|
||||
if ins.opcode == SET_STRING and ins.args and ins.args[0][0] == T_GLOBAL_STRING:
|
||||
name_base = ins.args[0][1]; break
|
||||
records, cur, prev_gstr, desc_slot, desc_bases = [], None, None, 0, {}
|
||||
for ins in scr.instructions:
|
||||
if ins.opcode == SET_STRING and ins.args and ins.args[0][0] == T_GLOBAL_STRING:
|
||||
addr = ins.args[0][1]
|
||||
txt = scr.strings.get(ins.args[1][1], (None,))[0] if len(ins.args) > 1 else None
|
||||
if prev_gstr is None or addr < prev_gstr:
|
||||
cur = {"id": addr - name_base, "name": txt, "fields": {}}
|
||||
records.append(cur); desc_slot = 0
|
||||
elif cur is not None:
|
||||
key = "desc" if desc_slot == 0 else f"desc{desc_slot}"
|
||||
cur[key] = txt; desc_bases.setdefault(key, addr - cur["id"]); desc_slot += 1
|
||||
prev_gstr = addr
|
||||
elif ins.opcode == MOV and cur is not None and ins.args and ins.args[0][0] == T_GLOBAL_INT:
|
||||
cur["fields"][f"0x{ins.args[0][1] - cur['id']:x}"] = _val(ins.args[1])
|
||||
return records, {"name_array_base": f"0x{name_base:x}",
|
||||
"desc_array_bases": {k: f"0x{v:x}" for k, v in sorted(desc_bases.items())}}
|
||||
|
||||
|
||||
def _int_writes(scr):
|
||||
"""Ordered (addr, value_arg) for global-int mov / copy-to-global."""
|
||||
out = []
|
||||
for ins in scr.instructions:
|
||||
if ins.opcode in (MOV, COPY_TO_GLOBAL) and ins.args and ins.args[0][0] == T_GLOBAL_INT:
|
||||
out.append((ins.args[0][1], ins.args[1]))
|
||||
return out
|
||||
|
||||
|
||||
def _longest_stride1_column(addrs):
|
||||
"""Pick the primary index array: the stride-1 arithmetic run covering the most records."""
|
||||
seen = set(addrs)
|
||||
best_base, best_len = None, 0
|
||||
for a in sorted(seen):
|
||||
if a - 1 in seen:
|
||||
continue # only start at a run's base
|
||||
n = 0
|
||||
while a + n in seen:
|
||||
n += 1
|
||||
if n > best_len:
|
||||
best_base, best_len = a, n
|
||||
return best_base, best_len
|
||||
|
||||
|
||||
def extract_numeric(scr):
|
||||
writes = _int_writes(scr)
|
||||
base, n = _longest_stride1_column([a for a, _ in writes])
|
||||
if base is None:
|
||||
return [], {}
|
||||
primary = set(range(base, base + n))
|
||||
records, buf = [], []
|
||||
for addr, varg in writes:
|
||||
buf.append((addr, varg))
|
||||
if addr in primary: # primary write closes the record
|
||||
rid = addr - base
|
||||
fields = {f"0x{a - rid:x}": _val(v) for a, v in buf}
|
||||
records.append({"id": rid, "fields": fields})
|
||||
buf = []
|
||||
return records, {"primary_index_base": f"0x{base:x}", "record_span": n}
|
||||
|
||||
|
||||
def extract_footer(scr):
|
||||
records = []
|
||||
for i, ins in enumerate(scr.instructions):
|
||||
if ins.opcode == COPY_LOCAL_ARRAY and ins.args and ins.args[0][0] == T_GLOBAL_INT:
|
||||
addr = ins.args[0][1]
|
||||
foff = ins.args[1][1]
|
||||
vals = read_footer_array(scr, foff)
|
||||
records.append({"id": i, "global_addr": f"0x{addr:x}",
|
||||
"footer_off": f"0x{foff:x}",
|
||||
"length": len(vals) if vals else 0,
|
||||
"values": vals if vals else []})
|
||||
return records, {}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
argv = [a for a in sys.argv[1:] if not a.startswith("--")]
|
||||
mode_arg = next((sys.argv[i + 1] for i, a in enumerate(sys.argv) if a == "--mode"), None)
|
||||
if not argv:
|
||||
raise SystemExit(__doc__)
|
||||
name = argv[0].upper().removesuffix(".BIN")
|
||||
outname = argv[1] if len(argv) > 1 else name
|
||||
scr = sys4load.load(resolve(name))
|
||||
|
||||
mode = mode_arg or detect_mode(scr)
|
||||
extractor = {"name": extract_name, "numeric": extract_numeric, "footer": extract_footer}[mode]
|
||||
recs, meta = extractor(scr)
|
||||
|
||||
cols = sorted({c for r in recs for c in r.get("fields", {})}, key=lambda h: int(h, 16))
|
||||
out = {"table": name, "source": scr.path.name, "magic": scr.magic, "mode": mode,
|
||||
"record_count": len(recs), **meta,
|
||||
"field_columns": cols if mode != "footer" else None, "records": recs}
|
||||
outpath = paths.BUILD / "data" / f"{outname}.json"
|
||||
outpath.parent.mkdir(parents=True, exist_ok=True)
|
||||
outpath.write_text(json.dumps(out, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
print(f"{name}: mode={mode}, {len(recs)} records"
|
||||
+ (f", {len(cols)} field-columns" if mode != 'footer' else "")
|
||||
+ f" -> build/data/{outname}.json")
|
||||
for r in recs[:4]:
|
||||
if mode == "footer":
|
||||
print(f" id {r['id']:>4} {r['global_addr']} <- footer {r['footer_off']} "
|
||||
f"len {r['length']} head={r['values'][:8]}")
|
||||
else:
|
||||
f4 = {k: r['fields'][k] for k in list(r['fields'])[:4]}
|
||||
print(f" id {r['id']:>4} {r.get('name','')!r:12} desc={r.get('desc','')!r} {f4}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
112
tools/extract_phase2.py
Normal file
112
tools/extract_phase2.py
Normal file
@@ -0,0 +1,112 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Phase 2 batch extraction: disassembly + text corpora from every SYS4 script.
|
||||
|
||||
Outputs (all under build/, workspace-root relative):
|
||||
build/disasm/<NAME>.asm full disassembly listing (one per script)
|
||||
build/text/<NAME>.strings.txt all inline strings in that script
|
||||
build/text/dialogue.jsonl show-text (0x6E) lines only — the translation corpus
|
||||
build/text/strings.jsonl every inline string, tagged by the opcode that references it
|
||||
build/manifest.json per-script stats (instructions, strings, dialogue, decode-clean)
|
||||
|
||||
Authoritative copies: a game-dir override shadows its extracted/DATA1 copy. Run:
|
||||
py -3.11 -X utf8 tools/extract_phase2.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE))
|
||||
import paths
|
||||
import sys4load
|
||||
|
||||
DATA1 = paths.DATA1
|
||||
BUILD = paths.BUILD
|
||||
DISASM = BUILD / "disasm"
|
||||
TEXT = BUILD / "text"
|
||||
|
||||
SHOW_TEXT = 0x6E # dialogue opcode; type-2 arg = displayed line
|
||||
|
||||
|
||||
def authoritative_scripts() -> dict[str, Path]:
|
||||
"""name -> path, game-dir overrides winning over extracted/DATA1."""
|
||||
return paths.scripts()
|
||||
|
||||
|
||||
def opcode_for_string_ref(scr, value_dword_index: int) -> tuple[int | None, str | None]:
|
||||
"""Given the body index of a string's *value* operand, find the owning instruction."""
|
||||
for ins in scr.instructions:
|
||||
base = ins.offset + 1
|
||||
for a in range(len(ins.args)):
|
||||
if base + 2 * a + 1 == value_dword_index:
|
||||
return ins.opcode, ins.label
|
||||
return None, None
|
||||
|
||||
|
||||
def main() -> int:
|
||||
for d in (DISASM, TEXT, BUILD / "data", BUILD / "scripts-json"):
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
files = authoritative_scripts()
|
||||
manifest = []
|
||||
n_dialogue = n_strings = 0
|
||||
skipped = []
|
||||
|
||||
with (TEXT / "dialogue.jsonl").open("w", encoding="utf-8") as dlg, \
|
||||
(TEXT / "strings.jsonl").open("w", encoding="utf-8") as allstr:
|
||||
for name, p in sorted(files.items()):
|
||||
try:
|
||||
scr = sys4load.load(p)
|
||||
except sys4load.Sys4Error as e:
|
||||
skipped.append((name, str(e)))
|
||||
continue
|
||||
|
||||
# 1) disassembly listing
|
||||
(DISASM / f"{Path(name).stem}.asm").write_text(
|
||||
sys4load.render_listing(scr), encoding="utf-8")
|
||||
|
||||
# 2) per-script strings file
|
||||
if scr.strings:
|
||||
lines = [f"0x{off:05x}\t{txt}" for off, (txt, _) in sorted(scr.strings.items())]
|
||||
(TEXT / f"{Path(name).stem}.strings.txt").write_text(
|
||||
"\n".join(lines) + "\n", encoding="utf-8")
|
||||
|
||||
# 3) combined corpora, tagged by referencing opcode
|
||||
for val_idx, off in sorted(scr.string_refs.items(), key=lambda kv: kv[1]):
|
||||
txt = scr.strings.get(off, (None,))[0]
|
||||
if txt is None:
|
||||
continue
|
||||
op, label = opcode_for_string_ref(scr, val_idx)
|
||||
rec = {"file": name, "off": f"0x{off:x}",
|
||||
"op": f"0x{op:x}" if op is not None else None,
|
||||
"label": label, "text": txt}
|
||||
allstr.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
||||
n_strings += 1
|
||||
if op == SHOW_TEXT:
|
||||
dlg.write(json.dumps({"file": name, "off": f"0x{off:x}", "text": txt},
|
||||
ensure_ascii=False) + "\n")
|
||||
n_dialogue += 1
|
||||
|
||||
n, unk, trunc, clean = sys4load.decode_stats(scr)
|
||||
manifest.append({"file": name, "instructions": n, "strings": len(scr.strings),
|
||||
"clean": clean, "unknown": unk, "truncated": trunc})
|
||||
|
||||
(BUILD / "manifest.json").write_text(
|
||||
json.dumps({"scripts": len(manifest), "skipped": skipped,
|
||||
"total_dialogue_lines": n_dialogue, "total_strings": n_strings,
|
||||
"per_script": manifest}, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
clean = sum(1 for m in manifest if m["clean"])
|
||||
print(f"scripts processed: {len(manifest)} (decode-clean: {clean}/{len(manifest)})")
|
||||
print(f"skipped (non-script magic): {len(skipped)} -> {[s[0] for s in skipped]}")
|
||||
print(f"disasm listings: build/disasm/*.asm")
|
||||
print(f"dialogue lines (show-text): {n_dialogue} -> build/text/dialogue.jsonl")
|
||||
print(f"all inline strings: {n_strings} -> build/text/strings.jsonl")
|
||||
print(f"manifest: build/manifest.json")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
199
tools/global_map.py
Normal file
199
tools/global_map.py
Normal file
@@ -0,0 +1,199 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Partial global-variable map — label raw global offsets by evidence (static, read-only).
|
||||
|
||||
Combines three static signals (see docs/name-resolution.md #2):
|
||||
1. *INIT writers — build/data/*.json name/desc/field bases ARE labelable global addresses.
|
||||
2. string anchors — set-string targets across the corpus = string tables.
|
||||
3. access shape — how each global is used: 2D-table base (+stride), 1D-array base,
|
||||
row-index (=> "current entity" pointer), or scalar.
|
||||
|
||||
Emits build/global-var-map.json (all evidence) + build/global-var-map.md (labelled subset).
|
||||
Usage: py -3.11 -X utf8 tools/global_map.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import json
|
||||
import sys
|
||||
import collections
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
ROOT = HERE.parent
|
||||
sys.path.insert(0, str(HERE))
|
||||
import paths
|
||||
import sys4load
|
||||
|
||||
CORPUS = paths.DATA1
|
||||
DATA = ROOT / "build" / "data"
|
||||
OUT = ROOT / "build"
|
||||
|
||||
LOOKUP = 0x61 # lookup-array: (dst, base1d, idx)
|
||||
LOOKUP2D = 0x12c # lookup-array-2d: (dst, base2d, rowidx, stride, col)
|
||||
SET_STRING = 0x192 # (strbase, text)
|
||||
GLOBAL_TYPES = {3: "int", 4: "float", 5: "string", 6: "ptr", 8: "string-ptr"}
|
||||
|
||||
INIT_IDENTITY = {
|
||||
"SKINIT": "skill", "ITINIT": "item", "EBINIT": "unit", "CGINIT": "cg-gallery",
|
||||
"MPINIT": "map", "STINIT": "stage",
|
||||
}
|
||||
|
||||
|
||||
def from_init_tables():
|
||||
"""Label name/desc/field base addresses from the extracted *INIT JSONs."""
|
||||
labels = {} # addr -> (label, table, confidence, kind)
|
||||
for jf in sorted(DATA.glob("*.json")):
|
||||
try:
|
||||
d = json.loads(jf.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
continue
|
||||
tbl = d.get("table", jf.stem)
|
||||
ent = INIT_IDENTITY.get(tbl, tbl.lower())
|
||||
if d.get("name_array_base"):
|
||||
a = int(d["name_array_base"], 16)
|
||||
labels[a] = (f"{ent}-name-table", tbl, "high", "string-table")
|
||||
for k, v in (d.get("desc_array_bases") or {}).items():
|
||||
labels[int(v, 16)] = (f"{ent}-{k}-table", tbl, "high", "string-table")
|
||||
# field columns: classify dense (shared) vs sparse (per-entity extras)
|
||||
recs = d.get("records", [])
|
||||
freq = collections.Counter()
|
||||
for r in recs:
|
||||
for c in r.get("fields", {}):
|
||||
freq[c] += 1
|
||||
n = max(1, len(recs))
|
||||
for col, c in freq.items():
|
||||
a = int(col, 16)
|
||||
if a in labels:
|
||||
continue
|
||||
dense = c >= 0.5 * n
|
||||
labels[a] = (f"{ent}-field" + ("" if dense else "?"), tbl,
|
||||
"med" if dense else "low", "entity-field-array")
|
||||
return labels
|
||||
|
||||
|
||||
def scan_corpus():
|
||||
scrs = []
|
||||
for p in sorted(CORPUS.glob("*.BIN")):
|
||||
try:
|
||||
scrs.append(sys4load.load(p))
|
||||
except Exception:
|
||||
pass
|
||||
roles = collections.defaultdict(collections.Counter) # addr -> role -> count
|
||||
strides = collections.defaultdict(collections.Counter) # addr -> stride -> count
|
||||
gtype = {} # addr -> global type name
|
||||
rowidx = collections.Counter() # addr -> times used as 2D row index
|
||||
strtable_writers = collections.defaultdict(collections.Counter) # straddr -> script -> count
|
||||
for scr in scrs:
|
||||
sname = scr.path.stem
|
||||
for ins in scr.instructions:
|
||||
op = ins.opcode
|
||||
for pos, (t, v) in enumerate(ins.args):
|
||||
if t not in GLOBAL_TYPES:
|
||||
continue
|
||||
gtype[v] = GLOBAL_TYPES[t]
|
||||
if op == LOOKUP2D and pos == 1:
|
||||
roles[v]["2d-base"] += 1
|
||||
if len(ins.args) > 3 and ins.args[3][0] == 0: # stride immediate
|
||||
strides[v][ins.args[3][1]] += 1
|
||||
elif op == LOOKUP2D and pos == 2:
|
||||
roles[v]["row-index"] += 1
|
||||
rowidx[v] += 1
|
||||
elif op == LOOKUP and pos == 1:
|
||||
roles[v]["1d-base"] += 1
|
||||
elif op == SET_STRING and pos == 0:
|
||||
roles[v]["string-table"] += 1
|
||||
strtable_writers[v][sname] += 1
|
||||
else:
|
||||
roles[v]["scalar"] += 1
|
||||
return roles, strides, gtype, rowidx, strtable_writers, len(scrs)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
init_labels = from_init_tables()
|
||||
roles, strides, gtype, rowidx, strwriters, nscr = scan_corpus()
|
||||
all_addrs = set(roles) | set(init_labels)
|
||||
|
||||
# merge into per-address records
|
||||
entries = {}
|
||||
for a in sorted(all_addrs):
|
||||
r = dict(roles.get(a, {}))
|
||||
total = sum(r.values())
|
||||
st = sorted(strides.get(a, {}), key=lambda s: -strides[a][s])
|
||||
label = conf = kind = None
|
||||
if a in init_labels:
|
||||
label, tbl, conf, kind = init_labels[a]
|
||||
elif r.get("string-table"):
|
||||
w = strwriters.get(a, {})
|
||||
top = max(w, key=w.get) if w else "?"
|
||||
label, conf, kind = f"string-table (written by {top})", "med", "string-table"
|
||||
elif r.get("2d-base"):
|
||||
stride = st[0] if st else "?"
|
||||
label, conf, kind = f"record-table[stride {stride}]", "med", "record-table-2d"
|
||||
elif r.get("1d-base"):
|
||||
label, conf, kind = "array", "low", "array-1d"
|
||||
elif r.get("row-index"): # used as a 2D row index but never a base -> an index var
|
||||
ri, sc = r["row-index"], r.get("scalar", 0)
|
||||
pure = ri / (ri + sc) if (ri + sc) else 0
|
||||
if pure >= 0.3:
|
||||
label, conf, kind = "current-entity-index?", "med", "index"
|
||||
else:
|
||||
label, conf, kind = "index/counter?", "low", "index"
|
||||
entries[a] = dict(addr=f"0x{a:x}", type=gtype.get(a, "?"), uses=total,
|
||||
roles=r, stride_candidates=[f"0x{s:x}" for s in st[:3]] or None,
|
||||
label=label, confidence=conf, kind=kind, table=(init_labels.get(a, (None, None))[1]))
|
||||
|
||||
# "current entity" pointers: globals used as 2D row index, ranked by purity
|
||||
def purity(a):
|
||||
ri, sc = rowidx[a], roles[a].get("scalar", 0)
|
||||
return ri / (ri + sc) if (ri + sc) else 0
|
||||
current_idx = [dict(addr=f"0x{a:x}", used_as_row_index=rowidx[a],
|
||||
also_scalar=roles[a].get("scalar", 0), purity=round(purity(a), 2))
|
||||
for a in sorted(rowidx, key=lambda a: (-round(purity(a), 2), -rowidx[a]))
|
||||
if rowidx[a] >= 10][:15]
|
||||
|
||||
labeled = {k: v for k, v in entries.items() if v["label"]}
|
||||
out = {
|
||||
"generated_from": f"extracted/DATA1 ({nscr} scripts) + build/data/*.json",
|
||||
"note": "Static partial map. Labels ending '?' are low-confidence. Column addresses "
|
||||
"are raw engine globals; Frida can confirm the ambiguous ones (see docs/name-resolution.md).",
|
||||
"totals": {"distinct_globals_seen": len(all_addrs), "labelled": len(labeled),
|
||||
"from_init_tables": len(init_labels)},
|
||||
"current_entity_index_candidates": current_idx,
|
||||
"globals": {v["addr"]: {k: val for k, val in v.items() if k != "addr"}
|
||||
for v in entries.values()},
|
||||
}
|
||||
(OUT / "global-var-map.json").write_text(json.dumps(out, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
# readable markdown of the labelled subset
|
||||
md = ["# Partial global-variable map", "",
|
||||
f"Static map from {nscr} scripts + `build/data/*.json`. "
|
||||
f"**{len(labeled)} of {len(all_addrs)} distinct globals labelled.** "
|
||||
"Regenerate: `py -3.11 -X utf8 tools/global_map.py`. See `docs/name-resolution.md`.", "",
|
||||
"## 'Current entity' index globals (dominant 2D row-index)", "",
|
||||
"The row index into per-entity tables — the VM's \"which unit/entity are we on\" pointers. "
|
||||
"Ranked by purity (fraction of uses that are row-index vs. general scalar); high purity = "
|
||||
"a dedicated index pointer, low = a general-purpose var reused as an index.", "",
|
||||
"| global | row-index uses | also scalar | purity |", "|---|---|---|---|"]
|
||||
for c in current_idx:
|
||||
md.append(f"| `{c['addr']}` | {c['used_as_row_index']} | {c['also_scalar']} | {c['purity']} |")
|
||||
for kind, title in [("string-table", "String tables (names / descriptions / messages)"),
|
||||
("entity-field-array", "Per-entity data-field arrays (from *INIT)"),
|
||||
("record-table-2d", "Row-major record tables (from access shape)")]:
|
||||
rows = sorted((v for v in labeled.values() if v["kind"] == kind), key=lambda v: -v["uses"])
|
||||
md += ["", f"## {title} ({len(rows)})", "", "| global | type | label | conf | uses | stride |",
|
||||
"|---|---|---|---|---|---|"]
|
||||
for v in rows[:40]:
|
||||
md.append(f"| `{v['addr']}` | {v['type']} | {v['label']} | {v['confidence']} | "
|
||||
f"{v['uses']} | {(v['stride_candidates'] or ['—'])[0]} |")
|
||||
if len(rows) > 40:
|
||||
md.append(f"| … | | +{len(rows)-40} more (see JSON) | | | |")
|
||||
(OUT / "global-var-map.md").write_text("\n".join(md), encoding="utf-8")
|
||||
|
||||
print(f"distinct globals seen: {len(all_addrs)}; labelled: {len(labeled)} "
|
||||
f"({len(init_labels)} from *INIT)")
|
||||
print(f"top 'current entity' index globals: " +
|
||||
", ".join(f"{c['addr']}({c['used_as_row_index']})" for c in current_idx[:5]))
|
||||
print(f"-> build/global-var-map.json, build/global-var-map.md")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
131
tools/opcode_context.py
Normal file
131
tools/opcode_context.py
Normal file
@@ -0,0 +1,131 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Evidence gatherer for classifying unnamed AGE opcodes.
|
||||
|
||||
For an opcode (or the top-N unnamed ones) prints: frequency + share + cumulative coverage,
|
||||
argc, operand-type signature histogram, most common predecessor/successor opcodes, a few
|
||||
real disassembly snippets, and Kelebek's inline comment (from the upstream cpp). Read-only.
|
||||
|
||||
Usage:
|
||||
py -3.11 -X utf8 tools/opcode_context.py --top 20 # ranked unnamed summary + coverage
|
||||
py -3.11 -X utf8 tools/opcode_context.py 0x1f4 0x71 0x7a # detailed evidence per opcode
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import re
|
||||
import sys
|
||||
import collections
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
ROOT = HERE.parent
|
||||
sys.path.insert(0, str(HERE))
|
||||
import paths
|
||||
import sys4load
|
||||
import age_opcodes as ao
|
||||
|
||||
CORPUS = paths.DATA1
|
||||
KELEBEK_CPP = ROOT / "vm-map" / "kelebek1-age-shared.cpp"
|
||||
|
||||
|
||||
def is_unnamed(op: int) -> bool:
|
||||
lbl = ao.OPCODES.get(op, ("?", 0))[0]
|
||||
return bool(re.fullmatch(r"u[0-9A-Fa-f]{8}", lbl)) or lbl.lower() == f"{op:x}" or lbl.startswith("dev_ukn")
|
||||
|
||||
|
||||
def label(op: int) -> str:
|
||||
return ao.OPCODES.get(op, (f"?{op:x}", 0))[0]
|
||||
|
||||
|
||||
def kelebek_comments() -> dict[int, str]:
|
||||
out = {}
|
||||
if not KELEBEK_CPP.exists():
|
||||
return out
|
||||
for m in re.finditer(r"\{\s*(0x[0-9A-Fa-f]+)\s*,\s*\"[^\"]*\"\s*,\s*0x[0-9A-Fa-f]+\s*\}\s*,?\s*//\s*(.*)",
|
||||
KELEBEK_CPP.read_text(encoding="utf-8")):
|
||||
out[int(m.group(1), 16)] = m.group(2).strip()
|
||||
return out
|
||||
|
||||
|
||||
def load_corpus():
|
||||
scrs = []
|
||||
for p in sorted(CORPUS.glob("*.BIN")):
|
||||
try:
|
||||
scrs.append(sys4load.load(p))
|
||||
except Exception:
|
||||
pass
|
||||
return scrs
|
||||
|
||||
|
||||
def fmt_instr(scr, ins) -> str:
|
||||
ops = " ".join(sys4load._fmt_operand(ins.opcode, i, t, v, scr.strings)
|
||||
for i, (t, v) in enumerate(ins.args))
|
||||
return f"{label(ins.opcode)}{(' ' + ops) if ops else ''}"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = sys.argv[1:]
|
||||
scrs = load_corpus()
|
||||
freq = collections.Counter()
|
||||
total = 0
|
||||
for scr in scrs:
|
||||
for ins in scr.instructions:
|
||||
freq[ins.opcode] += 1
|
||||
total += 1
|
||||
named_vol = sum(c for op, c in freq.items() if not is_unnamed(op))
|
||||
comments = kelebek_comments()
|
||||
|
||||
if args and args[0] == "--top":
|
||||
n = int(args[1]) if len(args) > 1 else 20
|
||||
unnamed = [(op, c) for op, c in freq.most_common() if is_unnamed(op)]
|
||||
print(f"corpus {len(scrs)} scripts, {total} instructions; "
|
||||
f"named coverage {100*named_vol/total:.2f}%; {len(unnamed)} distinct unnamed ops")
|
||||
print(f"{'#':>3} {'op':<7}{'argc':>5}{'count':>9}{'share':>8}{'cum-cov':>9} kelebek-comment")
|
||||
cum = named_vol
|
||||
for i, (op, c) in enumerate(unnamed[:n]):
|
||||
cum += c
|
||||
argc = ao.OPCODES.get(op, ("", 0))[1]
|
||||
print(f"{i+1:>3} 0x{op:<5x}{argc:>5}{c:>9}{100*c/total:>7.2f}%{100*cum/total:>8.2f}% {comments.get(op,'')[:48]}")
|
||||
return 0
|
||||
|
||||
# detailed per-op evidence
|
||||
targets = [int(a, 16) for a in args] if args else [op for op, _ in
|
||||
[(op, c) for op, c in freq.most_common() if is_unnamed(op)][:8]]
|
||||
# precompute neighbour + signature stats
|
||||
pred = collections.defaultdict(collections.Counter)
|
||||
succ = collections.defaultdict(collections.Counter)
|
||||
sig = collections.defaultdict(collections.Counter)
|
||||
locs = collections.defaultdict(list) # op -> [(scr, index)]
|
||||
for scr in scrs:
|
||||
ins = scr.instructions
|
||||
for i, x in enumerate(ins):
|
||||
if x.opcode in targets:
|
||||
if i > 0:
|
||||
pred[x.opcode][label(ins[i-1].opcode)] += 1
|
||||
if i+1 < len(ins):
|
||||
succ[x.opcode][label(ins[i+1].opcode)] += 1
|
||||
sig[x.opcode][tuple(ao.ARG_TYPES.get(t, hex(t)) for t, _ in x.args)] += 1
|
||||
if len(locs[x.opcode]) < 4:
|
||||
locs[x.opcode].append((scr, i))
|
||||
|
||||
for op in targets:
|
||||
c = freq.get(op, 0)
|
||||
argc = ao.OPCODES.get(op, ("", 0))[1]
|
||||
print(f"\n{'='*72}\nopcode 0x{op:x} label={label(op)} argc={argc} "
|
||||
f"count={c} ({100*c/total:.2f}% of instrs)")
|
||||
if comments.get(op):
|
||||
print(f" kelebek-comment: {comments[op]}")
|
||||
print(f" operand-type signatures: " +
|
||||
", ".join(f"{'/'.join(s) if s else 'none'}×{n}" for s, n in sig[op].most_common(4)))
|
||||
print(f" top predecessors: " + ", ".join(f"{k}×{v}" for k, v in pred[op].most_common(5)))
|
||||
print(f" top successors: " + ", ".join(f"{k}×{v}" for k, v in succ[op].most_common(5)))
|
||||
print(f" snippets:")
|
||||
for scr, i in locs[op]:
|
||||
lo, hi = max(0, i-2), min(len(scr.instructions), i+3)
|
||||
for j in range(lo, hi):
|
||||
mark = ">>" if j == i else " "
|
||||
print(f" {mark} [{scr.path.name}] {fmt_instr(scr, scr.instructions[j])}")
|
||||
print()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
41
tools/pack_check.py
Normal file
41
tools/pack_check.py
Normal file
@@ -0,0 +1,41 @@
|
||||
#!/usr/bin/env python3
|
||||
import os, sys, struct, math, re
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import paths
|
||||
data = paths.AGE_EXE.read_bytes()
|
||||
pe = struct.unpack_from("<I", data, 0x3C)[0]
|
||||
nsec = struct.unpack_from("<H", data, pe+6)[0]
|
||||
opt = pe+24
|
||||
nrva = struct.unpack_from("<I", data, opt+92)[0]
|
||||
# data directories: index 1 = import, 12 = IAT
|
||||
imp_rva, imp_sz = struct.unpack_from("<II", data, opt+96+1*8)
|
||||
iat_rva, iat_sz = struct.unpack_from("<II", data, opt+96+12*8)
|
||||
print(f"NumberOfRvaAndSizes {nrva}")
|
||||
print(f"ImportDir rva {imp_rva:#x} size {imp_sz:#x}")
|
||||
print(f"IAT rva {iat_rva:#x} size {iat_sz:#x}")
|
||||
sec_off = opt + struct.unpack_from("<H", data, pe+20)[0]
|
||||
print("\nsection entropy:")
|
||||
for i in range(nsec):
|
||||
o=sec_off+i*40
|
||||
name=data[o:o+8].rstrip(b"\0").decode('ascii','replace') or f"<blank{i}>"
|
||||
vsize,va,rsize,raw=struct.unpack_from("<IIII",data,o+8)
|
||||
chunk=data[raw:raw+rsize]
|
||||
if chunk:
|
||||
freq=[0]*256
|
||||
for b in chunk: freq[b]+=1
|
||||
ent=-sum((c/len(chunk))*math.log2(c/len(chunk)) for c in freq if c)
|
||||
else:
|
||||
ent=0
|
||||
print(f" {name:<10} va {va:#08x} vsize {vsize:#08x} raw {raw:#08x} rsize {rsize:#08x} entropy {ent:.2f}")
|
||||
|
||||
# UTF-16LE anchor search
|
||||
print("\nUTF-16 anchors:")
|
||||
for t in [b"SYS4422", b"DATA1", b"Eushully", b".BIN", b"AGE"]:
|
||||
u = t.decode().encode("utf-16le")
|
||||
n = data.count(u)
|
||||
print(f" {t.decode():10} utf16 hits {n}")
|
||||
# ascii import dll names still present?
|
||||
print("\nDLL-name-ish ascii:")
|
||||
for m in re.finditer(rb"[\x20-\x7e]{4,}\.dll", data, re.I):
|
||||
print(" ", m.group().decode(errors='replace'))
|
||||
50
tools/paths.py
Normal file
50
tools/paths.py
Normal file
@@ -0,0 +1,50 @@
|
||||
"""Central path anchor for the age-reimpl workspace.
|
||||
|
||||
Every path below is derived from this file's own location, so the whole tree can
|
||||
be relocated without editing any tool -- there are no hard-coded drive paths.
|
||||
|
||||
Workspace layout (siblings under the workspace root):
|
||||
|
||||
<workspace>/ e.g. S:\\Game Hacking\\Eushully\\Himegari
|
||||
姫狩りダンジョンマイスター/ pristine game install (AGE.EXE, *.ALF,
|
||||
loose *.BIN patch-overrides, DLLs)
|
||||
extracted/ extracted ALF data: DATA1 .. DATA5
|
||||
age-reimpl/ our work (this repo)
|
||||
tools/ build/ docs/ vm-map/ godot/ bin/
|
||||
|
||||
To point the tools at a different install, change GAME_DIR / EXTRACTED here only.
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent # age-reimpl/
|
||||
WORKSPACE = REPO.parent # workspace root
|
||||
GAME_DIR = WORKSPACE / "姫狩りダンジョンマイスター" # pristine game install
|
||||
EXTRACTED = WORKSPACE / "extracted" # extracted ALF archives
|
||||
DATA1 = EXTRACTED / "DATA1" # the .BIN script corpus
|
||||
BUILD = REPO / "build" # derived corpora (regenerable)
|
||||
VM_MAP = REPO / "vm-map"
|
||||
BIN = REPO / "bin" # 3rd-party tools (BinExtractALF, ...)
|
||||
AGE_EXE = GAME_DIR / "AGE.EXE"
|
||||
KELEBEK_CPP = VM_MAP / "kelebek1-age-shared.cpp"
|
||||
|
||||
|
||||
def add_self_to_syspath():
|
||||
"""Let a standalone script `import paths` / `import sys4load` from tools/."""
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
|
||||
def scripts():
|
||||
"""Authoritative {UPPERNAME.BIN -> path} map for the script corpus.
|
||||
|
||||
Loose *.BIN patch-overrides in the game dir shadow their extracted/DATA1
|
||||
copies (runtime behaviour), so they win on name collision.
|
||||
"""
|
||||
files = {}
|
||||
if DATA1.is_dir():
|
||||
for p in sorted(DATA1.glob("*.BIN")):
|
||||
files[p.name.upper()] = p
|
||||
for p in sorted(GAME_DIR.glob("*.BIN")): # overrides win
|
||||
files[p.name.upper()] = p
|
||||
return files
|
||||
69
tools/probe_header.py
Normal file
69
tools/probe_header.py
Normal file
@@ -0,0 +1,69 @@
|
||||
"""Probe SYS4 script headers across the DATA1 corpus.
|
||||
|
||||
Header hypothesis (offsets in bytes):
|
||||
0x00 char magic[8] "SYS4422 "
|
||||
0x08 u32 F0..F12 13 fields
|
||||
0x3C body (dword stream)
|
||||
|
||||
Tests invariants and prints field stats.
|
||||
"""
|
||||
import struct
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from collections import Counter
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2] / "extracted" / "DATA1"
|
||||
|
||||
def parse(path):
|
||||
data = path.read_bytes()
|
||||
magic = data[:8]
|
||||
fields = struct.unpack_from("<13I", data, 8)
|
||||
return data, magic, fields
|
||||
|
||||
def main():
|
||||
files = sorted(ROOT.glob("*.BIN"))
|
||||
print(f"files: {len(files)}")
|
||||
|
||||
magics = Counter()
|
||||
distinct = [Counter() for _ in range(13)]
|
||||
fail_f12 = [] # F12 != filedwords - 15
|
||||
fail_f6 = [] # F6 != 0x1C
|
||||
fail_f11 = [] # F11 != 0
|
||||
rel_a = Counter() # F8 + F9 vs F12
|
||||
rel_b = Counter() # F10 + F11 vs F12
|
||||
body_first = Counter()
|
||||
|
||||
for p in files:
|
||||
data, magic, f = parse(p)
|
||||
magics[magic] += 1
|
||||
nd = len(data) // 4
|
||||
for i, v in enumerate(f):
|
||||
distinct[i][v] += 1
|
||||
if f[12] != nd - 15:
|
||||
fail_f12.append((p.name, f[12], nd - 15))
|
||||
if f[6] != 0x1C:
|
||||
fail_f6.append((p.name, f[6]))
|
||||
if f[11] != 0:
|
||||
fail_f11.append((p.name, f[10], f[11], f[12]))
|
||||
rel_a["F8+F9==F12" if f[8] + f[9] == f[12] else
|
||||
("F8+F9<F12" if f[8] + f[9] < f[12] else "F8+F9>F12")] += 1
|
||||
rel_b["F10+F11==F12" if f[10] + f[11] == f[12] else
|
||||
("F10+F11<F12" if f[10] + f[11] < f[12] else "F10+F11>F12")] += 1
|
||||
if len(data) >= 0x40:
|
||||
body_first[struct.unpack_from("<I", data, 0x3C)[0]] += 1
|
||||
|
||||
print("magics:", dict(magics))
|
||||
print("\nfield value diversity (distinct count; top values):")
|
||||
for i, c in enumerate(distinct):
|
||||
top = ", ".join(f"{v:#x}x{n}" for v, n in c.most_common(4))
|
||||
print(f" F{i:<2} @0x{8+i*4:02X}: {len(c):4} distinct | {top}")
|
||||
print(f"\nF6==0x1C fails: {len(fail_f6)} {fail_f6[:5]}")
|
||||
print(f"F12==dwords-15 fails: {len(fail_f12)} {fail_f12[:5]}")
|
||||
print(f"F11!=0 count: {len(fail_f11)} {fail_f11[:5]}")
|
||||
print("rel F8+F9 vs F12:", dict(rel_a))
|
||||
print("rel F10+F11 vs F12:", dict(rel_b))
|
||||
print("\nfirst body dword (top 15):",
|
||||
", ".join(f"{v:#x}x{n}" for v, n in body_first.most_common(15)))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
74
tools/probe_leads.py
Normal file
74
tools/probe_leads.py
Normal file
@@ -0,0 +1,74 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Corpus-wide candidate-opcode histogram.
|
||||
|
||||
T3 entries point at [0x8F, 0, line#] records embedded in CODE. The dword right
|
||||
after each 3-dword T3 record is the lead of the next statement -> candidate
|
||||
opcode. Histogram those leads over all root-override + DATA1 scripts.
|
||||
Also: histogram the dword immediately preceding each `0x02 <off>` string ref
|
||||
(candidate "takes-a-string" opcodes), with example files.
|
||||
"""
|
||||
import os, sys, collections
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import paths
|
||||
import sys4load
|
||||
|
||||
# authoritative set: game-dir overrides shadow extracted/DATA1
|
||||
files = paths.scripts()
|
||||
print(f"{len(files)} scripts (overrides win)")
|
||||
|
||||
lead_hist = collections.Counter() # dword after each T3 record
|
||||
lead_examples = collections.defaultdict(set)
|
||||
pre_str = collections.Counter() # dword before each 0x02 <off> ref
|
||||
pre_str_ex = collections.defaultdict(set)
|
||||
first_dword = collections.Counter()
|
||||
t3_rec_shape_ok = 0
|
||||
t3_rec_total = 0
|
||||
bad = 0
|
||||
|
||||
for name, p in sorted(files.items()):
|
||||
try:
|
||||
scr = sys4load.load(p)
|
||||
except Exception as e:
|
||||
bad += 1
|
||||
continue
|
||||
dw = scr.dwords
|
||||
code_len = scr.code_len
|
||||
first_dword[dw[0]] += 1
|
||||
# statement leads via T3 records
|
||||
for e in scr.table_entries("T3"):
|
||||
if not (0 <= e < code_len):
|
||||
continue
|
||||
t3_rec_total += 1
|
||||
if e + 2 < code_len and dw[e] == 0x8F and dw[e+1] == 0:
|
||||
t3_rec_shape_ok += 1
|
||||
nxt = e + 3
|
||||
if nxt < code_len:
|
||||
v = dw[nxt]
|
||||
lead_hist[v] += 1
|
||||
if len(lead_examples[v]) < 3:
|
||||
lead_examples[v].add(name)
|
||||
# dword preceding string refs (ref index i+1 holds offset; tag at i)
|
||||
for ref_idx in scr.string_refs: # ref_idx points at the offset dword
|
||||
tag_idx = ref_idx - 1 # the 0x02
|
||||
if tag_idx - 1 >= 0:
|
||||
v = dw[tag_idx - 1]
|
||||
pre_str[v] += 1
|
||||
if len(pre_str_ex[v]) < 3:
|
||||
pre_str_ex[v].add(name)
|
||||
|
||||
print(f"parse failures: {bad}")
|
||||
print(f"T3 records with shape [0x8F,0,*]: {t3_rec_shape_ok}/{t3_rec_total}")
|
||||
print("\n== first body dword ==")
|
||||
for v, c in first_dword.most_common(8):
|
||||
print(f" {v:#06x} {c}")
|
||||
print(f"\n== statement leads after T3 records (top 40 of {len(lead_hist)}) ==")
|
||||
total = sum(lead_hist.values())
|
||||
for v, c in lead_hist.most_common(40):
|
||||
ex = ",".join(sorted(lead_examples[v]))
|
||||
print(f" {v:#06x} {c:>7} ({100*c/total:5.2f}%) e.g. {ex}")
|
||||
print(f"\n== dword preceding `0x02 <off>` string refs (top 25 of {len(pre_str)}) ==")
|
||||
tot2 = sum(pre_str.values())
|
||||
for v, c in pre_str.most_common(25):
|
||||
ex = ",".join(sorted(pre_str_ex[v]))
|
||||
print(f" {v:#06x} {c:>7} ({100*c/tot2:5.2f}%) e.g. {ex}")
|
||||
76
tools/probe_refs.py
Normal file
76
tools/probe_refs.py
Normal file
@@ -0,0 +1,76 @@
|
||||
"""What do T1/T2/T3 entries point at? Plus: extract XOR-0xFF strings from scripts.
|
||||
|
||||
Prints dword windows around table-entry targets, and scans for complement-encoded
|
||||
Shift-JIS strings.
|
||||
"""
|
||||
import struct
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2] / "extracted" / "DATA1"
|
||||
|
||||
def parse(path):
|
||||
data = path.read_bytes()
|
||||
f = struct.unpack_from("<13I", data, 8)
|
||||
body = data[0x3C:]
|
||||
return data, f, body
|
||||
|
||||
def dw(body, i):
|
||||
return struct.unpack_from("<I", body, i * 4)[0]
|
||||
|
||||
def try_string(body, i, max_dwords=40):
|
||||
"""Try to decode a XOR-0xFF cp932 string starting at dword i."""
|
||||
raw = bytearray()
|
||||
n = 0
|
||||
for j in range(i, min(i + max_dwords, len(body) // 4)):
|
||||
chunk = bytes(b ^ 0xFF for b in body[j*4:(j+1)*4])
|
||||
raw += chunk
|
||||
n += 1
|
||||
if 0 in chunk:
|
||||
break
|
||||
else:
|
||||
return None, 0
|
||||
s = raw.split(b"\0")[0]
|
||||
if len(s) == 0:
|
||||
return "", n
|
||||
try:
|
||||
dec = s.decode("cp932")
|
||||
except UnicodeDecodeError:
|
||||
return None, 0
|
||||
if all(0x20 <= b or b in (0x09,) for b in s):
|
||||
return dec, n
|
||||
return None, 0
|
||||
|
||||
def windows(name):
|
||||
data, f, body = parse(ROOT / name)
|
||||
nd = len(body) // 4
|
||||
print(f"\n=== {name} (body {nd} dw, code {f[8]} dw) ===")
|
||||
for label, cnt, off in (("T1", f[7], f[8]), ("T2", f[9], f[10]), ("T3", f[11], f[12])):
|
||||
entries = [dw(body, off + k) for k in range(min(cnt, 6))]
|
||||
print(f" {label}: {cnt} entries -> {[hex(e) for e in entries]}")
|
||||
for e in entries[:4]:
|
||||
ctx = [dw(body, i) for i in range(max(0, e - 2), min(nd, e + 5))]
|
||||
s, _ = try_string(body, e)
|
||||
tag = f" str@target: {s!r}" if s else ""
|
||||
print(f" target {e:#x}: [-2..+4] = {[hex(v) for v in ctx]}{tag}")
|
||||
|
||||
def scan_strings(name, limit=15, min_len=4):
|
||||
data, f, body = parse(ROOT / name)
|
||||
nd = len(body) // 4
|
||||
print(f"\n=== strings in {name} ===")
|
||||
found = 0
|
||||
i = 0
|
||||
while i < nd and found < limit:
|
||||
s, n = try_string(body, i)
|
||||
if s and len(s) >= min_len:
|
||||
print(f" [{i:#x}] {s!r}")
|
||||
found += 1
|
||||
i += n
|
||||
else:
|
||||
i += 1
|
||||
|
||||
if __name__ == "__main__":
|
||||
for n in ("MENU.BIN", "ADDEXP.BIN", "CALLBACK_LOST.BIN", "ALCHEMY.BIN"):
|
||||
windows(n)
|
||||
scan_strings("MENU.BIN")
|
||||
scan_strings("SC0030.BIN", limit=20, min_len=6)
|
||||
76
tools/probe_tables.py
Normal file
76
tools/probe_tables.py
Normal file
@@ -0,0 +1,76 @@
|
||||
"""Verify the three (count, offset) table descriptors and inspect table contents.
|
||||
|
||||
Layout hypothesis (all offsets in dwords, relative to body start at 0x3C):
|
||||
code: body[0 .. F8)
|
||||
T1: count=F7, offset=F8, entry size s1 (solve: F8 + s1*F7 == F10)
|
||||
T2: count=F9, offset=F10, entry size s2 (solve: F10 + s2*F9 == F12)
|
||||
T3: count=F11, offset=F12, entry size s3 (solve: F12 + s3*F11 == EOF)
|
||||
"""
|
||||
import struct
|
||||
from pathlib import Path
|
||||
from collections import Counter
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2] / "extracted" / "DATA1"
|
||||
|
||||
def parse(path):
|
||||
data = path.read_bytes()
|
||||
f = struct.unpack_from("<13I", data, 8)
|
||||
return data, f
|
||||
|
||||
def solve_entry_size(gap_pairs):
|
||||
"""gap_pairs: list of (count, gap_dwords). Return consistent entry size or None."""
|
||||
sizes = set()
|
||||
for count, gap in gap_pairs:
|
||||
if count:
|
||||
if gap % count:
|
||||
return f"non-integer ({count},{gap})"
|
||||
sizes.add(gap // count)
|
||||
return sizes
|
||||
|
||||
def main():
|
||||
files = sorted(ROOT.glob("*.BIN"))
|
||||
t1_pairs, t2_pairs, t3_pairs = [], [], []
|
||||
bad = []
|
||||
for p in files:
|
||||
data, f = parse(p)
|
||||
nd = len(data) // 4 - 15 # body dwords
|
||||
F7, F8, F9, F10, F11, F12 = f[7], f[8], f[9], f[10], f[11], f[12]
|
||||
if not (F8 <= F10 <= F12 <= nd):
|
||||
bad.append((p.name, F8, F10, F12, nd))
|
||||
continue
|
||||
t1_pairs.append((F7, F10 - F8))
|
||||
t2_pairs.append((F9, F12 - F10))
|
||||
t3_pairs.append((F11, nd - F12))
|
||||
print(f"ordering violations: {len(bad)} {bad[:5]}")
|
||||
print("T1 entry sizes:", solve_entry_size(t1_pairs))
|
||||
print("T2 entry sizes:", solve_entry_size(t2_pairs))
|
||||
print("T3 entry sizes:", solve_entry_size(t3_pairs))
|
||||
|
||||
# zero-count but nonzero-gap sanity
|
||||
for name, pairs in (("T1", t1_pairs), ("T2", t2_pairs), ("T3", t3_pairs)):
|
||||
odd = sum(1 for c, g in pairs if c == 0 and g != 0)
|
||||
print(f"{name}: count==0 but gap!=0 in {odd} files")
|
||||
|
||||
# F2 outliers
|
||||
print("\nF2 outliers:")
|
||||
for p in files:
|
||||
_, f = parse(p)
|
||||
if f[2] not in (1, 2):
|
||||
print(f" {p.name}: F2={f[2]:#x}")
|
||||
|
||||
# dump tables for a few files
|
||||
for name in ("MENU.BIN", "ADDEXP.BIN", "ADDILL.BIN", "ALCHEMY.BIN"):
|
||||
data, f = parse(ROOT / name)
|
||||
nd = len(data) // 4 - 15
|
||||
print(f"\n=== {name}: body={nd} F7-12: cnt/off T1={f[7]}/{f[8]} "
|
||||
f"T2={f[9]}/{f[10]} T3={f[11]}/{f[12]}")
|
||||
for label, cnt, off, end in (("T1", f[7], f[8], f[10]),
|
||||
("T2", f[9], f[10], f[12]),
|
||||
("T3", f[11], f[12], nd)):
|
||||
vals = struct.unpack_from(f"<{end-off}I", data, 0x3C + off*4)
|
||||
show = ", ".join(f"{v:#x}" for v in vals[:16])
|
||||
print(f" {label} ({cnt} entries, {end-off} dwords): {show}"
|
||||
+ (" ..." if end-off > 16 else ""))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
76
tools/probe_tags.py
Normal file
76
tools/probe_tags.py
Normal file
@@ -0,0 +1,76 @@
|
||||
"""Confirm operand tag hypothesis: string pointers are preceded by tag dword 2.
|
||||
Also profile the first body dword and the recurring 'type' dwords 0x55/0x6E/0x6F/0x71/0x72/0x8F.
|
||||
"""
|
||||
import struct
|
||||
from pathlib import Path
|
||||
from collections import Counter
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2] / "extracted" / "DATA1"
|
||||
|
||||
def parse(p):
|
||||
data = p.read_bytes()
|
||||
f = struct.unpack_from("<13I", data, 8)
|
||||
body = data[0x3C:]
|
||||
n = len(body) // 4
|
||||
dws = struct.unpack_from(f"<{n}I", body, 0)
|
||||
return f, dws, body
|
||||
|
||||
def is_string_at(body, i):
|
||||
n = len(body) // 4
|
||||
if i >= n:
|
||||
return False
|
||||
raw = bytearray()
|
||||
for j in range(i, min(i + 60, n)):
|
||||
chunk = bytes(b ^ 0xFF for b in body[j*4:(j+1)*4])
|
||||
raw += chunk
|
||||
if 0 in chunk:
|
||||
break
|
||||
else:
|
||||
return False
|
||||
s = bytes(raw).split(b"\0")[0]
|
||||
if len(s) < 2:
|
||||
return False
|
||||
i2, chars = 0, 0
|
||||
while i2 < len(s):
|
||||
b = s[i2]
|
||||
if 0x20 <= b <= 0x7E:
|
||||
i2 += 1; chars += 1
|
||||
elif 0x81 <= b <= 0x9F or 0xE0 <= b <= 0xEA:
|
||||
if i2 + 1 < len(s) and 0x40 <= s[i2+1] <= 0xFC and s[i2+1] != 0x7F:
|
||||
i2 += 2; chars += 1
|
||||
else:
|
||||
return False
|
||||
else:
|
||||
return False
|
||||
return chars >= 2
|
||||
|
||||
def main():
|
||||
files = sorted(ROOT.glob("*.BIN"))
|
||||
preceding = Counter() # dword right before a string-region start
|
||||
first_dword = Counter()
|
||||
total_strings = 0
|
||||
for p in files:
|
||||
f, dws, body = parse(p)
|
||||
n = len(dws)
|
||||
first_dword[dws[0]] += 1
|
||||
# find string regions in the string area (>= T3 end is where strings live typically)
|
||||
i = 1
|
||||
while i < n:
|
||||
if is_string_at(body, i) and not is_string_at(body, i - 1):
|
||||
total_strings += 1
|
||||
preceding[dws[i - 1]] += 1
|
||||
# skip the string
|
||||
while i < n and is_string_at(body, i):
|
||||
i += 1
|
||||
else:
|
||||
i += 1
|
||||
print(f"total string regions: {total_strings}")
|
||||
print("dword immediately preceding a string (top 10):")
|
||||
for v, c in preceding.most_common(10):
|
||||
print(f" {v:#x}: {c} ({100*c/total_strings:.1f}%)")
|
||||
print("\nfirst body dword (top 8):")
|
||||
for v, c in first_dword.most_common(8):
|
||||
print(f" {v:#x}: {c}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
101
tools/probe_types.py
Normal file
101
tools/probe_types.py
Normal file
@@ -0,0 +1,101 @@
|
||||
"""Corpus-wide: verify T1/T2/T3 target-dword types, test table completeness,
|
||||
and do a stricter string scan.
|
||||
"""
|
||||
import struct
|
||||
from pathlib import Path
|
||||
from collections import Counter
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2] / "extracted" / "DATA1"
|
||||
|
||||
def parse(path):
|
||||
data = path.read_bytes()
|
||||
f = struct.unpack_from("<13I", data, 8)
|
||||
body = data[0x3C:]
|
||||
n = len(body) // 4
|
||||
dws = struct.unpack_from(f"<{n}I", body, 0)
|
||||
return f, dws, body
|
||||
|
||||
def main():
|
||||
files = sorted(ROOT.glob("*.BIN"))
|
||||
type_hist = {1: Counter(), 2: Counter(), 3: Counter()}
|
||||
complete = {1: Counter(), 2: Counter(), 3: Counter()}
|
||||
for p in files:
|
||||
f, dws, body = parse(p)
|
||||
code_end = f[8]
|
||||
specs = {1: (f[7], f[8], 0x71), 2: (f[9], f[10], 0x03), 3: (f[11], f[12], 0x8F)}
|
||||
for t, (cnt, off, expect) in specs.items():
|
||||
targets = dws[off:off + cnt]
|
||||
for e in targets:
|
||||
type_hist[t][dws[e] if e < len(dws) else "OOB"] += 1
|
||||
# naive count of (expect, 0, X) triples in code, at any alignment
|
||||
naive = sum(1 for i in range(code_end - 2)
|
||||
if dws[i] == expect and dws[i + 1] == 0)
|
||||
if cnt == naive:
|
||||
complete[t]["exact"] += 1
|
||||
elif cnt < naive:
|
||||
complete[t]["table<naive"] += 1
|
||||
else:
|
||||
complete[t]["table>naive"] += 1
|
||||
for t in (1, 2, 3):
|
||||
print(f"T{t} target-dword histogram: "
|
||||
+ ", ".join(f"{v:#x}x{n}" for v, n in type_hist[t].most_common(6)))
|
||||
print(f"T{t} completeness vs naive scan: {dict(complete[t])}")
|
||||
|
||||
# value ranges of the operands referenced by each table (sample corpus-wide)
|
||||
for t, expect in ((1, 0x71), (2, 0x03), (3, 0x8F)):
|
||||
lo, hi = None, None
|
||||
vals = Counter()
|
||||
for p in files:
|
||||
f, dws, body = parse(p)
|
||||
cnt, off = (f[7], f[8]) if t == 1 else (f[9], f[10]) if t == 2 else (f[11], f[12])
|
||||
for e in dws[off:off + cnt]:
|
||||
if e + 2 < len(dws):
|
||||
v = dws[e + 2]
|
||||
vals[v] += 1
|
||||
lo = v if lo is None else min(lo, v)
|
||||
hi = v if hi is None else max(hi, v)
|
||||
print(f"T{t} operand values: min={lo and hex(lo)}, max={hi and hex(hi)}, "
|
||||
f"top: {', '.join(f'{v:#x}x{n}' for v, n in vals.most_common(8))}")
|
||||
|
||||
def good_string(bs):
|
||||
"""bs = decoded-candidate raw bytes (already XOR'd). Strict cp932 validity."""
|
||||
i, chars = 0, 0
|
||||
while i < len(bs):
|
||||
b = bs[i]
|
||||
if 0x20 <= b <= 0x7E:
|
||||
i += 1; chars += 1
|
||||
elif 0x81 <= b <= 0x9F or 0xE0 <= b <= 0xEA:
|
||||
if i + 1 < len(bs) and (0x40 <= bs[i+1] <= 0xFC) and bs[i+1] != 0x7F:
|
||||
i += 2; chars += 1
|
||||
else:
|
||||
return 0
|
||||
else:
|
||||
return 0
|
||||
return chars
|
||||
|
||||
def scan(name, limit=25, min_chars=3):
|
||||
f, dws, body = parse(ROOT / name)
|
||||
n = len(body) // 4
|
||||
print(f"\n=== strings in {name} (strict) ===")
|
||||
found, i = 0, 0
|
||||
while i < n and found < limit:
|
||||
raw = bytearray()
|
||||
j = i
|
||||
while j < n:
|
||||
chunk = bytes(b ^ 0xFF for b in body[j*4:(j+1)*4])
|
||||
raw += chunk
|
||||
j += 1
|
||||
if 0 in chunk:
|
||||
break
|
||||
s = bytes(raw).split(b"\0")[0]
|
||||
if len(s) >= 2 and good_string(s) >= min_chars:
|
||||
print(f" [{i:#x}] {s.decode('cp932')!r}")
|
||||
found += 1
|
||||
i = j
|
||||
else:
|
||||
i += 1
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
scan("SC0030.BIN")
|
||||
scan("MENU.BIN", limit=12)
|
||||
64
tools/probe_xref.py
Normal file
64
tools/probe_xref.py
Normal file
@@ -0,0 +1,64 @@
|
||||
"""Find how instructions reference string offsets, and inspect 0x71 operands
|
||||
in context. Uses T3 jump-target values as confirmed instruction starts.
|
||||
"""
|
||||
import struct
|
||||
from pathlib import Path
|
||||
from collections import Counter
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2] / "extracted" / "DATA1"
|
||||
|
||||
def parse(name):
|
||||
data = (ROOT / name).read_bytes()
|
||||
f = struct.unpack_from("<13I", data, 8)
|
||||
body = data[0x3C:]
|
||||
n = len(body) // 4
|
||||
dws = struct.unpack_from(f"<{n}I", body, 0)
|
||||
return f, dws, body
|
||||
|
||||
def decode_str(body, i):
|
||||
raw = bytearray()
|
||||
n = len(body) // 4
|
||||
for j in range(i, min(i + 60, n)):
|
||||
chunk = bytes(b ^ 0xFF for b in body[j*4:(j+1)*4])
|
||||
raw += chunk
|
||||
if 0 in chunk:
|
||||
break
|
||||
try:
|
||||
return bytes(raw).split(b"\0")[0].decode("cp932")
|
||||
except UnicodeDecodeError:
|
||||
return None
|
||||
|
||||
def xref(name, targets):
|
||||
f, dws, body = parse(name)
|
||||
print(f"\n=== {name}: xrefs to {[hex(t) for t in targets]} ===")
|
||||
for t in targets:
|
||||
hits = [i for i, v in enumerate(dws[:f[8]]) if v == t]
|
||||
for i in hits:
|
||||
lo = max(0, i - 6)
|
||||
ctx = " ".join(f"{v:x}" for v in dws[lo:i + 3])
|
||||
print(f" {t:#x} referenced at [{i:#x}]: ...{ctx}...")
|
||||
|
||||
def sample_71(name, k=6):
|
||||
f, dws, body = parse(name)
|
||||
print(f"\n=== {name}: T1 (0x71) operands in context ===")
|
||||
off, cnt = f[8], f[7]
|
||||
for e in list(dws[off:off + cnt])[:k]:
|
||||
lo = max(0, e - 8)
|
||||
ctx = " ".join(f"{v:x}" for v in dws[lo:e + 4])
|
||||
print(f" target [{e:#x}]: ...{ctx}...")
|
||||
|
||||
def first_instrs(name, count=20):
|
||||
"""Dump first N dwords raw, plus known instruction starts from T3 values."""
|
||||
f, dws, body = parse(name)
|
||||
starts = sorted(set(dws[f[12]:f[12] + f[11]]))
|
||||
print(f"\n=== {name}: first dwords ===")
|
||||
print(" " + " ".join(f"{v:x}" for v in dws[:count]))
|
||||
print(f" T3-confirmed instruction starts (first 12): {[hex(s) for s in starts[:12]]}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
xref("MENU.BIN", [0x303, 0x30B, 0x30E])
|
||||
xref("SC0030.BIN", [0xEF80, 0xEF89, 0xEF91])
|
||||
sample_71("SC0030.BIN")
|
||||
sample_71("MENU.BIN", k=2)
|
||||
first_instrs("MENU.BIN")
|
||||
first_instrs("SC0030.BIN")
|
||||
521
tools/sys4load.py
Normal file
521
tools/sys4load.py
Normal file
@@ -0,0 +1,521 @@
|
||||
#!/usr/bin/env python3
|
||||
"""sys4load — loader / disassembler-ish dumper for Eushully SYS4 `.BIN` scripts.
|
||||
|
||||
Parses the confirmed container format (see ../sys4-format-notes.md):
|
||||
* 60-byte header: magic "SYS4422 " + 13 little-endian u32 fields
|
||||
* body (dword stream) = CODE + three 1-dword pointer tables + inline strings
|
||||
* strings: XOR-0xFF cp932, NUL-terminated, referenced by a `0x02 <dword-off>` pair
|
||||
|
||||
The container format is byte-verified across all 481 DATA1 scripts. Opcodes are now
|
||||
DECODED using the AGE opcode table (age_opcodes.py, transcribed from Kelebek1's
|
||||
decompiler and validated 476/476 clean on Himegari): the code section is a flat stream
|
||||
of `<opcode:u32> + argc*(<argtype:u32><value:u32>)` instructions, length 1+2*argc dwords.
|
||||
Inline strings live after the code inside [0,F8), so decoding stops at the first string
|
||||
(type-2) or array (op 0x64) operand offset.
|
||||
|
||||
Usage:
|
||||
sys4load.py <file.BIN> full disassembly listing
|
||||
sys4load.py <file.BIN> --summary header + section sizes + table/string counts
|
||||
sys4load.py <file.BIN> --strings decoded string pool only
|
||||
sys4load.py <file.BIN> --json machine-readable structure (no code dump)
|
||||
sys4load.py <dir> --validate re-check format invariants across a folder
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from age_opcodes import (OPCODES, ARG_TYPES, CONTROL_FLOW, ARRAY_OPCODE,
|
||||
is_label_argument)
|
||||
except ImportError: # allow import from another cwd
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from age_opcodes import (OPCODES, ARG_TYPES, CONTROL_FLOW, ARRAY_OPCODE,
|
||||
is_label_argument)
|
||||
|
||||
# Himegari inference layer (optional): improves labels for unnamed opcodes. Keeps the
|
||||
# verbatim Kelebek table (age_opcodes) pristine; see age_opcodes_himegari.py.
|
||||
try:
|
||||
from age_opcodes_himegari import INFERRED
|
||||
except ImportError:
|
||||
INFERRED = {}
|
||||
|
||||
# Global-variable labels (optional): annotate global operands with the partial global-var
|
||||
# map (build/global-var-map.json, produced by tools/global_map.py). High/medium confidence
|
||||
# only — the low-confidence tail (~12k sparse guesses) is left out to keep listings readable.
|
||||
GLOBAL_ATYPES = {3, 4, 5, 6, 8} # global-int/float/string/ptr/string-ptr
|
||||
|
||||
|
||||
def _short_global_label(lbl: str) -> str:
|
||||
if lbl.startswith("record-table[stride "):
|
||||
return "rec[s" + lbl[len("record-table[stride "):-1] + "]"
|
||||
if lbl.startswith("string-table (written by "):
|
||||
return "str<" + lbl[len("string-table (written by "):-1] + ">"
|
||||
return lbl
|
||||
|
||||
|
||||
def _load_global_labels() -> dict:
|
||||
try:
|
||||
p = Path(__file__).resolve().parent.parent / "build" / "global-var-map.json"
|
||||
data = json.loads(p.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return {}
|
||||
out = {}
|
||||
for addr_s, e in data.get("globals", {}).items():
|
||||
lbl, conf = e.get("label"), e.get("confidence")
|
||||
if lbl and conf in ("high", "med"):
|
||||
out[int(addr_s, 16)] = _short_global_label(lbl)
|
||||
return out
|
||||
|
||||
|
||||
GLOBAL_LABELS = _load_global_labels()
|
||||
|
||||
|
||||
def display_label(op: int) -> str:
|
||||
"""Rendered mnemonic: Kelebek name if it has one, else the inferred name, else u00…."""
|
||||
lbl = OPCODES.get(op, (f"?{op:x}", 0))[0]
|
||||
is_unnamed = (lbl.startswith(("u00", "dev_ukn")) or lbl.lower() == f"{op:x}")
|
||||
if is_unnamed and op in INFERRED:
|
||||
return INFERRED[op]["name"]
|
||||
return lbl
|
||||
|
||||
MAGIC = b"SYS4422 " # canonical; also seen: SYS4424 (patch scripts). Both 0x3C headers.
|
||||
MAGIC_PREFIX = b"SYS4" # accept the whole SYS4 script family (SYS4422 / SYS4424 / ...)
|
||||
HEADER_SIZE = 0x3C # 8 magic + 13*4
|
||||
NUM_FIELDS = 13
|
||||
BODY_OFF = HEADER_SIZE
|
||||
|
||||
# Tag dword found at the target of each pointer table (100% pure across corpus).
|
||||
TABLE_TAGS = {"T1": 0x71, "T2": 0x03, "T3": 0x8F}
|
||||
STRING_REF_TAG = 0x02 # dword preceding an inline string offset
|
||||
|
||||
|
||||
class Sys4Error(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass
|
||||
class Instruction:
|
||||
offset: int # dword index of the opcode within the body
|
||||
opcode: int
|
||||
label: str | None # mnemonic from the opcode table, or None if unknown
|
||||
args: list # list of (type, value) operand pairs
|
||||
unknown: bool = False # opcode not in the table (decode desynced/stopped)
|
||||
truncated: bool = False # not enough dwords left for the declared arg count
|
||||
|
||||
@property
|
||||
def size(self): # length in dwords
|
||||
return 1 + 2 * len(self.args)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Sys4Script:
|
||||
path: Path
|
||||
fields: tuple # 13 header u32s (F0..F12)
|
||||
dwords: tuple # body as tuple[int], length = nbody
|
||||
magic: str = "SYS4422 " # actual 8-byte magic (SYS4422 / SYS4424 / ...)
|
||||
strings: dict = field(default_factory=dict) # start_dword -> (text, ndwords)
|
||||
string_refs: dict = field(default_factory=dict) # value-operand dword index -> string start
|
||||
instructions: list = field(default_factory=list) # decoded Instruction list
|
||||
code_end: int = 0 # dword where code stops and inline strings begin (<= code_len)
|
||||
|
||||
# ---- section geometry (all in dword units, relative to body start) ----
|
||||
@property
|
||||
def nbody(self):
|
||||
return len(self.dwords)
|
||||
|
||||
@property
|
||||
def code_len(self):
|
||||
return self.fields[8] # F8
|
||||
|
||||
@property
|
||||
def t1(self):
|
||||
return (self.fields[7], self.fields[8], self.fields[10]) # count, off, end
|
||||
|
||||
@property
|
||||
def t2(self):
|
||||
return (self.fields[9], self.fields[10], self.fields[12])
|
||||
|
||||
@property
|
||||
def t3(self):
|
||||
return (self.fields[11], self.fields[12], self.nbody)
|
||||
|
||||
def table_entries(self, which):
|
||||
count, off, _ = getattr(self, which.lower())
|
||||
return self.dwords[off:off + count]
|
||||
|
||||
# ---- byte offsets for reporting ----
|
||||
@staticmethod
|
||||
def dword_to_file_off(idx):
|
||||
return BODY_OFF + idx * 4
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# parsing
|
||||
# --------------------------------------------------------------------------- #
|
||||
def _decode_string(dwords, start, limit=4096):
|
||||
"""Decode a XOR-0xFF cp932 string beginning at dword `start`.
|
||||
|
||||
Returns (text, ndwords) or (None, 0) if it isn't a clean string.
|
||||
"""
|
||||
raw = bytearray()
|
||||
n = len(dwords)
|
||||
end = min(start + limit, n)
|
||||
for j in range(start, end):
|
||||
raw += struct.pack("<I", dwords[j] ^ 0xFFFFFFFF)
|
||||
if 0 in raw[-4:]:
|
||||
break
|
||||
else:
|
||||
return None, 0
|
||||
s = bytes(raw).split(b"\0")[0]
|
||||
if len(s) < 1:
|
||||
return "", 1
|
||||
# strict cp932 shape: ascii-printable or valid 2-byte lead+trail
|
||||
i, chars = 0, 0
|
||||
while i < len(s):
|
||||
b = s[i]
|
||||
if 0x20 <= b <= 0x7E:
|
||||
i += 1
|
||||
chars += 1
|
||||
elif 0x81 <= b <= 0x9F or 0xE0 <= b <= 0xEA:
|
||||
if i + 1 < len(s) and 0x40 <= s[i + 1] <= 0xFC and s[i + 1] != 0x7F:
|
||||
i += 2
|
||||
chars += 1
|
||||
else:
|
||||
return None, 0
|
||||
else:
|
||||
return None, 0
|
||||
if chars < 1:
|
||||
return None, 0
|
||||
try:
|
||||
text = s.decode("cp932")
|
||||
except UnicodeDecodeError:
|
||||
return None, 0
|
||||
ndwords = (len(bytes(raw).split(b"\0")[0]) // 4) + 1 # include the NUL dword
|
||||
return text, ndwords
|
||||
|
||||
|
||||
def load(path) -> Sys4Script:
|
||||
path = Path(path)
|
||||
data = path.read_bytes()
|
||||
if len(data) < HEADER_SIZE:
|
||||
raise Sys4Error(f"{path.name}: too small ({len(data)} bytes)")
|
||||
if data[:4] != MAGIC_PREFIX:
|
||||
raise Sys4Error(f"{path.name}: bad magic {data[:8]!r}")
|
||||
if len(data) % 4:
|
||||
raise Sys4Error(f"{path.name}: length {len(data)} not dword-aligned")
|
||||
fields = struct.unpack_from(f"<{NUM_FIELDS}I", data, 8)
|
||||
nbody = (len(data) - BODY_OFF) // 4
|
||||
dwords = struct.unpack_from(f"<{nbody}I", data, BODY_OFF)
|
||||
|
||||
scr = Sys4Script(path=path, fields=fields, dwords=dwords,
|
||||
magic=data[:8].decode("ascii", "replace"))
|
||||
_check_invariants(scr)
|
||||
decode_code(scr)
|
||||
return scr
|
||||
|
||||
|
||||
def decode_code(scr: Sys4Script):
|
||||
"""Walk the code section into instructions, resolving inline strings.
|
||||
|
||||
Model (age_opcodes.py): instruction = <opcode> + argc*(<type><value>), length
|
||||
1+2*argc dwords. Inline strings sit after the code inside [0,F8); a type-2 (string)
|
||||
or op-0x64 array operand offset marks where code ends, so we lower `code_end` to the
|
||||
smallest such offset seen and stop there. Populates scr.instructions / .strings /
|
||||
.string_refs / .code_end.
|
||||
"""
|
||||
dw = scr.dwords
|
||||
nbody = scr.nbody
|
||||
code_end = scr.code_len # F8; shrinks to first inline-string/array offset
|
||||
instrs, strings, refs = [], {}, {}
|
||||
i = 0
|
||||
while i < code_end:
|
||||
op = dw[i]
|
||||
info = OPCODES.get(op)
|
||||
if info is None:
|
||||
instrs.append(Instruction(i, op, None, [], unknown=True))
|
||||
break
|
||||
label, argc = info
|
||||
base = i + 1
|
||||
if base + 2 * argc > code_end:
|
||||
instrs.append(Instruction(i, op, label, [], truncated=True))
|
||||
break
|
||||
args = []
|
||||
for a in range(argc):
|
||||
atype = dw[base + 2 * a]
|
||||
aval = dw[base + 2 * a + 1]
|
||||
args.append((atype, aval))
|
||||
if atype == 2 and 0 <= aval < nbody: # inline string operand
|
||||
code_end = min(code_end, aval)
|
||||
if aval not in strings:
|
||||
text, nd = _decode_string(dw, aval)
|
||||
if text is not None:
|
||||
strings[aval] = (text, nd)
|
||||
refs[base + 2 * a + 1] = aval
|
||||
elif op == ARRAY_OPCODE and a == 1 and 0 <= aval < nbody: # footer array ref
|
||||
code_end = min(code_end, aval)
|
||||
instrs.append(Instruction(i, op, label, args))
|
||||
i = base + 2 * argc
|
||||
|
||||
scr.instructions = instrs
|
||||
scr.strings = strings
|
||||
scr.string_refs = refs
|
||||
scr.code_end = code_end
|
||||
return scr
|
||||
|
||||
|
||||
def _check_invariants(scr: Sys4Script):
|
||||
f = scr.fields
|
||||
nbody = scr.nbody
|
||||
F7, F8, F9, F10, F11, F12 = f[7], f[8], f[9], f[10], f[11], f[12]
|
||||
problems = []
|
||||
if f[6] != 0x1C:
|
||||
problems.append(f"F6={f[6]:#x} != 0x1C")
|
||||
if not (0 <= F8 <= F10 <= F12 <= nbody):
|
||||
problems.append(f"section ordering 0<=F8({F8})<=F10({F10})<=F12({F12})<=nbody({nbody})")
|
||||
else:
|
||||
if F10 - F8 != F7:
|
||||
problems.append(f"T1 size {(F10 - F8)} != count {F7}")
|
||||
if F12 - F10 != F9:
|
||||
problems.append(f"T2 size {(F12 - F10)} != count {F9}")
|
||||
if nbody - F12 != F11:
|
||||
problems.append(f"T3 size {(nbody - F12)} != count {F11}")
|
||||
if problems:
|
||||
raise Sys4Error(f"{scr.path.name}: " + "; ".join(problems))
|
||||
|
||||
|
||||
def check_table_tags(scr: Sys4Script):
|
||||
"""Return dict which -> (ok_count, total) for target-tag purity."""
|
||||
out = {}
|
||||
for which, tag in TABLE_TAGS.items():
|
||||
entries = scr.table_entries(which)
|
||||
ok = sum(1 for e in entries if 0 <= e < scr.nbody and scr.dwords[e] == tag)
|
||||
out[which] = (ok, len(entries))
|
||||
return out
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# rendering
|
||||
# --------------------------------------------------------------------------- #
|
||||
def _fmt_dwords(dwords, lo, hi, per_line=8):
|
||||
out = []
|
||||
for i in range(lo, hi, per_line):
|
||||
chunk = dwords[i:min(i + per_line, hi)]
|
||||
out.append(" " + " ".join(f"{v:08x}" for v in chunk))
|
||||
return out
|
||||
|
||||
|
||||
def decode_stats(scr: Sys4Script):
|
||||
"""(n_instructions, n_unknown, n_truncated, clean) for the decoded code."""
|
||||
n = len(scr.instructions)
|
||||
unk = sum(1 for ins in scr.instructions if ins.unknown)
|
||||
trunc = sum(1 for ins in scr.instructions if ins.truncated)
|
||||
clean = unk == 0 and trunc == 0 and scr.code_end == (
|
||||
scr.instructions[-1].offset + scr.instructions[-1].size if scr.instructions else 0)
|
||||
return n, unk, trunc, clean
|
||||
|
||||
|
||||
def render_summary(scr: Sys4Script) -> str:
|
||||
f = scr.fields
|
||||
tags = check_table_tags(scr)
|
||||
n, unk, trunc, clean = decode_stats(scr)
|
||||
lines = [
|
||||
f"file {scr.path.name}",
|
||||
f"body {scr.nbody} dwords ({scr.nbody * 4} bytes)",
|
||||
f"header F0={f[0]:#x} F1={f[1]} F2={f[2]:#x} F3={f[3]:#x} "
|
||||
f"F4={f[4]} F5={f[5]:#x} F6={f[6]:#x} (F0-F5 = local var counts)",
|
||||
f"code [0x00000 .. 0x{scr.code_end:05x}) {scr.code_end} dwords, "
|
||||
f"{n} instructions"
|
||||
+ (f", strings 0x{scr.code_end:05x}..0x{scr.code_len:05x}" if scr.code_end < scr.code_len else ""),
|
||||
f"decode {'CLEAN' if clean else 'INCOMPLETE'}"
|
||||
+ (f" ({unk} unknown, {trunc} truncated)" if (unk or trunc) else ""),
|
||||
f"table T1 off 0x{f[8]:05x} count {f[7]:<6} tag 0x71 "
|
||||
f"purity {tags['T1'][0]}/{tags['T1'][1]}",
|
||||
f"table T2 off 0x{f[10]:05x} count {f[9]:<6} tag 0x03 "
|
||||
f"purity {tags['T2'][0]}/{tags['T2'][1]}",
|
||||
f"table T3 off 0x{f[12]:05x} count {f[11]:<6} tag 0x8f "
|
||||
f"purity {tags['T3'][0]}/{tags['T3'][1]}",
|
||||
f"strings {len(scr.strings)} inline, {len(scr.string_refs)} references",
|
||||
]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def render_strings(scr: Sys4Script) -> str:
|
||||
out = []
|
||||
for off in sorted(scr.strings):
|
||||
text, nd = scr.strings[off]
|
||||
fo = scr.dword_to_file_off(off)
|
||||
out.append(f" [0x{off:05x} @file 0x{fo:06x}] ({nd}dw) {text!r}")
|
||||
return "\n".join(out) if out else " (no inline strings)"
|
||||
|
||||
|
||||
def _fmt_operand(op: int, arg_index: int, atype: int, aval: int, strings: dict) -> str:
|
||||
"""Render one (type, value) operand the way the disassembler labels them."""
|
||||
if atype == 2: # inline string
|
||||
text = strings.get(aval, ("?",))[0]
|
||||
return f'"{text}"'
|
||||
if is_label_argument(op, arg_index, aval): # code-offset jump/call target
|
||||
return f"label_{aval:x}"
|
||||
tlabel = ARG_TYPES.get(atype)
|
||||
if atype == 0 or tlabel is None: # immediate / unknown-tag: raw value
|
||||
if atype == 0:
|
||||
return f"{aval:#x}"
|
||||
return f"<t{atype:#x} {aval:#x}>"
|
||||
if tlabel == "float":
|
||||
return f"(float {aval:#x})"
|
||||
ann = ""
|
||||
if atype in GLOBAL_ATYPES and aval in GLOBAL_LABELS:
|
||||
ann = f" ={GLOBAL_LABELS[aval]}" # inferred global-var-map alias
|
||||
return f"({tlabel} {aval:#x}{ann})" # e.g. (global-int 17a =skill-name-table)
|
||||
|
||||
|
||||
def render_listing(scr: Sys4Script) -> str:
|
||||
"""Disassembly of the code section using the AGE opcode table."""
|
||||
out = [render_summary(scr), "", "; ---- CODE ----"]
|
||||
|
||||
# collect jump/call label targets so we can print label_XXXX: anchors
|
||||
label_targets = set()
|
||||
for ins in scr.instructions:
|
||||
for x, (atype, aval) in enumerate(ins.args):
|
||||
if is_label_argument(ins.opcode, x, aval):
|
||||
label_targets.add(aval)
|
||||
|
||||
for ins in scr.instructions:
|
||||
if ins.offset in label_targets:
|
||||
out.append(f"label_{ins.offset:x}:")
|
||||
if ins.unknown:
|
||||
out.append(f" 0x{ins.offset:05x}: ??? 0x{ins.opcode:x} ; unknown opcode — decode stopped")
|
||||
continue
|
||||
if ins.truncated:
|
||||
out.append(f" 0x{ins.offset:05x}: {ins.label} <truncated>")
|
||||
continue
|
||||
ops = " ".join(_fmt_operand(ins.opcode, x, t, v, scr.strings)
|
||||
for x, (t, v) in enumerate(ins.args))
|
||||
mnem = display_label(ins.opcode) # prefers Kelebek name, else inferred, else u00…
|
||||
# annotate unnamed/inferred ops with their raw value for grep-ability
|
||||
kelebek = ins.label
|
||||
unnamed = kelebek.startswith(("u00", "dev_ukn")) or kelebek[:1].isdigit()
|
||||
raw = f" ; op 0x{ins.opcode:x}" + (" inferred" if unnamed and ins.opcode in INFERRED else "") \
|
||||
if unnamed else ""
|
||||
out.append(f" 0x{ins.offset:05x}: {mnem}{(' ' + ops) if ops else ''}{raw}")
|
||||
|
||||
out.append("")
|
||||
out.append("; ---- TABLES ----")
|
||||
for which in ("T1", "T2", "T3"):
|
||||
count, off, end = getattr(scr, which.lower())
|
||||
entries = scr.table_entries(which)
|
||||
preview = " ".join(f"{e:x}" for e in entries[:16])
|
||||
more = " ..." if count > 16 else ""
|
||||
out.append(f"{which} (count {count}, off 0x{off:05x}): {preview}{more}")
|
||||
out.append("")
|
||||
out.append("; ---- STRINGS ----")
|
||||
out.append(render_strings(scr))
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def to_dict(scr: Sys4Script, with_code=False) -> dict:
|
||||
f = scr.fields
|
||||
n, unk, trunc, clean = decode_stats(scr)
|
||||
d = {
|
||||
"file": scr.path.name,
|
||||
"magic": scr.magic,
|
||||
"fields": {f"F{i}": f[i] for i in range(NUM_FIELDS)},
|
||||
"body_dwords": scr.nbody,
|
||||
"sections": {
|
||||
"code": {"off": 0, "len": scr.code_len, "code_end": scr.code_end},
|
||||
"T1": {"count": f[7], "off": f[8], "tag": 0x71},
|
||||
"T2": {"count": f[9], "off": f[10], "tag": 0x03},
|
||||
"T3": {"count": f[11], "off": f[12], "tag": 0x8F},
|
||||
},
|
||||
"decode": {"instructions": n, "unknown": unk, "truncated": trunc, "clean": clean},
|
||||
"table_tag_purity": {k: v for k, v in check_table_tags(scr).items()},
|
||||
"strings": {f"0x{off:x}": txt for off, (txt, _) in sorted(scr.strings.items())},
|
||||
}
|
||||
if with_code:
|
||||
d["code"] = [
|
||||
{"off": ins.offset, "op": f"0x{ins.opcode:x}", "label": ins.label,
|
||||
"args": [[f"0x{t:x}", f"0x{v:x}"] for t, v in ins.args]}
|
||||
for ins in scr.instructions
|
||||
]
|
||||
return d
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# validation pass over a directory
|
||||
# --------------------------------------------------------------------------- #
|
||||
def validate_dir(root: Path) -> int:
|
||||
files = sorted(root.glob("*.BIN"))
|
||||
if not files:
|
||||
print(f"no .BIN files in {root}")
|
||||
return 1
|
||||
ok, bad = 0, 0
|
||||
tag_fail = 0
|
||||
decode_clean = 0
|
||||
decode_dirty = []
|
||||
for p in files:
|
||||
try:
|
||||
scr = load(p)
|
||||
except Sys4Error as e:
|
||||
print(f" FAIL {e}")
|
||||
bad += 1
|
||||
continue
|
||||
purity = check_table_tags(scr)
|
||||
impure = [k for k, (o, t) in purity.items() if o != t]
|
||||
if impure:
|
||||
tag_fail += 1
|
||||
print(f" TAG {p.name}: impure tables {impure} {purity}")
|
||||
_, unk, trunc, clean = decode_stats(scr)
|
||||
if clean:
|
||||
decode_clean += 1
|
||||
else:
|
||||
decode_dirty.append((p.name, unk, trunc))
|
||||
ok += 1
|
||||
print(f"\n{len(files)} files: {ok} parsed clean, {bad} header/section failures, "
|
||||
f"{tag_fail} with impure table tags")
|
||||
print(f"opcode decode: {decode_clean}/{ok} fully clean (0 unknown/truncated)")
|
||||
for name, unk, trunc in decode_dirty[:20]:
|
||||
print(f" DECODE {name}: {unk} unknown, {trunc} truncated")
|
||||
if len(decode_dirty) > 20:
|
||||
print(f" ... and {len(decode_dirty) - 20} more")
|
||||
return 0 if bad == 0 and tag_fail == 0 else 2
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser(description="SYS4 .BIN loader / dumper")
|
||||
ap.add_argument("target", help="a .BIN file, or a directory with --validate")
|
||||
g = ap.add_mutually_exclusive_group()
|
||||
g.add_argument("--summary", action="store_true", help="header + section sizes only")
|
||||
g.add_argument("--strings", action="store_true", help="decoded string pool only")
|
||||
g.add_argument("--json", action="store_true", help="machine-readable structure")
|
||||
g.add_argument("--validate", action="store_true", help="check invariants over a folder")
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
target = Path(args.target)
|
||||
if args.validate or target.is_dir():
|
||||
return validate_dir(target)
|
||||
|
||||
try:
|
||||
scr = load(target)
|
||||
except Sys4Error as e:
|
||||
print(f"error: {e}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
if args.summary:
|
||||
print(render_summary(scr))
|
||||
elif args.strings:
|
||||
print(render_strings(scr))
|
||||
elif args.json:
|
||||
print(json.dumps(to_dict(scr), ensure_ascii=False, indent=2))
|
||||
else:
|
||||
print(render_listing(scr))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
58
tools/validate_opcode_table.py
Normal file
58
tools/validate_opcode_table.py
Normal file
@@ -0,0 +1,58 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Definitive test: replicate Kelebek's data_array_end shrinking (stop code at first
|
||||
string/array offset), then measure clean decode rate over all Himegari scripts."""
|
||||
import os, re, sys, collections
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import paths
|
||||
import sys4load
|
||||
|
||||
CPP=paths.KELEBEK_CPP.read_text(encoding="utf-8")
|
||||
TABLE={}; LABEL={}
|
||||
for m in re.finditer(r'\{\s*(0x[0-9A-Fa-f]+)\s*,\s*"([^"]*)"\s*,\s*(0x[0-9A-Fa-f]+)\s*\}', CPP):
|
||||
TABLE[int(m.group(1),16)]=int(m.group(3),16); LABEL[int(m.group(1),16)]=m.group(2)
|
||||
|
||||
files=paths.scripts()
|
||||
|
||||
clean=dirty=0; parsefail=0
|
||||
still_unknown=collections.Counter(); examples={}
|
||||
instr_total=0
|
||||
for name,p in sorted(files.items()):
|
||||
try: scr=sys4load.load(p)
|
||||
except Exception: parsefail+=1; continue
|
||||
dw=scr.dwords; cl=scr.code_len
|
||||
end=cl # data_array_end, starts at F8, shrinks to first string/array off
|
||||
i=0; ok=True; reason=""
|
||||
n_instr=0
|
||||
while i<end:
|
||||
op=dw[i]
|
||||
if op not in TABLE:
|
||||
ok=False; reason=f"unknown 0x{op:x}@{i}"
|
||||
if op<0x1000: still_unknown[op]+=1; examples.setdefault(op,name)
|
||||
break
|
||||
argc=TABLE[op]
|
||||
base=i+1
|
||||
if base+2*argc>end:
|
||||
ok=False; reason=f"overrun 0x{op:x}@{i}"; break
|
||||
for a in range(argc):
|
||||
atype=dw[base+2*a]; aval=dw[base+2*a+1]
|
||||
if atype==2 and 0<=aval<end: # inline string -> shrink code end
|
||||
end=min(end,aval)
|
||||
if op==0x64 and a==1 and 0<=aval<end: # copy-local-array footer ref
|
||||
end=min(end,aval)
|
||||
i=base+2*argc; n_instr+=1
|
||||
if ok and i==end:
|
||||
clean+=1; instr_total+=n_instr
|
||||
else:
|
||||
dirty+=1
|
||||
if len(examples)<20 and reason: pass
|
||||
|
||||
print(f"scripts: {len(files)} parsefail(container): {parsefail}")
|
||||
print(f"CLEAN decode (Kelebek table + string-pool boundary): {clean}")
|
||||
print(f"DIRTY: {dirty}")
|
||||
print(f"total instructions decoded in clean files: {instr_total}")
|
||||
print(f"\nremaining small unknown opcodes (genuine gaps):")
|
||||
for op,c in still_unknown.most_common():
|
||||
print(f" 0x{op:x} files={c} e.g. {examples.get(op)} known={op in TABLE}")
|
||||
if not still_unknown:
|
||||
print(" (none)")
|
||||
80
tools/validate_opcode_table_naive.py
Normal file
80
tools/validate_opcode_table_naive.py
Normal file
@@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate Kelebek1's AGE opcode table against Himegari SYS4 scripts.
|
||||
|
||||
Model (from Kelebek1 disassembler.cpp + age-shared.cpp):
|
||||
code stream = sequence of instructions.
|
||||
each instruction = <opcode:u32> then argument_count * <arg>, where each arg = <type:u32><value:u32>.
|
||||
=> instruction length in dwords = 1 + 2*argument_count (uniform; type-2/0x64 args seek elsewhere, don't consume inline)
|
||||
arg type 2 = inline string (value = dword offset into body). types: 0 imm,1 float,3 g-int,9 l-int, etc.
|
||||
A clean decode consumes exactly code_len dwords with no unknown opcode and no arg overrun.
|
||||
"""
|
||||
import os, re, sys, collections
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import paths
|
||||
import sys4load
|
||||
|
||||
CPP = paths.KELEBEK_CPP.read_text(encoding="utf-8")
|
||||
# parse {0x1F4, "label", 0x0},
|
||||
TABLE = {}
|
||||
LABEL = {}
|
||||
for m in re.finditer(r'\{\s*(0x[0-9A-Fa-f]+)\s*,\s*"([^"]*)"\s*,\s*(0x[0-9A-Fa-f]+)\s*\}', CPP):
|
||||
op = int(m.group(1), 16); lbl = m.group(2); argc = int(m.group(3), 16)
|
||||
TABLE[op] = argc; LABEL[op] = lbl
|
||||
print(f"parsed {len(TABLE)} opcode defs from Kelebek1 table (max op 0x{max(TABLE):x})")
|
||||
|
||||
files = paths.scripts()
|
||||
|
||||
clean = dirty = parsefail = 0
|
||||
unknown_ops = collections.Counter()
|
||||
fail_examples = []
|
||||
str_ok = str_bad = 0
|
||||
opcode_use = collections.Counter()
|
||||
|
||||
for name, p in sorted(files.items()):
|
||||
try:
|
||||
scr = sys4load.load(p)
|
||||
except Exception:
|
||||
parsefail += 1; continue
|
||||
dw = scr.dwords; cl = scr.code_len
|
||||
i = 0; ok = True; reason = ""
|
||||
while i < cl:
|
||||
op = dw[i]
|
||||
if op not in TABLE:
|
||||
unknown_ops[op] += 1; ok = False; reason = f"unknown op 0x{op:x} @{i}"; break
|
||||
opcode_use[op] += 1
|
||||
argc = TABLE[op]
|
||||
# check each arg's type; resolve strings
|
||||
base = i + 1
|
||||
if base + 2*argc > cl:
|
||||
ok = False; reason = f"arg overrun op 0x{op:x} @{i} needs {argc} args"; break
|
||||
for a in range(argc):
|
||||
atype = dw[base + 2*a]; aval = dw[base + 2*a + 1]
|
||||
if atype == 2: # inline string
|
||||
if 0 <= aval < scr.nbody:
|
||||
txt, nd = sys4load._decode_string(dw, aval)
|
||||
if txt is None: str_bad += 1
|
||||
else: str_ok += 1
|
||||
else:
|
||||
str_bad += 1
|
||||
i = base + 2*argc
|
||||
if ok and i == cl:
|
||||
clean += 1
|
||||
else:
|
||||
dirty += 1
|
||||
if len(fail_examples) < 15:
|
||||
fail_examples.append(f" {name}: {reason} (stopped @{i}/{cl})")
|
||||
|
||||
print(f"\n== decode result over {len(files)} scripts ==")
|
||||
print(f" clean (fully consumed, all opcodes known): {clean}")
|
||||
print(f" dirty: {dirty}")
|
||||
print(f" parse-fail (container): {parsefail}")
|
||||
print(f" string args resolved ok / bad: {str_ok} / {str_bad}")
|
||||
print(f"\ntop unknown opcodes (op: files affected):")
|
||||
for op, c in unknown_ops.most_common(25):
|
||||
print(f" 0x{op:x}: {c}")
|
||||
print(f"\nsample dirty files:")
|
||||
print("\n".join(fail_examples))
|
||||
print(f"\ntop 25 opcodes actually used in Himegari (op label count):")
|
||||
for op, c in opcode_use.most_common(25):
|
||||
print(f" 0x{op:<4x} {LABEL.get(op,'?'):22} {c}")
|
||||
425
tools/vm0.py
Normal file
425
tools/vm0.py
Normal file
@@ -0,0 +1,425 @@
|
||||
#!/usr/bin/env python3
|
||||
"""vm0 — headless AGE bytecode interpreter (Phase A0 prototype).
|
||||
|
||||
Executes one script's bytecode to validate the *execution model* before any C#/Godot work.
|
||||
Reuses tools/sys4load.py for all parsing/decoding. Effectful ops and call-script are STUBBED;
|
||||
show-text is captured. Named-op semantics come from Kelebek's table; the classified markers are
|
||||
treated as no-ops (this run TESTS that assumption).
|
||||
|
||||
Purpose (see docs/phase-a-slice-plan.md):
|
||||
* `--test` run the RECOVER unit test (pointer / 2D-array / loop / control-flow correctness).
|
||||
* <file> run a script; print captured show-text + an opcode-coverage/stub report.
|
||||
|
||||
Memory model:
|
||||
* one flat GLOBAL bank G (dict addr->int); globals are raw offsets into one space.
|
||||
* per-call local frame with sparse typed banks (int/float/string/ptr).
|
||||
* a `-ptr` variable holds an ADDRESS into G. lookup-array/2d with a ptr dst stores that
|
||||
address (take-reference); reading a ptr derefs (G[addr]); writing through a ptr writes G[addr].
|
||||
This is the model RECOVER forces; the unit test is its litmus.
|
||||
* jcc(cond, tA, tB): cond truthy -> goto tA else tB; 0xFFFFFFFF = fall through (from RECOVER+SCJUMP).
|
||||
* call/ret (0x8F/0x05) are intra-script subroutine calls (shared frame); call-script (0x03) is
|
||||
the inter-script one and is stubbed.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import sys
|
||||
import collections
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE))
|
||||
import paths
|
||||
import sys4load
|
||||
from age_opcodes import OPCODES
|
||||
|
||||
NOJUMP = 0xFFFFFFFF
|
||||
|
||||
# operand type tags
|
||||
T_IMM, T_STR = 0x0, 0x2
|
||||
T_GINT, T_GFLOAT, T_GSTR, T_GPTR = 0x3, 0x4, 0x5, 0x6
|
||||
T_LINT, T_LFLOAT, T_LSTR, T_LPTR = 0x9, 0xA, 0xB, 0xC
|
||||
|
||||
# opcodes treated as no-ops in v1 (classified markers; see age_opcodes_himegari.py)
|
||||
# 0x71 = label-definition pseudo-op (count == T1 table size) — structural, no runtime effect.
|
||||
MARKERS = {0x71, 0x1f4, 0x1f5, 0x1d5, 0x1bc, 0x1bf, 0x21b, 0x1d2, 0x258}
|
||||
|
||||
# loop-guard: halt a run once any single show-text line has been emitted this many times.
|
||||
# State-gated scenes run with zero initial state can spin a loop that re-emits the same block
|
||||
# forever; capping re-emission terminates them cleanly and flags them LOOPED (vs. blind step limit).
|
||||
EMIT_CAP = 2
|
||||
|
||||
|
||||
class Frame:
|
||||
def __init__(self):
|
||||
self.i = collections.defaultdict(int) # local-int
|
||||
self.f = collections.defaultdict(int) # local-float (stored raw)
|
||||
self.s = collections.defaultdict(str) # local-string
|
||||
self.p = collections.defaultdict(int) # local-ptr (holds a G address)
|
||||
|
||||
|
||||
class VM:
|
||||
def __init__(self, scr: sys4load.Sys4Script, verbose=False, emit_cap=EMIT_CAP):
|
||||
self.scr = scr
|
||||
self.verbose = verbose
|
||||
self.code = scr.instructions
|
||||
self.by_off = {ins.offset: idx for idx, ins in enumerate(self.code)}
|
||||
self.G = collections.defaultdict(int) # global-int bank (flat address space)
|
||||
self.Gs = collections.defaultdict(str) # global-string bank
|
||||
self.fr = Frame()
|
||||
self.callstack = [] # return indices for call/ret
|
||||
self.text = [] # captured show-text as (str_offset, text)
|
||||
self.emit_seen = collections.Counter() # per-offset emit count (loop-guard)
|
||||
self.emit_cap = emit_cap
|
||||
self.halt_reason = None # 'exit' | 'LOOP:...' | 'STEP-LIMIT' | 'ret-underflow'
|
||||
self.log = collections.Counter() # stub/unknown opcode hits
|
||||
self.exec_count = collections.Counter() # opcode coverage
|
||||
self.steps = 0
|
||||
|
||||
# ---- operand resolution --------------------------------------------------
|
||||
def _string(self, off):
|
||||
e = self.scr.strings.get(off)
|
||||
if e is not None:
|
||||
return e[0]
|
||||
txt, _ = sys4load._decode_string(self.scr.dwords, off)
|
||||
return txt if txt is not None else ""
|
||||
|
||||
def read(self, op):
|
||||
t, v = op
|
||||
if t == T_IMM: return v
|
||||
if t == T_STR: return self._string(v)
|
||||
if t == T_GINT: return self.G[v]
|
||||
if t == T_GFLOAT: return self.G[v]
|
||||
if t == T_GSTR: return self.Gs[v]
|
||||
if t == T_GPTR: return self.G[self.G[v]]
|
||||
if t == T_LINT: return self.fr.i[v]
|
||||
if t == T_LFLOAT: return self.fr.f[v]
|
||||
if t == T_LSTR: return self.fr.s[v]
|
||||
if t == T_LPTR: return self.G[self.fr.p[v]] # deref ptr
|
||||
self.log[f"read?t{t:#x}"] += 1
|
||||
return v
|
||||
|
||||
def write(self, op, val):
|
||||
t, v = op
|
||||
if t == T_GINT or t == T_GFLOAT: self.G[v] = val
|
||||
elif t == T_GSTR: self.Gs[v] = val
|
||||
elif t == T_GPTR: self.G[self.G[v]] = val
|
||||
elif t == T_LINT: self.fr.i[v] = val
|
||||
elif t == T_LFLOAT: self.fr.f[v] = val
|
||||
elif t == T_LSTR: self.fr.s[v] = val
|
||||
elif t == T_LPTR: self.G[self.fr.p[v]] = val # write through
|
||||
else: self.log[f"write?t{t:#x}"] += 1
|
||||
|
||||
def base_addr(self, op):
|
||||
"""The base ADDRESS an operand names, for array lookups."""
|
||||
t, v = op
|
||||
if t in (T_IMM, T_GINT, T_GFLOAT, T_GSTR, T_GPTR): return v # global's own offset
|
||||
if t == T_LINT: return self.fr.i[v]
|
||||
if t == T_LPTR: return self.fr.p[v]
|
||||
return v
|
||||
|
||||
def lookup_store(self, dst, addr):
|
||||
"""lookup result: ptr dst gets the reference (address); non-ptr gets the element value."""
|
||||
t, v = dst
|
||||
if t == T_LPTR: self.fr.p[v] = addr
|
||||
elif t == T_GPTR: self.G[v] = addr
|
||||
else: self.write(dst, self.G[addr])
|
||||
|
||||
# ---- execution -----------------------------------------------------------
|
||||
def run(self, entry_off=0, max_steps=2_000_000):
|
||||
pc = self.by_off.get(entry_off, 0)
|
||||
while 0 <= pc < len(self.code):
|
||||
if self.steps >= max_steps:
|
||||
self.log["STEP-LIMIT"] += 1
|
||||
self.halt_reason = self.halt_reason or "STEP-LIMIT"
|
||||
break
|
||||
self.steps += 1
|
||||
ins = self.code[pc]
|
||||
op = ins.opcode
|
||||
self.exec_count[op] += 1
|
||||
nxt = self.step(ins, pc)
|
||||
if nxt is None: # halt
|
||||
break
|
||||
pc = nxt
|
||||
else:
|
||||
self.halt_reason = self.halt_reason or "pc-out-of-range"
|
||||
return self
|
||||
|
||||
def step(self, ins, pc):
|
||||
op, a = ins.opcode, ins.args
|
||||
lbl = OPCODES.get(op, ("?", 0))[0]
|
||||
|
||||
# arithmetic / bitwise: dst = a1 <op> a2
|
||||
alu = {"add": lambda x, y: x + y, "sub": lambda x, y: x - y,
|
||||
"mul": lambda x, y: x * y, "div": lambda x, y: (x // y if y else 0),
|
||||
"mod": lambda x, y: (x % y if y else 0), "and": lambda x, y: x & y,
|
||||
"or": lambda x, y: x | y, "sar": lambda x, y: x >> (y & 31),
|
||||
"shl": lambda x, y: x << (y & 31)}
|
||||
cmp = {"eq": lambda x, y: int(x == y), "ne": lambda x, y: int(x != y),
|
||||
"lt": lambda x, y: int(x < y), "lte": lambda x, y: int(x <= y),
|
||||
"gr": lambda x, y: int(x > y), "gre": lambda x, y: int(x >= y)}
|
||||
|
||||
if lbl in alu:
|
||||
self.write(a[0], alu[lbl](self.read(a[1]), self.read(a[2]))); return pc + 1
|
||||
if lbl in cmp:
|
||||
self.write(a[0], cmp[lbl](self.read(a[1]), self.read(a[2]))); return pc + 1
|
||||
if lbl == "mov":
|
||||
self.write(a[0], self.read(a[1])); return pc + 1
|
||||
if lbl == "set-string":
|
||||
self.write(a[0], self.read(a[1])); return pc + 1
|
||||
if lbl == "lookup-array": # dst = base[idx]
|
||||
addr = self.base_addr(a[1]) + self.read(a[2])
|
||||
self.lookup_store(a[0], addr); return pc + 1
|
||||
if lbl == "lookup-array-2d": # dst = base[i*stride + col]
|
||||
addr = self.base_addr(a[1]) + self.read(a[2]) * self.read(a[3]) + self.read(a[4])
|
||||
self.lookup_store(a[0], addr); return pc + 1
|
||||
if lbl == "bit-set":
|
||||
self.write(a[0], self.read(a[0]) | self.read(a[1])); return pc + 1
|
||||
if lbl == "bit-reset":
|
||||
self.write(a[0], self.read(a[0]) & ~self.read(a[1])); return pc + 1
|
||||
if lbl == "check-bit": # p1 = (p2 >> p3) & 1
|
||||
self.write(a[0], (self.read(a[1]) >> (self.read(a[2]) & 31)) & 1); return pc + 1
|
||||
if lbl == "copy-to-global": # best-effort: p1 = p2 (single cell)
|
||||
self.write(a[0], self.read(a[1])); return pc + 1
|
||||
|
||||
# control flow
|
||||
if lbl == "jmp":
|
||||
return self.by_off.get(a[0][1], pc + 1)
|
||||
if lbl == "call": # intra-script subroutine
|
||||
self.callstack.append(pc + 1)
|
||||
return self.by_off.get(a[0][1], pc + 1)
|
||||
if lbl == "ret":
|
||||
if self.callstack:
|
||||
return self.callstack.pop()
|
||||
self.halt_reason = "ret-underflow"
|
||||
return None
|
||||
if lbl == "jcc": # (cond, tA, tB); 0xFFFFFFFF=fallthrough
|
||||
tgt = a[1][1] if self.read(a[0]) else a[2][1]
|
||||
return pc + 1 if tgt == NOJUMP else self.by_off.get(tgt, pc + 1)
|
||||
if lbl in ("exit", "exit-script"):
|
||||
self.halt_reason = "exit"
|
||||
return None
|
||||
if lbl == "call-script": # STUB (inter-script)
|
||||
self.log[f"call-script({self.read(a[0]) if a else '?'})"] += 1; return pc + 1
|
||||
|
||||
# ADV text: capture the string operand as (offset, text); loop-guard on re-emission
|
||||
if lbl == "show-text":
|
||||
for t, v in a:
|
||||
if t == T_STR:
|
||||
self.emit_seen[v] += 1
|
||||
if self.emit_seen[v] > self.emit_cap:
|
||||
self.halt_reason = f"LOOP:line@{v:#x}×{self.emit_seen[v]}"
|
||||
return None
|
||||
self.text.append((v, self._string(v)))
|
||||
return pc + 1
|
||||
if lbl in ("end-text-line", "wait-for-input", "set-font", "comment",
|
||||
"display-furigana", "dev_ukn"):
|
||||
return pc + 1
|
||||
|
||||
if op in MARKERS: # classified no-op markers
|
||||
return pc + 1
|
||||
|
||||
# everything else (effectful draw/audio/ui/input, unnamed) -> stub + continue
|
||||
self.log[f"stub:{lbl if not lbl.startswith('u00') else hex(op)}"] += 1
|
||||
return pc + 1
|
||||
|
||||
|
||||
def run_test():
|
||||
"""RECOVER unit test — pointer/2D-array/loop/control-flow correctness."""
|
||||
scr = sys4load.load(paths.DATA1 / "RECOVER.BIN")
|
||||
vm = VM(scr)
|
||||
unit = 0
|
||||
vm.G[0x152616] = unit
|
||||
A, B = 0x4e11b, 0x4e085 # block-1 tables (stride 14 / 3)
|
||||
C, E, F, FL = 0x52383, 0x52f3b, 0x5295f, 0xaacb4 # block-2 tables (stride 30) + flags
|
||||
# block 1 source: A[unit*14 + 11..13]
|
||||
for k in range(3):
|
||||
vm.G[A + unit * 14 + (11 + k)] = 100 + k
|
||||
# block 2: slot 5 active, slot 6 zero-C (skip), slot 7 flag-off (skip)
|
||||
vm.G[C + unit * 30 + 5] = 7; vm.G[FL + 5] = 1; vm.G[E + unit * 30 + 5] = 42
|
||||
vm.G[C + unit * 30 + 6] = 0; vm.G[FL + 6] = 1; vm.G[E + unit * 30 + 6] = 99
|
||||
vm.G[C + unit * 30 + 7] = 3; vm.G[FL + 7] = 0; vm.G[E + unit * 30 + 7] = 88
|
||||
vm.run()
|
||||
|
||||
checks = [
|
||||
("block1 B[0]=A[11]", vm.G[B + unit * 3 + 0], 100),
|
||||
("block1 B[1]=A[12]", vm.G[B + unit * 3 + 1], 101),
|
||||
("block1 B[2]=A[13]", vm.G[B + unit * 3 + 2], 102),
|
||||
("block2 s5 C:=E", vm.G[C + unit * 30 + 5], 42),
|
||||
("block2 s5 F:=-1", vm.G[F + unit * 30 + 5], -1),
|
||||
("block2 s6 C skip", vm.G[C + unit * 30 + 6], 0), # C stayed 0 (guard)
|
||||
("block2 s7 C skip", vm.G[C + unit * 30 + 7], 3), # flag off -> untouched
|
||||
]
|
||||
ok = True
|
||||
for name, got, want in checks:
|
||||
status = "OK " if got == want else "FAIL"
|
||||
if got != want:
|
||||
ok = False
|
||||
print(f" [{status}] {name}: got {got}, want {want}")
|
||||
print(f" steps={vm.steps} call-script stubs={sum(v for k,v in vm.log.items() if k.startswith('call-script'))}")
|
||||
print("RECOVER unit test:", "PASS" if ok else "FAIL")
|
||||
return 0 if ok else 1
|
||||
|
||||
|
||||
def run_file(path):
|
||||
scr = sys4load.load(path)
|
||||
vm = VM(scr).run()
|
||||
print(f"== {Path(path).name}: {vm.steps} steps, {len(vm.text)} show-text lines captured "
|
||||
f"(halt: {vm.halt_reason}) ==")
|
||||
for i, (off, t) in enumerate(vm.text[:20]):
|
||||
print(f" [{i}] {t}")
|
||||
if len(vm.text) > 20:
|
||||
print(f" ... (+{len(vm.text)-20} more)")
|
||||
stubs = [(k, v) for k, v in vm.log.most_common() if k.startswith("stub:")]
|
||||
if stubs:
|
||||
print(" top stubbed effectful/unknown ops:", ", ".join(f"{k[5:]}×{v}" for k, v in stubs[:10]))
|
||||
ncall = sum(v for k, v in vm.log.items() if k.startswith("call-script"))
|
||||
print(f" call-script stubs: {ncall} distinct opcodes executed: {len(vm.exec_count)}")
|
||||
return 0
|
||||
|
||||
|
||||
# ---- dialogue oracle ---------------------------------------------------------
|
||||
import json
|
||||
import re
|
||||
|
||||
SCENE_RE = re.compile(r"^S[CP]\d{4}\.BIN$")
|
||||
|
||||
|
||||
def load_oracle(path=None):
|
||||
"""file (UPPER) -> ordered list of (str_offset, text): the static show-text lines
|
||||
extract_phase2 dumped. This is the independent oracle the VM is validated against."""
|
||||
path = path or (paths.BUILD / "text" / "dialogue.jsonl")
|
||||
by = collections.defaultdict(list)
|
||||
with open(path, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
d = json.loads(line)
|
||||
by[d["file"].upper()].append((int(d["off"], 16), d["text"]))
|
||||
return by
|
||||
|
||||
|
||||
def subsequence_status(emitted_offs, static_offs):
|
||||
"""Classify the emitted show-text offset stream against the static ordered set.
|
||||
|
||||
Emitted may repeat offsets (loop re-emission); we validate the distinct stream in
|
||||
first-seen order as an in-order subsequence of the static lines. Returns (status,
|
||||
detail): OK | STRAY (offset never in static) | ORDER (in static but out of order)."""
|
||||
static_list = list(static_offs)
|
||||
static_set = set(static_list)
|
||||
seen, distinct = set(), []
|
||||
for o in emitted_offs:
|
||||
if o not in seen:
|
||||
seen.add(o)
|
||||
distinct.append(o)
|
||||
strays = [o for o in distinct if o not in static_set]
|
||||
if strays:
|
||||
return "STRAY", strays
|
||||
j = 0
|
||||
for o in distinct:
|
||||
while j < len(static_list) and static_list[j] != o:
|
||||
j += 1
|
||||
if j >= len(static_list):
|
||||
return "ORDER", [o]
|
||||
j += 1
|
||||
return "OK", []
|
||||
|
||||
|
||||
def verdict(status, halt, n_emit, n_static):
|
||||
"""Fold subsequence status + halt reason into one scene verdict."""
|
||||
if n_static == 0:
|
||||
return "NO-DIALOGUE" # not an ADV scene (no static show-text) — skip in scoring
|
||||
if n_emit == 0:
|
||||
return "EMPTY" # scene has dialogue but VM emitted none — investigate
|
||||
if status != "OK":
|
||||
return status # STRAY / ORDER — a real divergence
|
||||
return "CLEAN" if halt == "exit" else f"OK/{halt}" # OK subsequence; did it exit cleanly?
|
||||
|
||||
|
||||
def run_scene(name, path, oracle):
|
||||
"""Run one scene, return a result dict comparing emitted vs static show-text."""
|
||||
scr = sys4load.load(path)
|
||||
vm = VM(scr).run()
|
||||
emitted = [off for off, _ in vm.text]
|
||||
static = oracle.get(name.upper(), [])
|
||||
static_offs = [o for o, _ in static]
|
||||
status, detail = subsequence_status(emitted, static_offs)
|
||||
n_emit = len({o for o in emitted})
|
||||
return {"name": name, "vm": vm, "emitted": emitted, "static": static,
|
||||
"n_emit_distinct": n_emit, "n_static": len(static_offs),
|
||||
"status": status, "detail": detail,
|
||||
"verdict": verdict(status, vm.halt_reason, n_emit, len(static_offs))}
|
||||
|
||||
|
||||
def run_one_scene(name):
|
||||
"""Detailed single-scene oracle diff (for investigating one script)."""
|
||||
oracle = load_oracle()
|
||||
scripts = paths.scripts()
|
||||
name = name.upper()
|
||||
if not name.endswith(".BIN"):
|
||||
name += ".BIN"
|
||||
if name not in scripts:
|
||||
print(f"no such script: {name}")
|
||||
return 1
|
||||
r = run_scene(name, scripts[name], oracle)
|
||||
vm = r["vm"]
|
||||
print(f"== {name}: verdict {r['verdict']} ==")
|
||||
print(f" emitted {len(r['emitted'])} lines ({r['n_emit_distinct']} distinct), "
|
||||
f"static {r['n_static']}, steps {vm.steps}, halt {vm.halt_reason}")
|
||||
if r["status"] == "STRAY":
|
||||
stat_set = {o for o, _ in r["static"]}
|
||||
print(f" STRAY offsets (emitted, not in static): "
|
||||
f"{', '.join(hex(o) for o in r['detail'])}")
|
||||
for off, txt in vm.text:
|
||||
if off in set(r["detail"]):
|
||||
print(f" @ {off:#x}: {txt!r}")
|
||||
elif r["status"] == "ORDER":
|
||||
print(f" first out-of-order offset: {hex(r['detail'][0])}")
|
||||
return 0
|
||||
|
||||
|
||||
def run_sweep(limit=None):
|
||||
"""Run every SC####/SP#### scene against the dialogue oracle; print a coverage table."""
|
||||
oracle = load_oracle()
|
||||
scripts = paths.scripts()
|
||||
names = sorted(n for n in scripts if SCENE_RE.match(n))
|
||||
if limit:
|
||||
names = names[:limit]
|
||||
buckets = collections.Counter()
|
||||
rows = []
|
||||
for name in names:
|
||||
r = run_scene(name, scripts[name], oracle)
|
||||
buckets[r["verdict"]] += 1
|
||||
rows.append(r)
|
||||
|
||||
# non-clean scenes get listed for follow-up
|
||||
problem = [r for r in rows if r["verdict"] not in ("CLEAN", "NO-DIALOGUE")]
|
||||
if problem:
|
||||
print("non-clean scenes:")
|
||||
for r in sorted(problem, key=lambda r: r["verdict"]):
|
||||
d = ""
|
||||
if r["status"] in ("STRAY", "ORDER"):
|
||||
d = " " + " ".join(hex(o) for o in r["detail"][:4])
|
||||
print(f" {r['name']:<14} {r['verdict']:<16} "
|
||||
f"emit {r['n_emit_distinct']:>4}/{r['n_static']:<4} "
|
||||
f"steps {r['vm'].steps:>7} halt {r['vm'].halt_reason}{d}")
|
||||
print()
|
||||
scene_total = sum(v for k, v in buckets.items() if k != "NO-DIALOGUE")
|
||||
valid = sum(v for k, v in buckets.items()
|
||||
if k == "CLEAN" or k.startswith("OK/"))
|
||||
print(f"scenes with dialogue: {scene_total} (skipped {buckets['NO-DIALOGUE']} with no static show-text)")
|
||||
print("verdict breakdown:", dict(sorted(buckets.items())))
|
||||
print(f"DIALOGUE-VALID (clean in-order subsequence, no garbage): "
|
||||
f"{valid}/{scene_total} = {valid/scene_total*100:.1f}%")
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
argv = argv if argv is not None else sys.argv[1:]
|
||||
if not argv or argv[0] == "--test":
|
||||
return run_test()
|
||||
if argv[0] == "--sweep":
|
||||
return run_sweep(limit=int(argv[1]) if len(argv) > 1 else None)
|
||||
if argv[0] == "--scene":
|
||||
return run_one_scene(argv[1])
|
||||
return run_file(argv[0])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user