From df1e289a2de4a595e1f28093e4fa0e52b74c6664 Mon Sep 17 00:00:00 2001 From: gamer147 Date: Sun, 2 Aug 2026 15:17:08 -0400 Subject: [PATCH] Add layered validation driver --- docs/PROJECT-STRUCTURE.md | 2 + docs/tools-reference.md | 15 +- tools/test_validate.py | 101 +++++++++ tools/validate.py | 428 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 545 insertions(+), 1 deletion(-) create mode 100644 tools/test_validate.py create mode 100644 tools/validate.py diff --git a/docs/PROJECT-STRUCTURE.md b/docs/PROJECT-STRUCTURE.md index fe6846e..1a37125 100644 --- a/docs/PROJECT-STRUCTURE.md +++ b/docs/PROJECT-STRUCTURE.md @@ -35,6 +35,8 @@ S:\Game Hacking\Eushully\Himegari\ ← workspace root (three siblings) │ ├── paths.py ★ central path anchor — the ONLY place that knows │ │ where the game / extracted / build dirs are. All │ │ tools import it; relocatable with no other edits. + │ ├── validate.py layered core/workspace/runtime/full validation driver + │ ├── test_validate.py pure resolver + validation-plan regressions │ ├── sys4load.py loader + disassembler (opcode-decoding) │ ├── age_opcodes.py 548-entry Kelebek AGE opcode/arg-type table (PRISTINE; never edit) │ ├── opcodes_build.py generator/linter: vm-map/opcodes.toml -> the 4 artifacts below diff --git a/docs/tools-reference.md b/docs/tools-reference.md index 283badf..b48526e 100644 --- a/docs/tools-reference.md +++ b/docs/tools-reference.md @@ -22,6 +22,19 @@ whenever a tool's inputs/outputs change.** |---|---|---| | `paths.py` | ★ Single path anchor — derives all workspace dirs from its own location; `paths.scripts()` returns the override-aware `{NAME.BIN → path}` corpus map (loose game-folder patches shadow `extracted/DATA1`). | *Imported, not run.* | +## Project validation + +| Tool | Purpose | Run | Reads → Writes | +|---|---|---|---| +| `validate.py` | Layered project validation front door. `core` regenerates/runtime-checks opcode metadata, lints canonical registries, runs pure Python tooling tests and all .NET engine tests, checks generated opcode references, and runs `git diff --check`. `workspace` adds corpus-derived global generation, the real-data Python suites, full SYS4 decode, and Python RECOVER. `runtime` adds the Godot C# build and forced-portable threaded self-test. `full` combines all phases and adds the booted faithful-wait C# scene sweep. Selected prerequisites are strict: an unavailable game/corpus/Godot requirement fails before execution instead of becoming a green skip. Each gate has a timeout and UTF-8 log under `build/validation/validate-/`; the final table reports results/durations and a before/after Godot-process leak audit. | `validate.py` (defaults to `--level full`) · `--level core|workspace|runtime|full` · `--godot ` · `--game-root ` · `--verbose` · `--fail-fast` | sources + selected toolchain/game/corpus prerequisites → console summary + ⚙ `build/validation/validate-*/.log` | +| `test_validate.py` | Pure tests for launcher-equivalent explicit/environment/PATH/conventional resolution precedence, invalid-explicit hard failure, level composition, and final gate ordering. | `test_validate.py` | temporary files only | + +Levels are cumulative around `core`: `workspace` means core+workspace-corpus, `runtime` means core+Godot, +and `full` means every phase. `workspace`/`full` intentionally require the disposable generated inputs named +by a failed preflight; rebuild each through its owning tool in this reference. Runtime Godot resolution uses +`--godot`, then `AGE_GODOT_CONSOLE`, then `godot4`/`godot`/`godot-mono` on `PATH`. Game-root resolution uses +`--game-root`, then `AGE_GAME_ROOT`, then the conventional sibling install and always requires `SYS4INI.BIN`. + ## Container parse / disassemble | Tool | Purpose | Run | Reads → Writes | @@ -541,7 +554,7 @@ already exist. Prefer additions that produce reusable, offset-keyed evidence ove | P0 | **Scriptable 32-bit debugger** (x32dbg or WinDbg) | Breakpoints and memory snapshots for hot render/audio workers that are unsafe to hook densely with Frida. Keep Frida probes on known low-frequency handlers; never restore hot interpreter/glyph/render hooks merely for convenience. | | P1 | **Timestamped video capture** (ffmpeg desktop capture or command-controlled OBS) | Frame-by-frame native/port evidence for movie `0x236`, fades, and short animation boundaries without PNG-per-frame overhead. | | P1 | **WASAPI loopback/audio capture** | Objective SFX/BGM/voice start time, channel reuse, volume, stop, and waveform comparison. This is the main evidence upgrade for the pending SFX slice. | -| P1 | **One-command validation driver** | Run engine tests with shared compilation disabled, sweep, Godot build/selftest, Python suites, generated-reference lints, decode/RECOVER checks, `git diff --check`, and report leaked child processes in one summarized result. | +| DONE 2026-08-02 | **One-command validation driver** | Landed as `validate.py` with strict layered levels, per-gate logs/timeouts, generated-reference checks, Python/.NET/corpus/Godot gates, the faithful-wait sweep, whitespace checking, and a Godot-process leak audit. Canonical usage is under "Project validation" above. | | P1 | **Golden SC0000 checkpoint corpus** | Preserve the first 10-15 native pages as offset-keyed screenshots, click/wait events, retained-state summaries, and trace excerpts. Port regressions should be comparable without replaying the entire investigation. | | P2 | **Opcode dossier generator** | Combine corpus callsites/operands, native handler/worker addresses, runtime samples, Ghidra names, opcode provenance, and port coverage into a per-op investigation packet. | | P2 | **More typed Ghidra state** | Materialize retained-object, surface-slot, text-layout, and audio-channel structures so related handlers decompile against shared named fields. | diff --git a/tools/test_validate.py b/tools/test_validate.py new file mode 100644 index 0000000..c5529b2 --- /dev/null +++ b/tools/test_validate.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Pure tests for the layered validation driver.""" +from __future__ import annotations + +import os +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import validate + + +class EnvironmentResolutionTests(unittest.TestCase): + def test_explicit_godot_wins_over_environment_and_path(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + explicit = root / "explicit-godot" + explicit.write_bytes(b"") + with mock.patch.object(validate.shutil, "which", return_value=str(root / "path-godot")): + resolved = validate.resolve_godot( + str(explicit), {"AGE_GODOT_CONSOLE": str(root / "env-godot")} + ) + self.assertEqual(explicit.resolve(), resolved) + + def test_invalid_explicit_godot_does_not_fall_through(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + with mock.patch.object(validate.shutil, "which", return_value=str(root / "path-godot")): + with self.assertRaisesRegex(ValueError, "not found"): + validate.resolve_godot(str(root / "missing"), {}) + + def test_path_godot_is_used_after_empty_configuration(self) -> None: + with tempfile.TemporaryDirectory() as directory: + found = Path(directory) / "godot4" + found.write_bytes(b"") + with mock.patch.object( + validate.shutil, + "which", + side_effect=lambda name: str(found) if name == "godot4" else None, + ): + self.assertEqual(found.resolve(), validate.resolve_godot(None, {})) + + def test_game_root_precedence_and_sys4ini_validation(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + explicit = root / "explicit" + environment = root / "environment" + conventional = root / "conventional" + for candidate in (explicit, environment, conventional): + candidate.mkdir() + (candidate / "SYS4INI.BIN").write_bytes(b"SYS4") + resolved = validate.resolve_game_root( + str(explicit), {"AGE_GAME_ROOT": str(environment)}, conventional + ) + self.assertEqual(explicit.resolve(), resolved) + + def test_invalid_explicit_game_root_does_not_fall_through(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + conventional = root / "conventional" + conventional.mkdir() + (conventional / "SYS4INI.BIN").write_bytes(b"SYS4") + with self.assertRaisesRegex(ValueError, "not found"): + validate.resolve_game_root(str(root / "missing"), {}, conventional) + + +class GatePlanTests(unittest.TestCase): + def test_core_contains_no_workspace_or_runtime_gates(self) -> None: + keys = {gate.key for gate in validate.build_gate_plan("core")} + self.assertIn("engine-tests", keys) + self.assertIn("python-test_validate", keys) + self.assertNotIn("sys4-corpus-validate", keys) + self.assertNotIn("godot-selftest", keys) + self.assertNotIn("age-cli-sweep", keys) + + def test_workspace_extends_core(self) -> None: + core = {gate.key for gate in validate.build_gate_plan("core")} + workspace = {gate.key for gate in validate.build_gate_plan("workspace")} + self.assertLess(core, workspace) + self.assertIn("sys4-corpus-validate", workspace) + self.assertNotIn("godot-selftest", workspace) + + def test_runtime_requires_resolved_paths(self) -> None: + with self.assertRaisesRegex(ValueError, "resolved Godot"): + validate.build_gate_plan("runtime") + + def test_full_contains_every_phase(self) -> None: + fake_godot = Path("godot") + fake_root = Path("game") + keys = {gate.key for gate in validate.build_gate_plan("full", fake_godot, fake_root)} + self.assertIn("sys4-corpus-validate", keys) + self.assertIn("godot-selftest", keys) + self.assertIn("age-cli-sweep", keys) + self.assertEqual("diff-check", validate.build_gate_plan("full", fake_godot, fake_root)[-1].key) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/validate.py b/tools/validate.py new file mode 100644 index 0000000..479da35 --- /dev/null +++ b/tools/validate.py @@ -0,0 +1,428 @@ +#!/usr/bin/env python3 +"""Layered project validation driver. + +Run from the repository root: + py -3.11 -X utf8 tools/validate.py [--level core|workspace|runtime|full] + +The default is the complete validation level. Every selected prerequisite is required; +the driver never turns an unavailable required gate into a successful skip. +""" +from __future__ import annotations + +import argparse +import csv +import os +import shlex +import shutil +import signal +import subprocess +import sys +import time +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path + +HERE = Path(__file__).resolve().parent +REPO = HERE.parent +sys.path.insert(0, str(HERE)) +import paths + +LEVELS = ("core", "workspace", "runtime", "full") +CORE_TESTS = ( + "test_validate.py", + "test_diff_optrace.py", + "test_engine_ctx.py", + "test_ghidra_handler_map.py", + "test_init_table_profile.py", + "test_locate_page.py", + "test_opcodes.py", + "frida/test_map_imports.py", +) +WORKSPACE_TESTS = ( + "test_extract_init.py", + "test_globals.py", + "test_scjump.py", +) +WORKSPACE_GENERATED_INPUTS = ( + paths.BUILD / "global-var-map.json", + paths.BUILD / "callscript-names.json", + paths.BUILD / "scjump-decisions.json", +) + + +@dataclass(frozen=True) +class Gate: + key: str + label: str + command: tuple[str, ...] + timeout_seconds: int = 180 + + +@dataclass(frozen=True) +class GateResult: + label: str + status: str + duration_seconds: float + detail: str + log_path: Path | None = None + + +def _python_tool(relative: str, *arguments: str) -> tuple[str, ...]: + return (sys.executable, str(HERE / relative), *arguments) + + +def resolve_godot(explicit: str | None, environ: dict[str, str] | None = None) -> Path: + """Resolve explicit value, AGE_GODOT_CONSOLE, then known PATH commands.""" + env = os.environ if environ is None else environ + configured = explicit or env.get("AGE_GODOT_CONSOLE") + if configured: + candidate = Path(configured).expanduser() + if not candidate.is_file(): + raise ValueError(f"Godot .NET console executable not found: {candidate}") + return candidate.resolve() + for command in ("godot4", "godot", "godot-mono"): + if found := shutil.which(command): + return Path(found).resolve() + raise ValueError( + "Godot .NET console executable not found; pass --godot, set " + "AGE_GODOT_CONSOLE, or add godot4/godot/godot-mono to PATH" + ) + + +def resolve_game_root( + explicit: str | None, + environ: dict[str, str] | None = None, + conventional: Path | None = None, +) -> Path: + """Resolve explicit value, AGE_GAME_ROOT, then the conventional sibling install.""" + env = os.environ if environ is None else environ + candidate = Path( + explicit or env.get("AGE_GAME_ROOT") or conventional or paths.GAME_DIR + ).expanduser() + if not candidate.is_dir(): + raise ValueError(f"game root not found: {candidate}") + resolved = candidate.resolve() + if not (resolved / "SYS4INI.BIN").is_file(): + raise ValueError(f"game root does not contain SYS4INI.BIN: {resolved}") + return resolved + + +def selected_phases(level: str) -> set[str]: + if level not in LEVELS: + raise ValueError(f"unknown validation level: {level}") + phases = {"core"} + if level in ("workspace", "full"): + phases.add("workspace") + if level in ("runtime", "full"): + phases.add("runtime") + if level == "full": + phases.add("full") + return phases + + +def build_gate_plan( + level: str, + godot: Path | None = None, + game_root: Path | None = None, +) -> list[Gate]: + phases = selected_phases(level) + gates = [ + Gate("opcodes-build", "Opcode metadata build", _python_tool("opcodes_build.py", "--build")), + Gate("globals-lint", "Global registry lint", _python_tool("globals_build.py", "--lint")), + Gate("engine-ctx-lint", "Engine context lint", _python_tool("engine_ctx_build.py", "--lint")), + Gate( + "generated-opcode-diff", + "Generated opcode references", + ( + "git", "diff", "--exit-code", "--", + "tools/age_opcodes_himegari.py", "docs/opcode-reference.md", + ), + ), + ] + gates.extend( + Gate( + f"python-{Path(test).stem}", + f"Python {test}", + _python_tool(test), + ) + for test in CORE_TESTS + ) + gates.append( + Gate( + "engine-tests", + ".NET engine tests", + ( + "dotnet", "test", "engine/AgeEngine.sln", "--nologo", "--verbosity", "minimal", + "-p:UseSharedCompilation=false", + ), + 300, + ) + ) + + if "workspace" in phases: + gates.append( + Gate("globals-build", "Global registry build", _python_tool("globals_build.py", "--build")) + ) + gates.extend( + Gate( + f"python-{Path(test).stem}", + f"Python {test}", + _python_tool(test), + 300, + ) + for test in WORKSPACE_TESTS + ) + gates.extend(( + Gate( + "sys4-corpus-validate", + "SYS4 corpus decode", + _python_tool("sys4load.py", str(paths.DATA1), "--validate"), + 300, + ), + Gate("vm0-recover", "Python VM RECOVER", _python_tool("vm0.py", "--test")), + )) + + if "runtime" in phases: + if godot is None or game_root is None: + raise ValueError("runtime validation requires resolved Godot and game-root paths") + gates.extend(( + Gate( + "godot-build", + "Godot C# build", + ("dotnet", "build", "godot/Himegari.csproj", "--nologo", "--verbosity", "minimal"), + 300, + ), + Gate( + "godot-selftest", + "Godot threaded self-test", + ( + str(godot), "--headless", "--path", str(REPO / "godot"), "--", + "--selftest", "--game-root", str(game_root), "--text-backend", "portable", + ), + 180, + ), + )) + + if "full" in phases: + gates.append( + Gate( + "age-cli-sweep", + "C# VM scene sweep", + ( + "dotnet", "run", "--project", "engine/Age.Cli", "--configuration", "Debug", + "--", "sweep", "--boot", "--halt-at-wait", + ), + 600, + ) + ) + + gates.append(Gate("diff-check", "Git whitespace check", ("git", "diff", "--check"))) + return gates + + +def validate_prerequisites(level: str) -> list[str]: + errors = [] + for executable in ("git", "dotnet"): + if not shutil.which(executable): + errors.append(f"required executable not found on PATH: {executable}") + phases = selected_phases(level) + if "workspace" in phases: + if not paths.DATA1.is_dir(): + errors.append(f"extracted DATA1 corpus not found: {paths.DATA1}") + if not (paths.GAME_DIR / "SYS4INI.BIN").is_file(): + errors.append(f"conventional game install is unavailable: {paths.GAME_DIR}") + for required in WORKSPACE_GENERATED_INPUTS: + if not required.is_file(): + errors.append( + f"workspace-derived prerequisite not found: {required} " + "(rebuild it with the owning tool in docs/tools-reference.md)" + ) + return errors + + +def _command_text(command: tuple[str, ...]) -> str: + return subprocess.list2cmdline(command) if os.name == "nt" else shlex.join(command) + + +def _terminate_process_tree(process: subprocess.Popen[str]) -> None: + if process.poll() is not None: + return + if os.name == "nt": + subprocess.run( + ("taskkill", "/PID", str(process.pid), "/T", "/F"), + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + else: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + + +def run_gate(gate: Gate, log_dir: Path, verbose: bool) -> GateResult: + started = time.monotonic() + log_path = log_dir / f"{gate.key}.log" + environment = os.environ.copy() + environment.update({ + "PYTHONUTF8": "1", + "DOTNET_CLI_TELEMETRY_OPTOUT": "1", + "DOTNET_NOLOGO": "1", + }) + popen_arguments = { + "args": gate.command, + "cwd": REPO, + "env": environment, + "stdout": subprocess.PIPE, + "stderr": subprocess.STDOUT, + "text": True, + "encoding": "utf-8", + "errors": "replace", + } + if os.name == "nt": + popen_arguments["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP + else: + popen_arguments["start_new_session"] = True + + print(f"==> {gate.label}") + process = subprocess.Popen(**popen_arguments) + timed_out = False + try: + output, _ = process.communicate(timeout=gate.timeout_seconds) + except subprocess.TimeoutExpired: + timed_out = True + _terminate_process_tree(process) + output, _ = process.communicate() + duration = time.monotonic() - started + header = f"$ {_command_text(gate.command)}\n\n" + log_path.write_text(header + output, encoding="utf-8") + + if verbose and output: + print(output, end="" if output.endswith("\n") else "\n") + if timed_out: + detail = f"timeout after {gate.timeout_seconds}s" + _print_failure_excerpt(output) + return GateResult(gate.label, "FAIL", duration, detail, log_path) + if process.returncode != 0: + detail = f"exit {process.returncode}" + _print_failure_excerpt(output) + return GateResult(gate.label, "FAIL", duration, detail, log_path) + return GateResult(gate.label, "PASS", duration, "ok", log_path) + + +def _print_failure_excerpt(output: str, line_count: int = 30) -> None: + lines = output.rstrip().splitlines() + if lines: + print("--- failure excerpt ---") + print("\n".join(lines[-line_count:])) + + +def snapshot_godot_processes() -> dict[int, str]: + """Return live Godot-like process ids for a before/after leak audit.""" + processes: dict[int, str] = {} + try: + if os.name == "nt": + completed = subprocess.run( + ("tasklist", "/FO", "CSV", "/NH"), + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + check=False, + ) + for row in csv.reader(completed.stdout.splitlines()): + if len(row) >= 2 and "godot" in row[0].lower(): + processes[int(row[1])] = row[0] + else: + completed = subprocess.run( + ("ps", "-eo", "pid=,comm="), + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + check=False, + ) + for line in completed.stdout.splitlines(): + pid_text, _, name = line.strip().partition(" ") + if pid_text.isdigit() and "godot" in name.lower(): + processes[int(pid_text)] = name.strip() + except (OSError, ValueError): + return {} + return processes + + +def print_summary(results: list[GateResult], log_dir: Path) -> None: + label_width = max(len("Gate"), *(len(result.label) for result in results)) + print("\nValidation summary") + print(f"{'Gate':<{label_width}} Result Seconds Detail") + print(f"{'-' * label_width} ------ ------- ------") + for result in results: + print( + f"{result.label:<{label_width}} {result.status:<6} " + f"{result.duration_seconds:7.1f} {result.detail}" + ) + print(f"Logs: {log_dir}") + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--level", choices=LEVELS, default="full") + parser.add_argument("--godot", help="Godot 4.7 .NET console executable") + parser.add_argument("--game-root", help="AGE install containing SYS4INI.BIN") + parser.add_argument("--verbose", action="store_true", help="stream successful gate output") + parser.add_argument("--fail-fast", action="store_true", help="stop after the first failed gate") + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + arguments = parse_args(argv) + prerequisite_errors = validate_prerequisites(arguments.level) + phases = selected_phases(arguments.level) + godot = game_root = None + if "runtime" in phases: + try: + godot = resolve_godot(arguments.godot) + game_root = resolve_game_root(arguments.game_root) + except ValueError as error: + prerequisite_errors.append(str(error)) + if prerequisite_errors: + print("Validation prerequisites failed:", file=sys.stderr) + for error in prerequisite_errors: + print(f" - {error}", file=sys.stderr) + return 2 + + stamp = datetime.now().strftime("%Y%m%d-%H%M%S") + log_dir = paths.BUILD / "validation" / f"validate-{stamp}" + log_dir.mkdir(parents=True, exist_ok=True) + initial_godot = snapshot_godot_processes() + results: list[GateResult] = [] + for gate in build_gate_plan(arguments.level, godot, game_root): + result = run_gate(gate, log_dir, arguments.verbose) + results.append(result) + if result.status == "FAIL" and arguments.fail_fast: + break + + final_godot = snapshot_godot_processes() + leaked = final_godot.keys() - initial_godot.keys() + if leaked: + time.sleep(0.5) + final_godot = snapshot_godot_processes() + leaked = final_godot.keys() - initial_godot.keys() + leak_detail = ", ".join(f"{final_godot[pid]}({pid})" for pid in sorted(leaked)) + results.append(GateResult( + "Godot child-process audit", + "FAIL" if leaked else "PASS", + 0.0, + leak_detail or "no new Godot processes", + )) + print_summary(results, log_dir) + return 1 if any(result.status == "FAIL" for result in results) else 0 + + +if __name__ == "__main__": + raise SystemExit(main())