From 195c16fae6d5678b39d382492c647d17c69a270a Mon Sep 17 00:00:00 2001 From: gamer147 Date: Mon, 3 Aug 2026 16:35:27 -0400 Subject: [PATCH] Build verified Linux release packages --- README.md | 4 + docs/PROJECT-STRUCTURE.md | 11 + docs/platform-portability.md | 12 ++ docs/remake-architecture-and-roadmap.md | 12 ++ docs/tools-reference.md | 48 ++++- godot/FfmpegMovieDecoder.cs.uid | 1 - godot/FfmpegMovieNative.cs.uid | 1 - godot/GodotAdvHost.AdvText.cs.uid | 1 + godot/GodotAdvHost.Audio.cs.uid | 1 + godot/GodotAdvHost.Movies.cs.uid | 1 + godot/GodotAdvHost.PresentationInput.cs.uid | 1 + godot/GodotAdvHost.Surfaces.cs.uid | 1 + godot/GodotTimelineLog.cs.uid | 1 - godot/GodotTraceSink.cs.uid | 1 - godot/GodotVmOptions.cs.uid | 1 - godot/IMovieDecoder.cs.uid | 1 - godot/Main.Audio.cs.uid | 1 + godot/Main.Compositor.cs.uid | 1 + godot/Main.Input.cs.uid | 1 + godot/Main.Movie.cs.uid | 1 + godot/Main.PackageSmoke.cs | 32 +++ godot/Main.PackageSmoke.cs.uid | 1 + godot/Main.SelfTest.cs.uid | 1 + godot/Main.cs | 5 + godot/MovieAudioTimeline.cs.uid | 1 - godot/MovieRuntime.cs.uid | 1 - godot/MovieSurfaceRegistry.cs.uid | 1 - godot/PageLocatorState.cs.uid | 1 - godot/PerformanceFrameLog.cs.uid | 1 - godot/RiffWaveSanitizer.cs.uid | 1 - godot/StepLimitDiagnosticFormatter.cs.uid | 1 - godot/WindowLaunchOptions.cs.uid | 1 - tools/bootstrap-godot-linux-x64.sh | 71 +++++++ tools/build-linux-x64.sh | 63 ++++++ tools/export-linux-x64.ps1 | 29 +-- tools/godot-linux-x64.json | 27 +++ tools/install_godot_templates.py | 221 ++++++++++++++++++++ tools/package_linux_x64.py | 215 +++++++++++++++++++ tools/test_install_godot_templates.py | 79 +++++++ tools/test_package_linux_x64.py | 73 +++++++ tools/validate.py | 2 + 41 files changed, 886 insertions(+), 43 deletions(-) delete mode 100644 godot/FfmpegMovieDecoder.cs.uid delete mode 100644 godot/FfmpegMovieNative.cs.uid create mode 100644 godot/GodotAdvHost.AdvText.cs.uid create mode 100644 godot/GodotAdvHost.Audio.cs.uid create mode 100644 godot/GodotAdvHost.Movies.cs.uid create mode 100644 godot/GodotAdvHost.PresentationInput.cs.uid create mode 100644 godot/GodotAdvHost.Surfaces.cs.uid delete mode 100644 godot/GodotTimelineLog.cs.uid delete mode 100644 godot/GodotTraceSink.cs.uid delete mode 100644 godot/GodotVmOptions.cs.uid delete mode 100644 godot/IMovieDecoder.cs.uid create mode 100644 godot/Main.Audio.cs.uid create mode 100644 godot/Main.Compositor.cs.uid create mode 100644 godot/Main.Input.cs.uid create mode 100644 godot/Main.Movie.cs.uid create mode 100644 godot/Main.PackageSmoke.cs create mode 100644 godot/Main.PackageSmoke.cs.uid create mode 100644 godot/Main.SelfTest.cs.uid delete mode 100644 godot/MovieAudioTimeline.cs.uid delete mode 100644 godot/MovieRuntime.cs.uid delete mode 100644 godot/MovieSurfaceRegistry.cs.uid delete mode 100644 godot/PageLocatorState.cs.uid delete mode 100644 godot/PerformanceFrameLog.cs.uid delete mode 100644 godot/RiffWaveSanitizer.cs.uid delete mode 100644 godot/StepLimitDiagnosticFormatter.cs.uid delete mode 100644 godot/WindowLaunchOptions.cs.uid create mode 100755 tools/bootstrap-godot-linux-x64.sh create mode 100755 tools/build-linux-x64.sh create mode 100644 tools/godot-linux-x64.json create mode 100755 tools/install_godot_templates.py create mode 100755 tools/package_linux_x64.py create mode 100755 tools/test_install_godot_templates.py create mode 100755 tools/test_package_linux_x64.py diff --git a/README.md b/README.md index b282698..a9799b0 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,10 @@ py -3.11 -X utf8 tools\validate.py --level core .\run-godot.ps1 -GodotConsole -GameRoot ``` +Linux release exports are asset-free build products: from a Linux host, `./tools/build-linux-x64.sh` +bootstraps the pinned Godot/FFmpeg inputs, exports the runtime, runs its packaged smoke gate, and writes the +archive under `build/package/`. See the tools reference for exact inputs and output contents. + `workspace` and `full` validation additionally require the extracted script corpus and disposable derived inputs described by the tools reference. `runtime` and `full` require Godot plus a valid game root. A selected requirement that is unavailable is an error, not a silent skip. diff --git a/docs/PROJECT-STRUCTURE.md b/docs/PROJECT-STRUCTURE.md index eb1165a..792d6e8 100644 --- a/docs/PROJECT-STRUCTURE.md +++ b/docs/PROJECT-STRUCTURE.md @@ -44,6 +44,13 @@ S:\Game Hacking\Eushully\Himegari\ ← workspace root (three siblings) │ │ tools import it; relocatable with no other edits. │ ├── validate.py layered core/workspace/runtime/full validation driver │ ├── test_validate.py pure resolver + validation-plan regressions + │ ├── build-linux-x64.sh one-command pinned Linux export/package/smoke pipeline + │ ├── bootstrap-godot-linux-x64.sh repo-local Godot 4.7 Linux editor/template bootstrap + │ ├── godot-linux-x64.json immutable Godot editor/template source + SHA-256 manifest + │ ├── install_godot_templates.py selective HTTP-range installer for pinned template members + │ ├── package_linux_x64.py export verifier + notice/checksum/archive packager + │ ├── test_install_godot_templates.py, test_package_linux_x64.py + │ │ source-only bootstrap/package regressions │ ├── sys4load.py loader + disassembler (opcode-decoding) │ ├── age_opcodes.py GENERATED complete Python ABI view (do not hand-edit) │ ├── opcodes_build.py generator/linter: vm-map/opcodes.toml -> the 5 artifacts below @@ -93,6 +100,10 @@ S:\Game Hacking\Eushully\Himegari\ ← workspace root (three siblings) │ └── engine-re.md native-engine RE (Ghidra+MCP): dispatch table, decompiled ops │ ├── build/ DERIVED (our-work-side) — generated by tools/; disposable + │ ├── downloads/, toolchains/ immutable dependency caches and repo-local Godot toolchain + │ ├── export/linux-x64/ verified loose Godot Linux export + │ ├── native/linux-x64/ validated FFmpeg shim/shared-library bundle + │ ├── package/ staged release tree, smoke log, and `.tar.gz` artifact │ ├── disasm/ .asm — human-readable disassembly, one per script │ ├── text/ extracted text: │ │ ├── .strings.txt all inline strings in a script diff --git a/docs/platform-portability.md b/docs/platform-portability.md index 8b726b6..532a8bc 100644 --- a/docs/platform-portability.md +++ b/docs/platform-portability.md @@ -40,6 +40,18 @@ profile/save behavior remain to validate. The accepted DirectShow fallback was d LOGO/OP/CHAPTER gates passed. There are also softer Windows assumptions that should be tested or replaced before claiming portable exports. +The distributable Linux build path was completed on 2026-08-03. `tools/godot-linux-x64.json` pins the official +Godot 4.7 .NET Linux editor and release template independently; the bootstrap range-fetches only the required +73,591,000-byte template member instead of transferring the 1,200,753,503-byte all-platform archive. The single +`tools/build-linux-x64.sh` entry point regenerates runtime metadata, builds the pinned FFmpeg bundle, performs a +real release export, rejects incomplete/cross-target payloads, and creates a normalized `.tar.gz` carrying the +project license, third-party notices, build provenance, and a complete SHA-256 ledger. The final staged runtime +passes an asset-independent headless gate before packaging success is reported: the embedded 548-opcode table +loads and the project-owned shim resolves its local FFmpeg libraries at ABI 3. This proves artifact assembly and +native loading without weakening the separate installed-game/font/render/audio acceptance gates. A local Ubuntu +20.04 WSL proof passed using isolated Python 3.11/.NET 8.0.408 toolchains; the eventual Gitea job can provide those +through its existing setup actions. + ## Dependency inventory | Area | Current dependency | Runtime impact | Portability status / future action | diff --git a/docs/remake-architecture-and-roadmap.md b/docs/remake-architecture-and-roadmap.md index 07fd055..856c07c 100644 --- a/docs/remake-architecture-and-roadmap.md +++ b/docs/remake-architecture-and-roadmap.md @@ -987,6 +987,18 @@ do not mix mechanical moves with semantic changes. licensed third-party material. This closes the remaining repository-policy decision without attempting to relicense factual opcode provenance or prior research. + **Linux release-build foundation (2026-08-03):** the first CI/CD slice now has one Linux-native command, + `tools/build-linux-x64.sh`, for the complete release artifact. An immutable manifest pins the official Godot + 4.7 .NET Linux editor and release template; the bootstrap mirrors Godot's selective range downloader so a + cold build transfers only the Linux template member rather than the 1.2 GB all-platform set. The command also + rebuilds the pinned Linux FFmpeg shim, performs the real Godot release export, verifies target purity and + payload completeness, stages license/notices plus source/dependency metadata and per-file hashes, and emits a + timestamp/ownership/order-normalized `.tar.gz`. A new pre-startup `--package-smoke` path validates embedded + opcode metadata and dynamic loading of the bundled FFmpeg ABI without private game files. The full path passes + under Ubuntu/WSL with pinned Python 3.11 and .NET 8.0.408. The next CI/CD slice is a Gitea build workflow that + calls this command on `develop`, manual dispatch, and release tags, caches immutable downloads/toolchains, and + uploads the archive; release publication remains a later, tag-only promotion step after hosted artifact proof. + **Not cleanup targets:** generated `build/` output, the two intentional solution files, historical `docs/superpowers/` plans/specifications, and fidelity-specific complexity that is directly covered by the native ABI. Reorganization is successful when ownership and reproduction become clearer, not when the raw diff --git a/docs/tools-reference.md b/docs/tools-reference.md index 170779c..b851e11 100644 --- a/docs/tools-reference.md +++ b/docs/tools-reference.md @@ -356,15 +356,57 @@ change the process directory before managed code runs. **Linux x64 export** uses the committed `godot/export_presets.cfg` preset and `godot/Himegari.sln`: +For the complete Linux-native release path, run from `age-reimpl/`: + +```bash +./tools/build-linux-x64.sh +``` + +The command requires Python 3.11, the .NET SDK selected by `global.json`, a C compiler/binutils, `curl`, +`tar`, and normal ELF loader tools. `tools/godot-linux-x64.json` pins the Godot 4.7 .NET Linux editor and +Linux release template by URL, size, and SHA-256. `bootstrap-godot-linux-x64.sh` caches the editor below +`build/toolchains/` and uses `install_godot_templates.py` to range-fetch only the selected Linux template +member from Godot's all-platform archive; it does not download the complete 1.2 GB template set. + +The build regenerates embedded opcode metadata, bootstraps/builds the pinned Linux FFmpeg bundle, performs +the Godot release export, and delegates the shared required/forbidden payload checks to +`package_linux_x64.py`. The packager adds `LICENSE`, `README.md`, `THIRD_PARTY_NOTICES.md`, a source/dependency +`BUILD-INFO.json`, and per-file `SHA256SUMS` to a stable top-level directory. It normalizes archive ownership, +modes, ordering, and timestamps using `SOURCE_DATE_EPOCH` or the source commit time. Outputs are the loose +export under `build/export/linux-x64/`, the staged installation and `package-smoke.log` under +`build/package/`, and `build/package/OpenMaidEngine-Himegari-linux-x64.tar.gz`. + +Before reporting success, the command launches the staged executable as +`Himegari.x86_64 --headless -- --package-smoke`. This asset-independent mode runs before game-root discovery +and proves that the packaged managed assembly can read all 548 embedded opcode records and dynamically load +the bundled native movie shim plus its local FFmpeg dependencies at ABI version 3. It does not exercise game +archives, fonts, rendering, audio output, or a desktop window; retain the installed-game self-test below as a +separate trusted/runtime gate. + +The focused component commands are: + +```bash +./tools/bootstrap-godot-linux-x64.sh +python3 -X utf8 tools/package_linux_x64.py verify build/export/linux-x64 +python3 -X utf8 tools/package_linux_x64.py package build/export/linux-x64 +``` + +The bootstrap prints the pinned editor path. The verifier prints the accepted export directory; the packager +prints the staged root and archive path. Both Python components have source-only synthetic regressions in the +core validation level. + +The older Windows-hosted cross-export remains available for local development: + ```powershell .\tools\export-linux-x64.ps1 -GodotConsole ``` Install the Godot 4.7 .NET export templates first and build the pinned native bundle under `build/native/linux-x64` with the commands in “Native FFmpeg movie shim” above. The script safely replaces -only `build/export/linux-x64`, invokes the release preset with a bounded headless shutdown, verifies the -executable/PCK/self-contained managed payload plus all six project-owned FFmpeg files, and rejects the -Windows GDI adapter and Windows FFmpeg DLLs. Output is `build/export/linux-x64/Himegari.x86_64` with its +only `build/export/linux-x64`, invokes the release preset with a bounded headless shutdown, then calls the +same Python payload verifier used by the Linux pipeline. The verifier checks the executable/PCK/self-contained +managed payload plus all six project-owned FFmpeg files and rejects the Windows GDI adapter and Windows FFmpeg +DLLs. Output is `build/export/linux-x64/Himegari.x86_64` with its PCK and `data_Himegari_linuxbsd_x86_64/` runtime directory. Do not relocate only the executable; the three items are one artifact. diff --git a/godot/FfmpegMovieDecoder.cs.uid b/godot/FfmpegMovieDecoder.cs.uid deleted file mode 100644 index a3022c1..0000000 --- a/godot/FfmpegMovieDecoder.cs.uid +++ /dev/null @@ -1 +0,0 @@ -uid://vn771ej7jl5b diff --git a/godot/FfmpegMovieNative.cs.uid b/godot/FfmpegMovieNative.cs.uid deleted file mode 100644 index 5efb0c9..0000000 --- a/godot/FfmpegMovieNative.cs.uid +++ /dev/null @@ -1 +0,0 @@ -uid://bu4g2k4db17yj diff --git a/godot/GodotAdvHost.AdvText.cs.uid b/godot/GodotAdvHost.AdvText.cs.uid new file mode 100644 index 0000000..763ba2e --- /dev/null +++ b/godot/GodotAdvHost.AdvText.cs.uid @@ -0,0 +1 @@ +uid://crkylynsjje5u diff --git a/godot/GodotAdvHost.Audio.cs.uid b/godot/GodotAdvHost.Audio.cs.uid new file mode 100644 index 0000000..c9d525b --- /dev/null +++ b/godot/GodotAdvHost.Audio.cs.uid @@ -0,0 +1 @@ +uid://cvratxje7m4v2 diff --git a/godot/GodotAdvHost.Movies.cs.uid b/godot/GodotAdvHost.Movies.cs.uid new file mode 100644 index 0000000..8836bac --- /dev/null +++ b/godot/GodotAdvHost.Movies.cs.uid @@ -0,0 +1 @@ +uid://dthl1q7be414d diff --git a/godot/GodotAdvHost.PresentationInput.cs.uid b/godot/GodotAdvHost.PresentationInput.cs.uid new file mode 100644 index 0000000..7bf3f8f --- /dev/null +++ b/godot/GodotAdvHost.PresentationInput.cs.uid @@ -0,0 +1 @@ +uid://djhejlok24hqm diff --git a/godot/GodotAdvHost.Surfaces.cs.uid b/godot/GodotAdvHost.Surfaces.cs.uid new file mode 100644 index 0000000..dad0f62 --- /dev/null +++ b/godot/GodotAdvHost.Surfaces.cs.uid @@ -0,0 +1 @@ +uid://na8mjr1tkoel diff --git a/godot/GodotTimelineLog.cs.uid b/godot/GodotTimelineLog.cs.uid deleted file mode 100644 index af19ea5..0000000 --- a/godot/GodotTimelineLog.cs.uid +++ /dev/null @@ -1 +0,0 @@ -uid://dr372hrl1xv5o diff --git a/godot/GodotTraceSink.cs.uid b/godot/GodotTraceSink.cs.uid deleted file mode 100644 index e23ac80..0000000 --- a/godot/GodotTraceSink.cs.uid +++ /dev/null @@ -1 +0,0 @@ -uid://bj4w0ogg8o2qd diff --git a/godot/GodotVmOptions.cs.uid b/godot/GodotVmOptions.cs.uid deleted file mode 100644 index 8d42e2c..0000000 --- a/godot/GodotVmOptions.cs.uid +++ /dev/null @@ -1 +0,0 @@ -uid://dasroldd6i3nf diff --git a/godot/IMovieDecoder.cs.uid b/godot/IMovieDecoder.cs.uid deleted file mode 100644 index 38a1b3d..0000000 --- a/godot/IMovieDecoder.cs.uid +++ /dev/null @@ -1 +0,0 @@ -uid://ct34ufym8j2h0 diff --git a/godot/Main.Audio.cs.uid b/godot/Main.Audio.cs.uid new file mode 100644 index 0000000..7192ef9 --- /dev/null +++ b/godot/Main.Audio.cs.uid @@ -0,0 +1 @@ +uid://4lvidgobirdo diff --git a/godot/Main.Compositor.cs.uid b/godot/Main.Compositor.cs.uid new file mode 100644 index 0000000..5a08697 --- /dev/null +++ b/godot/Main.Compositor.cs.uid @@ -0,0 +1 @@ +uid://2485ahstyemb diff --git a/godot/Main.Input.cs.uid b/godot/Main.Input.cs.uid new file mode 100644 index 0000000..0c03138 --- /dev/null +++ b/godot/Main.Input.cs.uid @@ -0,0 +1 @@ +uid://deq0qdk5spfp4 diff --git a/godot/Main.Movie.cs.uid b/godot/Main.Movie.cs.uid new file mode 100644 index 0000000..3d8142f --- /dev/null +++ b/godot/Main.Movie.cs.uid @@ -0,0 +1 @@ +uid://b2hj6y7cjrraj diff --git a/godot/Main.PackageSmoke.cs b/godot/Main.PackageSmoke.cs new file mode 100644 index 0000000..0abdb1d --- /dev/null +++ b/godot/Main.PackageSmoke.cs @@ -0,0 +1,32 @@ +using System; +using Godot; + +public partial class Main +{ + private void RunPackageSmoke() + { + try + { + if (!OS.HasFeature("linux")) + throw new PlatformNotSupportedException("the packaged-runtime smoke gate requires Linux"); + var table = HimegariRuntimeMetadata.LoadOpcodeTable(); + if (table.Count != 548) + throw new InvalidOperationException( + $"embedded opcode table has {table.Count} entries; expected 548"); + if (!table.TryGet(0x55, out string label, out int argc) + || label != "mov" || argc != 2) + throw new InvalidOperationException("embedded opcode 0x55 metadata is invalid"); + uint ffmpegAbi = FfmpegMovieNative.AbiVersion(); + if (ffmpegAbi != 3) + throw new InvalidOperationException( + $"native FFmpeg shim ABI is {ffmpegAbi}; expected 3"); + GD.Print($"PACKAGE SMOKE OK: opcodes={table.Count} ffmpeg-abi={ffmpegAbi}"); + GetTree().Quit(0); + } + catch (Exception error) + { + GD.PushError($"[package-smoke] {error}"); + GetTree().Quit(1); + } + } +} diff --git a/godot/Main.PackageSmoke.cs.uid b/godot/Main.PackageSmoke.cs.uid new file mode 100644 index 0000000..eba76d7 --- /dev/null +++ b/godot/Main.PackageSmoke.cs.uid @@ -0,0 +1 @@ +uid://b7qfxcw51q1so diff --git a/godot/Main.SelfTest.cs.uid b/godot/Main.SelfTest.cs.uid new file mode 100644 index 0000000..102c2c2 --- /dev/null +++ b/godot/Main.SelfTest.cs.uid @@ -0,0 +1 @@ +uid://bemighlft6tva diff --git a/godot/Main.cs b/godot/Main.cs index 41df6a2..625c0a5 100644 --- a/godot/Main.cs +++ b/godot/Main.cs @@ -59,6 +59,11 @@ public partial class Main : Godot.Control public override void _Ready() { var userArgs = OS.GetCmdlineUserArgs(); + if (Array.IndexOf(userArgs, "--package-smoke") >= 0) + { + RunPackageSmoke(); + return; + } GameRootSelection gameRoot; Sys4AssetCatalog catalog; try diff --git a/godot/MovieAudioTimeline.cs.uid b/godot/MovieAudioTimeline.cs.uid deleted file mode 100644 index bae40be..0000000 --- a/godot/MovieAudioTimeline.cs.uid +++ /dev/null @@ -1 +0,0 @@ -uid://bqwevje8by8yn diff --git a/godot/MovieRuntime.cs.uid b/godot/MovieRuntime.cs.uid deleted file mode 100644 index 10bbee3..0000000 --- a/godot/MovieRuntime.cs.uid +++ /dev/null @@ -1 +0,0 @@ -uid://8cfjbhv1b5k7 diff --git a/godot/MovieSurfaceRegistry.cs.uid b/godot/MovieSurfaceRegistry.cs.uid deleted file mode 100644 index fa824df..0000000 --- a/godot/MovieSurfaceRegistry.cs.uid +++ /dev/null @@ -1 +0,0 @@ -uid://d1d60fk2538k5 diff --git a/godot/PageLocatorState.cs.uid b/godot/PageLocatorState.cs.uid deleted file mode 100644 index e2629b3..0000000 --- a/godot/PageLocatorState.cs.uid +++ /dev/null @@ -1 +0,0 @@ -uid://b8wh2a0rspxpp diff --git a/godot/PerformanceFrameLog.cs.uid b/godot/PerformanceFrameLog.cs.uid deleted file mode 100644 index 17ce616..0000000 --- a/godot/PerformanceFrameLog.cs.uid +++ /dev/null @@ -1 +0,0 @@ -uid://3hc1deciabid diff --git a/godot/RiffWaveSanitizer.cs.uid b/godot/RiffWaveSanitizer.cs.uid deleted file mode 100644 index 31bbf28..0000000 --- a/godot/RiffWaveSanitizer.cs.uid +++ /dev/null @@ -1 +0,0 @@ -uid://dl4vnduqdhes0 diff --git a/godot/StepLimitDiagnosticFormatter.cs.uid b/godot/StepLimitDiagnosticFormatter.cs.uid deleted file mode 100644 index dd6ea32..0000000 --- a/godot/StepLimitDiagnosticFormatter.cs.uid +++ /dev/null @@ -1 +0,0 @@ -uid://dbps8c8juv72d diff --git a/godot/WindowLaunchOptions.cs.uid b/godot/WindowLaunchOptions.cs.uid deleted file mode 100644 index 7813f19..0000000 --- a/godot/WindowLaunchOptions.cs.uid +++ /dev/null @@ -1 +0,0 @@ -uid://dwo5hjh8m6a3v diff --git a/tools/bootstrap-godot-linux-x64.sh b/tools/bootstrap-godot-linux-x64.sh new file mode 100755 index 0000000..6028726 --- /dev/null +++ b/tools/bootstrap-godot-linux-x64.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(cd -- "$script_dir/.." && pwd)" +manifest="$script_dir/godot-linux-x64.json" +toolchain_root="$repo_root/build/toolchains/godot-4.7-stable-mono-linux-x64" +xdg_data_home="$toolchain_root/xdg-data" +download_dir="$repo_root/build/downloads" + +manifest_value() { + python3 -c 'import json, sys; from functools import reduce; data=json.load(open(sys.argv[1], encoding="utf-8")); print(reduce(lambda value, key: value[key], sys.argv[2:], data))' \ + "$manifest" "$@" +} + +for command in python3 curl sha256sum find awk; do + command -v "$command" >/dev/null 2>&1 || { + echo "required command was not found: $command" >&2 + exit 1 + } +done + +archive="$(manifest_value editor archive)" +url="$(manifest_value editor url)" +expected_hash="$(manifest_value editor sha256)" +expected_size="$(manifest_value editor size)" +executable_name="$(manifest_value editor executable)" +template_version="$(manifest_value godot_version)" +archive_path="$download_dir/$archive" +editor_root="$toolchain_root/editor" +template_root="$xdg_data_home/godot/export_templates/$template_version" + +mkdir -p -- "$download_dir" "$editor_root" "$template_root" +if [[ ! -f "$archive_path" ]]; then + curl --fail --location --retry 3 --output "$archive_path" "$url" +fi +actual_size="$(wc -c < "$archive_path")" +if [[ "$actual_size" != "$expected_size" ]]; then + echo "Godot editor archive size mismatch: expected $expected_size, got $actual_size" >&2 + exit 1 +fi +actual_hash="$(sha256sum "$archive_path" | awk '{ print $1 }')" +if [[ "$actual_hash" != "$expected_hash" ]]; then + echo "Godot editor archive SHA-256 mismatch: expected $expected_hash, got $actual_hash" >&2 + exit 1 +fi + +mapfile -t editors < <(find "$editor_root" -type f -name "$executable_name" -print) +if [[ ${#editors[@]} -eq 0 ]]; then + python3 -c 'import pathlib, sys, zipfile; zipfile.ZipFile(sys.argv[1]).extractall(pathlib.Path(sys.argv[2]))' \ + "$archive_path" "$editor_root" + mapfile -t editors < <(find "$editor_root" -type f -name "$executable_name" -print) +fi +if [[ ${#editors[@]} -ne 1 ]]; then + echo "expected exactly one $executable_name under $editor_root, found ${#editors[@]}" >&2 + exit 1 +fi +editor="${editors[0]}" +chmod +x "$editor" + +python3 -X utf8 "$script_dir/install_godot_templates.py" \ + --manifest "$manifest" --destination "$template_root" >&2 + +reported="$(XDG_DATA_HOME="$xdg_data_home" "$editor" --headless --version)" +reported="${reported%%$'\n'*}" +if [[ "$reported" != 4.7.stable.mono* ]]; then + echo "unexpected Godot editor version: $reported" >&2 + exit 1 +fi + +printf '%s\n' "$editor" diff --git a/tools/build-linux-x64.sh b/tools/build-linux-x64.sh new file mode 100755 index 0000000..b8cfe89 --- /dev/null +++ b/tools/build-linux-x64.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(cd -- "$script_dir/.." && pwd)" +project_root="$repo_root/godot" +export_directory="$repo_root/build/export/linux-x64" +expected_export_directory="$repo_root/build/export/linux-x64" +package_directory="$repo_root/build/package" +toolchain_root="$repo_root/build/toolchains/godot-4.7-stable-mono-linux-x64" +xdg_data_home="$toolchain_root/xdg-data" + +for command in python3 dotnet cc readelf ldd curl sha256sum tar; do + command -v "$command" >/dev/null 2>&1 || { + echo "required command was not found: $command" >&2 + exit 1 + } +done + +python3 -X utf8 "$script_dir/opcodes_build.py" --build +godot_console="$("$script_dir/bootstrap-godot-linux-x64.sh")" +ffmpeg_sdk="$("$repo_root/native/age_movie_ffmpeg/bootstrap-linux-x64.sh")" +"$repo_root/native/age_movie_ffmpeg/build-linux-x64.sh" "$ffmpeg_sdk" + +if [[ "$export_directory" != "$expected_export_directory" || "$export_directory" == "/" ]]; then + echo "refusing to replace unexpected export directory: $export_directory" >&2 + exit 1 +fi +rm -rf -- "$export_directory" +mkdir -p -- "$export_directory" + +XDG_DATA_HOME="$xdg_data_home" "$godot_console" \ + --headless --quit-after 120 --path "$project_root" \ + --export-release "Linux x86_64" "$export_directory/Himegari.x86_64" + +python3 -X utf8 "$script_dir/package_linux_x64.py" verify "$export_directory" +mapfile -t package_outputs < <( + python3 -X utf8 "$script_dir/package_linux_x64.py" package "$export_directory" \ + --output-directory "$package_directory" +) +if [[ ${#package_outputs[@]} -ne 2 ]]; then + echo "packager returned an unexpected result" >&2 + exit 1 +fi +package_root="${package_outputs[0]}" +archive_path="${package_outputs[1]}" +smoke_log="$package_directory/package-smoke.log" +set +e +"$package_root/Himegari.x86_64" --headless -- --package-smoke >"$smoke_log" 2>&1 +smoke_status=$? +set -e +cat "$smoke_log" +if [[ $smoke_status -ne 0 ]]; then + echo "packaged Linux runtime smoke failed with exit code $smoke_status" >&2 + exit "$smoke_status" +fi +if ! grep -Fq "PACKAGE SMOKE OK: opcodes=548 ffmpeg-abi=3" "$smoke_log"; then + echo "packaged Linux runtime did not report its success marker" >&2 + exit 1 +fi + +archive_hash="$(sha256sum "$archive_path" | awk '{ print $1 }')" +printf 'Linux x64 package: %s\nSHA-256: %s\n' "$archive_path" "$archive_hash" diff --git a/tools/export-linux-x64.ps1 b/tools/export-linux-x64.ps1 index 9e1eb11..7aad0e4 100644 --- a/tools/export-linux-x64.ps1 +++ b/tools/export-linux-x64.ps1 @@ -37,32 +37,9 @@ if ($LASTEXITCODE -ne 0) { throw "Godot Linux x64 export failed with exit code $LASTEXITCODE. Install the Godot 4.7 .NET export templates and retry." } -$managedDirectory = Join-Path $outputDirectory 'data_Himegari_linuxbsd_x86_64' -foreach ($required in @( - $executable, - (Join-Path $outputDirectory 'Himegari.pck'), - (Join-Path $managedDirectory 'Himegari.dll'), - (Join-Path $managedDirectory 'Age.Engine.dll'), - (Join-Path $managedDirectory 'libage_movie_ffmpeg.so'), - (Join-Path $managedDirectory 'libavformat.so.62'), - (Join-Path $managedDirectory 'libavcodec.so.62'), - (Join-Path $managedDirectory 'libavutil.so.60'), - (Join-Path $managedDirectory 'libswscale.so.9'), - (Join-Path $managedDirectory 'libswresample.so.6'), - (Join-Path $managedDirectory 'FFmpeg-LICENSE.txt') -)) { - if (-not (Test-Path -LiteralPath $required -PathType Leaf)) { - throw "Linux export is incomplete; expected artifact was not found: $required" - } -} -foreach ($forbidden in @( - (Join-Path $managedDirectory 'Age.Engine.Text.Windows.dll'), - (Join-Path $managedDirectory 'age_movie_ffmpeg.dll'), - (Join-Path $managedDirectory 'avformat-62.dll') -)) { - if (Test-Path -LiteralPath $forbidden) { - throw "Linux export contains a Windows-only artifact: $forbidden" - } +py -3.11 -X utf8 (Join-Path $repoRoot 'tools\package_linux_x64.py') verify $outputDirectory +if ($LASTEXITCODE -ne 0) { + throw "Linux export payload verification failed with exit code $LASTEXITCODE." } Write-Output "Linux x64 export: $outputDirectory" diff --git a/tools/godot-linux-x64.json b/tools/godot-linux-x64.json new file mode 100644 index 0000000..38ad2f0 --- /dev/null +++ b/tools/godot-linux-x64.json @@ -0,0 +1,27 @@ +{ + "schema_version": 1, + "godot_version": "4.7.stable.mono", + "release_tag": "4.7-stable", + "editor": { + "archive": "Godot_v4.7-stable_mono_linux_x86_64.zip", + "url": "https://github.com/godotengine/godot-builds/releases/download/4.7-stable/Godot_v4.7-stable_mono_linux_x86_64.zip", + "sha256": "69e855001e34b108eb8124ff1eae8445026b2a30a83b6e6314f705ae963d0fe1", + "size": 105506785, + "executable": "Godot_v4.7-stable_mono_linux.x86_64" + }, + "templates": { + "archive": "Godot_v4.7-stable_mono_export_templates.tpz", + "url": "https://downloads.godotengine.org/?version=4.7&flavor=stable&slug=mono_export_templates.tpz&platform=templates", + "sha256": "4c02a0b99ad9c5bc243c2e79468628db3df89350d9db8fc995988f69d126e069", + "size": 1200753503, + "members": [ + { + "archive_path": "templates/linux_release.x86_64", + "install_name": "linux_release.x86_64", + "sha256": "26b1b5d076b78c02f1476dab78e183218a6aa30c8e614653eac11a0dd198456a", + "size": 73591000, + "mode": "0755" + } + ] + } +} diff --git a/tools/install_godot_templates.py b/tools/install_godot_templates.py new file mode 100755 index 0000000..ea17928 --- /dev/null +++ b/tools/install_godot_templates.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 +"""Install selected, pinned Godot export templates with HTTP range requests. + +Godot 4.7 stores every platform template in one archive. This reader mirrors the +editor's selective downloader: ZIP metadata and only the requested compressed +members cross the network. Each installed member is then checked against the +project manifest's size and SHA-256. +""" +from __future__ import annotations + +import argparse +import hashlib +import io +import json +import os +import time +import urllib.error +import urllib.request +import zipfile +from collections.abc import Callable +from pathlib import Path + +DEFAULT_WINDOW = 8 * 1024 * 1024 +HTTP_HEADERS = { + "User-Agent": "OpenMaidEngine-build/1.0 (+https://git.orfl.xyz/conco/OpenMaidEngine)", + "Accept": "application/octet-stream,*/*;q=0.8", +} + + +class RangeReader(io.RawIOBase): + """Seekable read-only view backed by a byte-range callback.""" + + def __init__( + self, + length: int, + fetch: Callable[[int, int], bytes], + window_size: int = DEFAULT_WINDOW, + ) -> None: + if length <= 0: + raise ValueError("range source length must be positive") + self._length = length + self._fetch = fetch + self._window_size = window_size + self._position = 0 + self._cache_start = 0 + self._cache = b"" + + def readable(self) -> bool: + return True + + def seekable(self) -> bool: + return True + + def tell(self) -> int: + return self._position + + def seek(self, offset: int, whence: int = io.SEEK_SET) -> int: + if whence == io.SEEK_SET: + position = offset + elif whence == io.SEEK_CUR: + position = self._position + offset + elif whence == io.SEEK_END: + position = self._length + offset + else: + raise ValueError(f"unsupported seek mode: {whence}") + if position < 0: + raise ValueError("negative seek position") + self._position = position + return position + + def read(self, size: int = -1) -> bytes: + if self._position >= self._length: + return b"" + if size is None or size < 0: + size = self._length - self._position + size = min(size, self._length - self._position) + if size == 0: + return b"" + + cache_end = self._cache_start + len(self._cache) + requested_end = self._position + size + if not ( + self._cache_start <= self._position + and requested_end <= cache_end + ): + fetch_size = max(size, self._window_size) + fetch_end = min(self._length, self._position + fetch_size) + self._cache_start = self._position + self._cache = self._fetch(self._position, fetch_end - 1) + expected = fetch_end - self._position + if len(self._cache) != expected: + raise OSError( + f"short range response: expected {expected} bytes, " + f"received {len(self._cache)}" + ) + cache_end = fetch_end + + offset = self._position - self._cache_start + data = self._cache[offset:offset + size] + self._position += len(data) + return data + + +def _open_with_retry(request: urllib.request.Request, attempts: int = 3): + last_error: Exception | None = None + for attempt in range(attempts): + try: + return urllib.request.urlopen(request, timeout=120) + except (OSError, urllib.error.URLError) as error: + last_error = error + if attempt + 1 < attempts: + time.sleep(2 ** attempt) + assert last_error is not None + raise last_error + + +def open_http_range_reader(url: str, expected_length: int) -> RangeReader: + head = urllib.request.Request(url, headers=HTTP_HEADERS, method="HEAD") + with _open_with_retry(head) as response: + final_url = response.geturl() + actual_length = int(response.headers.get("Content-Length", "0")) + if actual_length != expected_length: + raise ValueError( + f"template archive size mismatch: expected {expected_length}, " + f"server reported {actual_length}" + ) + + def fetch(start: int, end: int) -> bytes: + request_headers = dict(HTTP_HEADERS) + request_headers.update({ + "Range": f"bytes={start}-{end}", + "Accept-Encoding": "identity", + }) + request = urllib.request.Request(final_url, headers=request_headers) + with _open_with_retry(request) as response: + if response.status != 206: + raise OSError( + f"template server ignored byte range {start}-{end}: " + f"HTTP {response.status}" + ) + content_range = response.headers.get("Content-Range", "") + expected_range = f"bytes {start}-{end}/{expected_length}" + if content_range != expected_range: + raise OSError( + f"unexpected Content-Range: expected {expected_range!r}, " + f"received {content_range!r}" + ) + return response.read() + + return RangeReader(expected_length, fetch) + + +def file_matches(path: Path, expected_size: int, expected_hash: str) -> bool: + if not path.is_file() or path.stat().st_size != expected_size: + return False + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() == expected_hash.lower() + + +def install_members(manifest_path: Path, destination: Path) -> list[Path]: + with manifest_path.open(encoding="utf-8") as stream: + manifest = json.load(stream) + templates = manifest["templates"] + members = templates["members"] + destination.mkdir(parents=True, exist_ok=True) + + installed = [destination / member["install_name"] for member in members] + if all( + file_matches(path, member["size"], member["sha256"]) + for path, member in zip(installed, members, strict=True) + ): + return installed + + reader = open_http_range_reader(templates["url"], templates["size"]) + with zipfile.ZipFile(reader) as archive: + for path, member in zip(installed, members, strict=True): + if file_matches(path, member["size"], member["sha256"]): + continue + archive_path = member["archive_path"] + info = archive.getinfo(archive_path) + if info.file_size != member["size"]: + raise ValueError( + f"template member size mismatch for {archive_path}: " + f"expected {member['size']}, archive reports {info.file_size}" + ) + temporary = path.with_suffix(path.suffix + ".part") + digest = hashlib.sha256() + with archive.open(info) as source, temporary.open("wb") as target: + for block in iter(lambda: source.read(1024 * 1024), b""): + target.write(block) + digest.update(block) + if temporary.stat().st_size != member["size"]: + temporary.unlink(missing_ok=True) + raise ValueError(f"short extracted template member: {archive_path}") + actual_hash = digest.hexdigest() + if actual_hash != member["sha256"].lower(): + temporary.unlink(missing_ok=True) + raise ValueError( + f"template member SHA-256 mismatch for {archive_path}: " + f"expected {member['sha256']}, got {actual_hash}" + ) + os.chmod(temporary, int(member["mode"], 8)) + os.replace(temporary, path) + return installed + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--destination", type=Path, required=True) + args = parser.parse_args() + for installed in install_members(args.manifest.resolve(), args.destination.resolve()): + print(installed) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/package_linux_x64.py b/tools/package_linux_x64.py new file mode 100755 index 0000000..c817437 --- /dev/null +++ b/tools/package_linux_x64.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +"""Verify and package the Linux x64 Godot export.""" +from __future__ import annotations + +import argparse +import gzip +import hashlib +import json +import os +import shutil +import stat +import subprocess +import tarfile +from pathlib import Path + +HERE = Path(__file__).resolve().parent +REPO = HERE.parent +MANAGED_DIRECTORY = "data_Himegari_linuxbsd_x86_64" +PACKAGE_NAME = "OpenMaidEngine-Himegari-linux-x64" +REQUIRED_FILES = ( + "Himegari.x86_64", + "Himegari.pck", + f"{MANAGED_DIRECTORY}/Himegari.dll", + f"{MANAGED_DIRECTORY}/Age.Engine.dll", + f"{MANAGED_DIRECTORY}/Age.Engine.Frontend.dll", + f"{MANAGED_DIRECTORY}/libage_movie_ffmpeg.so", + f"{MANAGED_DIRECTORY}/libavformat.so.62", + f"{MANAGED_DIRECTORY}/libavcodec.so.62", + f"{MANAGED_DIRECTORY}/libavutil.so.60", + f"{MANAGED_DIRECTORY}/libswscale.so.9", + f"{MANAGED_DIRECTORY}/libswresample.so.6", + f"{MANAGED_DIRECTORY}/FFmpeg-LICENSE.txt", +) +FORBIDDEN_FILES = ( + f"{MANAGED_DIRECTORY}/Age.Engine.Text.Windows.dll", + f"{MANAGED_DIRECTORY}/age_movie_ffmpeg.dll", + f"{MANAGED_DIRECTORY}/avformat-62.dll", +) +PROJECT_FILES = ("LICENSE", "README.md", "THIRD_PARTY_NOTICES.md") + + +def _resolved_directory(path: Path, label: str) -> Path: + resolved = path.resolve() + if not resolved.is_dir(): + raise ValueError(f"{label} directory was not found: {resolved}") + return resolved + + +def verify_export(export_directory: Path) -> Path: + export_directory = _resolved_directory(export_directory, "Linux export") + missing = [name for name in REQUIRED_FILES if not (export_directory / name).is_file()] + if missing: + raise ValueError("Linux export is incomplete; missing: " + ", ".join(missing)) + forbidden = [name for name in FORBIDDEN_FILES if (export_directory / name).exists()] + if forbidden: + raise ValueError("Linux export contains Windows-only files: " + ", ".join(forbidden)) + executable = export_directory / "Himegari.x86_64" + if os.name != "nt" and not os.access(executable, os.X_OK): + raise ValueError(f"Linux export executable bit is not set: {executable}") + return export_directory + + +def _git(*arguments: str) -> str: + result = subprocess.run( + ("git", *arguments), cwd=REPO, check=True, capture_output=True, text=True + ) + return result.stdout.strip() + + +def source_date_epoch() -> int: + configured = os.environ.get("SOURCE_DATE_EPOCH") + value = configured or _git("show", "-s", "--format=%ct", "HEAD") + try: + epoch = int(value) + except ValueError as error: + raise ValueError(f"invalid SOURCE_DATE_EPOCH: {value!r}") from error + if epoch < 0: + raise ValueError("SOURCE_DATE_EPOCH must not be negative") + return epoch + + +def build_metadata(epoch: int) -> dict[str, object]: + with (HERE / "godot-linux-x64.json").open(encoding="utf-8") as stream: + godot = json.load(stream) + with (REPO / "native/age_movie_ffmpeg/dependency-linux-x64.json").open( + encoding="utf-8" + ) as stream: + ffmpeg = json.load(stream) + dirty = bool(_git("status", "--porcelain", "--untracked-files=normal")) + return { + "schema_version": 1, + "project": "OpenMaidEngine Himegari profile", + "target": "linux-x64", + "source_commit": _git("rev-parse", "HEAD"), + "source_dirty": dirty, + "source_date_epoch": epoch, + "godot": { + "version": godot["godot_version"], + "editor_archive_sha256": godot["editor"]["sha256"], + "release_template_sha256": godot["templates"]["members"][0]["sha256"], + }, + "ffmpeg": { + "provider": ffmpeg["provider"], + "version": ffmpeg["ffmpeg_version"], + "archive_sha256": ffmpeg["sha256"], + "minimum_glibc": ffmpeg["minimum_glibc"], + }, + } + + +def _hash(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _write_checksums(package_root: Path) -> None: + checksum_path = package_root / "SHA256SUMS" + files = sorted( + path for path in package_root.rglob("*") + if path.is_file() and path != checksum_path + ) + lines = [f"{_hash(path)} {path.relative_to(package_root).as_posix()}" for path in files] + checksum_path.write_text("\n".join(lines) + "\n", encoding="utf-8", newline="\n") + + +def _tar_info(info: tarfile.TarInfo, epoch: int) -> tarfile.TarInfo: + info.uid = 0 + info.gid = 0 + info.uname = "root" + info.gname = "root" + info.mtime = epoch + if info.isdir(): + info.mode = 0o755 + elif info.isfile(): + info.mode = 0o755 if info.mode & stat.S_IXUSR else 0o644 + return info + + +def create_package( + export_directory: Path, + output_directory: Path, + metadata: dict[str, object], + epoch: int, +) -> tuple[Path, Path]: + export_directory = verify_export(export_directory) + output_directory = output_directory.resolve() + expected_parent = (REPO / "build/package").resolve() + if output_directory != expected_parent: + raise ValueError( + f"refusing to replace unexpected package directory: {output_directory}" + ) + staging_parent = output_directory / "staging" + package_root = staging_parent / PACKAGE_NAME + archive_path = output_directory / f"{PACKAGE_NAME}.tar.gz" + if staging_parent.exists(): + shutil.rmtree(staging_parent) + staging_parent.mkdir(parents=True) + shutil.copytree(export_directory, package_root, copy_function=shutil.copy2) + for name in PROJECT_FILES: + source = REPO / name + if not source.is_file(): + raise ValueError(f"package notice file was not found: {source}") + shutil.copy2(source, package_root / name) + (package_root / "BUILD-INFO.json").write_text( + json.dumps(metadata, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + newline="\n", + ) + _write_checksums(package_root) + + output_directory.mkdir(parents=True, exist_ok=True) + temporary_archive = archive_path.with_suffix(archive_path.suffix + ".part") + with temporary_archive.open("wb") as raw: + with gzip.GzipFile(fileobj=raw, mode="wb", filename="", mtime=epoch) as compressed: + with tarfile.open(fileobj=compressed, mode="w", format=tarfile.PAX_FORMAT) as archive: + paths = [package_root, *sorted(package_root.rglob("*"))] + for path in paths: + arcname = path.relative_to(staging_parent).as_posix() + archive.add( + path, + arcname=arcname, + recursive=False, + filter=lambda info, fixed_epoch=epoch: _tar_info(info, fixed_epoch), + ) + os.replace(temporary_archive, archive_path) + return package_root, archive_path + + +def main() -> int: + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="command", required=True) + verify = subparsers.add_parser("verify") + verify.add_argument("export_directory", type=Path) + package = subparsers.add_parser("package") + package.add_argument("export_directory", type=Path) + package.add_argument("--output-directory", type=Path, default=REPO / "build/package") + args = parser.parse_args() + + if args.command == "verify": + print(verify_export(args.export_directory)) + return 0 + epoch = source_date_epoch() + package_root, archive_path = create_package( + args.export_directory, args.output_directory, build_metadata(epoch), epoch + ) + print(package_root) + print(archive_path) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/test_install_godot_templates.py b/tools/test_install_godot_templates.py new file mode 100755 index 0000000..0dbafcb --- /dev/null +++ b/tools/test_install_godot_templates.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import hashlib +import io +import json +import tempfile +import unittest +import zipfile +from pathlib import Path +from unittest.mock import patch + +import install_godot_templates + + +class RangeReaderTests(unittest.TestCase): + def test_seek_and_windowed_reads(self) -> None: + source = bytes(range(251)) * 100 + requests: list[tuple[int, int]] = [] + + def fetch(start: int, end: int) -> bytes: + requests.append((start, end)) + return source[start:end + 1] + + reader = install_godot_templates.RangeReader(len(source), fetch, window_size=64) + reader.seek(103) + self.assertEqual(source[103:113], reader.read(10)) + self.assertEqual(source[113:123], reader.read(10)) + reader.seek(-8, io.SEEK_END) + self.assertEqual(source[-8:], reader.read()) + self.assertEqual(2, len(requests)) + + def test_installs_only_selected_member_and_reuses_verified_file(self) -> None: + release = b"linux release template\0" * 200 + unused = b"other platform" * 100 + archive_buffer = io.BytesIO() + with zipfile.ZipFile(archive_buffer, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("templates/linux_release.x86_64", release) + archive.writestr("templates/windows_release_x86_64.exe", unused) + archive_bytes = archive_buffer.getvalue() + + def reader() -> install_godot_templates.RangeReader: + return install_godot_templates.RangeReader( + len(archive_bytes), + lambda start, end: archive_bytes[start:end + 1], + window_size=128, + ) + + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + manifest = root / "manifest.json" + manifest.write_text(json.dumps({ + "templates": { + "url": "https://invalid.example/templates.tpz", + "size": len(archive_bytes), + "members": [{ + "archive_path": "templates/linux_release.x86_64", + "install_name": "linux_release.x86_64", + "sha256": hashlib.sha256(release).hexdigest(), + "size": len(release), + "mode": "0755", + }], + } + }), encoding="utf-8") + destination = root / "templates" + with patch.object(install_godot_templates, "open_http_range_reader", return_value=reader()) as opened: + installed = install_godot_templates.install_members(manifest, destination) + opened.assert_called_once() + self.assertEqual(release, installed[0].read_bytes()) + self.assertFalse((destination / "windows_release_x86_64.exe").exists()) + + with patch.object(install_godot_templates, "open_http_range_reader") as opened: + reused = install_godot_templates.install_members(manifest, destination) + opened.assert_not_called() + self.assertEqual(installed, reused) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/test_package_linux_x64.py b/tools/test_package_linux_x64.py new file mode 100755 index 0000000..aaa6b96 --- /dev/null +++ b/tools/test_package_linux_x64.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import hashlib +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +import package_linux_x64 + + +def make_export(root: Path) -> Path: + export = root / "export" + for relative in package_linux_x64.REQUIRED_FILES: + path = export / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(f"fixture:{relative}".encode()) + (export / "Himegari.x86_64").chmod(0o755) + return export + + +class PackageLinuxX64Tests(unittest.TestCase): + def test_verify_rejects_missing_and_windows_files(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + export = make_export(Path(temporary)) + package_linux_x64.verify_export(export) + (export / package_linux_x64.REQUIRED_FILES[-1]).unlink() + with self.assertRaisesRegex(ValueError, "missing"): + package_linux_x64.verify_export(export) + (export / package_linux_x64.REQUIRED_FILES[-1]).write_bytes(b"restored") + forbidden = export / package_linux_x64.FORBIDDEN_FILES[0] + forbidden.write_bytes(b"windows") + with self.assertRaisesRegex(ValueError, "Windows-only"): + package_linux_x64.verify_export(export) + + @unittest.skipIf(package_linux_x64.os.name == "nt", "executable-bit gate is POSIX-only") + def test_verify_requires_executable_bit_on_posix(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + export = make_export(Path(temporary)) + (export / "Himegari.x86_64").chmod(0o644) + with self.assertRaisesRegex(ValueError, "executable bit"): + package_linux_x64.verify_export(export) + + def test_package_has_notices_checksums_and_deterministic_envelope(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + temporary_root = Path(temporary) + export = make_export(temporary_root) + output = temporary_root / "build/package" + metadata = {"schema_version": 1, "source_commit": "fixture"} + with patch.object(package_linux_x64, "REPO", temporary_root), patch.object( + package_linux_x64, "PROJECT_FILES", ("LICENSE", "THIRD_PARTY_NOTICES.md") + ): + (temporary_root / "LICENSE").write_text("MIT\n", encoding="utf-8") + (temporary_root / "THIRD_PARTY_NOTICES.md").write_text("notices\n", encoding="utf-8") + package_linux_x64.create_package(export, output, metadata, 123456789) + first = hashlib.sha256( + (output / f"{package_linux_x64.PACKAGE_NAME}.tar.gz").read_bytes() + ).hexdigest() + package_root, archive = package_linux_x64.create_package( + export, output, metadata, 123456789 + ) + second = hashlib.sha256(archive.read_bytes()).hexdigest() + self.assertEqual(first, second) + self.assertTrue((package_root / "LICENSE").is_file()) + self.assertTrue((package_root / "BUILD-INFO.json").is_file()) + checksums = (package_root / "SHA256SUMS").read_text(encoding="utf-8") + self.assertIn("Himegari.x86_64", checksums) + self.assertIn("THIRD_PARTY_NOTICES.md", checksums) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/validate.py b/tools/validate.py index 5ea8a3f..78c13f7 100644 --- a/tools/validate.py +++ b/tools/validate.py @@ -30,6 +30,8 @@ import paths LEVELS = ("core", "workspace", "runtime", "full") CORE_TESTS = ( "test_validate.py", + "test_install_godot_templates.py", + "test_package_linux_x64.py", "test_diff_optrace.py", "test_engine_ctx.py", "test_ghidra_handler_map.py",