From 02e9891baa99411d4e11beca9432558e58f52ccc Mon Sep 17 00:00:00 2001 From: gamer147 Date: Mon, 3 Aug 2026 21:59:52 -0400 Subject: [PATCH] Cross-export Windows release package on Linux --- README.md | 7 +- docs/PROJECT-STRUCTURE.md | 15 +- docs/platform-portability.md | 28 ++- docs/remake-architecture-and-roadmap.md | 10 + docs/tools-reference.md | 35 ++++ godot/Main.PackageSmoke.cs | 5 +- godot/export_presets.cfg | 34 ++++ tools/bootstrap-godot-linux-x64.sh | 12 +- tools/build-linux-x64.sh | 1 + tools/build-windows-x64.sh | 129 ++++++++++++ tools/dotnet_publish_proxy.py | 3 +- tools/godot-linux-x64.json | 7 + tools/install_godot_templates.py | 16 +- tools/package_windows_x64.py | 248 ++++++++++++++++++++++++ tools/test_dotnet_publish_proxy.py | 23 +++ tools/test_install_godot_templates.py | 43 +++- tools/test_package_windows_x64.py | 96 +++++++++ tools/validate.py | 1 + tools/verify_windows_native.py | 9 +- 19 files changed, 690 insertions(+), 32 deletions(-) create mode 100755 tools/build-windows-x64.sh create mode 100755 tools/package_windows_x64.py create mode 100755 tools/test_package_windows_x64.py diff --git a/README.md b/README.md index a9799b0..13001a7 100644 --- a/README.md +++ b/README.md @@ -27,9 +27,10 @@ py -3.11 -X utf8 tools\validate.py --level core .\run-godot.ps1 -GodotConsole -GameRoot ``` -Linux release exports are asset-free build products: from a Linux host, `./tools/build-linux-x64.sh` -bootstraps the pinned Godot/FFmpeg inputs, exports the runtime, runs its packaged smoke gate, and writes the -archive under `build/package/`. See the tools reference for exact inputs and output contents. +Linux and Windows release exports are asset-free build products. From a Linux host, +`./tools/build-linux-x64.sh` builds, packages, and smoke-tests the Linux runtime, while +`./tools/build-windows-x64.sh` cross-builds and structurally verifies the Windows runtime without Wine. +Both write archives under `build/package/`; see the tools reference for exact inputs and output contents. `workspace` and `full` validation additionally require the extracted script corpus and disposable derived inputs described by the tools reference. `runtime` and `full` require Godot plus a valid game root. A diff --git a/docs/PROJECT-STRUCTURE.md b/docs/PROJECT-STRUCTURE.md index e4f7a1a..587d45b 100644 --- a/docs/PROJECT-STRUCTURE.md +++ b/docs/PROJECT-STRUCTURE.md @@ -47,14 +47,17 @@ S:\Game Hacking\Eushully\Himegari\ ← workspace root (three siblings) │ ├── validate.py layered core/workspace/runtime/full validation driver │ ├── test_validate.py pure resolver + validation-plan regressions │ ├── build-linux-x64.sh one-command pinned Linux export/package/smoke pipeline + │ ├── build-windows-x64.sh Linux-hosted Windows cross-export/package pipeline │ ├── dotnet_publish_proxy.py guarded low-memory Godot export-time publish staging │ ├── publish_gitea_release.py guarded/retry-safe native Gitea release promotion │ ├── verify_windows_native.py static AMD64 PE/native-bundle contract verifier │ ├── bootstrap-godot-linux-x64.sh repo-local Godot 4.7 Linux editor/template bootstrap │ ├── godot-linux-x64.json immutable Godot editor/template source + SHA-256 manifest │ ├── install_godot_templates.py selective HTTP-range installer for pinned template members - │ ├── package_linux_x64.py export verifier + notice/checksum/archive packager + │ ├── package_linux_x64.py, + │ │ package_windows_x64.py target-specific export verifiers and normalized packagers │ ├── test_install_godot_templates.py, test_package_linux_x64.py, + │ │ test_package_windows_x64.py, │ │ test_dotnet_publish_proxy.py, │ │ test_publish_gitea_release.py, │ │ test_verify_windows_native.py source-only bootstrap/package/publish/native regressions @@ -109,9 +112,10 @@ S:\Game Hacking\Eushully\Himegari\ ← workspace root (three siblings) ├── build/ DERIVED (our-work-side) — generated by tools/; disposable │ ├── downloads/, toolchains/ immutable dependency caches and repo-local Godot toolchain │ ├── export/linux-x64/ verified loose Godot Linux export + │ ├── export/windows-x64/ structurally verified loose Godot Windows export │ ├── native/linux-x64/ validated Linux FFmpeg shim/shared-library bundle │ ├── native/win-x64/ validated Windows FFmpeg shim/DLL bundle - │ ├── package/ staged release tree, smoke log, and `.tar.gz` artifact + │ ├── package/ Linux staged tree/smoke/`.tar.gz` plus Windows x64 staged tree/ZIP │ ├── disasm/ .asm — human-readable disassembly, one per script │ ├── text/ extracted text: │ │ ├── .strings.txt all inline strings in a script @@ -160,9 +164,10 @@ S:\Game Hacking\Eushully\Himegari\ ← workspace root (three siblings) ``` The Godot deliverable includes `Himegari.sln` because Godot's .NET exporter requires a solution, -`export_presets.cfg` for the Linux x86_64 target, and `build/.gdignore` so disposable capture frames below -the project do not enter its import or export scan. `tools/export-linux-x64.ps1` produces and validates the -complete disposable artifact under `build/export/linux-x64/`. +`export_presets.cfg` for Linux and Windows x86_64, and `build/.gdignore` so disposable capture frames below +the project do not enter its import or export scan. `tools/build-linux-x64.sh` and +`tools/build-windows-x64.sh` produce their complete disposable artifacts under `build/export/`; the older +Windows-hosted `tools/export-linux-x64.ps1` remains a Linux-export development path. `godot/Main.cs` retains the front-end's startup and runtime coordination. Behavior-neutral partial-class companions keep cohesive surfaces independently navigable without changing the Godot node type or invocation diff --git a/docs/platform-portability.md b/docs/platform-portability.md index d997fb7..d661881 100644 --- a/docs/platform-portability.md +++ b/docs/platform-portability.md @@ -84,7 +84,7 @@ lightweight `v0.1.0` tag at `5fe3cd6` on 2026-08-03: the tag build and promotion the Gitea release carries the archive, archive checksum, build metadata, payload ledger, and smoke log. The Linux CI/CD path is therefore accepted end to end. -## Planned Windows x64 CI/CD artifact +## Windows x64 CI/CD artifact The Windows artifact will be cross-built on the existing Linux runner; neither a Windows runner nor Wine is part of its build contract. The .NET SDK publishes `win-x64`, the Linux Godot editor exports through the pinned Windows @@ -103,12 +103,26 @@ MinGW builds produced the identical shim SHA-256 `9852680ca32bc37200d0b63ad89acb5453ad913bdcf4441a15c8ac9356d4da8c`. This is a compiler-independent contract comparison, not an expectation that MSVC and MinGW produce byte-identical binaries. -The Windows build command will mirror the accepted Linux entry point while keeping target checks separate. It -will regenerate opcode metadata, selectively install the Windows template from the existing pinned Godot TPZ, -bootstrap/hash-check the pinned Windows FFmpeg SDK, cross-build and inspect `age_movie_ffmpeg.dll`, perform an -exact self-contained `ExportRelease/win-x64` publish, and run the real Godot Windows export. The low-memory publish -proxy will become target-parameterized rather than duplicating its fail-closed project/configuration/RID/output -checks. The accepted Linux path must remain byte- and behavior-compatible throughout this extension. +The cross-export/package contract was completed and locally accepted on 2026-08-03. The shared Godot manifest +now pins both release-template members while each build range-fetches only its selected target; Windows adds the +109,405,184-byte `windows_release_x86_64.exe` member with independently recorded SHA-256. The guarded publish +proxy requires its driver-selected RID, so the same low-memory sequence produces exact self-contained +`ExportRelease/linux-x64` and `ExportRelease/win-x64` payloads without weakening the project/configuration/RID/ +output-root checks. `tools/build-windows-x64.sh` composes the pinned editor/template, MinGW native bundle, .NET +publish, real Godot export, static verifier, and normalized ZIP. Its verifier requires the executable/PCK, +self-contained runtime, three project assemblies plus `Age.Engine.Text.Windows.dll`, native FFmpeg bundle and +license; it rejects Linux shared objects and checks the Godot EXE plus all six native DLLs as AMD64 PE. The ZIP +contains project notices, clean-source/dependency metadata, a complete payload checksum ledger, and the static +verification report under one stable root. A WSL cross-build completed in 91.3 seconds and two package passes +were byte-identical; the resulting packaged EXE also reported +`PACKAGE SMOKE OK: opcodes=548 ffmpeg-abi=3` when optionally run on the Windows host. CI remains structural and +will not execute that EXE. + +The Windows build command mirrors the accepted Linux entry point while keeping target checks separate. It +regenerates opcode metadata, selectively installs the Windows template from the existing pinned Godot TPZ, +bootstrap/hash-checks the pinned Windows FFmpeg SDK, cross-builds and inspects `age_movie_ffmpeg.dll`, performs an +exact self-contained `ExportRelease/win-x64` publish, and runs the real Godot Windows export. The accepted Linux +path retains its target selection and dynamic package-smoke behavior. CI acceptance is structural because the produced executable will not run on the Linux host. The verifier must require the Windows executable/PCK/self-contained managed directory, `Age.Engine.Text.Windows.dll`, the project diff --git a/docs/remake-architecture-and-roadmap.md b/docs/remake-architecture-and-roadmap.md index cb4e1c5..007c100 100644 --- a/docs/remake-architecture-and-roadmap.md +++ b/docs/remake-architecture-and-roadmap.md @@ -1063,6 +1063,16 @@ do not mix mechanical moves with semantic changes. next step: Windows Godot template/export, target-parameterized publish proxy, structural package contract, and normalized ZIP. + **Slice 2 completed 2026-08-03:** the shared Godot manifest now pins and selectively fetches either release + template, the guarded low-memory publish proxy requires the calling driver's exact Linux or Windows RID, and + the Windows preset includes the conditional exact-GDI assembly. `tools/build-windows-x64.sh` performs the + complete metadata/native/publish/Godot/export/verify/package sequence without Wine. Its packager rejects + incomplete or Linux-contaminated payloads, verifies the Godot EXE and six native DLLs as AMD64 PE, embeds + notices/build provenance/payload checksums/static verification, and emits one normalized ZIP. Synthetic + template/proxy/package tests pass; the 91.3-second WSL cross-export passed, repeated packaging was byte-for-byte + stable, and an optional Windows-host run of the packaged EXE reported the 548-opcode/FFmpeg-ABI-3 smoke marker. + Slice 3 is now active: add the read-only hosted Windows artifact job while retaining the accepted Linux job. + **Completion gate:** one deliberately selected future tag produces a single Gitea release containing the Linux `.tar.gz`, Windows `.zip`, and combined archive checksums; both archives bind to the tag commit, Linux retains its dynamic packaged smoke, Windows passes all structural PE/payload gates, and neither platform job diff --git a/docs/tools-reference.md b/docs/tools-reference.md index 7279ccc..4514136 100644 --- a/docs/tools-reference.md +++ b/docs/tools-reference.md @@ -429,6 +429,41 @@ The bootstrap prints the pinned editor path. The verifier prints the accepted ex prints the staged root and archive path. Those Python components and the guarded publish proxy have source-only synthetic regressions in the core validation level. +**Windows x64 cross-export** uses the same pinned Linux editor and low-memory publish boundary, with MinGW and +the selectively installed Windows template: + +```bash +./tools/build-windows-x64.sh +``` + +The command requires Python 3.11, the `global.json` .NET SDK, MinGW-w64 GCC/binutils, `curl`, and `sha256sum`. +It regenerates runtime metadata, builds and statically verifies the pinned Windows FFmpeg bundle, publishes the +exact self-contained `ExportRelease/win-x64` managed runtime, and performs the real Godot `Windows x86_64` +release export. No produced Windows program is executed. `package_windows_x64.py` requires the Godot AMD64 EXE, +PCK, self-contained runtime, project assemblies including `Age.Engine.Text.Windows.dll`, and FFmpeg DLL/license +set; it rejects Linux shared objects and embeds `BUILD-INFO.json`, `WINDOWS-VERIFICATION.json`, notices, and a +complete `SHA256SUMS`. The normalized output is +`build/package/windows-x64/OpenMaidEngine-Himegari-windows-x64.zip`. + +Focused commands are: + +```bash +./tools/bootstrap-godot-linux-x64.sh windows-x64 +python3 -X utf8 tools/package_windows_x64.py verify build/export/windows-x64 +python3 -X utf8 tools/package_windows_x64.py package build/export/windows-x64 +``` + +The bootstrap defaults to `linux-x64`; passing `windows-x64` changes only the selectively installed template. +The package verifier needs `x86_64-w64-mingw32-objdump` unless `--objdump` names an equivalent tool. On a real +Windows host, an extracted package can optionally run the same asset-independent dynamic gate as Linux: + +```powershell +.\Himegari.exe --headless -- --package-smoke +``` + +That command proves the embedded 548-opcode table and local FFmpeg ABI 3 load. It is a manual acceptance aid, +not part of Linux-hosted Windows CI. + The older Windows-hosted cross-export remains available for local development: ```powershell diff --git a/godot/Main.PackageSmoke.cs b/godot/Main.PackageSmoke.cs index 0abdb1d..4aa45ed 100644 --- a/godot/Main.PackageSmoke.cs +++ b/godot/Main.PackageSmoke.cs @@ -7,8 +7,9 @@ public partial class Main { try { - if (!OS.HasFeature("linux")) - throw new PlatformNotSupportedException("the packaged-runtime smoke gate requires Linux"); + if (!OS.HasFeature("linux") && !OS.HasFeature("windows")) + throw new PlatformNotSupportedException( + "the packaged-runtime smoke gate supports Linux and Windows x64"); var table = HimegariRuntimeMetadata.LoadOpcodeTable(); if (table.Count != 548) throw new InvalidOperationException( diff --git a/godot/export_presets.cfg b/godot/export_presets.cfg index 3cd1478..821e34a 100644 --- a/godot/export_presets.cfg +++ b/godot/export_presets.cfg @@ -30,3 +30,37 @@ ssh_remote_deploy/enabled=false dotnet/include_scripts_content=false dotnet/include_debug_symbols=false dotnet/embed_build_outputs=false + +[preset.1] + +name="Windows x86_64" +platform="Windows Desktop" +runnable=false +dedicated_server=false +custom_features="" +export_filter="all_resources" +include_filter="" +exclude_filter="" +export_path="../build/export/windows-x64/Himegari.exe" +patches=PackedStringArray() +encryption_include_filters="" +encryption_exclude_filters="" +seed=0 +encrypt_pck=false +encrypt_directory=false +script_export_mode=2 + +[preset.1.options] + +custom_template/debug="" +custom_template/release="" +debug/export_console_wrapper=1 +binary_format/embed_pck=false +texture_format/s3tc_bptc=true +texture_format/etc2_astc=false +binary_format/architecture="x86_64" +codesign/enable=false +ssh_remote_deploy/enabled=false +dotnet/include_scripts_content=false +dotnet/include_debug_symbols=false +dotnet/embed_build_outputs=false diff --git a/tools/bootstrap-godot-linux-x64.sh b/tools/bootstrap-godot-linux-x64.sh index 6028726..26e1722 100755 --- a/tools/bootstrap-godot-linux-x64.sh +++ b/tools/bootstrap-godot-linux-x64.sh @@ -4,6 +4,7 @@ set -euo pipefail script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" repo_root="$(cd -- "$script_dir/.." && pwd)" manifest="$script_dir/godot-linux-x64.json" +template_target="${1:-linux-x64}" toolchain_root="$repo_root/build/toolchains/godot-4.7-stable-mono-linux-x64" xdg_data_home="$toolchain_root/xdg-data" download_dir="$repo_root/build/downloads" @@ -30,6 +31,15 @@ archive_path="$download_dir/$archive" editor_root="$toolchain_root/editor" template_root="$xdg_data_home/godot/export_templates/$template_version" +case "$template_target" in + linux-x64) template_name="linux_release.x86_64" ;; + windows-x64) template_name="windows_release_x86_64.exe" ;; + *) + echo "unsupported Godot template target: $template_target" >&2 + exit 2 + ;; +esac + mkdir -p -- "$download_dir" "$editor_root" "$template_root" if [[ ! -f "$archive_path" ]]; then curl --fail --location --retry 3 --output "$archive_path" "$url" @@ -59,7 +69,7 @@ editor="${editors[0]}" chmod +x "$editor" python3 -X utf8 "$script_dir/install_godot_templates.py" \ - --manifest "$manifest" --destination "$template_root" >&2 + --manifest "$manifest" --destination "$template_root" --member "$template_name" >&2 reported="$(XDG_DATA_HOME="$xdg_data_home" "$editor" --headless --version)" reported="${reported%%$'\n'*}" diff --git a/tools/build-linux-x64.sh b/tools/build-linux-x64.sh index abe7a9a..aa839ee 100755 --- a/tools/build-linux-x64.sh +++ b/tools/build-linux-x64.sh @@ -94,6 +94,7 @@ AGE_PUBLISH_PROJECT="$project_root/Himegari.csproj" \ AGE_PREPUBLISHED_OUTPUT="$managed_proxy_directory" \ AGE_PUBLISH_OUTPUT_ROOT="${TMPDIR:-/tmp}/godot-publish-dotnet" \ AGE_PUBLISH_ASSEMBLY="Himegari.dll" \ +AGE_PUBLISH_RUNTIME="linux-x64" \ PATH="$dotnet_proxy_directory:$PATH" \ XDG_DATA_HOME="$xdg_data_home" "$godot_console" \ --headless --quit-after 120 --path "$project_root" \ diff --git a/tools/build-windows-x64.sh b/tools/build-windows-x64.sh new file mode 100755 index 0000000..1fa85cb --- /dev/null +++ b/tools/build-windows-x64.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(cd -- "$script_dir/.." && pwd)" +project_root="$repo_root/godot" +export_directory="$repo_root/build/export/windows-x64" +expected_export_directory="$repo_root/build/export/windows-x64" +package_directory="$repo_root/build/package/windows-x64" +managed_publish_directory="$repo_root/build/managed-publish/win-x64" +managed_proxy_directory="$repo_root/build/managed-publish-proxy/win-x64" +dotnet_proxy_directory="$repo_root/build/dotnet-export-proxy" +toolchain_root="$repo_root/build/toolchains/godot-4.7-stable-mono-linux-x64" +xdg_data_home="$toolchain_root/xdg-data" +compiler="${MINGW_CC:-x86_64-w64-mingw32-gcc}" +objdump="${MINGW_OBJDUMP:-x86_64-w64-mingw32-objdump}" + +for command in python3 dotnet "$compiler" "$objdump" curl sha256sum awk; do + command -v "$command" >/dev/null 2>&1 || { + echo "required command was not found: $command" >&2 + exit 1 + } +done + +python3 -X utf8 "$script_dir/opcodes_build.py" --build +godot_console="$("$script_dir/bootstrap-godot-linux-x64.sh" windows-x64)" +ffmpeg_sdk="$("$repo_root/native/age_movie_ffmpeg/bootstrap-win64.sh")" +"$repo_root/native/age_movie_ffmpeg/build-win64.sh" "$ffmpeg_sdk" + +# Keep the memory-heavy self-contained publish outside the resident Godot editor, as on Linux. +expected_managed_publish_directory="$repo_root/build/managed-publish/win-x64" +if [[ "$managed_publish_directory" != "$expected_managed_publish_directory" || "$managed_publish_directory" == "/" ]]; then + echo "refusing to replace unexpected managed publish directory: $managed_publish_directory" >&2 + exit 1 +fi +rm -rf -- "$managed_publish_directory" +mkdir -p -- "$managed_publish_directory" +DOTNET_CLI_USE_MSBUILD_SERVER=0 \ +MSBUILDDISABLENODEREUSE=1 \ +DOTNET_gcServer=0 \ +DOTNET_GCConserveMemory=9 \ +dotnet publish "$project_root/Himegari.csproj" \ + --configuration ExportRelease \ + --runtime win-x64 \ + --self-contained true \ + --output "$managed_publish_directory" \ + -p:GodotTargetPlatform=windows \ + -p:UseSharedCompilation=false \ + -p:BuildInParallel=false \ + -p:RestoreDisableParallel=true \ + -p:DebugType=None \ + -p:DebugSymbols=false + +expected_managed_proxy_directory="$repo_root/build/managed-publish-proxy/win-x64" +if [[ "$managed_proxy_directory" != "$expected_managed_proxy_directory" || "$managed_proxy_directory" == "/" ]]; then + echo "refusing to replace unexpected managed proxy directory: $managed_proxy_directory" >&2 + exit 1 +fi +rm -rf -- "$managed_proxy_directory" +mkdir -p -- "$managed_proxy_directory" +cp -- "$managed_publish_directory/Himegari.dll" "$managed_proxy_directory/Himegari.dll" + +real_dotnet="$(command -v dotnet)" +expected_dotnet_proxy_directory="$repo_root/build/dotnet-export-proxy" +if [[ "$dotnet_proxy_directory" != "$expected_dotnet_proxy_directory" || "$dotnet_proxy_directory" == "/" ]]; then + echo "refusing to replace unexpected dotnet proxy directory: $dotnet_proxy_directory" >&2 + exit 1 +fi +rm -rf -- "$dotnet_proxy_directory" +mkdir -p -- "$dotnet_proxy_directory" +cp -- "$script_dir/dotnet_publish_proxy.py" "$dotnet_proxy_directory/dotnet" +chmod +x "$dotnet_proxy_directory/dotnet" + +if [[ "$export_directory" != "$expected_export_directory" || "$export_directory" == "/" ]]; then + echo "refusing to replace unexpected export directory: $export_directory" >&2 + exit 1 +fi +rm -rf -- "$export_directory" +mkdir -p -- "$export_directory" + +set +e +DOTNET_CLI_USE_MSBUILD_SERVER=0 \ +MSBUILDDISABLENODEREUSE=1 \ +UseSharedCompilation=false \ +BuildInParallel=false \ +RestoreDisableParallel=true \ +DOTNET_gcServer=0 \ +DOTNET_GCConserveMemory=9 \ +AGE_REAL_DOTNET="$real_dotnet" \ +AGE_PUBLISH_PROJECT="$project_root/Himegari.csproj" \ +AGE_PREPUBLISHED_OUTPUT="$managed_proxy_directory" \ +AGE_PUBLISH_OUTPUT_ROOT="${TMPDIR:-/tmp}/godot-publish-dotnet" \ +AGE_PUBLISH_ASSEMBLY="Himegari.dll" \ +AGE_PUBLISH_RUNTIME="win-x64" \ +PATH="$dotnet_proxy_directory:$PATH" \ +XDG_DATA_HOME="$xdg_data_home" "$godot_console" \ + --headless --quit-after 120 --path "$project_root" \ + --export-release "Windows x86_64" "$export_directory/Himegari.exe" +export_status=$? +set -e +if [[ $export_status -ne 0 ]]; then + if [[ $export_status -eq 137 ]]; then + echo "Godot export was killed with SIGKILL (137); inspect the runner memory cgroup diagnostics." >&2 + fi + exit "$export_status" +fi + +managed_export_directory="$export_directory/data_Himegari_windows_x86_64" +expected_managed_export_directory="$repo_root/build/export/windows-x64/data_Himegari_windows_x86_64" +if [[ "$managed_export_directory" != "$expected_managed_export_directory" || "$managed_export_directory" == "/" ]]; then + echo "refusing to replace unexpected managed export directory: $managed_export_directory" >&2 + exit 1 +fi +rm -rf -- "$managed_export_directory" +mkdir -p -- "$managed_export_directory" +cp -a -- "$managed_publish_directory/." "$managed_export_directory/" + +python3 -X utf8 "$script_dir/package_windows_x64.py" verify "$export_directory" --objdump "$objdump" +mapfile -t package_outputs < <( + python3 -X utf8 "$script_dir/package_windows_x64.py" package "$export_directory" \ + --objdump "$objdump" --output-directory "$package_directory" +) +if [[ ${#package_outputs[@]} -ne 2 ]]; then + echo "packager returned an unexpected result" >&2 + exit 1 +fi +archive_path="${package_outputs[1]}" +archive_hash="$(sha256sum "$archive_path" | awk '{ print $1 }')" +printf 'Windows x64 package: %s\nSHA-256: %s\n' "$archive_path" "$archive_hash" diff --git a/tools/dotnet_publish_proxy.py b/tools/dotnet_publish_proxy.py index 63dc5be..fcf6961 100755 --- a/tools/dotnet_publish_proxy.py +++ b/tools/dotnet_publish_proxy.py @@ -76,13 +76,14 @@ def stage_publish(request: PublishRequest, environ: dict[str, str]) -> Path: source = Path(environ["AGE_PREPUBLISHED_OUTPUT"]).resolve() output_root = Path(environ["AGE_PUBLISH_OUTPUT_ROOT"]).resolve() assembly = environ["AGE_PUBLISH_ASSEMBLY"] + expected_runtime = environ["AGE_PUBLISH_RUNTIME"] output = request.output.resolve() if request.project.resolve() != expected_project: raise ValueError(f"unexpected publish project: {request.project}") if request.configuration != "ExportRelease": raise ValueError(f"unexpected publish configuration: {request.configuration}") - if request.runtime != "linux-x64": + if request.runtime != expected_runtime: raise ValueError(f"unexpected publish runtime: {request.runtime}") if request.self_contained.lower() != "true": raise ValueError(f"publish is not self-contained: {request.self_contained}") diff --git a/tools/godot-linux-x64.json b/tools/godot-linux-x64.json index 38ad2f0..cd0e122 100644 --- a/tools/godot-linux-x64.json +++ b/tools/godot-linux-x64.json @@ -21,6 +21,13 @@ "sha256": "26b1b5d076b78c02f1476dab78e183218a6aa30c8e614653eac11a0dd198456a", "size": 73591000, "mode": "0755" + }, + { + "archive_path": "templates/windows_release_x86_64.exe", + "install_name": "windows_release_x86_64.exe", + "sha256": "00c72494d58536b74b95d5044fa13c4cc54f2349d687b87f3d8801897741fac0", + "size": 109405184, + "mode": "0755" } ] } diff --git a/tools/install_godot_templates.py b/tools/install_godot_templates.py index ea17928..69b6158 100755 --- a/tools/install_godot_templates.py +++ b/tools/install_godot_templates.py @@ -160,11 +160,21 @@ def file_matches(path: Path, expected_size: int, expected_hash: str) -> bool: return digest.hexdigest() == expected_hash.lower() -def install_members(manifest_path: Path, destination: Path) -> list[Path]: +def install_members( + manifest_path: Path, + destination: Path, + selected_names: tuple[str, ...] | None = None, +) -> list[Path]: with manifest_path.open(encoding="utf-8") as stream: manifest = json.load(stream) templates = manifest["templates"] members = templates["members"] + if selected_names: + by_name = {member["install_name"]: member for member in members} + unknown = [name for name in selected_names if name not in by_name] + if unknown: + raise ValueError("template members were not found in the manifest: " + ", ".join(unknown)) + members = [by_name[name] for name in selected_names] destination.mkdir(parents=True, exist_ok=True) installed = [destination / member["install_name"] for member in members] @@ -211,8 +221,10 @@ def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--manifest", type=Path, required=True) parser.add_argument("--destination", type=Path, required=True) + parser.add_argument("--member", action="append", dest="members") args = parser.parse_args() - for installed in install_members(args.manifest.resolve(), args.destination.resolve()): + selected = tuple(args.members) if args.members else None + for installed in install_members(args.manifest.resolve(), args.destination.resolve(), selected): print(installed) return 0 diff --git a/tools/package_windows_x64.py b/tools/package_windows_x64.py new file mode 100755 index 0000000..5b5bde6 --- /dev/null +++ b/tools/package_windows_x64.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python3 +"""Verify and package the Windows x64 Godot export without executing it.""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shutil +import subprocess +import time +import zipfile +from pathlib import Path +from typing import Any + +import verify_windows_native + + +HERE = Path(__file__).resolve().parent +REPO = HERE.parent +MANAGED_DIRECTORY = "data_Himegari_windows_x86_64" +PACKAGE_NAME = "OpenMaidEngine-Himegari-windows-x64" +REQUIRED_FILES = ( + "Himegari.exe", + "Himegari.pck", + f"{MANAGED_DIRECTORY}/Himegari.dll", + f"{MANAGED_DIRECTORY}/Age.Engine.dll", + f"{MANAGED_DIRECTORY}/Age.Engine.Frontend.dll", + f"{MANAGED_DIRECTORY}/Age.Engine.Text.Windows.dll", + f"{MANAGED_DIRECTORY}/coreclr.dll", + f"{MANAGED_DIRECTORY}/hostfxr.dll", + f"{MANAGED_DIRECTORY}/hostpolicy.dll", + f"{MANAGED_DIRECTORY}/System.Private.CoreLib.dll", + *(f"{MANAGED_DIRECTORY}/{name}" for name in verify_windows_native.REQUIRED_FILES), +) +FORBIDDEN_FILES = ( + "Himegari.x86_64", + f"{MANAGED_DIRECTORY}/libage_movie_ffmpeg.so", + f"{MANAGED_DIRECTORY}/libavformat.so.62", +) +PROJECT_FILES = ("LICENSE", "README.md", "THIRD_PARTY_NOTICES.md") + + +def _resolved_directory(path: Path, label: str) -> Path: + resolved = path.resolve() + if not resolved.is_dir(): + raise ValueError(f"{label} directory was not found: {resolved}") + return resolved + + +def verify_export(export_directory: Path, objdump: str) -> tuple[Path, dict[str, Any]]: + export_directory = _resolved_directory(export_directory, "Windows export") + missing = [name for name in REQUIRED_FILES if not (export_directory / name).is_file()] + if missing: + raise ValueError("Windows export is incomplete; missing: " + ", ".join(missing)) + forbidden = [name for name in FORBIDDEN_FILES if (export_directory / name).exists()] + forbidden.extend( + path.relative_to(export_directory).as_posix() + for path in export_directory.rglob("*") + if path.is_file() and (path.name.endswith(".so") or ".so." in path.name) + ) + if forbidden: + raise ValueError("Windows export contains Linux-only files: " + ", ".join(sorted(set(forbidden)))) + executable_machine = verify_windows_native.pe_machine(export_directory / "Himegari.exe") + if executable_machine != verify_windows_native.PE_MACHINE_AMD64: + raise ValueError( + f"Windows executable is not AMD64 PE (0x{executable_machine:04x}): Himegari.exe" + ) + native_report = verify_windows_native.verify_bundle( + export_directory / MANAGED_DIRECTORY, + objdump, + exact_dlls=False, + ) + native_report["bundle"] = MANAGED_DIRECTORY + return export_directory, { + "schema_version": 1, + "target": "win-x64", + "executable_machine": "AMD64", + "native": native_report, + } + + +def _git(*arguments: str) -> str: + result = subprocess.run( + ("git", *arguments), cwd=REPO, check=True, capture_output=True, text=True + ) + return result.stdout.strip() + + +def source_date_epoch() -> int: + configured = os.environ.get("SOURCE_DATE_EPOCH") + value = configured or _git("show", "-s", "--format=%ct", "HEAD") + try: + epoch = int(value) + except ValueError as error: + raise ValueError(f"invalid SOURCE_DATE_EPOCH: {value!r}") from error + if epoch < 0: + raise ValueError("SOURCE_DATE_EPOCH must not be negative") + return epoch + + +def build_metadata(epoch: int) -> dict[str, object]: + with (HERE / "godot-linux-x64.json").open(encoding="utf-8") as stream: + godot = json.load(stream) + windows_template = next( + member + for member in godot["templates"]["members"] + if member["install_name"] == "windows_release_x86_64.exe" + ) + with (REPO / "native/age_movie_ffmpeg/dependency-win64.json").open( + encoding="utf-8" + ) as stream: + ffmpeg = json.load(stream) + return { + "schema_version": 1, + "project": "OpenMaidEngine Himegari profile", + "target": "win-x64", + "source_commit": _git("rev-parse", "HEAD"), + "source_dirty": bool(_git("status", "--porcelain", "--untracked-files=normal")), + "source_date_epoch": epoch, + "godot": { + "version": godot["godot_version"], + "editor_archive_sha256": godot["editor"]["sha256"], + "release_template_sha256": windows_template["sha256"], + }, + "ffmpeg": { + "provider": ffmpeg["provider"], + "version": ffmpeg["ffmpeg_version"], + "archive_sha256": ffmpeg["sha256"], + }, + } + + +def _hash(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _write_text(path: Path, content: str) -> None: + with path.open("w", encoding="utf-8", newline="\n") as stream: + stream.write(content) + + +def _write_checksums(package_root: Path) -> None: + checksum_path = package_root / "SHA256SUMS" + files = sorted( + path for path in package_root.rglob("*") + if path.is_file() and path != checksum_path + ) + lines = [f"{_hash(path)} {path.relative_to(package_root).as_posix()}" for path in files] + _write_text(checksum_path, "\n".join(lines) + "\n") + + +def _zip_timestamp(epoch: int) -> tuple[int, int, int, int, int, int]: + minimum = 315532800 # 1980-01-01, the earliest ZIP timestamp. + maximum = 4354819198 # 2107-12-31 23:59:58, the latest portable ZIP timestamp. + stamp = time.gmtime(max(minimum, min(epoch, maximum)))[:6] + return (*stamp[:5], stamp[5] - stamp[5] % 2) + + +def create_package( + export_directory: Path, + output_directory: Path, + metadata: dict[str, object], + verification: dict[str, Any], + epoch: int, +) -> tuple[Path, Path]: + export_directory = _resolved_directory(export_directory, "Windows export") + output_directory = output_directory.resolve() + expected_parent = (REPO / "build/package/windows-x64").resolve() + if output_directory != expected_parent: + raise ValueError(f"refusing to replace unexpected package directory: {output_directory}") + staging_parent = output_directory / "staging" + package_root = staging_parent / PACKAGE_NAME + archive_path = output_directory / f"{PACKAGE_NAME}.zip" + if staging_parent.exists(): + shutil.rmtree(staging_parent) + staging_parent.mkdir(parents=True) + shutil.copytree(export_directory, package_root, copy_function=shutil.copy2) + for name in PROJECT_FILES: + source = REPO / name + if not source.is_file(): + raise ValueError(f"package notice file was not found: {source}") + shutil.copy2(source, package_root / name) + _write_text( + package_root / "BUILD-INFO.json", + json.dumps(metadata, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + ) + _write_text( + package_root / "WINDOWS-VERIFICATION.json", + json.dumps(verification, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + ) + _write_checksums(package_root) + + output_directory.mkdir(parents=True, exist_ok=True) + temporary_archive = archive_path.with_suffix(archive_path.suffix + ".part") + timestamp = _zip_timestamp(epoch) + with zipfile.ZipFile( + temporary_archive, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=9 + ) as archive: + for path in sorted(item for item in package_root.rglob("*") if item.is_file()): + relative = path.relative_to(staging_parent).as_posix() + info = zipfile.ZipInfo(relative, timestamp) + info.create_system = 3 + info.external_attr = 0o100644 << 16 + info.compress_type = zipfile.ZIP_DEFLATED + with path.open("rb") as source, archive.open(info, "w", force_zip64=True) as target: + shutil.copyfileobj(source, target, length=1024 * 1024) + os.replace(temporary_archive, archive_path) + return package_root, archive_path + + +def main() -> int: + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="command", required=True) + verify = subparsers.add_parser("verify") + verify.add_argument("export_directory", type=Path) + verify.add_argument("--objdump", default="x86_64-w64-mingw32-objdump") + package = subparsers.add_parser("package") + package.add_argument("export_directory", type=Path) + package.add_argument("--objdump", default="x86_64-w64-mingw32-objdump") + package.add_argument( + "--output-directory", type=Path, default=REPO / "build/package/windows-x64" + ) + args = parser.parse_args() + + export_directory, verification = verify_export(args.export_directory, args.objdump) + if args.command == "verify": + print(export_directory) + return 0 + epoch = source_date_epoch() + package_root, archive_path = create_package( + export_directory, + args.output_directory, + build_metadata(epoch), + verification, + epoch, + ) + print(package_root) + print(archive_path) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/test_dotnet_publish_proxy.py b/tools/test_dotnet_publish_proxy.py index 4b9fee9..8cca306 100755 --- a/tools/test_dotnet_publish_proxy.py +++ b/tools/test_dotnet_publish_proxy.py @@ -30,6 +30,7 @@ class DotnetPublishProxyTests(unittest.TestCase): "AGE_PREPUBLISHED_OUTPUT": str(source), "AGE_PUBLISH_OUTPUT_ROOT": str(output_root), "AGE_PUBLISH_ASSEMBLY": "Himegari.dll", + "AGE_PUBLISH_RUNTIME": "linux-x64", }) self.assertEqual(output.resolve(), staged) self.assertEqual(b"managed", (staged / "Himegari.dll").read_bytes()) @@ -48,6 +49,7 @@ class DotnetPublishProxyTests(unittest.TestCase): "AGE_PREPUBLISHED_OUTPUT": str(source), "AGE_PUBLISH_OUTPUT_ROOT": str(root / "reserved"), "AGE_PUBLISH_ASSEMBLY": "Himegari.dll", + "AGE_PUBLISH_RUNTIME": "linux-x64", } wrong_runtime = dotnet_publish_proxy.PublishRequest( project, "ExportRelease", "win-x64", "true", root / "reserved/output" @@ -60,6 +62,27 @@ class DotnetPublishProxyTests(unittest.TestCase): with self.assertRaisesRegex(ValueError, "outside"): dotnet_publish_proxy.stage_publish(escaped, environment) + def test_stages_windows_publish_when_explicitly_selected(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + project = root / "Himegari.csproj" + project.write_text("", encoding="utf-8") + source = root / "prepublished" + source.mkdir() + (source / "Himegari.dll").write_bytes(b"managed") + output = root / "reserved/123-ExportRelease-win-x64" + request = dotnet_publish_proxy.PublishRequest( + project, "ExportRelease", "win-x64", "true", output + ) + staged = dotnet_publish_proxy.stage_publish(request, { + "AGE_PUBLISH_PROJECT": str(project), + "AGE_PREPUBLISHED_OUTPUT": str(source), + "AGE_PUBLISH_OUTPUT_ROOT": str(root / "reserved"), + "AGE_PUBLISH_ASSEMBLY": "Himegari.dll", + "AGE_PUBLISH_RUNTIME": "win-x64", + }) + self.assertEqual(output.resolve(), staged) + if __name__ == "__main__": unittest.main() diff --git a/tools/test_install_godot_templates.py b/tools/test_install_godot_templates.py index 0dbafcb..b548526 100755 --- a/tools/test_install_godot_templates.py +++ b/tools/test_install_godot_templates.py @@ -53,27 +53,52 @@ class RangeReaderTests(unittest.TestCase): "templates": { "url": "https://invalid.example/templates.tpz", "size": len(archive_bytes), - "members": [{ - "archive_path": "templates/linux_release.x86_64", - "install_name": "linux_release.x86_64", - "sha256": hashlib.sha256(release).hexdigest(), - "size": len(release), - "mode": "0755", - }], + "members": [ + { + "archive_path": "templates/linux_release.x86_64", + "install_name": "linux_release.x86_64", + "sha256": hashlib.sha256(release).hexdigest(), + "size": len(release), + "mode": "0755", + }, + { + "archive_path": "templates/windows_release_x86_64.exe", + "install_name": "windows_release_x86_64.exe", + "sha256": hashlib.sha256(unused).hexdigest(), + "size": len(unused), + "mode": "0755", + }, + ], } }), encoding="utf-8") destination = root / "templates" with patch.object(install_godot_templates, "open_http_range_reader", return_value=reader()) as opened: - installed = install_godot_templates.install_members(manifest, destination) + installed = install_godot_templates.install_members( + manifest, destination, ("linux_release.x86_64",) + ) opened.assert_called_once() self.assertEqual(release, installed[0].read_bytes()) self.assertFalse((destination / "windows_release_x86_64.exe").exists()) with patch.object(install_godot_templates, "open_http_range_reader") as opened: - reused = install_godot_templates.install_members(manifest, destination) + reused = install_godot_templates.install_members( + manifest, destination, ("linux_release.x86_64",) + ) opened.assert_not_called() self.assertEqual(installed, reused) + def test_rejects_unknown_selected_member(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + manifest = root / "manifest.json" + manifest.write_text(json.dumps({ + "templates": {"members": [], "url": "unused", "size": 1} + }), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "not found in the manifest"): + install_godot_templates.install_members( + manifest, root / "templates", ("missing-template",) + ) + if __name__ == "__main__": unittest.main() diff --git a/tools/test_package_windows_x64.py b/tools/test_package_windows_x64.py new file mode 100755 index 0000000..fb64ded --- /dev/null +++ b/tools/test_package_windows_x64.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import hashlib +import struct +import tempfile +import unittest +from pathlib import Path +from unittest.mock import Mock, patch + +import package_windows_x64 +import verify_windows_native + + +def write_pe(path: Path, machine: int = verify_windows_native.PE_MACHINE_AMD64) -> None: + data = bytearray(128) + data[:2] = b"MZ" + struct.pack_into(" str: + lines = [f"DLL Name: {name}" for name in verify_windows_native.RUNTIME_DLLS] + lines.extend(f"[ 0] {name}" for name in verify_windows_native.REQUIRED_EXPORTS) + return "\n".join(lines) + + +def make_export(root: Path) -> Path: + export = root / "export" + for relative in package_windows_x64.REQUIRED_FILES: + path = export / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(f"fixture:{relative}".encode()) + write_pe(export / "Himegari.exe") + managed = export / package_windows_x64.MANAGED_DIRECTORY + for name in (verify_windows_native.SHIM, *verify_windows_native.RUNTIME_DLLS): + write_pe(managed / name) + return export + + +class PackageWindowsX64Tests(unittest.TestCase): + def test_verify_rejects_missing_linux_and_wrong_machine_files(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + export = make_export(Path(temporary)) + completed = Mock(returncode=0, stdout=objdump_text(), stderr="") + with patch.object(verify_windows_native.subprocess, "run", return_value=completed): + package_windows_x64.verify_export(export, "objdump") + (export / package_windows_x64.REQUIRED_FILES[-1]).unlink() + with self.assertRaisesRegex(ValueError, "missing"): + package_windows_x64.verify_export(export, "objdump") + (export / package_windows_x64.REQUIRED_FILES[-1]).write_bytes(b"restored") + linux = export / package_windows_x64.MANAGED_DIRECTORY / "stale.so" + linux.write_bytes(b"linux") + with self.assertRaisesRegex(ValueError, "Linux-only"): + package_windows_x64.verify_export(export, "objdump") + linux.unlink() + write_pe(export / "Himegari.exe", machine=0x014C) + with self.assertRaisesRegex(ValueError, "not AMD64"): + package_windows_x64.verify_export(export, "objdump") + + def test_package_has_notices_reports_checksums_and_deterministic_zip(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + temporary_root = Path(temporary) + export = make_export(temporary_root) + output = temporary_root / "build/package/windows-x64" + metadata = {"schema_version": 1, "source_commit": "fixture"} + verification = {"schema_version": 1, "target": "win-x64"} + with patch.object(package_windows_x64, "REPO", temporary_root), patch.object( + package_windows_x64, "PROJECT_FILES", ("LICENSE", "THIRD_PARTY_NOTICES.md") + ): + (temporary_root / "LICENSE").write_text("MIT\n", encoding="utf-8") + (temporary_root / "THIRD_PARTY_NOTICES.md").write_text( + "notices\n", encoding="utf-8" + ) + package_windows_x64.create_package( + export, output, metadata, verification, 123456789 + ) + archive = output / f"{package_windows_x64.PACKAGE_NAME}.zip" + first = hashlib.sha256(archive.read_bytes()).hexdigest() + package_root, archive = package_windows_x64.create_package( + export, output, metadata, verification, 123456789 + ) + second = hashlib.sha256(archive.read_bytes()).hexdigest() + self.assertEqual(first, second) + self.assertTrue((package_root / "LICENSE").is_file()) + self.assertTrue((package_root / "BUILD-INFO.json").is_file()) + self.assertTrue((package_root / "WINDOWS-VERIFICATION.json").is_file()) + checksums = (package_root / "SHA256SUMS").read_text(encoding="utf-8") + self.assertIn("Himegari.exe", checksums) + self.assertIn("THIRD_PARTY_NOTICES.md", checksums) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/validate.py b/tools/validate.py index a201ddb..6ff46f1 100644 --- a/tools/validate.py +++ b/tools/validate.py @@ -32,6 +32,7 @@ CORE_TESTS = ( "test_validate.py", "test_install_godot_templates.py", "test_package_linux_x64.py", + "test_package_windows_x64.py", "test_dotnet_publish_proxy.py", "test_publish_gitea_release.py", "test_verify_windows_native.py", diff --git a/tools/verify_windows_native.py b/tools/verify_windows_native.py index 8b85723..1e39c60 100755 --- a/tools/verify_windows_native.py +++ b/tools/verify_windows_native.py @@ -59,7 +59,12 @@ def parse_objdump(output: str) -> tuple[set[str], set[str]]: return imports, exports -def verify_bundle(bundle: Path, objdump: str) -> dict[str, Any]: +def verify_bundle( + bundle: Path, + objdump: str, + *, + exact_dlls: bool = True, +) -> dict[str, Any]: bundle = bundle.resolve() if not bundle.is_dir(): raise ValueError(f"Windows native bundle directory was not found: {bundle}") @@ -73,7 +78,7 @@ def verify_bundle(bundle: Path, objdump: str) -> dict[str, Any]: for path in bundle.iterdir() if path.is_file() and path.suffix.lower() == ".dll" and path.name.lower() not in expected_dlls ) - if unexpected_dlls: + if exact_dlls and unexpected_dlls: raise ValueError("Windows native bundle has unexpected DLLs: " + ", ".join(unexpected_dlls)) machines: dict[str, str] = {}