diff --git a/.gitea/workflows/linux-release-build.yml b/.gitea/workflows/linux-release-build.yml index 415daa0..632fc24 100644 --- a/.gitea/workflows/linux-release-build.yml +++ b/.gitea/workflows/linux-release-build.yml @@ -57,6 +57,11 @@ jobs: dotnet --version cc --version readelf --version + free -h + if [ -r /sys/fs/cgroup/memory.max ]; then + echo "cgroup memory.max=$(cat /sys/fs/cgroup/memory.max)" + echo "cgroup memory.current=$(cat /sys/fs/cgroup/memory.current)" + fi - name: Build and smoke-test Linux package run: ./tools/build-linux-x64.sh diff --git a/docs/PROJECT-STRUCTURE.md b/docs/PROJECT-STRUCTURE.md index fa3b4e8..c17bd8a 100644 --- a/docs/PROJECT-STRUCTURE.md +++ b/docs/PROJECT-STRUCTURE.md @@ -47,12 +47,13 @@ S:\Game Hacking\Eushully\Himegari\ ← workspace root (three siblings) │ ├── validate.py layered core/workspace/runtime/full validation driver │ ├── test_validate.py pure resolver + validation-plan regressions │ ├── build-linux-x64.sh one-command pinned Linux export/package/smoke pipeline + │ ├── dotnet_publish_proxy.py guarded low-memory Godot export-time publish staging │ ├── bootstrap-godot-linux-x64.sh repo-local Godot 4.7 Linux editor/template bootstrap │ ├── godot-linux-x64.json immutable Godot editor/template source + SHA-256 manifest │ ├── install_godot_templates.py selective HTTP-range installer for pinned template members │ ├── package_linux_x64.py export verifier + notice/checksum/archive packager - │ ├── test_install_godot_templates.py, test_package_linux_x64.py - │ │ source-only bootstrap/package regressions + │ ├── test_install_godot_templates.py, test_package_linux_x64.py, + │ │ test_dotnet_publish_proxy.py source-only bootstrap/package/publish regressions │ ├── sys4load.py loader + disassembler (opcode-decoding) │ ├── age_opcodes.py GENERATED complete Python ABI view (do not hand-edit) │ ├── opcodes_build.py generator/linter: vm-map/opcodes.toml -> the 5 artifacts below diff --git a/docs/platform-portability.md b/docs/platform-portability.md index 52773de..05af9e5 100644 --- a/docs/platform-portability.md +++ b/docs/platform-portability.md @@ -58,8 +58,15 @@ fast integration gate. The cache key is derived from both immutable dependency m prefix; cached archives/templates are still size/hash checked by their owning bootstrap before use. A successful job retains the archive, external build/checksum manifests, and smoke log for 30 days. It has only read access to repository contents and does not use private data, secrets, registry credentials, or release-write permission. -The first hosted execution remains the acceptance gate for runner image dependencies and cache/artifact action -behavior; release publication is intentionally deferred until that artifact run succeeds. +The first hosted execution reached Godot's managed publish but was killed with status 137. Local stage-level +measurement reproduced the pressure: the resident Godot export peaks at 772,476 KiB and an isolated managed +publish peaks at 223,764 KiB, so nesting the latter under the former can cross a roughly 1 GiB runner cgroup. +The build now serializes those peaks: it creates the exact self-contained publish first with compiler/build +servers disabled, lets the real Godot export consume a strictly validated one-assembly proxy publish, and stages +the complete external managed payload only after Godot exits. The complete revised path passes locally, including +payload verification and the packaged opcode/FFmpeg smoke gate. The hosted retry remains the acceptance gate for +the mitigation plus cache/artifact action behavior; release publication is intentionally deferred until it +succeeds. The workflow reports both host memory and cgroup limits to make any further runner constraint explicit. ## Dependency inventory diff --git a/docs/remake-architecture-and-roadmap.md b/docs/remake-architecture-and-roadmap.md index 5bf597f..6db223a 100644 --- a/docs/remake-architecture-and-roadmap.md +++ b/docs/remake-architecture-and-roadmap.md @@ -1005,9 +1005,14 @@ do not mix mechanical moves with semantic changes. template, calls the same locally accepted build command, and uploads the archive with its external build info, checksum ledger, and smoke log. Cache entries have no broad fallback and remain subject to the bootstrap's size/SHA checks. The workflow has no game corpus, secrets, registry login, or release-write authority; tags - still produce ordinary retained workflow artifacts. The immediate gate is the first hosted build on this - server. If it passes, the next bounded slice can promote an already-verified tag artifact to a Gitea release - without rebuilding it; if it fails, fix only the observed runner/action dependency before adding publication. + still produce ordinary retained workflow artifacts. The first hosted build reached Godot's managed publish + and was killed with status 137. Stage-level measurement found a 772,476 KiB Godot peak and a 223,764 KiB + isolated publish peak, explaining why the nested processes can exceed a roughly 1 GiB cgroup. The build now + runs the exact self-contained publish first, substitutes a tightly validated one-assembly publish while Godot + creates the real PCK/executable, and stages the full external managed payload after the editor exits. The + revised complete build, payload verification, and packaged smoke gate pass locally; a hosted retry is the + immediate acceptance gate. Once that succeeds, the next bounded slice can promote an already-verified tag + artifact to a Gitea release without rebuilding it. **Not cleanup targets:** generated `build/` output, the two intentional solution files, historical `docs/superpowers/` plans/specifications, and fidelity-specific complexity that is directly covered by the diff --git a/docs/tools-reference.md b/docs/tools-reference.md index b0a73b4..d3e416e 100644 --- a/docs/tools-reference.md +++ b/docs/tools-reference.md @@ -380,7 +380,15 @@ member from Godot's all-platform archive; it does not download the complete 1.2 The build regenerates embedded opcode metadata, bootstraps/builds the pinned Linux FFmpeg bundle, performs the Godot release export, and delegates the shared required/forbidden payload checks to -`package_linux_x64.py`. The packager adds `LICENSE`, `README.md`, `THIRD_PARTY_NOTICES.md`, a source/dependency +`package_linux_x64.py`. To remain inside constrained CI memory cgroups, it first runs the exact self-contained +`ExportRelease` publish with build/compiler servers and debug symbols disabled. Godot then performs its real +PCK/executable export against a one-assembly staging payload supplied by `dotnet_publish_proxy.py`; the proxy +refuses any project, configuration, runtime, self-contained flag, or output root other than the invocation +reserved by the build driver. Because managed outputs are external to the PCK, the complete prepublished +payload replaces that temporary staging directory only after Godot exits. The final payload verifier and +packaged smoke test therefore validate the same complete runtime that is archived. + +The packager adds `LICENSE`, `README.md`, `THIRD_PARTY_NOTICES.md`, a source/dependency `BUILD-INFO.json`, and per-file `SHA256SUMS` to a stable top-level directory. It normalizes archive ownership, modes, ordering, and timestamps using `SOURCE_DATE_EPOCH` or the source commit time. Outputs are the loose export under `build/export/linux-x64/`, the staged installation and `package-smoke.log` under @@ -402,8 +410,8 @@ python3 -X utf8 tools/package_linux_x64.py package build/export/linux-x64 ``` The bootstrap prints the pinned editor path. The verifier prints the accepted export directory; the packager -prints the staged root and archive path. Both Python components have source-only synthetic regressions in the -core validation level. +prints the staged root and archive path. Those Python components and the guarded publish proxy have source-only +synthetic regressions in the core validation level. The older Windows-hosted cross-export remains available for local development: diff --git a/godot/export_presets.cfg b/godot/export_presets.cfg index 0b95821..3cd1478 100644 --- a/godot/export_presets.cfg +++ b/godot/export_presets.cfg @@ -27,3 +27,6 @@ texture_format/s3tc_bptc=true texture_format/etc2_astc=false binary_format/architecture="x86_64" ssh_remote_deploy/enabled=false +dotnet/include_scripts_content=false +dotnet/include_debug_symbols=false +dotnet/embed_build_outputs=false diff --git a/tools/build-linux-x64.sh b/tools/build-linux-x64.sh index b8cfe89..abe7a9a 100755 --- a/tools/build-linux-x64.sh +++ b/tools/build-linux-x64.sh @@ -7,6 +7,9 @@ project_root="$repo_root/godot" export_directory="$repo_root/build/export/linux-x64" expected_export_directory="$repo_root/build/export/linux-x64" package_directory="$repo_root/build/package" +managed_publish_directory="$repo_root/build/managed-publish/linux-x64" +managed_proxy_directory="$repo_root/build/managed-publish-proxy/linux-x64" +dotnet_proxy_directory="$repo_root/build/dotnet-export-proxy" toolchain_root="$repo_root/build/toolchains/godot-4.7-stable-mono-linux-x64" xdg_data_home="$toolchain_root/xdg-data" @@ -22,6 +25,55 @@ godot_console="$("$script_dir/bootstrap-godot-linux-x64.sh")" ffmpeg_sdk="$("$repo_root/native/age_movie_ffmpeg/bootstrap-linux-x64.sh")" "$repo_root/native/age_movie_ffmpeg/build-linux-x64.sh" "$ffmpeg_sdk" +# Godot keeps the editor resident while it launches dotnet publish. On constrained CI runners, that process +# pair can exceed the job's memory cgroup. Produce the exact ExportRelease/linux-x64 payload first, without +# compiler/build servers, so the memory-heavy compile and editor stages run sequentially. +expected_managed_publish_directory="$repo_root/build/managed-publish/linux-x64" +if [[ "$managed_publish_directory" != "$expected_managed_publish_directory" || "$managed_publish_directory" == "/" ]]; then + echo "refusing to replace unexpected managed publish directory: $managed_publish_directory" >&2 + exit 1 +fi +rm -rf -- "$managed_publish_directory" +mkdir -p -- "$managed_publish_directory" +DOTNET_CLI_USE_MSBUILD_SERVER=0 \ +MSBUILDDISABLENODEREUSE=1 \ +DOTNET_gcServer=0 \ +DOTNET_GCConserveMemory=9 \ +dotnet publish "$project_root/Himegari.csproj" \ + --configuration ExportRelease \ + --runtime linux-x64 \ + --self-contained true \ + --output "$managed_publish_directory" \ + -p:GodotTargetPlatform=linuxbsd \ + -p:UseSharedCompilation=false \ + -p:BuildInParallel=false \ + -p:RestoreDisableParallel=true \ + -p:DebugType=None \ + -p:DebugSymbols=false + +# With dotnet/embed_build_outputs=false, Godot places publish files outside the PCK. Give the editor only the +# required profile assembly while it creates the real PCK, then stage the complete prepublish after the editor +# exits. This avoids making the memory-heavy editor traverse/hash the self-contained runtime and FFmpeg bundle. +expected_managed_proxy_directory="$repo_root/build/managed-publish-proxy/linux-x64" +if [[ "$managed_proxy_directory" != "$expected_managed_proxy_directory" || "$managed_proxy_directory" == "/" ]]; then + echo "refusing to replace unexpected managed proxy directory: $managed_proxy_directory" >&2 + exit 1 +fi +rm -rf -- "$managed_proxy_directory" +mkdir -p -- "$managed_proxy_directory" +cp -- "$managed_publish_directory/Himegari.dll" "$managed_proxy_directory/Himegari.dll" + +real_dotnet="$(command -v dotnet)" +expected_dotnet_proxy_directory="$repo_root/build/dotnet-export-proxy" +if [[ "$dotnet_proxy_directory" != "$expected_dotnet_proxy_directory" || "$dotnet_proxy_directory" == "/" ]]; then + echo "refusing to replace unexpected dotnet proxy directory: $dotnet_proxy_directory" >&2 + exit 1 +fi +rm -rf -- "$dotnet_proxy_directory" +mkdir -p -- "$dotnet_proxy_directory" +cp -- "$script_dir/dotnet_publish_proxy.py" "$dotnet_proxy_directory/dotnet" +chmod +x "$dotnet_proxy_directory/dotnet" + if [[ "$export_directory" != "$expected_export_directory" || "$export_directory" == "/" ]]; then echo "refusing to replace unexpected export directory: $export_directory" >&2 exit 1 @@ -29,9 +81,41 @@ fi rm -rf -- "$export_directory" mkdir -p -- "$export_directory" +set +e +DOTNET_CLI_USE_MSBUILD_SERVER=0 \ +MSBUILDDISABLENODEREUSE=1 \ +UseSharedCompilation=false \ +BuildInParallel=false \ +RestoreDisableParallel=true \ +DOTNET_gcServer=0 \ +DOTNET_GCConserveMemory=9 \ +AGE_REAL_DOTNET="$real_dotnet" \ +AGE_PUBLISH_PROJECT="$project_root/Himegari.csproj" \ +AGE_PREPUBLISHED_OUTPUT="$managed_proxy_directory" \ +AGE_PUBLISH_OUTPUT_ROOT="${TMPDIR:-/tmp}/godot-publish-dotnet" \ +AGE_PUBLISH_ASSEMBLY="Himegari.dll" \ +PATH="$dotnet_proxy_directory:$PATH" \ XDG_DATA_HOME="$xdg_data_home" "$godot_console" \ --headless --quit-after 120 --path "$project_root" \ --export-release "Linux x86_64" "$export_directory/Himegari.x86_64" +export_status=$? +set -e +if [[ $export_status -ne 0 ]]; then + if [[ $export_status -eq 137 ]]; then + echo "Godot export was killed with SIGKILL (137); inspect the runner memory cgroup diagnostics." >&2 + fi + exit "$export_status" +fi + +managed_export_directory="$export_directory/data_Himegari_linuxbsd_x86_64" +expected_managed_export_directory="$repo_root/build/export/linux-x64/data_Himegari_linuxbsd_x86_64" +if [[ "$managed_export_directory" != "$expected_managed_export_directory" || "$managed_export_directory" == "/" ]]; then + echo "refusing to replace unexpected managed export directory: $managed_export_directory" >&2 + exit 1 +fi +rm -rf -- "$managed_export_directory" +mkdir -p -- "$managed_export_directory" +cp -a -- "$managed_publish_directory/." "$managed_export_directory/" python3 -X utf8 "$script_dir/package_linux_x64.py" verify "$export_directory" mapfile -t package_outputs < <( diff --git a/tools/dotnet_publish_proxy.py b/tools/dotnet_publish_proxy.py new file mode 100755 index 0000000..63dc5be --- /dev/null +++ b/tools/dotnet_publish_proxy.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""Stage an exact prepublished payload for Godot's redundant export-time publish. + +All non-publish dotnet commands are delegated to AGE_REAL_DOTNET. A publish is +accepted only when its configuration, RID, self-contained flag, project, output +root, and prepublished assembly match the values supplied by the build driver. +""" +from __future__ import annotations + +import os +import shutil +import sys +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class PublishRequest: + project: Path + configuration: str + runtime: str + self_contained: str + output: Path + + +def _option(arguments: list[str], *names: str) -> str | None: + for index, argument in enumerate(arguments): + if argument in names: + if index + 1 >= len(arguments): + raise ValueError(f"missing value after {argument}") + return arguments[index + 1] + for name in names: + prefix = name + "=" + if argument.startswith(prefix): + return argument[len(prefix):] + return None + + +def parse_publish(arguments: list[str]) -> PublishRequest: + if not arguments or arguments[0] != "publish": + raise ValueError("arguments are not a dotnet publish command") + project = next( + (Path(argument) for argument in arguments[1:] if argument.lower().endswith(".csproj")), + None, + ) + configuration = _option(arguments, "-c", "--configuration") + runtime = _option(arguments, "-r", "--runtime") + self_contained = _option(arguments, "--self-contained") + output = _option(arguments, "-o", "--output") + missing = [ + name for name, value in ( + ("project", project), + ("configuration", configuration), + ("runtime", runtime), + ("self-contained", self_contained), + ("output", output), + ) if value is None + ] + if missing: + raise ValueError("publish command is missing: " + ", ".join(missing)) + assert project is not None and configuration is not None and runtime is not None + assert self_contained is not None and output is not None + return PublishRequest(project, configuration, runtime, self_contained, Path(output)) + + +def _is_within(path: Path, root: Path) -> bool: + try: + path.relative_to(root) + return True + except ValueError: + return False + + +def stage_publish(request: PublishRequest, environ: dict[str, str]) -> Path: + expected_project = Path(environ["AGE_PUBLISH_PROJECT"]).resolve() + source = Path(environ["AGE_PREPUBLISHED_OUTPUT"]).resolve() + output_root = Path(environ["AGE_PUBLISH_OUTPUT_ROOT"]).resolve() + assembly = environ["AGE_PUBLISH_ASSEMBLY"] + output = request.output.resolve() + + if request.project.resolve() != expected_project: + raise ValueError(f"unexpected publish project: {request.project}") + if request.configuration != "ExportRelease": + raise ValueError(f"unexpected publish configuration: {request.configuration}") + if request.runtime != "linux-x64": + raise ValueError(f"unexpected publish runtime: {request.runtime}") + if request.self_contained.lower() != "true": + raise ValueError(f"publish is not self-contained: {request.self_contained}") + if output == output_root or not _is_within(output, output_root): + raise ValueError(f"publish output is outside the reserved root: {output}") + if not (source / assembly).is_file(): + raise ValueError(f"prepublished assembly was not found: {source / assembly}") + + output.mkdir(parents=True, exist_ok=True) + if any(output.iterdir()): + raise ValueError(f"publish output directory is not empty: {output}") + for child in source.iterdir(): + target = output / child.name + if child.is_dir(): + shutil.copytree(child, target, symlinks=True) + else: + shutil.copy2(child, target, follow_symlinks=False) + return output + + +def main(arguments: list[str] | None = None) -> int: + args = list(sys.argv[1:] if arguments is None else arguments) + real_dotnet = os.environ.get("AGE_REAL_DOTNET") + if not real_dotnet: + raise ValueError("AGE_REAL_DOTNET is required") + if not args or args[0] != "publish": + os.execv(real_dotnet, [real_dotnet, *args]) + request = parse_publish(args) + output = stage_publish(request, dict(os.environ)) + print(f"publish proxy: staged {output}") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (KeyError, OSError, ValueError) as error: + print(f"dotnet publish proxy refused request: {error}", file=sys.stderr) + raise SystemExit(2) diff --git a/tools/test_dotnet_publish_proxy.py b/tools/test_dotnet_publish_proxy.py new file mode 100755 index 0000000..4b9fee9 --- /dev/null +++ b/tools/test_dotnet_publish_proxy.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path + +import dotnet_publish_proxy + + +class DotnetPublishProxyTests(unittest.TestCase): + def test_stages_exact_expected_publish(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + project = root / "Himegari.csproj" + project.write_text("", encoding="utf-8") + source = root / "prepublished" + source.mkdir() + (source / "Himegari.dll").write_bytes(b"managed") + (source / "libhostfxr.so").write_bytes(b"runtime") + output_root = root / "godot-publish-dotnet" + output = output_root / "123-ExportRelease-linux-x64" + request = dotnet_publish_proxy.parse_publish([ + "publish", str(project), "--self-contained", "true", + "-c", "ExportRelease", "-r", "linux-x64", "-o", str(output), + "-p:GodotTargetPlatform=linuxbsd", + ]) + staged = dotnet_publish_proxy.stage_publish(request, { + "AGE_PUBLISH_PROJECT": str(project), + "AGE_PREPUBLISHED_OUTPUT": str(source), + "AGE_PUBLISH_OUTPUT_ROOT": str(output_root), + "AGE_PUBLISH_ASSEMBLY": "Himegari.dll", + }) + self.assertEqual(output.resolve(), staged) + self.assertEqual(b"managed", (staged / "Himegari.dll").read_bytes()) + self.assertEqual(b"runtime", (staged / "libhostfxr.so").read_bytes()) + + def test_rejects_drift_and_output_escape(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + project = root / "Himegari.csproj" + project.write_text("", encoding="utf-8") + source = root / "prepublished" + source.mkdir() + (source / "Himegari.dll").write_bytes(b"managed") + environment = { + "AGE_PUBLISH_PROJECT": str(project), + "AGE_PREPUBLISHED_OUTPUT": str(source), + "AGE_PUBLISH_OUTPUT_ROOT": str(root / "reserved"), + "AGE_PUBLISH_ASSEMBLY": "Himegari.dll", + } + wrong_runtime = dotnet_publish_proxy.PublishRequest( + project, "ExportRelease", "win-x64", "true", root / "reserved/output" + ) + with self.assertRaisesRegex(ValueError, "runtime"): + dotnet_publish_proxy.stage_publish(wrong_runtime, environment) + escaped = dotnet_publish_proxy.PublishRequest( + project, "ExportRelease", "linux-x64", "true", root / "outside" + ) + with self.assertRaisesRegex(ValueError, "outside"): + dotnet_publish_proxy.stage_publish(escaped, environment) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/validate.py b/tools/validate.py index 78c13f7..3e28a95 100644 --- a/tools/validate.py +++ b/tools/validate.py @@ -32,6 +32,7 @@ CORE_TESTS = ( "test_validate.py", "test_install_godot_templates.py", "test_package_linux_x64.py", + "test_dotnet_publish_proxy.py", "test_diff_optrace.py", "test_engine_ctx.py", "test_ghidra_handler_map.py",