Avoid nested publish OOM in Linux export

tools/build-linux-x64.sh

@@ -7,6 +7,9 @@ project_root="$repo_root/godot"
 export_directory="$repo_root/build/export/linux-x64"
 expected_export_directory="$repo_root/build/export/linux-x64"
 package_directory="$repo_root/build/package"
+managed_publish_directory="$repo_root/build/managed-publish/linux-x64"
+managed_proxy_directory="$repo_root/build/managed-publish-proxy/linux-x64"
+dotnet_proxy_directory="$repo_root/build/dotnet-export-proxy"
 toolchain_root="$repo_root/build/toolchains/godot-4.7-stable-mono-linux-x64"
 xdg_data_home="$toolchain_root/xdg-data"
 
@@ -22,6 +25,55 @@ godot_console="$("$script_dir/bootstrap-godot-linux-x64.sh")"
 ffmpeg_sdk="$("$repo_root/native/age_movie_ffmpeg/bootstrap-linux-x64.sh")"
 "$repo_root/native/age_movie_ffmpeg/build-linux-x64.sh" "$ffmpeg_sdk"
 
+# Godot keeps the editor resident while it launches dotnet publish. On constrained CI runners, that process
+# pair can exceed the job's memory cgroup. Produce the exact ExportRelease/linux-x64 payload first, without
+# compiler/build servers, so the memory-heavy compile and editor stages run sequentially.
+expected_managed_publish_directory="$repo_root/build/managed-publish/linux-x64"
+if [[ "$managed_publish_directory" != "$expected_managed_publish_directory" || "$managed_publish_directory" == "/" ]]; then
+    echo "refusing to replace unexpected managed publish directory: $managed_publish_directory" >&2
+    exit 1
+fi
+rm -rf -- "$managed_publish_directory"
+mkdir -p -- "$managed_publish_directory"
+DOTNET_CLI_USE_MSBUILD_SERVER=0 \
+MSBUILDDISABLENODEREUSE=1 \
+DOTNET_gcServer=0 \
+DOTNET_GCConserveMemory=9 \
+dotnet publish "$project_root/Himegari.csproj" \
+    --configuration ExportRelease \
+    --runtime linux-x64 \
+    --self-contained true \
+    --output "$managed_publish_directory" \
+    -p:GodotTargetPlatform=linuxbsd \
+    -p:UseSharedCompilation=false \
+    -p:BuildInParallel=false \
+    -p:RestoreDisableParallel=true \
+    -p:DebugType=None \
+    -p:DebugSymbols=false
+
+# With dotnet/embed_build_outputs=false, Godot places publish files outside the PCK. Give the editor only the
+# required profile assembly while it creates the real PCK, then stage the complete prepublish after the editor
+# exits. This avoids making the memory-heavy editor traverse/hash the self-contained runtime and FFmpeg bundle.
+expected_managed_proxy_directory="$repo_root/build/managed-publish-proxy/linux-x64"
+if [[ "$managed_proxy_directory" != "$expected_managed_proxy_directory" || "$managed_proxy_directory" == "/" ]]; then
+    echo "refusing to replace unexpected managed proxy directory: $managed_proxy_directory" >&2
+    exit 1
+fi
+rm -rf -- "$managed_proxy_directory"
+mkdir -p -- "$managed_proxy_directory"
+cp -- "$managed_publish_directory/Himegari.dll" "$managed_proxy_directory/Himegari.dll"
+
+real_dotnet="$(command -v dotnet)"
+expected_dotnet_proxy_directory="$repo_root/build/dotnet-export-proxy"
+if [[ "$dotnet_proxy_directory" != "$expected_dotnet_proxy_directory" || "$dotnet_proxy_directory" == "/" ]]; then
+    echo "refusing to replace unexpected dotnet proxy directory: $dotnet_proxy_directory" >&2
+    exit 1
+fi
+rm -rf -- "$dotnet_proxy_directory"
+mkdir -p -- "$dotnet_proxy_directory"
+cp -- "$script_dir/dotnet_publish_proxy.py" "$dotnet_proxy_directory/dotnet"
+chmod +x "$dotnet_proxy_directory/dotnet"
+
 if [[ "$export_directory" != "$expected_export_directory" || "$export_directory" == "/" ]]; then
     echo "refusing to replace unexpected export directory: $export_directory" >&2
     exit 1
@@ -29,9 +81,41 @@ fi
 rm -rf -- "$export_directory"
 mkdir -p -- "$export_directory"
 
+set +e
+DOTNET_CLI_USE_MSBUILD_SERVER=0 \
+MSBUILDDISABLENODEREUSE=1 \
+UseSharedCompilation=false \
+BuildInParallel=false \
+RestoreDisableParallel=true \
+DOTNET_gcServer=0 \
+DOTNET_GCConserveMemory=9 \
+AGE_REAL_DOTNET="$real_dotnet" \
+AGE_PUBLISH_PROJECT="$project_root/Himegari.csproj" \
+AGE_PREPUBLISHED_OUTPUT="$managed_proxy_directory" \
+AGE_PUBLISH_OUTPUT_ROOT="${TMPDIR:-/tmp}/godot-publish-dotnet" \
+AGE_PUBLISH_ASSEMBLY="Himegari.dll" \
+PATH="$dotnet_proxy_directory:$PATH" \
 XDG_DATA_HOME="$xdg_data_home" "$godot_console" \
     --headless --quit-after 120 --path "$project_root" \
     --export-release "Linux x86_64" "$export_directory/Himegari.x86_64"
+export_status=$?
+set -e
+if [[ $export_status -ne 0 ]]; then
+    if [[ $export_status -eq 137 ]]; then
+        echo "Godot export was killed with SIGKILL (137); inspect the runner memory cgroup diagnostics." >&2
+    fi
+    exit "$export_status"
+fi
+
+managed_export_directory="$export_directory/data_Himegari_linuxbsd_x86_64"
+expected_managed_export_directory="$repo_root/build/export/linux-x64/data_Himegari_linuxbsd_x86_64"
+if [[ "$managed_export_directory" != "$expected_managed_export_directory" || "$managed_export_directory" == "/" ]]; then
+    echo "refusing to replace unexpected managed export directory: $managed_export_directory" >&2
+    exit 1
+fi
+rm -rf -- "$managed_export_directory"
+mkdir -p -- "$managed_export_directory"
+cp -a -- "$managed_publish_directory/." "$managed_export_directory/"
 
 python3 -X utf8 "$script_dir/package_linux_x64.py" verify "$export_directory"
 mapfile -t package_outputs < <(

tools/dotnet_publish_proxy.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+"""Stage an exact prepublished payload for Godot's redundant export-time publish.
+
+All non-publish dotnet commands are delegated to AGE_REAL_DOTNET. A publish is
+accepted only when its configuration, RID, self-contained flag, project, output
+root, and prepublished assembly match the values supplied by the build driver.
+"""
+from __future__ import annotations
+
+import os
+import shutil
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass(frozen=True)
+class PublishRequest:
+    project: Path
+    configuration: str
+    runtime: str
+    self_contained: str
+    output: Path
+
+
+def _option(arguments: list[str], *names: str) -> str | None:
+    for index, argument in enumerate(arguments):
+        if argument in names:
+            if index + 1 >= len(arguments):
+                raise ValueError(f"missing value after {argument}")
+            return arguments[index + 1]
+        for name in names:
+            prefix = name + "="
+            if argument.startswith(prefix):
+                return argument[len(prefix):]
+    return None
+
+
+def parse_publish(arguments: list[str]) -> PublishRequest:
+    if not arguments or arguments[0] != "publish":
+        raise ValueError("arguments are not a dotnet publish command")
+    project = next(
+        (Path(argument) for argument in arguments[1:] if argument.lower().endswith(".csproj")),
+        None,
+    )
+    configuration = _option(arguments, "-c", "--configuration")
+    runtime = _option(arguments, "-r", "--runtime")
+    self_contained = _option(arguments, "--self-contained")
+    output = _option(arguments, "-o", "--output")
+    missing = [
+        name for name, value in (
+            ("project", project),
+            ("configuration", configuration),
+            ("runtime", runtime),
+            ("self-contained", self_contained),
+            ("output", output),
+        ) if value is None
+    ]
+    if missing:
+        raise ValueError("publish command is missing: " + ", ".join(missing))
+    assert project is not None and configuration is not None and runtime is not None
+    assert self_contained is not None and output is not None
+    return PublishRequest(project, configuration, runtime, self_contained, Path(output))
+
+
+def _is_within(path: Path, root: Path) -> bool:
+    try:
+        path.relative_to(root)
+        return True
+    except ValueError:
+        return False
+
+
+def stage_publish(request: PublishRequest, environ: dict[str, str]) -> Path:
+    expected_project = Path(environ["AGE_PUBLISH_PROJECT"]).resolve()
+    source = Path(environ["AGE_PREPUBLISHED_OUTPUT"]).resolve()
+    output_root = Path(environ["AGE_PUBLISH_OUTPUT_ROOT"]).resolve()
+    assembly = environ["AGE_PUBLISH_ASSEMBLY"]
+    output = request.output.resolve()
+
+    if request.project.resolve() != expected_project:
+        raise ValueError(f"unexpected publish project: {request.project}")
+    if request.configuration != "ExportRelease":
+        raise ValueError(f"unexpected publish configuration: {request.configuration}")
+    if request.runtime != "linux-x64":
+        raise ValueError(f"unexpected publish runtime: {request.runtime}")
+    if request.self_contained.lower() != "true":
+        raise ValueError(f"publish is not self-contained: {request.self_contained}")
+    if output == output_root or not _is_within(output, output_root):
+        raise ValueError(f"publish output is outside the reserved root: {output}")
+    if not (source / assembly).is_file():
+        raise ValueError(f"prepublished assembly was not found: {source / assembly}")
+
+    output.mkdir(parents=True, exist_ok=True)
+    if any(output.iterdir()):
+        raise ValueError(f"publish output directory is not empty: {output}")
+    for child in source.iterdir():
+        target = output / child.name
+        if child.is_dir():
+            shutil.copytree(child, target, symlinks=True)
+        else:
+            shutil.copy2(child, target, follow_symlinks=False)
+    return output
+
+
+def main(arguments: list[str] | None = None) -> int:
+    args = list(sys.argv[1:] if arguments is None else arguments)
+    real_dotnet = os.environ.get("AGE_REAL_DOTNET")
+    if not real_dotnet:
+        raise ValueError("AGE_REAL_DOTNET is required")
+    if not args or args[0] != "publish":
+        os.execv(real_dotnet, [real_dotnet, *args])
+    request = parse_publish(args)
+    output = stage_publish(request, dict(os.environ))
+    print(f"publish proxy: staged {output}")
+    return 0
+
+
+if __name__ == "__main__":
+    try:
+        raise SystemExit(main())
+    except (KeyError, OSError, ValueError) as error:
+        print(f"dotnet publish proxy refused request: {error}", file=sys.stderr)
+        raise SystemExit(2)

tools/test_dotnet_publish_proxy.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import tempfile
+import unittest
+from pathlib import Path
+
+import dotnet_publish_proxy
+
+
+class DotnetPublishProxyTests(unittest.TestCase):
+    def test_stages_exact_expected_publish(self) -> None:
+        with tempfile.TemporaryDirectory() as temporary:
+            root = Path(temporary)
+            project = root / "Himegari.csproj"
+            project.write_text("<Project />", encoding="utf-8")
+            source = root / "prepublished"
+            source.mkdir()
+            (source / "Himegari.dll").write_bytes(b"managed")
+            (source / "libhostfxr.so").write_bytes(b"runtime")
+            output_root = root / "godot-publish-dotnet"
+            output = output_root / "123-ExportRelease-linux-x64"
+            request = dotnet_publish_proxy.parse_publish([
+                "publish", str(project), "--self-contained", "true",
+                "-c", "ExportRelease", "-r", "linux-x64", "-o", str(output),
+                "-p:GodotTargetPlatform=linuxbsd",
+            ])
+            staged = dotnet_publish_proxy.stage_publish(request, {
+                "AGE_PUBLISH_PROJECT": str(project),
+                "AGE_PREPUBLISHED_OUTPUT": str(source),
+                "AGE_PUBLISH_OUTPUT_ROOT": str(output_root),
+                "AGE_PUBLISH_ASSEMBLY": "Himegari.dll",
+            })
+            self.assertEqual(output.resolve(), staged)
+            self.assertEqual(b"managed", (staged / "Himegari.dll").read_bytes())
+            self.assertEqual(b"runtime", (staged / "libhostfxr.so").read_bytes())
+
+    def test_rejects_drift_and_output_escape(self) -> None:
+        with tempfile.TemporaryDirectory() as temporary:
+            root = Path(temporary)
+            project = root / "Himegari.csproj"
+            project.write_text("<Project />", encoding="utf-8")
+            source = root / "prepublished"
+            source.mkdir()
+            (source / "Himegari.dll").write_bytes(b"managed")
+            environment = {
+                "AGE_PUBLISH_PROJECT": str(project),
+                "AGE_PREPUBLISHED_OUTPUT": str(source),
+                "AGE_PUBLISH_OUTPUT_ROOT": str(root / "reserved"),
+                "AGE_PUBLISH_ASSEMBLY": "Himegari.dll",
+            }
+            wrong_runtime = dotnet_publish_proxy.PublishRequest(
+                project, "ExportRelease", "win-x64", "true", root / "reserved/output"
+            )
+            with self.assertRaisesRegex(ValueError, "runtime"):
+                dotnet_publish_proxy.stage_publish(wrong_runtime, environment)
+            escaped = dotnet_publish_proxy.PublishRequest(
+                project, "ExportRelease", "linux-x64", "true", root / "outside"
+            )
+            with self.assertRaisesRegex(ValueError, "outside"):
+                dotnet_publish_proxy.stage_publish(escaped, environment)
+
+
+if __name__ == "__main__":
+    unittest.main()

tools/validate.py

@@ -32,6 +32,7 @@ CORE_TESTS = (
     "test_validate.py",
     "test_install_godot_templates.py",
     "test_package_linux_x64.py",
+    "test_dotnet_publish_proxy.py",
     "test_diff_optrace.py",
     "test_engine_ctx.py",
     "test_ghidra_handler_map.py",