Build verified Linux release packages
All checks were successful
Core validation / Linux core gate (push) Successful in 1m17s
All checks were successful
Core validation / Linux core gate (push) Successful in 1m17s
This commit is contained in:
71
tools/bootstrap-godot-linux-x64.sh
Executable file
71
tools/bootstrap-godot-linux-x64.sh
Executable file
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
repo_root="$(cd -- "$script_dir/.." && pwd)"
|
||||
manifest="$script_dir/godot-linux-x64.json"
|
||||
toolchain_root="$repo_root/build/toolchains/godot-4.7-stable-mono-linux-x64"
|
||||
xdg_data_home="$toolchain_root/xdg-data"
|
||||
download_dir="$repo_root/build/downloads"
|
||||
|
||||
manifest_value() {
|
||||
python3 -c 'import json, sys; from functools import reduce; data=json.load(open(sys.argv[1], encoding="utf-8")); print(reduce(lambda value, key: value[key], sys.argv[2:], data))' \
|
||||
"$manifest" "$@"
|
||||
}
|
||||
|
||||
for command in python3 curl sha256sum find awk; do
|
||||
command -v "$command" >/dev/null 2>&1 || {
|
||||
echo "required command was not found: $command" >&2
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
|
||||
archive="$(manifest_value editor archive)"
|
||||
url="$(manifest_value editor url)"
|
||||
expected_hash="$(manifest_value editor sha256)"
|
||||
expected_size="$(manifest_value editor size)"
|
||||
executable_name="$(manifest_value editor executable)"
|
||||
template_version="$(manifest_value godot_version)"
|
||||
archive_path="$download_dir/$archive"
|
||||
editor_root="$toolchain_root/editor"
|
||||
template_root="$xdg_data_home/godot/export_templates/$template_version"
|
||||
|
||||
mkdir -p -- "$download_dir" "$editor_root" "$template_root"
|
||||
if [[ ! -f "$archive_path" ]]; then
|
||||
curl --fail --location --retry 3 --output "$archive_path" "$url"
|
||||
fi
|
||||
actual_size="$(wc -c < "$archive_path")"
|
||||
if [[ "$actual_size" != "$expected_size" ]]; then
|
||||
echo "Godot editor archive size mismatch: expected $expected_size, got $actual_size" >&2
|
||||
exit 1
|
||||
fi
|
||||
actual_hash="$(sha256sum "$archive_path" | awk '{ print $1 }')"
|
||||
if [[ "$actual_hash" != "$expected_hash" ]]; then
|
||||
echo "Godot editor archive SHA-256 mismatch: expected $expected_hash, got $actual_hash" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mapfile -t editors < <(find "$editor_root" -type f -name "$executable_name" -print)
|
||||
if [[ ${#editors[@]} -eq 0 ]]; then
|
||||
python3 -c 'import pathlib, sys, zipfile; zipfile.ZipFile(sys.argv[1]).extractall(pathlib.Path(sys.argv[2]))' \
|
||||
"$archive_path" "$editor_root"
|
||||
mapfile -t editors < <(find "$editor_root" -type f -name "$executable_name" -print)
|
||||
fi
|
||||
if [[ ${#editors[@]} -ne 1 ]]; then
|
||||
echo "expected exactly one $executable_name under $editor_root, found ${#editors[@]}" >&2
|
||||
exit 1
|
||||
fi
|
||||
editor="${editors[0]}"
|
||||
chmod +x "$editor"
|
||||
|
||||
python3 -X utf8 "$script_dir/install_godot_templates.py" \
|
||||
--manifest "$manifest" --destination "$template_root" >&2
|
||||
|
||||
reported="$(XDG_DATA_HOME="$xdg_data_home" "$editor" --headless --version)"
|
||||
reported="${reported%%$'\n'*}"
|
||||
if [[ "$reported" != 4.7.stable.mono* ]]; then
|
||||
echo "unexpected Godot editor version: $reported" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf '%s\n' "$editor"
|
||||
63
tools/build-linux-x64.sh
Executable file
63
tools/build-linux-x64.sh
Executable file
@@ -0,0 +1,63 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
repo_root="$(cd -- "$script_dir/.." && pwd)"
|
||||
project_root="$repo_root/godot"
|
||||
export_directory="$repo_root/build/export/linux-x64"
|
||||
expected_export_directory="$repo_root/build/export/linux-x64"
|
||||
package_directory="$repo_root/build/package"
|
||||
toolchain_root="$repo_root/build/toolchains/godot-4.7-stable-mono-linux-x64"
|
||||
xdg_data_home="$toolchain_root/xdg-data"
|
||||
|
||||
for command in python3 dotnet cc readelf ldd curl sha256sum tar; do
|
||||
command -v "$command" >/dev/null 2>&1 || {
|
||||
echo "required command was not found: $command" >&2
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
|
||||
python3 -X utf8 "$script_dir/opcodes_build.py" --build
|
||||
godot_console="$("$script_dir/bootstrap-godot-linux-x64.sh")"
|
||||
ffmpeg_sdk="$("$repo_root/native/age_movie_ffmpeg/bootstrap-linux-x64.sh")"
|
||||
"$repo_root/native/age_movie_ffmpeg/build-linux-x64.sh" "$ffmpeg_sdk"
|
||||
|
||||
if [[ "$export_directory" != "$expected_export_directory" || "$export_directory" == "/" ]]; then
|
||||
echo "refusing to replace unexpected export directory: $export_directory" >&2
|
||||
exit 1
|
||||
fi
|
||||
rm -rf -- "$export_directory"
|
||||
mkdir -p -- "$export_directory"
|
||||
|
||||
XDG_DATA_HOME="$xdg_data_home" "$godot_console" \
|
||||
--headless --quit-after 120 --path "$project_root" \
|
||||
--export-release "Linux x86_64" "$export_directory/Himegari.x86_64"
|
||||
|
||||
python3 -X utf8 "$script_dir/package_linux_x64.py" verify "$export_directory"
|
||||
mapfile -t package_outputs < <(
|
||||
python3 -X utf8 "$script_dir/package_linux_x64.py" package "$export_directory" \
|
||||
--output-directory "$package_directory"
|
||||
)
|
||||
if [[ ${#package_outputs[@]} -ne 2 ]]; then
|
||||
echo "packager returned an unexpected result" >&2
|
||||
exit 1
|
||||
fi
|
||||
package_root="${package_outputs[0]}"
|
||||
archive_path="${package_outputs[1]}"
|
||||
smoke_log="$package_directory/package-smoke.log"
|
||||
set +e
|
||||
"$package_root/Himegari.x86_64" --headless -- --package-smoke >"$smoke_log" 2>&1
|
||||
smoke_status=$?
|
||||
set -e
|
||||
cat "$smoke_log"
|
||||
if [[ $smoke_status -ne 0 ]]; then
|
||||
echo "packaged Linux runtime smoke failed with exit code $smoke_status" >&2
|
||||
exit "$smoke_status"
|
||||
fi
|
||||
if ! grep -Fq "PACKAGE SMOKE OK: opcodes=548 ffmpeg-abi=3" "$smoke_log"; then
|
||||
echo "packaged Linux runtime did not report its success marker" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
archive_hash="$(sha256sum "$archive_path" | awk '{ print $1 }')"
|
||||
printf 'Linux x64 package: %s\nSHA-256: %s\n' "$archive_path" "$archive_hash"
|
||||
@@ -37,32 +37,9 @@ if ($LASTEXITCODE -ne 0) {
|
||||
throw "Godot Linux x64 export failed with exit code $LASTEXITCODE. Install the Godot 4.7 .NET export templates and retry."
|
||||
}
|
||||
|
||||
$managedDirectory = Join-Path $outputDirectory 'data_Himegari_linuxbsd_x86_64'
|
||||
foreach ($required in @(
|
||||
$executable,
|
||||
(Join-Path $outputDirectory 'Himegari.pck'),
|
||||
(Join-Path $managedDirectory 'Himegari.dll'),
|
||||
(Join-Path $managedDirectory 'Age.Engine.dll'),
|
||||
(Join-Path $managedDirectory 'libage_movie_ffmpeg.so'),
|
||||
(Join-Path $managedDirectory 'libavformat.so.62'),
|
||||
(Join-Path $managedDirectory 'libavcodec.so.62'),
|
||||
(Join-Path $managedDirectory 'libavutil.so.60'),
|
||||
(Join-Path $managedDirectory 'libswscale.so.9'),
|
||||
(Join-Path $managedDirectory 'libswresample.so.6'),
|
||||
(Join-Path $managedDirectory 'FFmpeg-LICENSE.txt')
|
||||
)) {
|
||||
if (-not (Test-Path -LiteralPath $required -PathType Leaf)) {
|
||||
throw "Linux export is incomplete; expected artifact was not found: $required"
|
||||
}
|
||||
}
|
||||
foreach ($forbidden in @(
|
||||
(Join-Path $managedDirectory 'Age.Engine.Text.Windows.dll'),
|
||||
(Join-Path $managedDirectory 'age_movie_ffmpeg.dll'),
|
||||
(Join-Path $managedDirectory 'avformat-62.dll')
|
||||
)) {
|
||||
if (Test-Path -LiteralPath $forbidden) {
|
||||
throw "Linux export contains a Windows-only artifact: $forbidden"
|
||||
}
|
||||
py -3.11 -X utf8 (Join-Path $repoRoot 'tools\package_linux_x64.py') verify $outputDirectory
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Linux export payload verification failed with exit code $LASTEXITCODE."
|
||||
}
|
||||
|
||||
Write-Output "Linux x64 export: $outputDirectory"
|
||||
|
||||
27
tools/godot-linux-x64.json
Normal file
27
tools/godot-linux-x64.json
Normal file
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"godot_version": "4.7.stable.mono",
|
||||
"release_tag": "4.7-stable",
|
||||
"editor": {
|
||||
"archive": "Godot_v4.7-stable_mono_linux_x86_64.zip",
|
||||
"url": "https://github.com/godotengine/godot-builds/releases/download/4.7-stable/Godot_v4.7-stable_mono_linux_x86_64.zip",
|
||||
"sha256": "69e855001e34b108eb8124ff1eae8445026b2a30a83b6e6314f705ae963d0fe1",
|
||||
"size": 105506785,
|
||||
"executable": "Godot_v4.7-stable_mono_linux.x86_64"
|
||||
},
|
||||
"templates": {
|
||||
"archive": "Godot_v4.7-stable_mono_export_templates.tpz",
|
||||
"url": "https://downloads.godotengine.org/?version=4.7&flavor=stable&slug=mono_export_templates.tpz&platform=templates",
|
||||
"sha256": "4c02a0b99ad9c5bc243c2e79468628db3df89350d9db8fc995988f69d126e069",
|
||||
"size": 1200753503,
|
||||
"members": [
|
||||
{
|
||||
"archive_path": "templates/linux_release.x86_64",
|
||||
"install_name": "linux_release.x86_64",
|
||||
"sha256": "26b1b5d076b78c02f1476dab78e183218a6aa30c8e614653eac11a0dd198456a",
|
||||
"size": 73591000,
|
||||
"mode": "0755"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
221
tools/install_godot_templates.py
Executable file
221
tools/install_godot_templates.py
Executable file
@@ -0,0 +1,221 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Install selected, pinned Godot export templates with HTTP range requests.
|
||||
|
||||
Godot 4.7 stores every platform template in one archive. This reader mirrors the
|
||||
editor's selective downloader: ZIP metadata and only the requested compressed
|
||||
members cross the network. Each installed member is then checked against the
|
||||
project manifest's size and SHA-256.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import zipfile
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
|
||||
DEFAULT_WINDOW = 8 * 1024 * 1024
|
||||
HTTP_HEADERS = {
|
||||
"User-Agent": "OpenMaidEngine-build/1.0 (+https://git.orfl.xyz/conco/OpenMaidEngine)",
|
||||
"Accept": "application/octet-stream,*/*;q=0.8",
|
||||
}
|
||||
|
||||
|
||||
class RangeReader(io.RawIOBase):
|
||||
"""Seekable read-only view backed by a byte-range callback."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
length: int,
|
||||
fetch: Callable[[int, int], bytes],
|
||||
window_size: int = DEFAULT_WINDOW,
|
||||
) -> None:
|
||||
if length <= 0:
|
||||
raise ValueError("range source length must be positive")
|
||||
self._length = length
|
||||
self._fetch = fetch
|
||||
self._window_size = window_size
|
||||
self._position = 0
|
||||
self._cache_start = 0
|
||||
self._cache = b""
|
||||
|
||||
def readable(self) -> bool:
|
||||
return True
|
||||
|
||||
def seekable(self) -> bool:
|
||||
return True
|
||||
|
||||
def tell(self) -> int:
|
||||
return self._position
|
||||
|
||||
def seek(self, offset: int, whence: int = io.SEEK_SET) -> int:
|
||||
if whence == io.SEEK_SET:
|
||||
position = offset
|
||||
elif whence == io.SEEK_CUR:
|
||||
position = self._position + offset
|
||||
elif whence == io.SEEK_END:
|
||||
position = self._length + offset
|
||||
else:
|
||||
raise ValueError(f"unsupported seek mode: {whence}")
|
||||
if position < 0:
|
||||
raise ValueError("negative seek position")
|
||||
self._position = position
|
||||
return position
|
||||
|
||||
def read(self, size: int = -1) -> bytes:
|
||||
if self._position >= self._length:
|
||||
return b""
|
||||
if size is None or size < 0:
|
||||
size = self._length - self._position
|
||||
size = min(size, self._length - self._position)
|
||||
if size == 0:
|
||||
return b""
|
||||
|
||||
cache_end = self._cache_start + len(self._cache)
|
||||
requested_end = self._position + size
|
||||
if not (
|
||||
self._cache_start <= self._position
|
||||
and requested_end <= cache_end
|
||||
):
|
||||
fetch_size = max(size, self._window_size)
|
||||
fetch_end = min(self._length, self._position + fetch_size)
|
||||
self._cache_start = self._position
|
||||
self._cache = self._fetch(self._position, fetch_end - 1)
|
||||
expected = fetch_end - self._position
|
||||
if len(self._cache) != expected:
|
||||
raise OSError(
|
||||
f"short range response: expected {expected} bytes, "
|
||||
f"received {len(self._cache)}"
|
||||
)
|
||||
cache_end = fetch_end
|
||||
|
||||
offset = self._position - self._cache_start
|
||||
data = self._cache[offset:offset + size]
|
||||
self._position += len(data)
|
||||
return data
|
||||
|
||||
|
||||
def _open_with_retry(request: urllib.request.Request, attempts: int = 3):
|
||||
last_error: Exception | None = None
|
||||
for attempt in range(attempts):
|
||||
try:
|
||||
return urllib.request.urlopen(request, timeout=120)
|
||||
except (OSError, urllib.error.URLError) as error:
|
||||
last_error = error
|
||||
if attempt + 1 < attempts:
|
||||
time.sleep(2 ** attempt)
|
||||
assert last_error is not None
|
||||
raise last_error
|
||||
|
||||
|
||||
def open_http_range_reader(url: str, expected_length: int) -> RangeReader:
|
||||
head = urllib.request.Request(url, headers=HTTP_HEADERS, method="HEAD")
|
||||
with _open_with_retry(head) as response:
|
||||
final_url = response.geturl()
|
||||
actual_length = int(response.headers.get("Content-Length", "0"))
|
||||
if actual_length != expected_length:
|
||||
raise ValueError(
|
||||
f"template archive size mismatch: expected {expected_length}, "
|
||||
f"server reported {actual_length}"
|
||||
)
|
||||
|
||||
def fetch(start: int, end: int) -> bytes:
|
||||
request_headers = dict(HTTP_HEADERS)
|
||||
request_headers.update({
|
||||
"Range": f"bytes={start}-{end}",
|
||||
"Accept-Encoding": "identity",
|
||||
})
|
||||
request = urllib.request.Request(final_url, headers=request_headers)
|
||||
with _open_with_retry(request) as response:
|
||||
if response.status != 206:
|
||||
raise OSError(
|
||||
f"template server ignored byte range {start}-{end}: "
|
||||
f"HTTP {response.status}"
|
||||
)
|
||||
content_range = response.headers.get("Content-Range", "")
|
||||
expected_range = f"bytes {start}-{end}/{expected_length}"
|
||||
if content_range != expected_range:
|
||||
raise OSError(
|
||||
f"unexpected Content-Range: expected {expected_range!r}, "
|
||||
f"received {content_range!r}"
|
||||
)
|
||||
return response.read()
|
||||
|
||||
return RangeReader(expected_length, fetch)
|
||||
|
||||
|
||||
def file_matches(path: Path, expected_size: int, expected_hash: str) -> bool:
|
||||
if not path.is_file() or path.stat().st_size != expected_size:
|
||||
return False
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for block in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(block)
|
||||
return digest.hexdigest() == expected_hash.lower()
|
||||
|
||||
|
||||
def install_members(manifest_path: Path, destination: Path) -> list[Path]:
|
||||
with manifest_path.open(encoding="utf-8") as stream:
|
||||
manifest = json.load(stream)
|
||||
templates = manifest["templates"]
|
||||
members = templates["members"]
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
installed = [destination / member["install_name"] for member in members]
|
||||
if all(
|
||||
file_matches(path, member["size"], member["sha256"])
|
||||
for path, member in zip(installed, members, strict=True)
|
||||
):
|
||||
return installed
|
||||
|
||||
reader = open_http_range_reader(templates["url"], templates["size"])
|
||||
with zipfile.ZipFile(reader) as archive:
|
||||
for path, member in zip(installed, members, strict=True):
|
||||
if file_matches(path, member["size"], member["sha256"]):
|
||||
continue
|
||||
archive_path = member["archive_path"]
|
||||
info = archive.getinfo(archive_path)
|
||||
if info.file_size != member["size"]:
|
||||
raise ValueError(
|
||||
f"template member size mismatch for {archive_path}: "
|
||||
f"expected {member['size']}, archive reports {info.file_size}"
|
||||
)
|
||||
temporary = path.with_suffix(path.suffix + ".part")
|
||||
digest = hashlib.sha256()
|
||||
with archive.open(info) as source, temporary.open("wb") as target:
|
||||
for block in iter(lambda: source.read(1024 * 1024), b""):
|
||||
target.write(block)
|
||||
digest.update(block)
|
||||
if temporary.stat().st_size != member["size"]:
|
||||
temporary.unlink(missing_ok=True)
|
||||
raise ValueError(f"short extracted template member: {archive_path}")
|
||||
actual_hash = digest.hexdigest()
|
||||
if actual_hash != member["sha256"].lower():
|
||||
temporary.unlink(missing_ok=True)
|
||||
raise ValueError(
|
||||
f"template member SHA-256 mismatch for {archive_path}: "
|
||||
f"expected {member['sha256']}, got {actual_hash}"
|
||||
)
|
||||
os.chmod(temporary, int(member["mode"], 8))
|
||||
os.replace(temporary, path)
|
||||
return installed
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--manifest", type=Path, required=True)
|
||||
parser.add_argument("--destination", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
for installed in install_members(args.manifest.resolve(), args.destination.resolve()):
|
||||
print(installed)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
215
tools/package_linux_x64.py
Executable file
215
tools/package_linux_x64.py
Executable file
@@ -0,0 +1,215 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify and package the Linux x64 Godot export."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import tarfile
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
REPO = HERE.parent
|
||||
MANAGED_DIRECTORY = "data_Himegari_linuxbsd_x86_64"
|
||||
PACKAGE_NAME = "OpenMaidEngine-Himegari-linux-x64"
|
||||
REQUIRED_FILES = (
|
||||
"Himegari.x86_64",
|
||||
"Himegari.pck",
|
||||
f"{MANAGED_DIRECTORY}/Himegari.dll",
|
||||
f"{MANAGED_DIRECTORY}/Age.Engine.dll",
|
||||
f"{MANAGED_DIRECTORY}/Age.Engine.Frontend.dll",
|
||||
f"{MANAGED_DIRECTORY}/libage_movie_ffmpeg.so",
|
||||
f"{MANAGED_DIRECTORY}/libavformat.so.62",
|
||||
f"{MANAGED_DIRECTORY}/libavcodec.so.62",
|
||||
f"{MANAGED_DIRECTORY}/libavutil.so.60",
|
||||
f"{MANAGED_DIRECTORY}/libswscale.so.9",
|
||||
f"{MANAGED_DIRECTORY}/libswresample.so.6",
|
||||
f"{MANAGED_DIRECTORY}/FFmpeg-LICENSE.txt",
|
||||
)
|
||||
FORBIDDEN_FILES = (
|
||||
f"{MANAGED_DIRECTORY}/Age.Engine.Text.Windows.dll",
|
||||
f"{MANAGED_DIRECTORY}/age_movie_ffmpeg.dll",
|
||||
f"{MANAGED_DIRECTORY}/avformat-62.dll",
|
||||
)
|
||||
PROJECT_FILES = ("LICENSE", "README.md", "THIRD_PARTY_NOTICES.md")
|
||||
|
||||
|
||||
def _resolved_directory(path: Path, label: str) -> Path:
|
||||
resolved = path.resolve()
|
||||
if not resolved.is_dir():
|
||||
raise ValueError(f"{label} directory was not found: {resolved}")
|
||||
return resolved
|
||||
|
||||
|
||||
def verify_export(export_directory: Path) -> Path:
|
||||
export_directory = _resolved_directory(export_directory, "Linux export")
|
||||
missing = [name for name in REQUIRED_FILES if not (export_directory / name).is_file()]
|
||||
if missing:
|
||||
raise ValueError("Linux export is incomplete; missing: " + ", ".join(missing))
|
||||
forbidden = [name for name in FORBIDDEN_FILES if (export_directory / name).exists()]
|
||||
if forbidden:
|
||||
raise ValueError("Linux export contains Windows-only files: " + ", ".join(forbidden))
|
||||
executable = export_directory / "Himegari.x86_64"
|
||||
if os.name != "nt" and not os.access(executable, os.X_OK):
|
||||
raise ValueError(f"Linux export executable bit is not set: {executable}")
|
||||
return export_directory
|
||||
|
||||
|
||||
def _git(*arguments: str) -> str:
|
||||
result = subprocess.run(
|
||||
("git", *arguments), cwd=REPO, check=True, capture_output=True, text=True
|
||||
)
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def source_date_epoch() -> int:
|
||||
configured = os.environ.get("SOURCE_DATE_EPOCH")
|
||||
value = configured or _git("show", "-s", "--format=%ct", "HEAD")
|
||||
try:
|
||||
epoch = int(value)
|
||||
except ValueError as error:
|
||||
raise ValueError(f"invalid SOURCE_DATE_EPOCH: {value!r}") from error
|
||||
if epoch < 0:
|
||||
raise ValueError("SOURCE_DATE_EPOCH must not be negative")
|
||||
return epoch
|
||||
|
||||
|
||||
def build_metadata(epoch: int) -> dict[str, object]:
|
||||
with (HERE / "godot-linux-x64.json").open(encoding="utf-8") as stream:
|
||||
godot = json.load(stream)
|
||||
with (REPO / "native/age_movie_ffmpeg/dependency-linux-x64.json").open(
|
||||
encoding="utf-8"
|
||||
) as stream:
|
||||
ffmpeg = json.load(stream)
|
||||
dirty = bool(_git("status", "--porcelain", "--untracked-files=normal"))
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"project": "OpenMaidEngine Himegari profile",
|
||||
"target": "linux-x64",
|
||||
"source_commit": _git("rev-parse", "HEAD"),
|
||||
"source_dirty": dirty,
|
||||
"source_date_epoch": epoch,
|
||||
"godot": {
|
||||
"version": godot["godot_version"],
|
||||
"editor_archive_sha256": godot["editor"]["sha256"],
|
||||
"release_template_sha256": godot["templates"]["members"][0]["sha256"],
|
||||
},
|
||||
"ffmpeg": {
|
||||
"provider": ffmpeg["provider"],
|
||||
"version": ffmpeg["ffmpeg_version"],
|
||||
"archive_sha256": ffmpeg["sha256"],
|
||||
"minimum_glibc": ffmpeg["minimum_glibc"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _hash(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for block in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _write_checksums(package_root: Path) -> None:
|
||||
checksum_path = package_root / "SHA256SUMS"
|
||||
files = sorted(
|
||||
path for path in package_root.rglob("*")
|
||||
if path.is_file() and path != checksum_path
|
||||
)
|
||||
lines = [f"{_hash(path)} {path.relative_to(package_root).as_posix()}" for path in files]
|
||||
checksum_path.write_text("\n".join(lines) + "\n", encoding="utf-8", newline="\n")
|
||||
|
||||
|
||||
def _tar_info(info: tarfile.TarInfo, epoch: int) -> tarfile.TarInfo:
|
||||
info.uid = 0
|
||||
info.gid = 0
|
||||
info.uname = "root"
|
||||
info.gname = "root"
|
||||
info.mtime = epoch
|
||||
if info.isdir():
|
||||
info.mode = 0o755
|
||||
elif info.isfile():
|
||||
info.mode = 0o755 if info.mode & stat.S_IXUSR else 0o644
|
||||
return info
|
||||
|
||||
|
||||
def create_package(
|
||||
export_directory: Path,
|
||||
output_directory: Path,
|
||||
metadata: dict[str, object],
|
||||
epoch: int,
|
||||
) -> tuple[Path, Path]:
|
||||
export_directory = verify_export(export_directory)
|
||||
output_directory = output_directory.resolve()
|
||||
expected_parent = (REPO / "build/package").resolve()
|
||||
if output_directory != expected_parent:
|
||||
raise ValueError(
|
||||
f"refusing to replace unexpected package directory: {output_directory}"
|
||||
)
|
||||
staging_parent = output_directory / "staging"
|
||||
package_root = staging_parent / PACKAGE_NAME
|
||||
archive_path = output_directory / f"{PACKAGE_NAME}.tar.gz"
|
||||
if staging_parent.exists():
|
||||
shutil.rmtree(staging_parent)
|
||||
staging_parent.mkdir(parents=True)
|
||||
shutil.copytree(export_directory, package_root, copy_function=shutil.copy2)
|
||||
for name in PROJECT_FILES:
|
||||
source = REPO / name
|
||||
if not source.is_file():
|
||||
raise ValueError(f"package notice file was not found: {source}")
|
||||
shutil.copy2(source, package_root / name)
|
||||
(package_root / "BUILD-INFO.json").write_text(
|
||||
json.dumps(metadata, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
newline="\n",
|
||||
)
|
||||
_write_checksums(package_root)
|
||||
|
||||
output_directory.mkdir(parents=True, exist_ok=True)
|
||||
temporary_archive = archive_path.with_suffix(archive_path.suffix + ".part")
|
||||
with temporary_archive.open("wb") as raw:
|
||||
with gzip.GzipFile(fileobj=raw, mode="wb", filename="", mtime=epoch) as compressed:
|
||||
with tarfile.open(fileobj=compressed, mode="w", format=tarfile.PAX_FORMAT) as archive:
|
||||
paths = [package_root, *sorted(package_root.rglob("*"))]
|
||||
for path in paths:
|
||||
arcname = path.relative_to(staging_parent).as_posix()
|
||||
archive.add(
|
||||
path,
|
||||
arcname=arcname,
|
||||
recursive=False,
|
||||
filter=lambda info, fixed_epoch=epoch: _tar_info(info, fixed_epoch),
|
||||
)
|
||||
os.replace(temporary_archive, archive_path)
|
||||
return package_root, archive_path
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
verify = subparsers.add_parser("verify")
|
||||
verify.add_argument("export_directory", type=Path)
|
||||
package = subparsers.add_parser("package")
|
||||
package.add_argument("export_directory", type=Path)
|
||||
package.add_argument("--output-directory", type=Path, default=REPO / "build/package")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == "verify":
|
||||
print(verify_export(args.export_directory))
|
||||
return 0
|
||||
epoch = source_date_epoch()
|
||||
package_root, archive_path = create_package(
|
||||
args.export_directory, args.output_directory, build_metadata(epoch), epoch
|
||||
)
|
||||
print(package_root)
|
||||
print(archive_path)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
79
tools/test_install_godot_templates.py
Executable file
79
tools/test_install_godot_templates.py
Executable file
@@ -0,0 +1,79 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import install_godot_templates
|
||||
|
||||
|
||||
class RangeReaderTests(unittest.TestCase):
|
||||
def test_seek_and_windowed_reads(self) -> None:
|
||||
source = bytes(range(251)) * 100
|
||||
requests: list[tuple[int, int]] = []
|
||||
|
||||
def fetch(start: int, end: int) -> bytes:
|
||||
requests.append((start, end))
|
||||
return source[start:end + 1]
|
||||
|
||||
reader = install_godot_templates.RangeReader(len(source), fetch, window_size=64)
|
||||
reader.seek(103)
|
||||
self.assertEqual(source[103:113], reader.read(10))
|
||||
self.assertEqual(source[113:123], reader.read(10))
|
||||
reader.seek(-8, io.SEEK_END)
|
||||
self.assertEqual(source[-8:], reader.read())
|
||||
self.assertEqual(2, len(requests))
|
||||
|
||||
def test_installs_only_selected_member_and_reuses_verified_file(self) -> None:
|
||||
release = b"linux release template\0" * 200
|
||||
unused = b"other platform" * 100
|
||||
archive_buffer = io.BytesIO()
|
||||
with zipfile.ZipFile(archive_buffer, "w", zipfile.ZIP_DEFLATED) as archive:
|
||||
archive.writestr("templates/linux_release.x86_64", release)
|
||||
archive.writestr("templates/windows_release_x86_64.exe", unused)
|
||||
archive_bytes = archive_buffer.getvalue()
|
||||
|
||||
def reader() -> install_godot_templates.RangeReader:
|
||||
return install_godot_templates.RangeReader(
|
||||
len(archive_bytes),
|
||||
lambda start, end: archive_bytes[start:end + 1],
|
||||
window_size=128,
|
||||
)
|
||||
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
manifest = root / "manifest.json"
|
||||
manifest.write_text(json.dumps({
|
||||
"templates": {
|
||||
"url": "https://invalid.example/templates.tpz",
|
||||
"size": len(archive_bytes),
|
||||
"members": [{
|
||||
"archive_path": "templates/linux_release.x86_64",
|
||||
"install_name": "linux_release.x86_64",
|
||||
"sha256": hashlib.sha256(release).hexdigest(),
|
||||
"size": len(release),
|
||||
"mode": "0755",
|
||||
}],
|
||||
}
|
||||
}), encoding="utf-8")
|
||||
destination = root / "templates"
|
||||
with patch.object(install_godot_templates, "open_http_range_reader", return_value=reader()) as opened:
|
||||
installed = install_godot_templates.install_members(manifest, destination)
|
||||
opened.assert_called_once()
|
||||
self.assertEqual(release, installed[0].read_bytes())
|
||||
self.assertFalse((destination / "windows_release_x86_64.exe").exists())
|
||||
|
||||
with patch.object(install_godot_templates, "open_http_range_reader") as opened:
|
||||
reused = install_godot_templates.install_members(manifest, destination)
|
||||
opened.assert_not_called()
|
||||
self.assertEqual(installed, reused)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
73
tools/test_package_linux_x64.py
Executable file
73
tools/test_package_linux_x64.py
Executable file
@@ -0,0 +1,73 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import package_linux_x64
|
||||
|
||||
|
||||
def make_export(root: Path) -> Path:
|
||||
export = root / "export"
|
||||
for relative in package_linux_x64.REQUIRED_FILES:
|
||||
path = export / relative
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(f"fixture:{relative}".encode())
|
||||
(export / "Himegari.x86_64").chmod(0o755)
|
||||
return export
|
||||
|
||||
|
||||
class PackageLinuxX64Tests(unittest.TestCase):
|
||||
def test_verify_rejects_missing_and_windows_files(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
export = make_export(Path(temporary))
|
||||
package_linux_x64.verify_export(export)
|
||||
(export / package_linux_x64.REQUIRED_FILES[-1]).unlink()
|
||||
with self.assertRaisesRegex(ValueError, "missing"):
|
||||
package_linux_x64.verify_export(export)
|
||||
(export / package_linux_x64.REQUIRED_FILES[-1]).write_bytes(b"restored")
|
||||
forbidden = export / package_linux_x64.FORBIDDEN_FILES[0]
|
||||
forbidden.write_bytes(b"windows")
|
||||
with self.assertRaisesRegex(ValueError, "Windows-only"):
|
||||
package_linux_x64.verify_export(export)
|
||||
|
||||
@unittest.skipIf(package_linux_x64.os.name == "nt", "executable-bit gate is POSIX-only")
|
||||
def test_verify_requires_executable_bit_on_posix(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
export = make_export(Path(temporary))
|
||||
(export / "Himegari.x86_64").chmod(0o644)
|
||||
with self.assertRaisesRegex(ValueError, "executable bit"):
|
||||
package_linux_x64.verify_export(export)
|
||||
|
||||
def test_package_has_notices_checksums_and_deterministic_envelope(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
temporary_root = Path(temporary)
|
||||
export = make_export(temporary_root)
|
||||
output = temporary_root / "build/package"
|
||||
metadata = {"schema_version": 1, "source_commit": "fixture"}
|
||||
with patch.object(package_linux_x64, "REPO", temporary_root), patch.object(
|
||||
package_linux_x64, "PROJECT_FILES", ("LICENSE", "THIRD_PARTY_NOTICES.md")
|
||||
):
|
||||
(temporary_root / "LICENSE").write_text("MIT\n", encoding="utf-8")
|
||||
(temporary_root / "THIRD_PARTY_NOTICES.md").write_text("notices\n", encoding="utf-8")
|
||||
package_linux_x64.create_package(export, output, metadata, 123456789)
|
||||
first = hashlib.sha256(
|
||||
(output / f"{package_linux_x64.PACKAGE_NAME}.tar.gz").read_bytes()
|
||||
).hexdigest()
|
||||
package_root, archive = package_linux_x64.create_package(
|
||||
export, output, metadata, 123456789
|
||||
)
|
||||
second = hashlib.sha256(archive.read_bytes()).hexdigest()
|
||||
self.assertEqual(first, second)
|
||||
self.assertTrue((package_root / "LICENSE").is_file())
|
||||
self.assertTrue((package_root / "BUILD-INFO.json").is_file())
|
||||
checksums = (package_root / "SHA256SUMS").read_text(encoding="utf-8")
|
||||
self.assertIn("Himegari.x86_64", checksums)
|
||||
self.assertIn("THIRD_PARTY_NOTICES.md", checksums)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -30,6 +30,8 @@ import paths
|
||||
LEVELS = ("core", "workspace", "runtime", "full")
|
||||
CORE_TESTS = (
|
||||
"test_validate.py",
|
||||
"test_install_godot_templates.py",
|
||||
"test_package_linux_x64.py",
|
||||
"test_diff_optrace.py",
|
||||
"test_engine_ctx.py",
|
||||
"test_ghidra_handler_map.py",
|
||||
|
||||
Reference in New Issue
Block a user