conco/OpenMaidEngine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Mount SYS4 append asset catalogs

Browse Source
This commit is contained in:
gamer147
2026-07-11 10:42:33 -04:00
parent a1f6d0ab51
commit e0ca7936e2
9 changed files with 235 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
engine/Age.Engine.Tests/Sys4AssetStoreTests.cs
View File

@@ -1,9 +1,91 @@
using System.Text.Json;
using System.Security.Cryptography;
using System.Text;
using System.Diagnostics;
using Age.Engine.Sys4;
using Xunit;
public class Sys4AssetStoreTests
{
[Fact]
public void InstalledAppendCatalogHasNativePackSelectionAndStableDirectory()
{
var catalog = Sys4AssetCatalog.Load(Paths.Sys4Ini);
var append = Assert.Single(catalog.AppendPacks).Value;
Assert.Equal(1, append.PackId);
Assert.Equal("S4AC422 ", append.Magic);
Assert.Equal(catalog.Title, append.Title);
Assert.Equal(new[] { "APPEND01.ALF" }, append.Archives);
Assert.Equal(81, append.RawSlots.Count);
Assert.Equal(81, append.Files.Count);
Assert.All(append.Files, entry =>
{
Assert.Equal(1, entry.PackId);
Assert.StartsWith("$1$", entry.Name);
Assert.Equal("APPEND01.ALF", entry.Archive);
});
Assert.Same(append.RawSlots[0], catalog.ResolvePacked(0x01000000));
Assert.Same(append.RawSlots[^1], catalog.ResolvePacked(0x01000050));
Assert.Null(catalog.ResolvePacked(0x02000000));
Assert.Null(catalog.ResolvePacked(0x80000000));
Assert.Null(catalog.ResolveName(append.RawSlots[0].Name));
string directory = string.Join("\n", append.RawSlots.Select(e =>
$"{e.RawIndex}|{e.Name}|{e.ArchiveId}|{e.Archive}|{e.FileNumber}|{e.Offset}|{e.Size}"));
string digest = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(directory)));
Assert.Equal("23F0C104A45C099CEFB7D333362716EDE6F20B9EC53E4C3705A8E3A87063708E", digest);
}
[Fact]
public void CompleteAppendDirectoryAndPayloadsMatchBinExtractAlf()
{
string temp = Path.Combine(Path.GetTempPath(), "age-vfsb-oracle-" + Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(temp);
try
{
var start = new ProcessStartInfo(Paths.BinExtractAlf)
{
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
CreateNoWindow = true,
};
start.ArgumentList.Add(Paths.Append01Aai);
start.ArgumentList.Add(temp);
using var process = Process.Start(start)!;
string stdout = process.StandardOutput.ReadToEnd();
string stderr = process.StandardError.ReadToEnd();
process.WaitForExit();
Assert.True(process.ExitCode == 0, stdout + stderr);
var catalog = Sys4AssetCatalog.Load(Paths.Sys4Ini);
var append = catalog.AppendPacks[1];
var store = new Sys4AssetStore(catalog, Paths.GameDir, Paths.GameDir);
string output = Path.Combine(temp, "APPEND01");
var oracleNames = Directory.EnumerateFiles(output).Select(Path.GetFileName)
.Order(StringComparer.OrdinalIgnoreCase).ToArray();
var actualNames = append.Files.Select(e => e.Name)
.Order(StringComparer.OrdinalIgnoreCase).ToArray();
Assert.Equal(actualNames, oracleNames);
long archiveLength = new FileInfo(Path.Combine(Paths.GameDir, "APPEND01.ALF")).Length;
foreach (var entry in append.Files)
{
Assert.InRange(entry.Offset, 0, archiveLength);
Assert.InRange(entry.Size, 0, archiveLength - entry.Offset);
byte[] oracle = File.ReadAllBytes(Path.Combine(output, entry.Name));
Assert.Equal(entry.Size, oracle.LongLength);
Assert.Equal(oracle, store.ReadAll(entry));
}
}
finally
{
Directory.Delete(temp, recursive: true);
}
}
[Fact]
public void RuntimeCatalogMatchesDiagnosticCatalogAndSceneViews()
{

15
engine/Age.Engine.Tests/Sys4ScriptProviderTests.cs
View File

@@ -18,6 +18,21 @@ public class Sys4ScriptProviderTests
Assert.Null(provider.GetById(long.MaxValue)); // unknown id
}
[Fact]
public void HighByteSelectsAppendPackWithoutReplacingBaseNames()
{
var table = OpcodeTableJson.Load(Paths.OpcodesJson);
var provider = Sys4ScriptProvider.Load(table);
var append = provider.Catalog.AppendPacks[1];
var entry = append.Files.Single(e => e.Name == "$1$SC1260.BIN");
long packedId = 0x01000000L | (uint)entry.RawIndex;
var script = provider.GetById(packedId);
Assert.NotNull(script);
Assert.True(script!.Instructions.Count > 0);
Assert.Null(provider.GetByName("$1$SC1260.BIN"));
}
[Fact]
public void RootScriptLoadingUsesTheSameAssetStoreAndLoosePrecedence()
{

2
engine/Age.Engine/Sys4/Paths.cs
View File

@@ -13,6 +13,8 @@ public static class Paths
public static string CallscriptNamesJson => Path.Combine(Build, "callscript-names.json");
public static string Textures => Path.Combine(Build, "textures");
public static string Sys4Ini => Path.Combine(GameDir, "SYS4INI.BIN");
public static string Append01Aai => Path.Combine(GameDir, "APPEND01.AAI");
public static string BinExtractAlf => Path.Combine(Repo, "bin", "BinExtractALF.exe");
private static string FindRepo()
{

84
engine/Age.Engine/Sys4/Sys4AssetCatalog.cs
View File

@@ -13,27 +13,33 @@ public sealed record AssetEntry(
int RawIndex = -1,
int ArchiveId = -1,
int FileNumber = -1,
bool IsPlaceholder = false);
bool IsPlaceholder = false,
int PackId = 0);
/// <summary>Runtime parser and lookup views for a base S4IC SYS4INI catalog.</summary>
/// <summary>Runtime parser and lookup views for a base S4IC SYS4INI catalog and its S4AC append mounts.</summary>
public sealed class Sys4AssetCatalog
{
private const int PackedSizeOffset = 0x134;
private const int ExpandedSizeOffset = 0x12c;
private const int ArchiveNameSize = 256;
private const int RecordSize = 80;
private readonly Dictionary<string, AssetEntry> _byName;
private readonly Dictionary<string, (int Start, int End)> _sceneRanges;
private readonly Dictionary<int, Sys4AssetCatalog> _appendPacks = new();
public string Magic { get; }
public string Title { get; }
public int PackId { get; }
public IReadOnlyList<string> Archives { get; }
public IReadOnlyList<AssetEntry> RawSlots { get; }
public IReadOnlyList<AssetEntry> Files { get; }
public IReadOnlyDictionary<int, Sys4AssetCatalog> AppendPacks => _appendPacks;
private Sys4AssetCatalog(string magic, List<string> archives, List<AssetEntry> rawSlots)
private Sys4AssetCatalog(string magic, string title, int packId,
List<string> archives, List<AssetEntry> rawSlots)
{
Magic = magic;
Title = title;
PackId = packId;
Archives = archives;
RawSlots = rawSlots;
Files = rawSlots.Where(r => !r.IsPlaceholder).ToArray();
@@ -41,21 +47,38 @@ public sealed class Sys4AssetCatalog
_sceneRanges = BuildSceneRanges(Files);
}
public static Sys4AssetCatalog Load(string path) => Parse(File.ReadAllBytes(path), Path.GetFileName(path));
public static Sys4AssetCatalog Load(string path)
{
var catalog = Parse(File.ReadAllBytes(path), Path.GetFileName(path));
if (!catalog.Magic.StartsWith("S4IC", StringComparison.Ordinal)) return catalog;
string? root = Path.GetDirectoryName(Path.GetFullPath(path));
if (root == null) return catalog;
foreach (string appendPath in Directory.EnumerateFiles(root, "*.AAI"))
catalog.MountAppend(Parse(File.ReadAllBytes(appendPath), Path.GetFileName(appendPath)));
return catalog;
}
public static Sys4AssetCatalog Parse(byte[] data, string name = "SYS4INI.BIN")
{
if (data.Length < PackedSizeOffset + 4 || !data.AsSpan(0, 4).SequenceEqual("S4IC"u8))
throw new InvalidDataException($"{name}: expected an S4IC catalog");
bool isBase = data.Length >= 4 && data.AsSpan(0, 4).SequenceEqual("S4IC"u8);
bool isAppend = data.Length >= 4 && data.AsSpan(0, 4).SequenceEqual("S4AC"u8);
if (!isBase && !isAppend)
throw new InvalidDataException($"{name}: expected an S4IC or S4AC catalog");
uint expandedSize = BinaryPrimitives.ReadUInt32LittleEndian(data.AsSpan(ExpandedSizeOffset, 4));
uint packedSize = BinaryPrimitives.ReadUInt32LittleEndian(data.AsSpan(PackedSizeOffset, 4));
if (packedSize > data.Length - (PackedSizeOffset + 4))
int expandedSizeOffset = isAppend ? 0x110 : 0x12c;
int packedSizeOffset = isAppend ? 0x114 : 0x134;
if (data.Length < packedSizeOffset + 4)
throw new InvalidDataException($"{name}: catalog header is truncated");
uint expandedSize = BinaryPrimitives.ReadUInt32LittleEndian(data.AsSpan(expandedSizeOffset, 4));
uint packedSize = BinaryPrimitives.ReadUInt32LittleEndian(data.AsSpan(packedSizeOffset, 4));
if (packedSize > data.Length - (packedSizeOffset + 4))
throw new InvalidDataException($"{name}: packed directory is truncated");
if (expandedSize == 0 || expandedSize > int.MaxValue)
throw new InvalidDataException($"{name}: invalid expanded size {expandedSize}");
var blob = LzssDecoder.Decode(data.AsSpan(PackedSizeOffset + 4, checked((int)packedSize)),
var blob = LzssDecoder.Decode(data.AsSpan(packedSizeOffset + 4, checked((int)packedSize)),
checked((int)expandedSize), name);
int p = 0;
uint ReadU32()
@@ -76,6 +99,13 @@ public sealed class Sys4AssetCatalog
uint fileCount = ReadU32();
if (fileCount is 0 or >= 0x400000 || fileCount > (blob.Length - p) / RecordSize)
throw new InvalidDataException($"{name}: invalid file count {fileCount}");
int packId = isAppend
? checked((int)BinaryPrimitives.ReadUInt32LittleEndian(data.AsSpan(0x108, 4))) : 0;
// AGE uses SAR 24 before indexing its mount table, so selectors with the sign bit set do
// not address slots 0x80..0xff. Reject them instead of inventing unsigned behavior.
if (isAppend && packId is not (> 0 and < 0x80))
throw new InvalidDataException($"{name}: invalid append pack selector {packId}");
var slots = new List<AssetEntry>(checked((int)fileCount));
for (int i = 0; i < fileCount; i++, p += RecordSize)
{
@@ -87,17 +117,43 @@ public sealed class Sys4AssetCatalog
long size = BinaryPrimitives.ReadUInt32LittleEndian(row.Slice(76, 4));
string archive = archiveId >= 0 && archiveId < archives.Count ? archives[archiveId] : "";
bool placeholder = fileName is "" or "@";
slots.Add(new AssetEntry(fileName, archive, offset, size, i, archiveId, fileNumber, placeholder));
slots.Add(new AssetEntry(fileName, archive, offset, size, i, archiveId, fileNumber,
placeholder, packId));
}
string magic = ReadCString(data.AsSpan(0, Math.Min(8, data.Length)));
return new Sys4AssetCatalog(magic, archives, slots);
string title = ReadCString(data.AsSpan(8, Math.Min(256, data.Length - 8)));
return new Sys4AssetCatalog(magic, title, packId, archives, slots);
}
/// <summary>Mount an append catalog by its native header selector. A later mount of the same
/// selector replaces the earlier one, matching AGE's FindNextFile/store loop.</summary>
public void MountAppend(Sys4AssetCatalog append)
{
if (PackId != 0) throw new InvalidOperationException("append catalogs cannot own append mounts");
if (!append.Magic.StartsWith("S4AC", StringComparison.Ordinal) || append.PackId == 0)
throw new InvalidDataException("mounted catalog is not an S4AC append pack");
if (!string.Equals(Title, append.Title, StringComparison.Ordinal))
throw new InvalidDataException($"append title mismatch: {append.Title}");
_appendPacks[append.PackId] = append;
}
/// <summary>Universal raw-id lookup. Placeholder slots are returned, not collapsed.</summary>
public AssetEntry? ResolveRaw(long rawId)
=> rawId >= 0 && rawId < RawSlots.Count ? RawSlots[(int)rawId] : null;
/// <summary>Native packed-id lookup: zero high byte selects SYS4INI; otherwise the high byte
/// selects an S4AC mount and the low 24 bits index that pack.</summary>
public AssetEntry? ResolvePacked(long id)
{
if (id < 0 || id > uint.MaxValue) return null;
int pack = (int)((id >> 24) & 0xff);
if (pack >= 0x80) return null;
long index = id & 0xffffff;
return pack == 0 ? ResolveRaw(index)
: _appendPacks.TryGetValue(pack, out var append) ? append.ResolveRaw(index) : null;
}
/// <summary>Case-insensitive exact-name lookup over real records.</summary>
public AssetEntry? ResolveName(string name)
=> _byName.TryGetValue(Path.GetFileName(name), out var entry) && Path.GetFileName(name) == name

2
engine/Age.Engine/Sys4/Sys4ScriptProvider.cs
View File

@@ -27,7 +27,7 @@ public sealed class Sys4ScriptProvider : IScriptProvider
public Script? GetById(long id)
{
if (_cache.TryGetValue(id, out var cached)) return cached;
var entry = Catalog.ResolveRaw(id);
var entry = Catalog.ResolvePacked(id);
Script? script = entry is { IsPlaceholder: false }
&& entry.Name.EndsWith(".BIN", StringComparison.OrdinalIgnoreCase)
? Parse(entry) : null;