Implement base SYS4 asset store

engine/Age.Engine/Sys4/Paths.cs

@@ -12,6 +12,7 @@ public static class Paths
     public static string AssetIndexJson => Path.Combine(Build, "asset-index.json");
     public static string CallscriptNamesJson => Path.Combine(Build, "callscript-names.json");
     public static string Textures => Path.Combine(Build, "textures");
+    public static string Sys4Ini => Path.Combine(GameDir, "SYS4INI.BIN");
 
     private static string FindRepo()
     {

engine/Age.Engine/Sys4/ResourceMap.cs

@@ -1,56 +1,23 @@
-using System.Text.Json;
-
 namespace Age.Engine.Sys4;
 
-/// <summary>One SYS4INI asset entry.</summary>
-public sealed record AssetEntry(string Name, string Archive, long Offset, long Size);
-
 /// <summary>
-/// Static asset resolver. SYS4INI's file list is sectioned (one per scene: SCxxxx.BIN + its
+/// Compatibility facade over the runtime SYS4 catalog. SYS4INI's file list is sectioned (one per scene: SCxxxx.BIN + its
 /// cross-archive asset manifest); file_number is the index within a section. So a bytecode
 /// resId resolves as files[section_base(scene) + resId] -- unified for graphics and audio.
-/// See docs/asset-resolution-re.md. Built from build/asset-index.json + build/asset-sections.json.
+/// See docs/asset-resolution-re.md. Extracted paths remain temporary graphics/audio backends only.
 /// </summary>
 public sealed class ResourceMap
 {
-    private readonly IReadOnlyList<AssetEntry> _files;
-    private readonly IReadOnlyDictionary<string, int> _sceneBase;   // "SC0000" -> section base index
+    private readonly Sys4AssetCatalog _catalog;
 
-    public ResourceMap(IReadOnlyList<AssetEntry> files, IReadOnlyDictionary<string, int> sceneBase)
-    {
-        _files = files;
-        _sceneBase = sceneBase;
-    }
+    public ResourceMap(Sys4AssetCatalog catalog) => _catalog = catalog;
 
-    public static ResourceMap Load(string indexPath, string sectionsPath)
-    {
-        var files = new List<AssetEntry>();
-        using (var idx = JsonDocument.Parse(File.ReadAllText(indexPath)))
-            foreach (var f in idx.RootElement.GetProperty("files").EnumerateArray())
-                files.Add(new AssetEntry(
-                    f.GetProperty("name").GetString()!,
-                    f.GetProperty("archive").GetString()!,
-                    f.GetProperty("offset").GetInt64(),
-                    f.GetProperty("size").GetInt64()));
-
-        var sceneBase = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);
-        using (var sec = JsonDocument.Parse(File.ReadAllText(sectionsPath)))
-            foreach (var p in sec.RootElement.GetProperty("scene_base").EnumerateObject())
-                sceneBase[p.Name] = p.Value.GetInt32();
-
-        return new ResourceMap(files, sceneBase);
-    }
-
-    public static ResourceMap Load() => Load(Paths.AssetIndexJson, Paths.AssetSectionsJson);
+    public static ResourceMap Load() => new(Sys4AssetCatalog.Load(Paths.Sys4Ini));
 
     /// <summary>Resolve a scene-local resId to its asset, or null if out of range / unknown scene.</summary>
     public AssetEntry? Resolve(string scene, long resId)
     {
-        var key = scene.EndsWith(".BIN", StringComparison.OrdinalIgnoreCase)
-            ? scene[..^4] : scene;
-        if (!_sceneBase.TryGetValue(key, out var b)) return null;
-        long p = b + resId;
-        return p >= 0 && p < _files.Count ? _files[(int)p] : null;
+        return _catalog.ResolveScene(scene, resId);
     }
 
     /// <summary>Pre-converted BMP path for an AGF asset (see tools/convert_agf.py).</summary>
@@ -70,10 +37,8 @@ public sealed class ResourceMap
     public string? BgmPathById(long id)
     {
         var name = $"BGM{id:D3}.OGG";
-        foreach (var f in _files)
-            if (f.Name.Equals(name, StringComparison.OrdinalIgnoreCase))
-                return AudioPath(f);
-        return null;
+        var f = _catalog.ResolveName(name);
+        return f == null ? null : AudioPath(f);
     }
 
     /// <summary>Loose extracted OGG/WAV path for an audio asset (extracted/DATA{n}/{name}), or null.

engine/Age.Engine/Sys4/Sys4AssetCatalog.cs

@@ -0,0 +1,185 @@
+using System.Buffers.Binary;
+using System.Text;
+
+namespace Age.Engine.Sys4;
+
+/// <summary>One raw SYS4INI file record. Placeholder records remain addressable by
+/// <see cref="RawIndex"/> but are excluded from scene and name views.</summary>
+public sealed record AssetEntry(
+    string Name,
+    string Archive,
+    long Offset,
+    long Size,
+    int RawIndex = -1,
+    int ArchiveId = -1,
+    int FileNumber = -1,
+    bool IsPlaceholder = false);
+
+/// <summary>Runtime parser and lookup views for a base S4IC SYS4INI catalog.</summary>
+public sealed class Sys4AssetCatalog
+{
+    private const int PackedSizeOffset = 0x134;
+    private const int ExpandedSizeOffset = 0x12c;
+    private const int ArchiveNameSize = 256;
+    private const int RecordSize = 80;
+
+    private readonly Dictionary<string, AssetEntry> _byName;
+    private readonly Dictionary<string, (int Start, int End)> _sceneRanges;
+
+    public string Magic { get; }
+    public IReadOnlyList<string> Archives { get; }
+    public IReadOnlyList<AssetEntry> RawSlots { get; }
+    public IReadOnlyList<AssetEntry> Files { get; }
+
+    private Sys4AssetCatalog(string magic, List<string> archives, List<AssetEntry> rawSlots)
+    {
+        Magic = magic;
+        Archives = archives;
+        RawSlots = rawSlots;
+        Files = rawSlots.Where(r => !r.IsPlaceholder).ToArray();
+        _byName = Files.ToDictionary(r => r.Name, StringComparer.OrdinalIgnoreCase);
+        _sceneRanges = BuildSceneRanges(Files);
+    }
+
+    public static Sys4AssetCatalog Load(string path) => Parse(File.ReadAllBytes(path), Path.GetFileName(path));
+
+    public static Sys4AssetCatalog Parse(byte[] data, string name = "SYS4INI.BIN")
+    {
+        if (data.Length < PackedSizeOffset + 4 || !data.AsSpan(0, 4).SequenceEqual("S4IC"u8))
+            throw new InvalidDataException($"{name}: expected an S4IC catalog");
+
+        uint expandedSize = BinaryPrimitives.ReadUInt32LittleEndian(data.AsSpan(ExpandedSizeOffset, 4));
+        uint packedSize = BinaryPrimitives.ReadUInt32LittleEndian(data.AsSpan(PackedSizeOffset, 4));
+        if (packedSize > data.Length - (PackedSizeOffset + 4))
+            throw new InvalidDataException($"{name}: packed directory is truncated");
+        if (expandedSize == 0 || expandedSize > int.MaxValue)
+            throw new InvalidDataException($"{name}: invalid expanded size {expandedSize}");
+
+        var blob = DecompressLzss(data.AsSpan(PackedSizeOffset + 4, checked((int)packedSize)),
+                                  checked((int)expandedSize), name);
+        int p = 0;
+        uint ReadU32()
+        {
+            if (p > blob.Length - 4) throw new InvalidDataException($"{name}: directory is truncated");
+            uint value = BinaryPrimitives.ReadUInt32LittleEndian(blob.AsSpan(p, 4));
+            p += 4;
+            return value;
+        }
+
+        uint archiveCount = ReadU32();
+        if (archiveCount is 0 or >= 0x1000 || archiveCount > (blob.Length - p) / ArchiveNameSize)
+            throw new InvalidDataException($"{name}: invalid archive count {archiveCount}");
+        var archives = new List<string>(checked((int)archiveCount));
+        for (int i = 0; i < archiveCount; i++, p += ArchiveNameSize)
+            archives.Add(ReadCString(blob.AsSpan(p, ArchiveNameSize)));
+
+        uint fileCount = ReadU32();
+        if (fileCount is 0 or >= 0x400000 || fileCount > (blob.Length - p) / RecordSize)
+            throw new InvalidDataException($"{name}: invalid file count {fileCount}");
+        var slots = new List<AssetEntry>(checked((int)fileCount));
+        for (int i = 0; i < fileCount; i++, p += RecordSize)
+        {
+            var row = blob.AsSpan(p, RecordSize);
+            string fileName = ReadCString(row[..64]);
+            int archiveId = checked((int)BinaryPrimitives.ReadUInt32LittleEndian(row.Slice(64, 4)));
+            int fileNumber = checked((int)BinaryPrimitives.ReadUInt32LittleEndian(row.Slice(68, 4)));
+            long offset = BinaryPrimitives.ReadUInt32LittleEndian(row.Slice(72, 4));
+            long size = BinaryPrimitives.ReadUInt32LittleEndian(row.Slice(76, 4));
+            string archive = archiveId >= 0 && archiveId < archives.Count ? archives[archiveId] : "";
+            bool placeholder = fileName is "" or "@";
+            slots.Add(new AssetEntry(fileName, archive, offset, size, i, archiveId, fileNumber, placeholder));
+        }
+
+        string magic = ReadCString(data.AsSpan(0, Math.Min(8, data.Length)));
+        return new Sys4AssetCatalog(magic, archives, slots);
+    }
+
+    /// <summary>Universal raw-id lookup. Placeholder slots are returned, not collapsed.</summary>
+    public AssetEntry? ResolveRaw(long rawId)
+        => rawId >= 0 && rawId < RawSlots.Count ? RawSlots[(int)rawId] : null;
+
+    /// <summary>Case-insensitive exact-name lookup over real records.</summary>
+    public AssetEntry? ResolveName(string name)
+        => _byName.TryGetValue(Path.GetFileName(name), out var entry) && Path.GetFileName(name) == name
+            ? entry : null;
+
+    /// <summary>Resolve within the owning scene section; ids cannot spill into the next section.</summary>
+    public AssetEntry? ResolveScene(string scene, long localId)
+    {
+        string key = Path.GetFileNameWithoutExtension(scene);
+        if (!_sceneRanges.TryGetValue(key, out var range)) return null;
+        long pos = range.Start + localId;
+        return localId >= 0 && pos <= range.End ? Files[(int)pos] : null;
+    }
+
+    public IReadOnlyList<string> ScriptNames => Files
+        .Where(f => f.Name.EndsWith(".BIN", StringComparison.OrdinalIgnoreCase))
+        .Select(f => f.Name.ToUpperInvariant()).ToArray();
+
+    private static Dictionary<string, (int Start, int End)> BuildSceneRanges(IReadOnlyList<AssetEntry> files)
+    {
+        var ranges = new Dictionary<string, (int Start, int End)>(StringComparer.OrdinalIgnoreCase);
+        int start = 0;
+        for (int i = 1; i <= files.Count; i++)
+        {
+            bool end = i == files.Count || files[i].FileNumber <= files[i - 1].FileNumber;
+            if (!end) continue;
+            for (int k = start; k < i; k++)
+                if (files[k].Name.Length == 10 && files[k].Name.StartsWith("SC", StringComparison.OrdinalIgnoreCase)
+                    && files[k].Name.EndsWith(".BIN", StringComparison.OrdinalIgnoreCase)
+                    && files[k].Name.AsSpan(2, 4).ToString().All(char.IsDigit))
+                {
+                    ranges[Path.GetFileNameWithoutExtension(files[k].Name)] = (start, i - 1);
+                    break;
+                }
+            start = i;
+        }
+        return ranges;
+    }
+
+    private static string ReadCString(ReadOnlySpan<byte> bytes)
+    {
+        int zero = bytes.IndexOf((byte)0);
+        if (zero >= 0) bytes = bytes[..zero];
+        Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
+        return Encoding.GetEncoding(932).GetString(bytes);
+    }
+
+    private static byte[] DecompressLzss(ReadOnlySpan<byte> source, int expectedSize, string name)
+    {
+        var frame = new byte[0x1000];
+        int framePos = 0xfee, input = 0, output = 0;
+        var result = new byte[expectedSize];
+        while (output < expectedSize)
+        {
+            if (input >= source.Length) throw new InvalidDataException($"{name}: LZSS stream ended early");
+            int control = source[input++];
+            for (int bit = 1; bit <= 0x80 && output < expectedSize; bit <<= 1)
+            {
+                if ((control & bit) != 0)
+                {
+                    if (input >= source.Length) throw new InvalidDataException($"{name}: truncated LZSS literal");
+                    byte value = source[input++];
+                    result[output++] = value;
+                    frame[framePos] = value;
+                    framePos = (framePos + 1) & 0xfff;
+                }
+                else
+                {
+                    if (input > source.Length - 2) throw new InvalidDataException($"{name}: truncated LZSS back-reference");
+                    int lo = source[input++], hi = source[input++];
+                    int readPos = ((hi & 0xf0) << 4) | lo;
+                    int length = 3 + (hi & 0x0f);
+                    for (int j = 0; j < length && output < expectedSize; j++)
+                    {
+                        byte value = frame[readPos++ & 0xfff];
+                        result[output++] = value;
+                        frame[framePos] = value;
+                        framePos = (framePos + 1) & 0xfff;
+                    }
+                }
+            }
+        }
+        return result;
+    }
+}

engine/Age.Engine/Sys4/Sys4AssetStore.cs

@@ -0,0 +1,129 @@
+namespace Age.Engine.Sys4;
+
+/// <summary>Read-only byte seam after catalog resolution.</summary>
+public interface IAssetStore
+{
+    Stream Open(AssetEntry entry);
+    byte[] ReadAll(AssetEntry entry);
+}
+
+/// <summary>Native base-game precedence: exact-basename loose roots first, indexed ALF range second.</summary>
+public sealed class Sys4AssetStore : IAssetStore
+{
+    private readonly string _archiveRoot;
+    private readonly string[] _looseRoots;
+
+    public Sys4AssetCatalog Catalog { get; }
+
+    public Sys4AssetStore(Sys4AssetCatalog catalog, string archiveRoot, params string[] looseRoots)
+    {
+        Catalog = catalog;
+        _archiveRoot = Path.GetFullPath(archiveRoot);
+        _looseRoots = looseRoots.Select(Path.GetFullPath).ToArray();
+    }
+
+    public Stream Open(AssetEntry entry)
+    {
+        ValidateBasename(entry.Name, "asset");
+        if (entry.IsPlaceholder) throw new FileNotFoundException("SYS4INI placeholder has no payload", entry.Name);
+
+        foreach (string root in _looseRoots)
+        {
+            string candidate = Path.GetFullPath(Path.Combine(root, entry.Name));
+            if (!IsDirectChild(root, candidate)) throw new InvalidDataException($"unsafe asset name: {entry.Name}");
+            try
+            {
+                return new FileStream(candidate, FileMode.Open, FileAccess.Read, FileShare.Read,
+                                      64 * 1024, FileOptions.RandomAccess);
+            }
+            catch (FileNotFoundException) { }
+            catch (DirectoryNotFoundException) { }
+        }
+
+        ValidateBasename(entry.Archive, "archive");
+        string archivePath = Path.GetFullPath(Path.Combine(_archiveRoot, entry.Archive));
+        if (!IsDirectChild(_archiveRoot, archivePath))
+            throw new InvalidDataException($"unsafe archive name: {entry.Archive}");
+        var file = new FileStream(archivePath, FileMode.Open, FileAccess.Read, FileShare.Read,
+                                  64 * 1024, FileOptions.RandomAccess);
+        try
+        {
+            if (entry.Offset < 0 || entry.Size < 0 || entry.Offset > file.Length
+                || entry.Size > file.Length - entry.Offset)
+                throw new InvalidDataException($"{entry.Name}: ALF range {entry.Offset}+{entry.Size} exceeds {entry.Archive} ({file.Length})");
+            return new BoundedReadStream(file, entry.Offset, entry.Size);
+        }
+        catch
+        {
+            file.Dispose();
+            throw;
+        }
+    }
+
+    public byte[] ReadAll(AssetEntry entry)
+    {
+        using Stream stream = Open(entry);
+        if (stream.Length > int.MaxValue) throw new InvalidDataException($"{entry.Name}: payload is too large");
+        var bytes = new byte[checked((int)stream.Length)];
+        stream.ReadExactly(bytes);
+        return bytes;
+    }
+
+    private static void ValidateBasename(string value, string kind)
+    {
+        if (string.IsNullOrWhiteSpace(value) || Path.IsPathRooted(value)
+            || value.Contains('/') || value.Contains('\\') || value is "." or "..")
+            throw new InvalidDataException($"unsafe {kind} name: {value}");
+    }
+
+    private static bool IsDirectChild(string root, string child)
+        => string.Equals(Path.GetDirectoryName(child)?.TrimEnd(Path.DirectorySeparatorChar),
+                         root.TrimEnd(Path.DirectorySeparatorChar), StringComparison.OrdinalIgnoreCase);
+
+    private sealed class BoundedReadStream : Stream
+    {
+        private readonly FileStream _file;
+        private readonly long _start;
+        private readonly long _length;
+        private long _position;
+
+        public BoundedReadStream(FileStream file, long start, long length)
+        {
+            _file = file; _start = start; _length = length;
+            _file.Position = start;
+        }
+
+        public override bool CanRead => true;
+        public override bool CanSeek => true;
+        public override bool CanWrite => false;
+        public override long Length => _length;
+        public override long Position { get => _position; set => Seek(value, SeekOrigin.Begin); }
+        public override void Flush() { }
+        public override int Read(byte[] buffer, int offset, int count)
+            => Read(buffer.AsSpan(offset, count));
+        public override int Read(Span<byte> buffer)
+        {
+            int wanted = (int)Math.Min(buffer.Length, _length - _position);
+            if (wanted <= 0) return 0;
+            int read = _file.Read(buffer[..wanted]);
+            _position += read;
+            return read;
+        }
+        public override long Seek(long offset, SeekOrigin origin)
+        {
+            long target = origin switch
+            {
+                SeekOrigin.Begin => offset,
+                SeekOrigin.Current => checked(_position + offset),
+                SeekOrigin.End => checked(_length + offset),
+                _ => throw new ArgumentOutOfRangeException(nameof(origin)),
+            };
+            if (target < 0 || target > _length) throw new IOException("seek outside asset range");
+            _file.Position = _start + target;
+            return _position = target;
+        }
+        public override void SetLength(long value) => throw new NotSupportedException();
+        public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException();
+        protected override void Dispose(bool disposing) { if (disposing) _file.Dispose(); base.Dispose(disposing); }
+    }
+}

engine/Age.Engine/Sys4/Sys4ScriptProvider.cs

@@ -1,39 +1,54 @@
-using System.Text.Json;
 using Age.Engine.Hosting;
 using Age.Engine.Model;
+
 namespace Age.Engine.Sys4;
 
-/// <summary>Resolves call-script ids (raw SYS4INI file indices) to loaded scripts, using
-/// build/callscript-names.json (id→name) + Paths.Scripts() (name→path). Cached per id.
-/// The native resolver prefers a loose override before the archive; Paths.Scripts() already
-/// shadows extracted/DATA1 with root overrides, so that behavior is preserved.</summary>
+/// <summary>Loads root and call-script bytecode through the native loose-first asset-store seam.</summary>
 public sealed class Sys4ScriptProvider : IScriptProvider
 {
     private readonly OpcodeTable _table;
-    private readonly IReadOnlyDictionary<long, string> _idToName;
-    private readonly Dictionary<string, string> _byName;      // NAME(UPPER) -> path
+    private readonly IAssetStore _store;
     private readonly Dictionary<long, Script?> _cache = new();
+    private readonly Dictionary<string, Script?> _nameCache = new(StringComparer.OrdinalIgnoreCase);
 
-    public Sys4ScriptProvider(OpcodeTable table, IReadOnlyDictionary<long, string> idToName,
-                              Dictionary<string, string> byName)
-    { _table = table; _idToName = idToName; _byName = byName; }
+    public Sys4AssetCatalog Catalog { get; }
+    public IReadOnlyList<string> ScriptNames => Catalog.ScriptNames;
+
+    public Sys4ScriptProvider(OpcodeTable table, Sys4AssetCatalog catalog, IAssetStore store)
+    { _table = table; Catalog = catalog; _store = store; }
 
     public static Sys4ScriptProvider Load(OpcodeTable table)
     {
-        var raw = JsonSerializer.Deserialize<Dictionary<string, string>>(
-            File.ReadAllText(Paths.CallscriptNamesJson)) ?? new();
-        var idToName = raw.ToDictionary(kv => long.Parse(kv.Key), kv => kv.Value);
-        return new Sys4ScriptProvider(table, idToName, Paths.Scripts());
+        var catalog = Sys4AssetCatalog.Load(Paths.Sys4Ini);
+        return new Sys4ScriptProvider(table, catalog,
+            new Sys4AssetStore(catalog, Paths.GameDir, Paths.GameDir));
     }
 
     public Script? GetById(long id)
     {
         if (_cache.TryGetValue(id, out var cached)) return cached;
-        Script? s = null;
-        if (_idToName.TryGetValue(id, out var name) &&
-            _byName.TryGetValue(name.ToUpperInvariant(), out var path))
-            s = Sys4Loader.Load(path, _table);
-        _cache[id] = s;
-        return s;
+        var entry = Catalog.ResolveRaw(id);
+        Script? script = entry is { IsPlaceholder: false }
+                        && entry.Name.EndsWith(".BIN", StringComparison.OrdinalIgnoreCase)
+            ? Parse(entry) : null;
+        _cache[id] = script;
+        return script;
     }
+
+    public Script? GetByName(string name)
+    {
+        string key = name;
+        if (_nameCache.TryGetValue(key, out var cached)) return cached;
+        var entry = Catalog.ResolveName(key);
+        Script? script = entry != null && entry.Name.EndsWith(".BIN", StringComparison.OrdinalIgnoreCase)
+            ? Parse(entry) : null;
+        _nameCache[key] = script;
+        return script;
+    }
+
+    public Script RequireByName(string name)
+        => GetByName(name) ?? throw new FileNotFoundException($"script is not in SYS4INI: {name}", name);
+
+    private Script Parse(AssetEntry entry)
+        => Sys4Loader.Parse(_store.ReadAll(entry), _table, entry.Name);
 }