diff --git a/docs/PROJECT-STRUCTURE.md b/docs/PROJECT-STRUCTURE.md index c0b42c4..9e9f6c6 100644 --- a/docs/PROJECT-STRUCTURE.md +++ b/docs/PROJECT-STRUCTURE.md @@ -101,8 +101,9 @@ S:\Game Hacking\Eushully\Himegari\ ← workspace root (three siblings) │ └── manifest.json, opcode-coverage.md (opcode-coverage.md GENERATED from opcodes.toml) │ ├── engine/ DELIVERABLE — the .NET VM core (AgeEngine.sln: Age.Engine / Age.Cli / tests) - │ └── Age.Engine/Sys4/ runtime catalog parser, loose-first bounded ALF asset store, + │ ├── Age.Engine/Sys4/ runtime catalog parser, loose-first bounded ALF asset store, │ script provider, AGF/LZSS and Windows CUR decoders, and resource facade + │ └── Age.Engine/Persistence/ native S3SD/S4SD container codec and profile/numbered DAT lifecycle seam ├── native/ authored native runtime boundaries │ └── age_movie_ffmpeg/ project-owned FFmpeg C ABI, immutable Windows dependency manifest, │ and bootstrap/build scripts (outputs stay under disposable build/) diff --git a/docs/engine-re.md b/docs/engine-re.md index c5ba6ce..6d2cc01 100644 --- a/docs/engine-re.md +++ b/docs/engine-re.md @@ -1722,6 +1722,16 @@ inventing a second save container. a profile/save service so extended mode can later add JSON inspection/export, namespaced mod state, migrations, or a friendlier editor without changing compatibility-mode opcode semantics or the native import/export path. +**Port correspondence (2026-07-24, codec foundation implemented):** +`Age.Engine.Persistence.NativeSaveContainerCodec` now reads and writes the common header, Shift-JIS game id, +SYSTEMTIME/playtime/version metadata, both CRC layers, version-2 compression wrapper, and exact reversible +DWORD transform. `Sys4.LzssEncoder` emits the same 4 KiB-ring token dialect already consumed by +`LzssDecoder`, falling back to native verbatim storage when compression does not shrink. The +`INativeDatStore` boundary and `DirectoryNativeDatStore` own shared `$$SAVE.DAT` → `SAVE.DAT` / +`SAVE.BAK` replacement/fallback and direct numbered `SAVE##.DAT` writes. Payload schemas, `RT.DAT`, +thumbnails, and opcode wiring remain deliberately above or after this layer; the existing `GameSession` +JSON snapshot is unchanged. + ### Opcode `0xae` continues numbered-save stack restoration (2026-07-20) Opcode `0xae` is the load-side rendezvous paired with serialized script-frame state. Its handler, diff --git a/docs/phase-a-slice-plan.md b/docs/phase-a-slice-plan.md index 23ab53b..229af55 100644 --- a/docs/phase-a-slice-plan.md +++ b/docs/phase-a-slice-plan.md @@ -3355,6 +3355,32 @@ No runtime persistence code changed in this reconnaissance slice. The canonical references now describe the native ABI; the corresponding `/v2` handlers and codec helpers are named, commented, and saved. +### Persistence implementation step 1 — native DAT codec and store boundary (2026-07-24) + +The reusable compatibility floor is now implemented without prematurely inventing either shared-profile or +numbered-state payload objects. `NativeSaveContainerCodec` owns the exact `0x124` header and `0x14` codec +frame, Shift-JIS game identity, SYSTEMTIME/playtime/version fields, inner and outer CRC pairs, rolling +seed/odd-multiplier DWORD expansion, its exact-division inverse, and the SaveVersion2>=2 wrapper. The new +`LzssEncoder` shares the existing native 4096-byte-ring dialect and uses the native equal-length verbatim +fallback when compression is not beneficial. + +`INativeDatStore` is the payload-agnostic runtime seam. Its directory implementation performs shared +`$$SAVE.DAT` → `SAVE.DAT` replacement with `SAVE.BAK` fallback and direct `SAVE##.DAT` reads/writes, while +validating generation, compatibility id, game id, and the native layout-version exception. It does not +serialize the VM's whole global bank, change `GameSession.ToJson`, wire persistence opcodes, or claim +`RT.DAT`/`.STH` support. + +Seven focused tests cover independent standard CRC vectors, compressed and verbatim LZSS round trips, +S3SD/S4SD header/frame offsets, deterministic transform products, both codec-version branches, corruption +rejection after outer-CRC repair, layout-2 version compatibility, shared backup recovery, and direct numbered +files. The full engine suite passes 359 tests. The opcode table count regression now distinguishes 248 +Himegari-observed opcodes from the mapped-but-unused shared ABI opcode `0x19f`. + +**Next persistence step:** implement the typed shared `SAVE.DAT` logical payload and profile-owned selected +integer/string cell maps, then connect `0x1a2`/`0x1a3` and `0x1a9`/`0x1aa`. Add native `RT.DAT` and its +read-message lifecycle after that shared ownership is live; numbered active-frame layouts remain the later, +larger payload. + ## Data-semantics sidebar: focused append EBINIT inspection (2026-07-24) The static INIT surface now accepts a universal packed script id for focused append inspection. diff --git a/docs/phase-b-framework.md b/docs/phase-b-framework.md index 3364b6d..4e97e43 100644 --- a/docs/phase-b-framework.md +++ b/docs/phase-b-framework.md @@ -361,7 +361,8 @@ In scope: Deferred to bounded follow-ups unless the happy path requires them: - Full configuration UI and every setting. -- Load/save implementation and save-format reversal. +- Full load/save opcode and logical-payload implementation. Native format reversal and the common DAT + codec/store foundation landed on 2026-07-24. - Extras, galleries, replay modes, and unrelated submenus. - Menu visual polish that does not obstruct correct selection or state production. diff --git a/docs/remake-architecture-and-roadmap.md b/docs/remake-architecture-and-roadmap.md index 9b8899f..2f61829 100644 --- a/docs/remake-architecture-and-roadmap.md +++ b/docs/remake-architecture-and-roadmap.md @@ -466,7 +466,8 @@ domains and lifecycle—shared `SAVE.DAT`/`SAVE.BAK`, `RT.DAT`/`RT.BAK`, numbere paired BMP `.STH` thumbnails. Keep the codec behind a profile/save-service boundary. Human-readable JSON inspection/export, migrations, and namespaced mod state are additive extended-mode work, not a replacement for compatibility-mode import/export. The recovered native contract lives in -`docs/engine-re.md`. +`docs/engine-re.md`. The common container codec and payload-agnostic shared/numbered DAT store boundary +landed on 2026-07-24; logical payload services and opcode wiring remain Phase B work. ### Phase C — Externalize & modding foundation - Add **editable named data overlays** mapped explicitly onto the VM's `*INIT`-produced state; external diff --git a/engine/Age.Engine.Tests/NativeSaveContainerTests.cs b/engine/Age.Engine.Tests/NativeSaveContainerTests.cs new file mode 100644 index 0000000..2c1eba7 --- /dev/null +++ b/engine/Age.Engine.Tests/NativeSaveContainerTests.cs @@ -0,0 +1,177 @@ +using System.Buffers.Binary; +using System.Text; +using Age.Engine.Persistence; +using Age.Engine.Sys4; + +public class NativeSaveContainerTests +{ + private static readonly NativeSystemTime Timestamp = + new(2026, 7, 5, 24, 13, 42, 17, 321); + + [Fact] + public void NativeCrcVariantsMatchIndependentCheckVectors() + { + byte[] input = Encoding.ASCII.GetBytes("123456789"); + Assert.Equal(0xfc891918u, NativeSaveContainerCodec.Crc32Msb(input)); + Assert.Equal(0xcbf43926u, NativeSaveContainerCodec.Crc32Reflected(input)); + } + + [Fact] + public void LzssEncoderRoundTripsTheNativeRingDialect() + { + byte[] input = Encoding.ASCII.GetBytes(string.Concat( + Enumerable.Repeat("HIMEGARI-HIMEGARI-0000000000000000-", 80))); + byte[] encoded = LzssEncoder.EncodeOrVerbatim(input); + + Assert.True(encoded.Length < input.Length); + Assert.Equal(input, LzssDecoder.Decode(encoded, input.Length, "save test")); + + byte[] incompressible = [1, 9, 2, 8, 3, 7, 4, 6]; + Assert.Equal(incompressible, LzssEncoder.EncodeOrVerbatim(incompressible)); + } + + [Fact] + public void VersionTwoContainerWritesNativeHeaderFrameAndRoundTrips() + { + byte[] payload = Enumerable.Range(0, 256) + .SelectMany(i => BitConverter.GetBytes(i % 7)) + .ToArray(); + var metadata = new NativeSaveMetadata( + NativeSaveMagic.S4SD, 0x10203040, "姫狩りDM", Timestamp, 54321, 3, 2); + var options = new NativeSaveEncodingOptions(0x78563412, 0x1357); + + byte[] file = NativeSaveContainerCodec.Encode(payload, metadata, options); + + Assert.Equal("S4SD", Encoding.ASCII.GetString(file, 0, 4)); + Assert.Equal(0x10203040u, BinaryPrimitives.ReadUInt32LittleEndian(file.AsSpan(4))); + Assert.Equal((ushort)2026, BinaryPrimitives.ReadUInt16LittleEndian(file.AsSpan(0x108))); + Assert.Equal((ushort)7, BinaryPrimitives.ReadUInt16LittleEndian(file.AsSpan(0x10a))); + Assert.Equal((ushort)24, BinaryPrimitives.ReadUInt16LittleEndian(file.AsSpan(0x10e))); + Assert.Equal(54321u, BinaryPrimitives.ReadUInt32LittleEndian(file.AsSpan(0x118))); + Assert.Equal(3, BinaryPrimitives.ReadInt32LittleEndian(file.AsSpan(0x11c))); + Assert.Equal(2, BinaryPrimitives.ReadInt32LittleEndian(file.AsSpan(0x120))); + Assert.Equal(0x78563412u, BinaryPrimitives.ReadUInt32LittleEndian(file.AsSpan(0x130))); + Assert.Equal(0x1357u, BinaryPrimitives.ReadUInt32LittleEndian(file.AsSpan(0x134))); + + byte[] checkedLogical = new byte[payload.Length + 8]; + payload.CopyTo(checkedLogical.AsSpan(8)); + BinaryPrimitives.WriteUInt32LittleEndian( + checkedLogical, NativeSaveContainerCodec.Crc32Msb(payload)); + BinaryPrimitives.WriteUInt32LittleEndian( + checkedLogical.AsSpan(4), NativeSaveContainerCodec.Crc32Reflected(payload)); + byte[] stored = LzssEncoder.EncodeOrVerbatim(checkedLogical); + int expectedTransformDwords = stored.Length / 4 + 0x0d; + Assert.Equal( + checked((uint)(expectedTransformDwords * 2)), + BinaryPrimitives.ReadUInt32LittleEndian(file.AsSpan(0x124))); + + NativeSaveDocument decoded = NativeSaveContainerCodec.Decode(file); + Assert.Equal(metadata, decoded.Metadata); + Assert.Equal(payload, decoded.Payload); + Assert.Equal(file.Length, decoded.BytesConsumed); + Assert.Equal(metadata, NativeSaveContainerCodec.ReadMetadata(file.AsSpan(0, 0x124))); + } + + [Fact] + public void LegacyUncompressedContainerRoundTrips() + { + byte[] payload = Enumerable.Range(0, 64).Select(i => (byte)(i * 37)).ToArray(); + var metadata = new NativeSaveMetadata( + NativeSaveMagic.S3SD, 77, "legacy", Timestamp, 9, 1, 1); + + byte[] file = NativeSaveContainerCodec.Encode( + payload, metadata, new NativeSaveEncodingOptions(0xabcdef01, 3)); + NativeSaveDocument decoded = NativeSaveContainerCodec.Decode(file); + + Assert.Equal(metadata, decoded.Metadata); + Assert.Equal(payload, decoded.Payload); + Assert.Equal((uint)((payload.Length + 8) / 4 * 2), + BinaryPrimitives.ReadUInt32LittleEndian(file.AsSpan(0x124))); + + uint mixed = NativeSaveContainerCodec.Crc32Msb(payload) ^ 0xabcdef01; + Assert.Equal((mixed >> 16) * 3, + BinaryPrimitives.ReadUInt32LittleEndian( + file.AsSpan(NativeSaveContainerCodec.FixedPrefixSize))); + Assert.Equal((mixed & 0xffff) * 3, + BinaryPrimitives.ReadUInt32LittleEndian( + file.AsSpan(NativeSaveContainerCodec.FixedPrefixSize + 4))); + } + + [Fact] + public void EncodedChecksumAndExactDivisionRejectCorruption() + { + byte[] payload = Enumerable.Range(0, 16).SelectMany(BitConverter.GetBytes).ToArray(); + var metadata = new NativeSaveMetadata( + NativeSaveMagic.S4SD, 1, "test", Timestamp, 0, 1, 1); + byte[] file = NativeSaveContainerCodec.Encode( + payload, metadata, new NativeSaveEncodingOptions(0x12345678, 3)); + + byte[] crcFailure = file.ToArray(); + crcFailure[^1] ^= 0x80; + Assert.Contains("checksum", Assert.Throws( + () => NativeSaveContainerCodec.Decode(crcFailure)).Message); + + byte[] divisionFailure = file.ToArray(); + int encodedOffset = NativeSaveContainerCodec.FixedPrefixSize; + uint firstProduct = BinaryPrimitives.ReadUInt32LittleEndian(divisionFailure.AsSpan(encodedOffset)); + BinaryPrimitives.WriteUInt32LittleEndian(divisionFailure.AsSpan(encodedOffset), firstProduct + 1); + ReadOnlySpan encoded = divisionFailure.AsSpan(encodedOffset); + BinaryPrimitives.WriteUInt32LittleEndian( + divisionFailure.AsSpan(0x128), NativeSaveContainerCodec.Crc32Msb(encoded)); + BinaryPrimitives.WriteUInt32LittleEndian( + divisionFailure.AsSpan(0x12c), NativeSaveContainerCodec.Crc32Reflected(encoded)); + + Assert.Contains("divisible", Assert.Throws( + () => NativeSaveContainerCodec.Decode(divisionFailure)).Message); + } + + [Fact] + public void IdentityUsesNativeLayoutTwoVersionCompatibility() + { + var identity = new NativeSaveIdentity(NativeSaveMagic.S4SD, 9, "game", 2, 10); + identity.Validate(identity.CreateMetadata(Timestamp, 0) with { SaveVersion2 = 20 }); + + Assert.Throws(() => + identity.Validate(identity.CreateMetadata(Timestamp, 0) with { SaveVersion1 = 3 })); + Assert.Throws(() => + identity.Validate(identity.CreateMetadata(Timestamp, 0) with { GameId = "other" })); + } + + [Fact] + public void DirectoryStoreKeepsSharedBackupFallbackAndDirectNumberedFiles() + { + string root = Path.Combine(Path.GetTempPath(), "age-native-save-" + Guid.NewGuid().ToString("N")); + try + { + var identity = new NativeSaveIdentity(NativeSaveMagic.S4SD, 123, "himegari-test", 3, 2); + var store = new DirectoryNativeDatStore(root, identity); + byte[] first = Enumerable.Repeat((byte)0x11, 64).ToArray(); + byte[] second = Enumerable.Repeat((byte)0x22, 64).ToArray(); + byte[] numbered = Enumerable.Repeat((byte)0x33, 64).ToArray(); + + store.SaveShared(first, Timestamp, 10); + Assert.True(File.Exists(Path.Combine(root, "SAVE.DAT"))); + Assert.False(File.Exists(Path.Combine(root, "$$SAVE.DAT"))); + Assert.False(File.Exists(Path.Combine(root, "SAVE.BAK"))); + + store.SaveShared(second, Timestamp, 20); + Assert.Equal(second, store.LoadShared()!.Payload); + Assert.True(File.Exists(Path.Combine(root, "SAVE.BAK"))); + + string primary = Path.Combine(root, "SAVE.DAT"); + byte[] corrupt = File.ReadAllBytes(primary); + corrupt[^1] ^= 1; + File.WriteAllBytes(primary, corrupt); + Assert.Equal(first, store.LoadShared()!.Payload); + + store.SaveNumbered(4, numbered, Timestamp, 30); + Assert.True(File.Exists(Path.Combine(root, "SAVE04.DAT"))); + Assert.Equal(numbered, store.LoadNumbered(4)!.Payload); + Assert.Null(store.LoadNumbered(5)); + } + finally + { + if (Directory.Exists(root)) Directory.Delete(root, recursive: true); + } + } +} diff --git a/engine/Age.Engine.Tests/OpcodeTableTests.cs b/engine/Age.Engine.Tests/OpcodeTableTests.cs index 321ddff..8f5587f 100644 --- a/engine/Age.Engine.Tests/OpcodeTableTests.cs +++ b/engine/Age.Engine.Tests/OpcodeTableTests.cs @@ -4,15 +4,16 @@ using Xunit; public class OpcodeTableTests { [Fact] - public void LoadsAll248FromJson() + public void LoadsObservedOpcodesPlusMappedUnusedAbiEntries() { var t = OpcodeTableJson.Load(Paths.OpcodesJson); - Assert.Equal(248, t.Count); + Assert.Equal(249, t.Count); // 248 corpus-observed + unused persistence ABI opcode 0x19f Assert.True(t.TryGet(0x55, out var label, out var argc)); Assert.Equal("mov", label); Assert.Equal(2, argc); Assert.Equal("u0041BEB0", t.Label(0x90)); Assert.Equal(7, t.Argc(0x90)); + Assert.Equal(2, t.Argc(0x19f)); Assert.Equal(-1, t.Argc(0x9999)); // absent -> -1 } } diff --git a/engine/Age.Engine/Persistence/NativeDatStore.cs b/engine/Age.Engine/Persistence/NativeDatStore.cs new file mode 100644 index 0000000..d883918 --- /dev/null +++ b/engine/Age.Engine/Persistence/NativeDatStore.cs @@ -0,0 +1,151 @@ +using System.Globalization; + +namespace Age.Engine.Persistence; + +public sealed record NativeSaveIdentity( + NativeSaveMagic Magic, + uint CompatibilityId, + string GameId, + int SaveVersion1, + int SaveVersion2) +{ + public NativeSaveMetadata CreateMetadata(NativeSystemTime timestamp, uint accumulatedPlaySeconds) + => new(Magic, CompatibilityId, GameId, timestamp, accumulatedPlaySeconds, SaveVersion1, SaveVersion2); + + public void Validate(NativeSaveMetadata metadata) + { + if (metadata.Magic != Magic) + throw new InvalidDataException($"Native save generation mismatch: expected {Magic}, got {metadata.Magic}."); + if (metadata.CompatibilityId != CompatibilityId) + throw new InvalidDataException("Native save compatibility id mismatch."); + if (!StringComparer.Ordinal.Equals(metadata.GameId, GameId)) + throw new InvalidDataException("Native save game id mismatch."); + + bool versionsMatch = + metadata.SaveVersion1 == SaveVersion1 && + metadata.SaveVersion2 == SaveVersion2; + bool layoutTwoCompatibility = + metadata.SaveVersion1 == 2 && + SaveVersion1 == 2; + if (!versionsMatch && !layoutTwoCompatibility) + throw new InvalidDataException( + $"Native save version mismatch: expected {SaveVersion1}.{SaveVersion2}, " + + $"got {metadata.SaveVersion1}.{metadata.SaveVersion2}."); + } +} + +public interface INativeDatStore +{ + NativeSaveDocument? LoadShared(); + void SaveShared(ReadOnlySpan payload, NativeSystemTime timestamp, uint accumulatedPlaySeconds); + NativeSaveDocument? LoadNumbered(int slot); + void SaveNumbered(int slot, ReadOnlySpan payload, NativeSystemTime timestamp, uint accumulatedPlaySeconds); +} + +/// +/// Directory-backed native DAT lifecycle. Shared state uses $$SAVE.DAT -> SAVE.DAT with SAVE.BAK +/// fallback; numbered slots are written directly as SAVE##.DAT, matching AGE's separate behavior. +/// Payload ownership remains above this boundary. +/// +public sealed class DirectoryNativeDatStore : INativeDatStore +{ + public const string SharedFileName = "SAVE.DAT"; + public const string SharedTemporaryFileName = "$$SAVE.DAT"; + public const string SharedBackupFileName = "SAVE.BAK"; + + private readonly string _root; + private readonly NativeSaveIdentity _identity; + + public DirectoryNativeDatStore(string root, NativeSaveIdentity identity) + { + ArgumentException.ThrowIfNullOrWhiteSpace(root); + _root = Path.GetFullPath(root); + _identity = identity; + } + + public NativeSaveDocument? LoadShared() + { + string primary = Path.Combine(_root, SharedFileName); + string backup = Path.Combine(_root, SharedBackupFileName); + if (!File.Exists(primary)) + return File.Exists(backup) ? LoadAndValidate(backup) : null; + + try + { + return LoadAndValidate(primary); + } + catch (Exception primaryError) when ( + primaryError is IOException or UnauthorizedAccessException or InvalidDataException) + { + if (!File.Exists(backup)) throw; + try + { + return LoadAndValidate(backup); + } + catch (Exception backupError) when ( + backupError is IOException or UnauthorizedAccessException or InvalidDataException) + { + throw new InvalidDataException( + "Both SAVE.DAT and SAVE.BAK failed native container validation.", + new AggregateException(primaryError, backupError)); + } + } + } + + public void SaveShared( + ReadOnlySpan payload, + NativeSystemTime timestamp, + uint accumulatedPlaySeconds) + { + byte[] encoded = NativeSaveContainerCodec.Encode( + payload, _identity.CreateMetadata(timestamp, accumulatedPlaySeconds)); + Directory.CreateDirectory(_root); + + string temporary = Path.Combine(_root, SharedTemporaryFileName); + string primary = Path.Combine(_root, SharedFileName); + string backup = Path.Combine(_root, SharedBackupFileName); + WriteThrough(temporary, encoded); + if (File.Exists(backup)) File.Delete(backup); + if (File.Exists(primary)) File.Move(primary, backup); + File.Move(temporary, primary); + } + + public NativeSaveDocument? LoadNumbered(int slot) + { + string path = Path.Combine(_root, NumberedFileName(slot)); + return File.Exists(path) ? LoadAndValidate(path) : null; + } + + public void SaveNumbered( + int slot, + ReadOnlySpan payload, + NativeSystemTime timestamp, + uint accumulatedPlaySeconds) + { + byte[] encoded = NativeSaveContainerCodec.Encode( + payload, _identity.CreateMetadata(timestamp, accumulatedPlaySeconds)); + Directory.CreateDirectory(_root); + WriteThrough(Path.Combine(_root, NumberedFileName(slot)), encoded); + } + + public static string NumberedFileName(int slot) + { + if (slot < 0) throw new ArgumentOutOfRangeException(nameof(slot)); + return "SAVE" + slot.ToString("00", CultureInfo.InvariantCulture) + ".DAT"; + } + + private NativeSaveDocument LoadAndValidate(string path) + { + NativeSaveDocument document = NativeSaveContainerCodec.Decode(File.ReadAllBytes(path)); + _identity.Validate(document.Metadata); + return document; + } + + private static void WriteThrough(string path, ReadOnlySpan data) + { + using var stream = new FileStream( + path, FileMode.Create, FileAccess.Write, FileShare.None, 64 * 1024, FileOptions.SequentialScan); + stream.Write(data); + stream.Flush(flushToDisk: true); + } +} diff --git a/engine/Age.Engine/Persistence/NativeSaveContainerCodec.cs b/engine/Age.Engine/Persistence/NativeSaveContainerCodec.cs new file mode 100644 index 0000000..46a4c6b --- /dev/null +++ b/engine/Age.Engine/Persistence/NativeSaveContainerCodec.cs @@ -0,0 +1,365 @@ +using System.Buffers.Binary; +using System.Security.Cryptography; +using System.Text; +using Age.Engine.Sys4; + +namespace Age.Engine.Persistence; + +public enum NativeSaveMagic +{ + S3SD, + S4SD, +} + +public readonly record struct NativeSystemTime( + ushort Year, + ushort Month, + ushort DayOfWeek, + ushort Day, + ushort Hour, + ushort Minute, + ushort Second, + ushort Milliseconds) +{ + public static NativeSystemTime FromLocalDateTime(DateTime value) + { + DateTime local = value.Kind == DateTimeKind.Local ? value : value.ToLocalTime(); + return new NativeSystemTime( + (ushort)local.Year, (ushort)local.Month, (ushort)local.DayOfWeek, (ushort)local.Day, + (ushort)local.Hour, (ushort)local.Minute, (ushort)local.Second, (ushort)local.Millisecond); + } +} + +public sealed record NativeSaveMetadata( + NativeSaveMagic Magic, + uint CompatibilityId, + string GameId, + NativeSystemTime Timestamp, + uint AccumulatedPlaySeconds, + int SaveVersion1, + int SaveVersion2); + +public sealed record NativeSaveDocument( + NativeSaveMetadata Metadata, + byte[] Payload, + int BytesConsumed); + +public sealed record NativeSaveEncodingOptions(uint XorSeed, ushort Multiplier) +{ + public static NativeSaveEncodingOptions Random() + { + Span random = stackalloc byte[6]; + RandomNumberGenerator.Fill(random); + uint seed = BinaryPrimitives.ReadUInt32LittleEndian(random); + ushort multiplier = (ushort)(BinaryPrimitives.ReadUInt16LittleEndian(random[4..]) | 1); + return new NativeSaveEncodingOptions(seed, multiplier); + } +} + +/// +/// AGE's common S3SD/S4SD wrapper used by shared SAVE.DAT and numbered SAVE##.DAT files. +/// Payload serializers own the bytes inside this wrapper; this class owns the native header, +/// integrity checks, optional LZSS layer, and reversible DWORD transform. +/// +public static class NativeSaveContainerCodec +{ + public const int HeaderSize = 0x124; + public const int CodecFrameSize = 0x14; + public const int FixedPrefixSize = HeaderSize + CodecFrameSize; + + private const uint SeedIncrement = 0x0b0b0b0b; + private const ushort MultiplierIncrement = 0x0b02; + private const uint MsbPolynomial = 0x04c11db7; + private const uint ReflectedPolynomial = 0xedb88320; + private static readonly Encoding ShiftJis = CreateShiftJis(); + + public static byte[] Encode( + ReadOnlySpan payload, + NativeSaveMetadata metadata, + NativeSaveEncodingOptions? options = null) + { + if ((payload.Length & 3) != 0) + throw new ArgumentException("Native save payload length must be DWORD-aligned.", nameof(payload)); + if (metadata.SaveVersion2 < 0) + throw new ArgumentOutOfRangeException(nameof(metadata), "SaveVersion2 cannot be negative."); + + options ??= NativeSaveEncodingOptions.Random(); + ValidateMultiplier(options.Multiplier); + + byte[] checkedLogical = new byte[payload.Length + 8]; + payload.CopyTo(checkedLogical.AsSpan(8)); + WriteLogicalChecksums(checkedLogical); + + byte[] transformInput = metadata.SaveVersion2 >= 2 + ? BuildCompressedWrapper(checkedLogical) + : checkedLogical; + byte[] encoded = ExpandTransform(transformInput, options.XorSeed, options.Multiplier); + + byte[] result = new byte[checked(HeaderSize + CodecFrameSize + encoded.Length)]; + WriteHeader(result.AsSpan(0, HeaderSize), metadata); + Span frame = result.AsSpan(HeaderSize, CodecFrameSize); + BinaryPrimitives.WriteUInt32LittleEndian(frame, checked((uint)(encoded.Length / 4))); + BinaryPrimitives.WriteUInt32LittleEndian(frame[4..], Crc32Msb(encoded)); + BinaryPrimitives.WriteUInt32LittleEndian(frame[8..], Crc32Reflected(encoded)); + BinaryPrimitives.WriteUInt32LittleEndian(frame[12..], options.XorSeed); + BinaryPrimitives.WriteUInt32LittleEndian(frame[16..], options.Multiplier); + encoded.CopyTo(result.AsSpan(FixedPrefixSize)); + return result; + } + + public static NativeSaveDocument Decode(ReadOnlySpan source) + { + if (source.Length < FixedPrefixSize) + throw new InvalidDataException("Native save container is shorter than its fixed header."); + + NativeSaveMetadata metadata = ReadMetadata(source); + ReadOnlySpan frame = source.Slice(HeaderSize, CodecFrameSize); + uint encodedDwordCount = BinaryPrimitives.ReadUInt32LittleEndian(frame); + if ((encodedDwordCount & 1) != 0) + throw new InvalidDataException("Native save encoded DWORD count is not even."); + if (encodedDwordCount > int.MaxValue / 4) + throw new InvalidDataException("Native save encoded DWORD count is too large."); + int encodedLength = (int)encodedDwordCount * 4; + if (encodedLength > source.Length - FixedPrefixSize) + throw new InvalidDataException("Native save encoded payload is truncated."); + + ReadOnlySpan encoded = source.Slice(FixedPrefixSize, encodedLength); + uint expectedMsb = BinaryPrimitives.ReadUInt32LittleEndian(frame[4..]); + uint expectedReflected = BinaryPrimitives.ReadUInt32LittleEndian(frame[8..]); + if (Crc32Msb(encoded) != expectedMsb || Crc32Reflected(encoded) != expectedReflected) + throw new InvalidDataException("Native save encoded payload checksum mismatch."); + + uint seed = BinaryPrimitives.ReadUInt32LittleEndian(frame[12..]); + uint storedMultiplier = BinaryPrimitives.ReadUInt32LittleEndian(frame[16..]); + if ((storedMultiplier & 0xffff0000) != 0) + throw new InvalidDataException("Native save multiplier contains unexpected high bits."); + ushort multiplier = (ushort)storedMultiplier; + ValidateMultiplier(multiplier); + + byte[] transformInput = InverseTransform(encoded, seed, multiplier); + byte[] checkedLogical = metadata.SaveVersion2 >= 2 + ? DecodeCompressedWrapper(transformInput) + : transformInput; + if (checkedLogical.Length < 8 || (checkedLogical.Length & 3) != 0) + throw new InvalidDataException("Native save logical payload has an invalid length."); + + ReadOnlySpan payload = checkedLogical.AsSpan(8); + uint logicalMsb = BinaryPrimitives.ReadUInt32LittleEndian(checkedLogical); + uint logicalReflected = BinaryPrimitives.ReadUInt32LittleEndian(checkedLogical.AsSpan(4)); + if (Crc32Msb(payload) != logicalMsb || Crc32Reflected(payload) != logicalReflected) + throw new InvalidDataException("Native save logical payload checksum mismatch."); + + return new NativeSaveDocument( + metadata, + payload.ToArray(), + checked(FixedPrefixSize + encodedLength)); + } + + /// Read only the fixed native header, matching metadata-query opcode 0x1a0. + public static NativeSaveMetadata ReadMetadata(ReadOnlySpan source) + { + if (source.Length < HeaderSize) + throw new InvalidDataException("Native save container is shorter than its metadata header."); + return ReadHeader(source[..HeaderSize]); + } + + public static uint Crc32Msb(ReadOnlySpan data) + { + uint crc = uint.MaxValue; + foreach (byte value in data) + { + crc ^= (uint)value << 24; + for (int bit = 0; bit < 8; bit++) + crc = (crc & 0x80000000) != 0 ? crc << 1 ^ MsbPolynomial : crc << 1; + } + return ~crc; + } + + public static uint Crc32Reflected(ReadOnlySpan data) + { + uint crc = uint.MaxValue; + foreach (byte value in data) + { + crc ^= value; + for (int bit = 0; bit < 8; bit++) + crc = (crc & 1) != 0 ? crc >> 1 ^ ReflectedPolynomial : crc >> 1; + } + return ~crc; + } + + private static void WriteHeader(Span header, NativeSaveMetadata metadata) + { + header.Clear(); + WriteMagic(header, metadata.Magic); + BinaryPrimitives.WriteUInt32LittleEndian(header[4..], metadata.CompatibilityId); + + if (metadata.GameId.Contains('\0')) + throw new ArgumentException("Native save game id cannot contain NUL.", nameof(metadata)); + byte[] gameId = ShiftJis.GetBytes(metadata.GameId); + if (gameId.Length >= 0x100) + throw new ArgumentException("Native save game id must fit in 255 Shift-JIS bytes.", nameof(metadata)); + gameId.CopyTo(header[8..]); + + Span systemTime = header[0x108..0x118]; + BinaryPrimitives.WriteUInt16LittleEndian(systemTime, metadata.Timestamp.Year); + BinaryPrimitives.WriteUInt16LittleEndian(systemTime[2..], metadata.Timestamp.Month); + BinaryPrimitives.WriteUInt16LittleEndian(systemTime[4..], metadata.Timestamp.DayOfWeek); + BinaryPrimitives.WriteUInt16LittleEndian(systemTime[6..], metadata.Timestamp.Day); + BinaryPrimitives.WriteUInt16LittleEndian(systemTime[8..], metadata.Timestamp.Hour); + BinaryPrimitives.WriteUInt16LittleEndian(systemTime[10..], metadata.Timestamp.Minute); + BinaryPrimitives.WriteUInt16LittleEndian(systemTime[12..], metadata.Timestamp.Second); + BinaryPrimitives.WriteUInt16LittleEndian(systemTime[14..], metadata.Timestamp.Milliseconds); + BinaryPrimitives.WriteUInt32LittleEndian(header[0x118..], metadata.AccumulatedPlaySeconds); + BinaryPrimitives.WriteInt32LittleEndian(header[0x11c..], metadata.SaveVersion1); + BinaryPrimitives.WriteInt32LittleEndian(header[0x120..], metadata.SaveVersion2); + } + + private static NativeSaveMetadata ReadHeader(ReadOnlySpan header) + { + NativeSaveMagic magic = ReadMagic(header); + uint compatibilityId = BinaryPrimitives.ReadUInt32LittleEndian(header[4..]); + ReadOnlySpan gameIdBytes = header.Slice(8, 0x100); + int terminator = gameIdBytes.IndexOf((byte)0); + if (terminator < 0) + throw new InvalidDataException("Native save game id is not NUL-terminated."); + string gameId; + try + { + gameId = ShiftJis.GetString(gameIdBytes[..terminator]); + } + catch (DecoderFallbackException error) + { + throw new InvalidDataException("Native save game id is not valid Shift-JIS.", error); + } + + ReadOnlySpan systemTime = header[0x108..0x118]; + var timestamp = new NativeSystemTime( + BinaryPrimitives.ReadUInt16LittleEndian(systemTime), + BinaryPrimitives.ReadUInt16LittleEndian(systemTime[2..]), + BinaryPrimitives.ReadUInt16LittleEndian(systemTime[4..]), + BinaryPrimitives.ReadUInt16LittleEndian(systemTime[6..]), + BinaryPrimitives.ReadUInt16LittleEndian(systemTime[8..]), + BinaryPrimitives.ReadUInt16LittleEndian(systemTime[10..]), + BinaryPrimitives.ReadUInt16LittleEndian(systemTime[12..]), + BinaryPrimitives.ReadUInt16LittleEndian(systemTime[14..])); + + return new NativeSaveMetadata( + magic, + compatibilityId, + gameId, + timestamp, + BinaryPrimitives.ReadUInt32LittleEndian(header[0x118..]), + BinaryPrimitives.ReadInt32LittleEndian(header[0x11c..]), + BinaryPrimitives.ReadInt32LittleEndian(header[0x120..])); + } + + private static byte[] BuildCompressedWrapper(ReadOnlySpan checkedLogical) + { + byte[] stored = LzssEncoder.EncodeOrVerbatim(checkedLogical); + int nativeDwordCount = checked(stored.Length / 4 + 0x0d); + byte[] wrapper = new byte[checked(nativeDwordCount * 4)]; + BinaryPrimitives.WriteUInt32LittleEndian(wrapper, checked((uint)checkedLogical.Length)); + BinaryPrimitives.WriteUInt32LittleEndian(wrapper.AsSpan(4), checked((uint)checkedLogical.Length)); + BinaryPrimitives.WriteUInt32LittleEndian(wrapper.AsSpan(8), checked((uint)stored.Length)); + stored.CopyTo(wrapper.AsSpan(12)); + return wrapper; + } + + private static byte[] DecodeCompressedWrapper(ReadOnlySpan wrapper) + { + if (wrapper.Length < 12) + throw new InvalidDataException("Native save compressed wrapper is truncated."); + uint originalLength = BinaryPrimitives.ReadUInt32LittleEndian(wrapper); + uint consumedLength = BinaryPrimitives.ReadUInt32LittleEndian(wrapper[4..]); + uint storedLength = BinaryPrimitives.ReadUInt32LittleEndian(wrapper[8..]); + if (originalLength != consumedLength) + throw new InvalidDataException("Native save LZSS wrapper length fields disagree."); + if (originalLength > int.MaxValue || storedLength > int.MaxValue || + storedLength > (uint)(wrapper.Length - 12)) + throw new InvalidDataException("Native save LZSS wrapper length is invalid."); + + ReadOnlySpan stored = wrapper.Slice(12, (int)storedLength); + return storedLength == originalLength + ? stored.ToArray() + : LzssDecoder.Decode(stored, (int)originalLength, "native save LZSS"); + } + + private static byte[] ExpandTransform(ReadOnlySpan source, uint seed, ushort multiplier) + { + if ((source.Length & 3) != 0) + throw new InvalidDataException("Native save transform input is not DWORD-aligned."); + byte[] encoded = new byte[checked(source.Length * 2)]; + for (int input = 0, output = 0; input < source.Length; input += 4, output += 8) + { + uint mixed = BinaryPrimitives.ReadUInt32LittleEndian(source[input..]) ^ seed; + BinaryPrimitives.WriteUInt32LittleEndian(encoded.AsSpan(output), (mixed >> 16) * multiplier); + BinaryPrimitives.WriteUInt32LittleEndian(encoded.AsSpan(output + 4), (mixed & 0xffff) * multiplier); + seed = unchecked(seed + SeedIncrement); + multiplier = unchecked((ushort)(multiplier + MultiplierIncrement)); + } + return encoded; + } + + private static byte[] InverseTransform(ReadOnlySpan encoded, uint seed, ushort multiplier) + { + if ((encoded.Length & 7) != 0) + throw new InvalidDataException("Native save encoded transform length is invalid."); + byte[] source = new byte[encoded.Length / 2]; + for (int input = 0, output = 0; input < encoded.Length; input += 8, output += 4) + { + uint highProduct = BinaryPrimitives.ReadUInt32LittleEndian(encoded[input..]); + uint lowProduct = BinaryPrimitives.ReadUInt32LittleEndian(encoded[(input + 4)..]); + if (highProduct % multiplier != 0 || lowProduct % multiplier != 0) + throw new InvalidDataException("Native save transform product is not exactly divisible."); + uint high = highProduct / multiplier; + uint low = lowProduct / multiplier; + if (high > ushort.MaxValue || low > ushort.MaxValue) + throw new InvalidDataException("Native save transform quotient exceeds 16 bits."); + + uint mixed = high << 16 | low; + BinaryPrimitives.WriteUInt32LittleEndian(source.AsSpan(output), mixed ^ seed); + seed = unchecked(seed + SeedIncrement); + multiplier = unchecked((ushort)(multiplier + MultiplierIncrement)); + } + return source; + } + + private static void WriteLogicalChecksums(Span checkedLogical) + { + ReadOnlySpan payload = checkedLogical[8..]; + BinaryPrimitives.WriteUInt32LittleEndian(checkedLogical, Crc32Msb(payload)); + BinaryPrimitives.WriteUInt32LittleEndian(checkedLogical[4..], Crc32Reflected(payload)); + } + + private static void WriteMagic(Span destination, NativeSaveMagic magic) + { + destination[0] = (byte)'S'; + destination[1] = magic == NativeSaveMagic.S3SD ? (byte)'3' : (byte)'4'; + destination[2] = (byte)'S'; + destination[3] = (byte)'D'; + } + + private static NativeSaveMagic ReadMagic(ReadOnlySpan source) + { + if (source[0] != 'S' || source[2] != 'S' || source[3] != 'D') + throw new InvalidDataException("Native save magic is invalid."); + return source[1] switch + { + (byte)'3' => NativeSaveMagic.S3SD, + (byte)'4' => NativeSaveMagic.S4SD, + _ => throw new InvalidDataException("Native save generation is unsupported."), + }; + } + + private static void ValidateMultiplier(ushort multiplier) + { + if (multiplier == 0 || (multiplier & 1) == 0) + throw new InvalidDataException("Native save multiplier must be a nonzero odd value."); + } + + private static Encoding CreateShiftJis() + { + Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); + return Encoding.GetEncoding(932, EncoderFallback.ExceptionFallback, DecoderFallback.ExceptionFallback); + } +} diff --git a/engine/Age.Engine/Sys4/LzssEncoder.cs b/engine/Age.Engine/Sys4/LzssEncoder.cs new file mode 100644 index 0000000..60f1aa3 --- /dev/null +++ b/engine/Age.Engine/Sys4/LzssEncoder.cs @@ -0,0 +1,118 @@ +namespace Age.Engine.Sys4; + +/// +/// Encoder for Eushully's 4 KiB-ring LZSS stream. Tokens are grouped under an LSB-first flag byte; +/// one bits are literals and zero bits are 12-bit ring offsets plus a four-bit length-minus-three. +/// +public static class LzssEncoder +{ + private const int RingSize = 0x1000; + private const int RingMask = RingSize - 1; + private const int InitialRingPosition = 0xfee; + private const int MaxMatchLength = 18; + private const int CandidateLimit = 128; + + /// + /// Compress . When the encoded stream would not be smaller, return the + /// original bytes; the native container marks that case by storing equal source/stored lengths. + /// + public static byte[] EncodeOrVerbatim(ReadOnlySpan source) + { + if (source.IsEmpty) return []; + + var output = new List(source.Length); + var positionsByPrefix = new Dictionary>(); + int position = 0; + + while (position < source.Length) + { + int flagIndex = output.Count; + output.Add(0); + byte flags = 0; + + for (int bit = 0; bit < 8 && position < source.Length; bit++) + { + (int candidate, int length) = FindMatch(source, position, positionsByPrefix); + if (length >= 3) + { + int ringOffset = (InitialRingPosition + candidate) & RingMask; + output.Add((byte)ringOffset); + output.Add((byte)(((ringOffset >> 4) & 0xf0) | (length - 3))); + AddPositions(source, position, length, positionsByPrefix); + position += length; + } + else + { + flags |= (byte)(1 << bit); + output.Add(source[position]); + AddPositions(source, position, 1, positionsByPrefix); + position++; + } + } + + output[flagIndex] = flags; + } + + return output.Count < source.Length ? output.ToArray() : source.ToArray(); + } + + private static (int Candidate, int Length) FindMatch( + ReadOnlySpan source, + int position, + Dictionary> positionsByPrefix) + { + if (position > source.Length - 3) return (-1, 0); + int key = PrefixKey(source, position); + if (!positionsByPrefix.TryGetValue(key, out var candidates)) return (-1, 0); + + int oldestAllowed = position - RingSize; + while (candidates.First is { } first && first.Value < oldestAllowed) + candidates.RemoveFirst(); + + int bestCandidate = -1, bestLength = 0, inspected = 0; + for (LinkedListNode? node = candidates.Last; + node is not null && inspected < CandidateLimit; + node = node.Previous, inspected++) + { + int candidate = node.Value; + int distance = position - candidate; + if (distance <= 0 || distance > RingSize) continue; + + int limit = Math.Min(MaxMatchLength, source.Length - position); + int length = 0; + while (length < limit && + source[position + length] == source[candidate + (length % distance)]) + length++; + + if (length > bestLength) + { + bestCandidate = candidate; + bestLength = length; + if (length == MaxMatchLength) break; + } + } + return (bestCandidate, bestLength); + } + + private static void AddPositions( + ReadOnlySpan source, + int start, + int count, + Dictionary> positionsByPrefix) + { + int end = Math.Min(start + count, source.Length - 2); + for (int position = start; position < end; position++) + { + int key = PrefixKey(source, position); + if (!positionsByPrefix.TryGetValue(key, out var positions)) + { + positions = new LinkedList(); + positionsByPrefix.Add(key, positions); + } + positions.AddLast(position); + } + } + + private static int PrefixKey(ReadOnlySpan source, int position) + => source[position] | source[position + 1] << 8 | source[position + 2] << 16; +}