Implement native DAT container codec
This commit is contained in:
gamer147
@@ -101,8 +101,9 @@ S:\Game Hacking\Eushully\Himegari\ ← workspace root (three siblings)
|
||||
│ └── manifest.json, opcode-coverage.md (opcode-coverage.md GENERATED from opcodes.toml)
|
||||
│
|
||||
├── engine/ DELIVERABLE — the .NET VM core (AgeEngine.sln: Age.Engine / Age.Cli / tests)
|
||||
│ └── Age.Engine/Sys4/ runtime catalog parser, loose-first bounded ALF asset store,
|
||||
│ ├── Age.Engine/Sys4/ runtime catalog parser, loose-first bounded ALF asset store,
|
||||
│ script provider, AGF/LZSS and Windows CUR decoders, and resource facade
|
||||
│ └── Age.Engine/Persistence/ native S3SD/S4SD container codec and profile/numbered DAT lifecycle seam
|
||||
├── native/ authored native runtime boundaries
|
||||
│ └── age_movie_ffmpeg/ project-owned FFmpeg C ABI, immutable Windows dependency manifest,
|
||||
│ and bootstrap/build scripts (outputs stay under disposable build/)
|
||||
|
||||
@@ -1722,6 +1722,16 @@ inventing a second save container.
|
||||
a profile/save service so extended mode can later add JSON inspection/export, namespaced mod state, migrations,
|
||||
or a friendlier editor without changing compatibility-mode opcode semantics or the native import/export path.
|
||||
|
||||
**Port correspondence (2026-07-24, codec foundation implemented):**
|
||||
`Age.Engine.Persistence.NativeSaveContainerCodec` now reads and writes the common header, Shift-JIS game id,
|
||||
SYSTEMTIME/playtime/version metadata, both CRC layers, version-2 compression wrapper, and exact reversible
|
||||
DWORD transform. `Sys4.LzssEncoder` emits the same 4 KiB-ring token dialect already consumed by
|
||||
`LzssDecoder`, falling back to native verbatim storage when compression does not shrink. The
|
||||
`INativeDatStore` boundary and `DirectoryNativeDatStore` own shared `$$SAVE.DAT` → `SAVE.DAT` /
|
||||
`SAVE.BAK` replacement/fallback and direct numbered `SAVE##.DAT` writes. Payload schemas, `RT.DAT`,
|
||||
thumbnails, and opcode wiring remain deliberately above or after this layer; the existing `GameSession`
|
||||
JSON snapshot is unchanged.
|
||||
|
||||
### Opcode `0xae` continues numbered-save stack restoration (2026-07-20)
|
||||
|
||||
Opcode `0xae` is the load-side rendezvous paired with serialized script-frame state. Its handler,
|
||||
|
||||
@@ -3355,6 +3355,32 @@ No runtime persistence code changed in this reconnaissance slice. The canonical
|
||||
references now describe the native ABI; the corresponding `/v2` handlers and codec helpers are named,
|
||||
commented, and saved.
|
||||
|
||||
### Persistence implementation step 1 — native DAT codec and store boundary (2026-07-24)
|
||||
|
||||
The reusable compatibility floor is now implemented without prematurely inventing either shared-profile or
|
||||
numbered-state payload objects. `NativeSaveContainerCodec` owns the exact `0x124` header and `0x14` codec
|
||||
frame, Shift-JIS game identity, SYSTEMTIME/playtime/version fields, inner and outer CRC pairs, rolling
|
||||
seed/odd-multiplier DWORD expansion, its exact-division inverse, and the SaveVersion2>=2 wrapper. The new
|
||||
`LzssEncoder` shares the existing native 4096-byte-ring dialect and uses the native equal-length verbatim
|
||||
fallback when compression is not beneficial.
|
||||
|
||||
`INativeDatStore` is the payload-agnostic runtime seam. Its directory implementation performs shared
|
||||
`$$SAVE.DAT` → `SAVE.DAT` replacement with `SAVE.BAK` fallback and direct `SAVE##.DAT` reads/writes, while
|
||||
validating generation, compatibility id, game id, and the native layout-version exception. It does not
|
||||
serialize the VM's whole global bank, change `GameSession.ToJson`, wire persistence opcodes, or claim
|
||||
`RT.DAT`/`.STH` support.
|
||||
|
||||
Seven focused tests cover independent standard CRC vectors, compressed and verbatim LZSS round trips,
|
||||
S3SD/S4SD header/frame offsets, deterministic transform products, both codec-version branches, corruption
|
||||
rejection after outer-CRC repair, layout-2 version compatibility, shared backup recovery, and direct numbered
|
||||
files. The full engine suite passes 359 tests. The opcode table count regression now distinguishes 248
|
||||
Himegari-observed opcodes from the mapped-but-unused shared ABI opcode `0x19f`.
|
||||
|
||||
**Next persistence step:** implement the typed shared `SAVE.DAT` logical payload and profile-owned selected
|
||||
integer/string cell maps, then connect `0x1a2`/`0x1a3` and `0x1a9`/`0x1aa`. Add native `RT.DAT` and its
|
||||
read-message lifecycle after that shared ownership is live; numbered active-frame layouts remain the later,
|
||||
larger payload.
|
||||
|
||||
## Data-semantics sidebar: focused append EBINIT inspection (2026-07-24)
|
||||
|
||||
The static INIT surface now accepts a universal packed script id for focused append inspection.
|
||||
|
||||
@@ -361,7 +361,8 @@ In scope:
|
||||
Deferred to bounded follow-ups unless the happy path requires them:
|
||||
|
||||
- Full configuration UI and every setting.
|
||||
- Load/save implementation and save-format reversal.
|
||||
- Full load/save opcode and logical-payload implementation. Native format reversal and the common DAT
|
||||
codec/store foundation landed on 2026-07-24.
|
||||
- Extras, galleries, replay modes, and unrelated submenus.
|
||||
- Menu visual polish that does not obstruct correct selection or state production.
|
||||
|
||||
|
||||
@@ -466,7 +466,8 @@ domains and lifecycle—shared `SAVE.DAT`/`SAVE.BAK`, `RT.DAT`/`RT.BAK`, numbere
|
||||
paired BMP `.STH` thumbnails. Keep the codec behind a profile/save-service boundary. Human-readable
|
||||
JSON inspection/export, migrations, and namespaced mod state are additive extended-mode work, not a
|
||||
replacement for compatibility-mode import/export. The recovered native contract lives in
|
||||
`docs/engine-re.md`.
|
||||
`docs/engine-re.md`. The common container codec and payload-agnostic shared/numbered DAT store boundary
|
||||
landed on 2026-07-24; logical payload services and opcode wiring remain Phase B work.
|
||||
|
||||
### Phase C — Externalize & modding foundation
|
||||
- Add **editable named data overlays** mapped explicitly onto the VM's `*INIT`-produced state; external
|
||||
|
||||
177
engine/Age.Engine.Tests/NativeSaveContainerTests.cs
Normal file
177
engine/Age.Engine.Tests/NativeSaveContainerTests.cs
Normal file
@@ -0,0 +1,177 @@
|
||||
using System.Buffers.Binary;
|
||||
using System.Text;
|
||||
using Age.Engine.Persistence;
|
||||
using Age.Engine.Sys4;
|
||||
|
||||
public class NativeSaveContainerTests
|
||||
{
|
||||
private static readonly NativeSystemTime Timestamp =
|
||||
new(2026, 7, 5, 24, 13, 42, 17, 321);
|
||||
|
||||
[Fact]
|
||||
public void NativeCrcVariantsMatchIndependentCheckVectors()
|
||||
{
|
||||
byte[] input = Encoding.ASCII.GetBytes("123456789");
|
||||
Assert.Equal(0xfc891918u, NativeSaveContainerCodec.Crc32Msb(input));
|
||||
Assert.Equal(0xcbf43926u, NativeSaveContainerCodec.Crc32Reflected(input));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void LzssEncoderRoundTripsTheNativeRingDialect()
|
||||
{
|
||||
byte[] input = Encoding.ASCII.GetBytes(string.Concat(
|
||||
Enumerable.Repeat("HIMEGARI-HIMEGARI-0000000000000000-", 80)));
|
||||
byte[] encoded = LzssEncoder.EncodeOrVerbatim(input);
|
||||
|
||||
Assert.True(encoded.Length < input.Length);
|
||||
Assert.Equal(input, LzssDecoder.Decode(encoded, input.Length, "save test"));
|
||||
|
||||
byte[] incompressible = [1, 9, 2, 8, 3, 7, 4, 6];
|
||||
Assert.Equal(incompressible, LzssEncoder.EncodeOrVerbatim(incompressible));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void VersionTwoContainerWritesNativeHeaderFrameAndRoundTrips()
|
||||
{
|
||||
byte[] payload = Enumerable.Range(0, 256)
|
||||
.SelectMany(i => BitConverter.GetBytes(i % 7))
|
||||
.ToArray();
|
||||
var metadata = new NativeSaveMetadata(
|
||||
NativeSaveMagic.S4SD, 0x10203040, "姫狩りDM", Timestamp, 54321, 3, 2);
|
||||
var options = new NativeSaveEncodingOptions(0x78563412, 0x1357);
|
||||
|
||||
byte[] file = NativeSaveContainerCodec.Encode(payload, metadata, options);
|
||||
|
||||
Assert.Equal("S4SD", Encoding.ASCII.GetString(file, 0, 4));
|
||||
Assert.Equal(0x10203040u, BinaryPrimitives.ReadUInt32LittleEndian(file.AsSpan(4)));
|
||||
Assert.Equal((ushort)2026, BinaryPrimitives.ReadUInt16LittleEndian(file.AsSpan(0x108)));
|
||||
Assert.Equal((ushort)7, BinaryPrimitives.ReadUInt16LittleEndian(file.AsSpan(0x10a)));
|
||||
Assert.Equal((ushort)24, BinaryPrimitives.ReadUInt16LittleEndian(file.AsSpan(0x10e)));
|
||||
Assert.Equal(54321u, BinaryPrimitives.ReadUInt32LittleEndian(file.AsSpan(0x118)));
|
||||
Assert.Equal(3, BinaryPrimitives.ReadInt32LittleEndian(file.AsSpan(0x11c)));
|
||||
Assert.Equal(2, BinaryPrimitives.ReadInt32LittleEndian(file.AsSpan(0x120)));
|
||||
Assert.Equal(0x78563412u, BinaryPrimitives.ReadUInt32LittleEndian(file.AsSpan(0x130)));
|
||||
Assert.Equal(0x1357u, BinaryPrimitives.ReadUInt32LittleEndian(file.AsSpan(0x134)));
|
||||
|
||||
byte[] checkedLogical = new byte[payload.Length + 8];
|
||||
payload.CopyTo(checkedLogical.AsSpan(8));
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(
|
||||
checkedLogical, NativeSaveContainerCodec.Crc32Msb(payload));
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(
|
||||
checkedLogical.AsSpan(4), NativeSaveContainerCodec.Crc32Reflected(payload));
|
||||
byte[] stored = LzssEncoder.EncodeOrVerbatim(checkedLogical);
|
||||
int expectedTransformDwords = stored.Length / 4 + 0x0d;
|
||||
Assert.Equal(
|
||||
checked((uint)(expectedTransformDwords * 2)),
|
||||
BinaryPrimitives.ReadUInt32LittleEndian(file.AsSpan(0x124)));
|
||||
|
||||
NativeSaveDocument decoded = NativeSaveContainerCodec.Decode(file);
|
||||
Assert.Equal(metadata, decoded.Metadata);
|
||||
Assert.Equal(payload, decoded.Payload);
|
||||
Assert.Equal(file.Length, decoded.BytesConsumed);
|
||||
Assert.Equal(metadata, NativeSaveContainerCodec.ReadMetadata(file.AsSpan(0, 0x124)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void LegacyUncompressedContainerRoundTrips()
|
||||
{
|
||||
byte[] payload = Enumerable.Range(0, 64).Select(i => (byte)(i * 37)).ToArray();
|
||||
var metadata = new NativeSaveMetadata(
|
||||
NativeSaveMagic.S3SD, 77, "legacy", Timestamp, 9, 1, 1);
|
||||
|
||||
byte[] file = NativeSaveContainerCodec.Encode(
|
||||
payload, metadata, new NativeSaveEncodingOptions(0xabcdef01, 3));
|
||||
NativeSaveDocument decoded = NativeSaveContainerCodec.Decode(file);
|
||||
|
||||
Assert.Equal(metadata, decoded.Metadata);
|
||||
Assert.Equal(payload, decoded.Payload);
|
||||
Assert.Equal((uint)((payload.Length + 8) / 4 * 2),
|
||||
BinaryPrimitives.ReadUInt32LittleEndian(file.AsSpan(0x124)));
|
||||
|
||||
uint mixed = NativeSaveContainerCodec.Crc32Msb(payload) ^ 0xabcdef01;
|
||||
Assert.Equal((mixed >> 16) * 3,
|
||||
BinaryPrimitives.ReadUInt32LittleEndian(
|
||||
file.AsSpan(NativeSaveContainerCodec.FixedPrefixSize)));
|
||||
Assert.Equal((mixed & 0xffff) * 3,
|
||||
BinaryPrimitives.ReadUInt32LittleEndian(
|
||||
file.AsSpan(NativeSaveContainerCodec.FixedPrefixSize + 4)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EncodedChecksumAndExactDivisionRejectCorruption()
|
||||
{
|
||||
byte[] payload = Enumerable.Range(0, 16).SelectMany(BitConverter.GetBytes).ToArray();
|
||||
var metadata = new NativeSaveMetadata(
|
||||
NativeSaveMagic.S4SD, 1, "test", Timestamp, 0, 1, 1);
|
||||
byte[] file = NativeSaveContainerCodec.Encode(
|
||||
payload, metadata, new NativeSaveEncodingOptions(0x12345678, 3));
|
||||
|
||||
byte[] crcFailure = file.ToArray();
|
||||
crcFailure[^1] ^= 0x80;
|
||||
Assert.Contains("checksum", Assert.Throws<InvalidDataException>(
|
||||
() => NativeSaveContainerCodec.Decode(crcFailure)).Message);
|
||||
|
||||
byte[] divisionFailure = file.ToArray();
|
||||
int encodedOffset = NativeSaveContainerCodec.FixedPrefixSize;
|
||||
uint firstProduct = BinaryPrimitives.ReadUInt32LittleEndian(divisionFailure.AsSpan(encodedOffset));
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(divisionFailure.AsSpan(encodedOffset), firstProduct + 1);
|
||||
ReadOnlySpan<byte> encoded = divisionFailure.AsSpan(encodedOffset);
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(
|
||||
divisionFailure.AsSpan(0x128), NativeSaveContainerCodec.Crc32Msb(encoded));
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(
|
||||
divisionFailure.AsSpan(0x12c), NativeSaveContainerCodec.Crc32Reflected(encoded));
|
||||
|
||||
Assert.Contains("divisible", Assert.Throws<InvalidDataException>(
|
||||
() => NativeSaveContainerCodec.Decode(divisionFailure)).Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void IdentityUsesNativeLayoutTwoVersionCompatibility()
|
||||
{
|
||||
var identity = new NativeSaveIdentity(NativeSaveMagic.S4SD, 9, "game", 2, 10);
|
||||
identity.Validate(identity.CreateMetadata(Timestamp, 0) with { SaveVersion2 = 20 });
|
||||
|
||||
Assert.Throws<InvalidDataException>(() =>
|
||||
identity.Validate(identity.CreateMetadata(Timestamp, 0) with { SaveVersion1 = 3 }));
|
||||
Assert.Throws<InvalidDataException>(() =>
|
||||
identity.Validate(identity.CreateMetadata(Timestamp, 0) with { GameId = "other" }));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DirectoryStoreKeepsSharedBackupFallbackAndDirectNumberedFiles()
|
||||
{
|
||||
string root = Path.Combine(Path.GetTempPath(), "age-native-save-" + Guid.NewGuid().ToString("N"));
|
||||
try
|
||||
{
|
||||
var identity = new NativeSaveIdentity(NativeSaveMagic.S4SD, 123, "himegari-test", 3, 2);
|
||||
var store = new DirectoryNativeDatStore(root, identity);
|
||||
byte[] first = Enumerable.Repeat((byte)0x11, 64).ToArray();
|
||||
byte[] second = Enumerable.Repeat((byte)0x22, 64).ToArray();
|
||||
byte[] numbered = Enumerable.Repeat((byte)0x33, 64).ToArray();
|
||||
|
||||
store.SaveShared(first, Timestamp, 10);
|
||||
Assert.True(File.Exists(Path.Combine(root, "SAVE.DAT")));
|
||||
Assert.False(File.Exists(Path.Combine(root, "$$SAVE.DAT")));
|
||||
Assert.False(File.Exists(Path.Combine(root, "SAVE.BAK")));
|
||||
|
||||
store.SaveShared(second, Timestamp, 20);
|
||||
Assert.Equal(second, store.LoadShared()!.Payload);
|
||||
Assert.True(File.Exists(Path.Combine(root, "SAVE.BAK")));
|
||||
|
||||
string primary = Path.Combine(root, "SAVE.DAT");
|
||||
byte[] corrupt = File.ReadAllBytes(primary);
|
||||
corrupt[^1] ^= 1;
|
||||
File.WriteAllBytes(primary, corrupt);
|
||||
Assert.Equal(first, store.LoadShared()!.Payload);
|
||||
|
||||
store.SaveNumbered(4, numbered, Timestamp, 30);
|
||||
Assert.True(File.Exists(Path.Combine(root, "SAVE04.DAT")));
|
||||
Assert.Equal(numbered, store.LoadNumbered(4)!.Payload);
|
||||
Assert.Null(store.LoadNumbered(5));
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (Directory.Exists(root)) Directory.Delete(root, recursive: true);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,15 +4,16 @@ using Xunit;
|
||||
public class OpcodeTableTests
|
||||
{
|
||||
[Fact]
|
||||
public void LoadsAll248FromJson()
|
||||
public void LoadsObservedOpcodesPlusMappedUnusedAbiEntries()
|
||||
{
|
||||
var t = OpcodeTableJson.Load(Paths.OpcodesJson);
|
||||
Assert.Equal(248, t.Count);
|
||||
Assert.Equal(249, t.Count); // 248 corpus-observed + unused persistence ABI opcode 0x19f
|
||||
Assert.True(t.TryGet(0x55, out var label, out var argc));
|
||||
Assert.Equal("mov", label);
|
||||
Assert.Equal(2, argc);
|
||||
Assert.Equal("u0041BEB0", t.Label(0x90));
|
||||
Assert.Equal(7, t.Argc(0x90));
|
||||
Assert.Equal(2, t.Argc(0x19f));
|
||||
Assert.Equal(-1, t.Argc(0x9999)); // absent -> -1
|
||||
}
|
||||
}
|
||||
|
||||
151
engine/Age.Engine/Persistence/NativeDatStore.cs
Normal file
151
engine/Age.Engine/Persistence/NativeDatStore.cs
Normal file
@@ -0,0 +1,151 @@
|
||||
using System.Globalization;
|
||||
|
||||
namespace Age.Engine.Persistence;
|
||||
|
||||
public sealed record NativeSaveIdentity(
|
||||
NativeSaveMagic Magic,
|
||||
uint CompatibilityId,
|
||||
string GameId,
|
||||
int SaveVersion1,
|
||||
int SaveVersion2)
|
||||
{
|
||||
public NativeSaveMetadata CreateMetadata(NativeSystemTime timestamp, uint accumulatedPlaySeconds)
|
||||
=> new(Magic, CompatibilityId, GameId, timestamp, accumulatedPlaySeconds, SaveVersion1, SaveVersion2);
|
||||
|
||||
public void Validate(NativeSaveMetadata metadata)
|
||||
{
|
||||
if (metadata.Magic != Magic)
|
||||
throw new InvalidDataException($"Native save generation mismatch: expected {Magic}, got {metadata.Magic}.");
|
||||
if (metadata.CompatibilityId != CompatibilityId)
|
||||
throw new InvalidDataException("Native save compatibility id mismatch.");
|
||||
if (!StringComparer.Ordinal.Equals(metadata.GameId, GameId))
|
||||
throw new InvalidDataException("Native save game id mismatch.");
|
||||
|
||||
bool versionsMatch =
|
||||
metadata.SaveVersion1 == SaveVersion1 &&
|
||||
metadata.SaveVersion2 == SaveVersion2;
|
||||
bool layoutTwoCompatibility =
|
||||
metadata.SaveVersion1 == 2 &&
|
||||
SaveVersion1 == 2;
|
||||
if (!versionsMatch && !layoutTwoCompatibility)
|
||||
throw new InvalidDataException(
|
||||
$"Native save version mismatch: expected {SaveVersion1}.{SaveVersion2}, " +
|
||||
$"got {metadata.SaveVersion1}.{metadata.SaveVersion2}.");
|
||||
}
|
||||
}
|
||||
|
||||
public interface INativeDatStore
|
||||
{
|
||||
NativeSaveDocument? LoadShared();
|
||||
void SaveShared(ReadOnlySpan<byte> payload, NativeSystemTime timestamp, uint accumulatedPlaySeconds);
|
||||
NativeSaveDocument? LoadNumbered(int slot);
|
||||
void SaveNumbered(int slot, ReadOnlySpan<byte> payload, NativeSystemTime timestamp, uint accumulatedPlaySeconds);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Directory-backed native DAT lifecycle. Shared state uses $$SAVE.DAT -> SAVE.DAT with SAVE.BAK
|
||||
/// fallback; numbered slots are written directly as SAVE##.DAT, matching AGE's separate behavior.
|
||||
/// Payload ownership remains above this boundary.
|
||||
/// </summary>
|
||||
public sealed class DirectoryNativeDatStore : INativeDatStore
|
||||
{
|
||||
public const string SharedFileName = "SAVE.DAT";
|
||||
public const string SharedTemporaryFileName = "$$SAVE.DAT";
|
||||
public const string SharedBackupFileName = "SAVE.BAK";
|
||||
|
||||
private readonly string _root;
|
||||
private readonly NativeSaveIdentity _identity;
|
||||
|
||||
public DirectoryNativeDatStore(string root, NativeSaveIdentity identity)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(root);
|
||||
_root = Path.GetFullPath(root);
|
||||
_identity = identity;
|
||||
}
|
||||
|
||||
public NativeSaveDocument? LoadShared()
|
||||
{
|
||||
string primary = Path.Combine(_root, SharedFileName);
|
||||
string backup = Path.Combine(_root, SharedBackupFileName);
|
||||
if (!File.Exists(primary))
|
||||
return File.Exists(backup) ? LoadAndValidate(backup) : null;
|
||||
|
||||
try
|
||||
{
|
||||
return LoadAndValidate(primary);
|
||||
}
|
||||
catch (Exception primaryError) when (
|
||||
primaryError is IOException or UnauthorizedAccessException or InvalidDataException)
|
||||
{
|
||||
if (!File.Exists(backup)) throw;
|
||||
try
|
||||
{
|
||||
return LoadAndValidate(backup);
|
||||
}
|
||||
catch (Exception backupError) when (
|
||||
backupError is IOException or UnauthorizedAccessException or InvalidDataException)
|
||||
{
|
||||
throw new InvalidDataException(
|
||||
"Both SAVE.DAT and SAVE.BAK failed native container validation.",
|
||||
new AggregateException(primaryError, backupError));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void SaveShared(
|
||||
ReadOnlySpan<byte> payload,
|
||||
NativeSystemTime timestamp,
|
||||
uint accumulatedPlaySeconds)
|
||||
{
|
||||
byte[] encoded = NativeSaveContainerCodec.Encode(
|
||||
payload, _identity.CreateMetadata(timestamp, accumulatedPlaySeconds));
|
||||
Directory.CreateDirectory(_root);
|
||||
|
||||
string temporary = Path.Combine(_root, SharedTemporaryFileName);
|
||||
string primary = Path.Combine(_root, SharedFileName);
|
||||
string backup = Path.Combine(_root, SharedBackupFileName);
|
||||
WriteThrough(temporary, encoded);
|
||||
if (File.Exists(backup)) File.Delete(backup);
|
||||
if (File.Exists(primary)) File.Move(primary, backup);
|
||||
File.Move(temporary, primary);
|
||||
}
|
||||
|
||||
public NativeSaveDocument? LoadNumbered(int slot)
|
||||
{
|
||||
string path = Path.Combine(_root, NumberedFileName(slot));
|
||||
return File.Exists(path) ? LoadAndValidate(path) : null;
|
||||
}
|
||||
|
||||
public void SaveNumbered(
|
||||
int slot,
|
||||
ReadOnlySpan<byte> payload,
|
||||
NativeSystemTime timestamp,
|
||||
uint accumulatedPlaySeconds)
|
||||
{
|
||||
byte[] encoded = NativeSaveContainerCodec.Encode(
|
||||
payload, _identity.CreateMetadata(timestamp, accumulatedPlaySeconds));
|
||||
Directory.CreateDirectory(_root);
|
||||
WriteThrough(Path.Combine(_root, NumberedFileName(slot)), encoded);
|
||||
}
|
||||
|
||||
public static string NumberedFileName(int slot)
|
||||
{
|
||||
if (slot < 0) throw new ArgumentOutOfRangeException(nameof(slot));
|
||||
return "SAVE" + slot.ToString("00", CultureInfo.InvariantCulture) + ".DAT";
|
||||
}
|
||||
|
||||
private NativeSaveDocument LoadAndValidate(string path)
|
||||
{
|
||||
NativeSaveDocument document = NativeSaveContainerCodec.Decode(File.ReadAllBytes(path));
|
||||
_identity.Validate(document.Metadata);
|
||||
return document;
|
||||
}
|
||||
|
||||
private static void WriteThrough(string path, ReadOnlySpan<byte> data)
|
||||
{
|
||||
using var stream = new FileStream(
|
||||
path, FileMode.Create, FileAccess.Write, FileShare.None, 64 * 1024, FileOptions.SequentialScan);
|
||||
stream.Write(data);
|
||||
stream.Flush(flushToDisk: true);
|
||||
}
|
||||
}
|
||||
365
engine/Age.Engine/Persistence/NativeSaveContainerCodec.cs
Normal file
365
engine/Age.Engine/Persistence/NativeSaveContainerCodec.cs
Normal file
@@ -0,0 +1,365 @@
|
||||
using System.Buffers.Binary;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using Age.Engine.Sys4;
|
||||
|
||||
namespace Age.Engine.Persistence;
|
||||
|
||||
public enum NativeSaveMagic
|
||||
{
|
||||
S3SD,
|
||||
S4SD,
|
||||
}
|
||||
|
||||
public readonly record struct NativeSystemTime(
|
||||
ushort Year,
|
||||
ushort Month,
|
||||
ushort DayOfWeek,
|
||||
ushort Day,
|
||||
ushort Hour,
|
||||
ushort Minute,
|
||||
ushort Second,
|
||||
ushort Milliseconds)
|
||||
{
|
||||
public static NativeSystemTime FromLocalDateTime(DateTime value)
|
||||
{
|
||||
DateTime local = value.Kind == DateTimeKind.Local ? value : value.ToLocalTime();
|
||||
return new NativeSystemTime(
|
||||
(ushort)local.Year, (ushort)local.Month, (ushort)local.DayOfWeek, (ushort)local.Day,
|
||||
(ushort)local.Hour, (ushort)local.Minute, (ushort)local.Second, (ushort)local.Millisecond);
|
||||
}
|
||||
}
|
||||
|
||||
public sealed record NativeSaveMetadata(
|
||||
NativeSaveMagic Magic,
|
||||
uint CompatibilityId,
|
||||
string GameId,
|
||||
NativeSystemTime Timestamp,
|
||||
uint AccumulatedPlaySeconds,
|
||||
int SaveVersion1,
|
||||
int SaveVersion2);
|
||||
|
||||
public sealed record NativeSaveDocument(
|
||||
NativeSaveMetadata Metadata,
|
||||
byte[] Payload,
|
||||
int BytesConsumed);
|
||||
|
||||
public sealed record NativeSaveEncodingOptions(uint XorSeed, ushort Multiplier)
|
||||
{
|
||||
public static NativeSaveEncodingOptions Random()
|
||||
{
|
||||
Span<byte> random = stackalloc byte[6];
|
||||
RandomNumberGenerator.Fill(random);
|
||||
uint seed = BinaryPrimitives.ReadUInt32LittleEndian(random);
|
||||
ushort multiplier = (ushort)(BinaryPrimitives.ReadUInt16LittleEndian(random[4..]) | 1);
|
||||
return new NativeSaveEncodingOptions(seed, multiplier);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// AGE's common S3SD/S4SD wrapper used by shared SAVE.DAT and numbered SAVE##.DAT files.
|
||||
/// Payload serializers own the bytes inside this wrapper; this class owns the native header,
|
||||
/// integrity checks, optional LZSS layer, and reversible DWORD transform.
|
||||
/// </summary>
|
||||
public static class NativeSaveContainerCodec
|
||||
{
|
||||
public const int HeaderSize = 0x124;
|
||||
public const int CodecFrameSize = 0x14;
|
||||
public const int FixedPrefixSize = HeaderSize + CodecFrameSize;
|
||||
|
||||
private const uint SeedIncrement = 0x0b0b0b0b;
|
||||
private const ushort MultiplierIncrement = 0x0b02;
|
||||
private const uint MsbPolynomial = 0x04c11db7;
|
||||
private const uint ReflectedPolynomial = 0xedb88320;
|
||||
private static readonly Encoding ShiftJis = CreateShiftJis();
|
||||
|
||||
public static byte[] Encode(
|
||||
ReadOnlySpan<byte> payload,
|
||||
NativeSaveMetadata metadata,
|
||||
NativeSaveEncodingOptions? options = null)
|
||||
{
|
||||
if ((payload.Length & 3) != 0)
|
||||
throw new ArgumentException("Native save payload length must be DWORD-aligned.", nameof(payload));
|
||||
if (metadata.SaveVersion2 < 0)
|
||||
throw new ArgumentOutOfRangeException(nameof(metadata), "SaveVersion2 cannot be negative.");
|
||||
|
||||
options ??= NativeSaveEncodingOptions.Random();
|
||||
ValidateMultiplier(options.Multiplier);
|
||||
|
||||
byte[] checkedLogical = new byte[payload.Length + 8];
|
||||
payload.CopyTo(checkedLogical.AsSpan(8));
|
||||
WriteLogicalChecksums(checkedLogical);
|
||||
|
||||
byte[] transformInput = metadata.SaveVersion2 >= 2
|
||||
? BuildCompressedWrapper(checkedLogical)
|
||||
: checkedLogical;
|
||||
byte[] encoded = ExpandTransform(transformInput, options.XorSeed, options.Multiplier);
|
||||
|
||||
byte[] result = new byte[checked(HeaderSize + CodecFrameSize + encoded.Length)];
|
||||
WriteHeader(result.AsSpan(0, HeaderSize), metadata);
|
||||
Span<byte> frame = result.AsSpan(HeaderSize, CodecFrameSize);
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(frame, checked((uint)(encoded.Length / 4)));
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(frame[4..], Crc32Msb(encoded));
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(frame[8..], Crc32Reflected(encoded));
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(frame[12..], options.XorSeed);
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(frame[16..], options.Multiplier);
|
||||
encoded.CopyTo(result.AsSpan(FixedPrefixSize));
|
||||
return result;
|
||||
}
|
||||
|
||||
public static NativeSaveDocument Decode(ReadOnlySpan<byte> source)
|
||||
{
|
||||
if (source.Length < FixedPrefixSize)
|
||||
throw new InvalidDataException("Native save container is shorter than its fixed header.");
|
||||
|
||||
NativeSaveMetadata metadata = ReadMetadata(source);
|
||||
ReadOnlySpan<byte> frame = source.Slice(HeaderSize, CodecFrameSize);
|
||||
uint encodedDwordCount = BinaryPrimitives.ReadUInt32LittleEndian(frame);
|
||||
if ((encodedDwordCount & 1) != 0)
|
||||
throw new InvalidDataException("Native save encoded DWORD count is not even.");
|
||||
if (encodedDwordCount > int.MaxValue / 4)
|
||||
throw new InvalidDataException("Native save encoded DWORD count is too large.");
|
||||
int encodedLength = (int)encodedDwordCount * 4;
|
||||
if (encodedLength > source.Length - FixedPrefixSize)
|
||||
throw new InvalidDataException("Native save encoded payload is truncated.");
|
||||
|
||||
ReadOnlySpan<byte> encoded = source.Slice(FixedPrefixSize, encodedLength);
|
||||
uint expectedMsb = BinaryPrimitives.ReadUInt32LittleEndian(frame[4..]);
|
||||
uint expectedReflected = BinaryPrimitives.ReadUInt32LittleEndian(frame[8..]);
|
||||
if (Crc32Msb(encoded) != expectedMsb || Crc32Reflected(encoded) != expectedReflected)
|
||||
throw new InvalidDataException("Native save encoded payload checksum mismatch.");
|
||||
|
||||
uint seed = BinaryPrimitives.ReadUInt32LittleEndian(frame[12..]);
|
||||
uint storedMultiplier = BinaryPrimitives.ReadUInt32LittleEndian(frame[16..]);
|
||||
if ((storedMultiplier & 0xffff0000) != 0)
|
||||
throw new InvalidDataException("Native save multiplier contains unexpected high bits.");
|
||||
ushort multiplier = (ushort)storedMultiplier;
|
||||
ValidateMultiplier(multiplier);
|
||||
|
||||
byte[] transformInput = InverseTransform(encoded, seed, multiplier);
|
||||
byte[] checkedLogical = metadata.SaveVersion2 >= 2
|
||||
? DecodeCompressedWrapper(transformInput)
|
||||
: transformInput;
|
||||
if (checkedLogical.Length < 8 || (checkedLogical.Length & 3) != 0)
|
||||
throw new InvalidDataException("Native save logical payload has an invalid length.");
|
||||
|
||||
ReadOnlySpan<byte> payload = checkedLogical.AsSpan(8);
|
||||
uint logicalMsb = BinaryPrimitives.ReadUInt32LittleEndian(checkedLogical);
|
||||
uint logicalReflected = BinaryPrimitives.ReadUInt32LittleEndian(checkedLogical.AsSpan(4));
|
||||
if (Crc32Msb(payload) != logicalMsb || Crc32Reflected(payload) != logicalReflected)
|
||||
throw new InvalidDataException("Native save logical payload checksum mismatch.");
|
||||
|
||||
return new NativeSaveDocument(
|
||||
metadata,
|
||||
payload.ToArray(),
|
||||
checked(FixedPrefixSize + encodedLength));
|
||||
}
|
||||
|
||||
/// <summary>Read only the fixed native header, matching metadata-query opcode 0x1a0.</summary>
|
||||
public static NativeSaveMetadata ReadMetadata(ReadOnlySpan<byte> source)
|
||||
{
|
||||
if (source.Length < HeaderSize)
|
||||
throw new InvalidDataException("Native save container is shorter than its metadata header.");
|
||||
return ReadHeader(source[..HeaderSize]);
|
||||
}
|
||||
|
||||
public static uint Crc32Msb(ReadOnlySpan<byte> data)
|
||||
{
|
||||
uint crc = uint.MaxValue;
|
||||
foreach (byte value in data)
|
||||
{
|
||||
crc ^= (uint)value << 24;
|
||||
for (int bit = 0; bit < 8; bit++)
|
||||
crc = (crc & 0x80000000) != 0 ? crc << 1 ^ MsbPolynomial : crc << 1;
|
||||
}
|
||||
return ~crc;
|
||||
}
|
||||
|
||||
public static uint Crc32Reflected(ReadOnlySpan<byte> data)
|
||||
{
|
||||
uint crc = uint.MaxValue;
|
||||
foreach (byte value in data)
|
||||
{
|
||||
crc ^= value;
|
||||
for (int bit = 0; bit < 8; bit++)
|
||||
crc = (crc & 1) != 0 ? crc >> 1 ^ ReflectedPolynomial : crc >> 1;
|
||||
}
|
||||
return ~crc;
|
||||
}
|
||||
|
||||
private static void WriteHeader(Span<byte> header, NativeSaveMetadata metadata)
|
||||
{
|
||||
header.Clear();
|
||||
WriteMagic(header, metadata.Magic);
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(header[4..], metadata.CompatibilityId);
|
||||
|
||||
if (metadata.GameId.Contains('\0'))
|
||||
throw new ArgumentException("Native save game id cannot contain NUL.", nameof(metadata));
|
||||
byte[] gameId = ShiftJis.GetBytes(metadata.GameId);
|
||||
if (gameId.Length >= 0x100)
|
||||
throw new ArgumentException("Native save game id must fit in 255 Shift-JIS bytes.", nameof(metadata));
|
||||
gameId.CopyTo(header[8..]);
|
||||
|
||||
Span<byte> systemTime = header[0x108..0x118];
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(systemTime, metadata.Timestamp.Year);
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(systemTime[2..], metadata.Timestamp.Month);
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(systemTime[4..], metadata.Timestamp.DayOfWeek);
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(systemTime[6..], metadata.Timestamp.Day);
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(systemTime[8..], metadata.Timestamp.Hour);
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(systemTime[10..], metadata.Timestamp.Minute);
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(systemTime[12..], metadata.Timestamp.Second);
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(systemTime[14..], metadata.Timestamp.Milliseconds);
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(header[0x118..], metadata.AccumulatedPlaySeconds);
|
||||
BinaryPrimitives.WriteInt32LittleEndian(header[0x11c..], metadata.SaveVersion1);
|
||||
BinaryPrimitives.WriteInt32LittleEndian(header[0x120..], metadata.SaveVersion2);
|
||||
}
|
||||
|
||||
private static NativeSaveMetadata ReadHeader(ReadOnlySpan<byte> header)
|
||||
{
|
||||
NativeSaveMagic magic = ReadMagic(header);
|
||||
uint compatibilityId = BinaryPrimitives.ReadUInt32LittleEndian(header[4..]);
|
||||
ReadOnlySpan<byte> gameIdBytes = header.Slice(8, 0x100);
|
||||
int terminator = gameIdBytes.IndexOf((byte)0);
|
||||
if (terminator < 0)
|
||||
throw new InvalidDataException("Native save game id is not NUL-terminated.");
|
||||
string gameId;
|
||||
try
|
||||
{
|
||||
gameId = ShiftJis.GetString(gameIdBytes[..terminator]);
|
||||
}
|
||||
catch (DecoderFallbackException error)
|
||||
{
|
||||
throw new InvalidDataException("Native save game id is not valid Shift-JIS.", error);
|
||||
}
|
||||
|
||||
ReadOnlySpan<byte> systemTime = header[0x108..0x118];
|
||||
var timestamp = new NativeSystemTime(
|
||||
BinaryPrimitives.ReadUInt16LittleEndian(systemTime),
|
||||
BinaryPrimitives.ReadUInt16LittleEndian(systemTime[2..]),
|
||||
BinaryPrimitives.ReadUInt16LittleEndian(systemTime[4..]),
|
||||
BinaryPrimitives.ReadUInt16LittleEndian(systemTime[6..]),
|
||||
BinaryPrimitives.ReadUInt16LittleEndian(systemTime[8..]),
|
||||
BinaryPrimitives.ReadUInt16LittleEndian(systemTime[10..]),
|
||||
BinaryPrimitives.ReadUInt16LittleEndian(systemTime[12..]),
|
||||
BinaryPrimitives.ReadUInt16LittleEndian(systemTime[14..]));
|
||||
|
||||
return new NativeSaveMetadata(
|
||||
magic,
|
||||
compatibilityId,
|
||||
gameId,
|
||||
timestamp,
|
||||
BinaryPrimitives.ReadUInt32LittleEndian(header[0x118..]),
|
||||
BinaryPrimitives.ReadInt32LittleEndian(header[0x11c..]),
|
||||
BinaryPrimitives.ReadInt32LittleEndian(header[0x120..]));
|
||||
}
|
||||
|
||||
private static byte[] BuildCompressedWrapper(ReadOnlySpan<byte> checkedLogical)
|
||||
{
|
||||
byte[] stored = LzssEncoder.EncodeOrVerbatim(checkedLogical);
|
||||
int nativeDwordCount = checked(stored.Length / 4 + 0x0d);
|
||||
byte[] wrapper = new byte[checked(nativeDwordCount * 4)];
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(wrapper, checked((uint)checkedLogical.Length));
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(wrapper.AsSpan(4), checked((uint)checkedLogical.Length));
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(wrapper.AsSpan(8), checked((uint)stored.Length));
|
||||
stored.CopyTo(wrapper.AsSpan(12));
|
||||
return wrapper;
|
||||
}
|
||||
|
||||
private static byte[] DecodeCompressedWrapper(ReadOnlySpan<byte> wrapper)
|
||||
{
|
||||
if (wrapper.Length < 12)
|
||||
throw new InvalidDataException("Native save compressed wrapper is truncated.");
|
||||
uint originalLength = BinaryPrimitives.ReadUInt32LittleEndian(wrapper);
|
||||
uint consumedLength = BinaryPrimitives.ReadUInt32LittleEndian(wrapper[4..]);
|
||||
uint storedLength = BinaryPrimitives.ReadUInt32LittleEndian(wrapper[8..]);
|
||||
if (originalLength != consumedLength)
|
||||
throw new InvalidDataException("Native save LZSS wrapper length fields disagree.");
|
||||
if (originalLength > int.MaxValue || storedLength > int.MaxValue ||
|
||||
storedLength > (uint)(wrapper.Length - 12))
|
||||
throw new InvalidDataException("Native save LZSS wrapper length is invalid.");
|
||||
|
||||
ReadOnlySpan<byte> stored = wrapper.Slice(12, (int)storedLength);
|
||||
return storedLength == originalLength
|
||||
? stored.ToArray()
|
||||
: LzssDecoder.Decode(stored, (int)originalLength, "native save LZSS");
|
||||
}
|
||||
|
||||
private static byte[] ExpandTransform(ReadOnlySpan<byte> source, uint seed, ushort multiplier)
|
||||
{
|
||||
if ((source.Length & 3) != 0)
|
||||
throw new InvalidDataException("Native save transform input is not DWORD-aligned.");
|
||||
byte[] encoded = new byte[checked(source.Length * 2)];
|
||||
for (int input = 0, output = 0; input < source.Length; input += 4, output += 8)
|
||||
{
|
||||
uint mixed = BinaryPrimitives.ReadUInt32LittleEndian(source[input..]) ^ seed;
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(encoded.AsSpan(output), (mixed >> 16) * multiplier);
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(encoded.AsSpan(output + 4), (mixed & 0xffff) * multiplier);
|
||||
seed = unchecked(seed + SeedIncrement);
|
||||
multiplier = unchecked((ushort)(multiplier + MultiplierIncrement));
|
||||
}
|
||||
return encoded;
|
||||
}
|
||||
|
||||
private static byte[] InverseTransform(ReadOnlySpan<byte> encoded, uint seed, ushort multiplier)
|
||||
{
|
||||
if ((encoded.Length & 7) != 0)
|
||||
throw new InvalidDataException("Native save encoded transform length is invalid.");
|
||||
byte[] source = new byte[encoded.Length / 2];
|
||||
for (int input = 0, output = 0; input < encoded.Length; input += 8, output += 4)
|
||||
{
|
||||
uint highProduct = BinaryPrimitives.ReadUInt32LittleEndian(encoded[input..]);
|
||||
uint lowProduct = BinaryPrimitives.ReadUInt32LittleEndian(encoded[(input + 4)..]);
|
||||
if (highProduct % multiplier != 0 || lowProduct % multiplier != 0)
|
||||
throw new InvalidDataException("Native save transform product is not exactly divisible.");
|
||||
uint high = highProduct / multiplier;
|
||||
uint low = lowProduct / multiplier;
|
||||
if (high > ushort.MaxValue || low > ushort.MaxValue)
|
||||
throw new InvalidDataException("Native save transform quotient exceeds 16 bits.");
|
||||
|
||||
uint mixed = high << 16 | low;
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(source.AsSpan(output), mixed ^ seed);
|
||||
seed = unchecked(seed + SeedIncrement);
|
||||
multiplier = unchecked((ushort)(multiplier + MultiplierIncrement));
|
||||
}
|
||||
return source;
|
||||
}
|
||||
|
||||
private static void WriteLogicalChecksums(Span<byte> checkedLogical)
|
||||
{
|
||||
ReadOnlySpan<byte> payload = checkedLogical[8..];
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(checkedLogical, Crc32Msb(payload));
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(checkedLogical[4..], Crc32Reflected(payload));
|
||||
}
|
||||
|
||||
private static void WriteMagic(Span<byte> destination, NativeSaveMagic magic)
|
||||
{
|
||||
destination[0] = (byte)'S';
|
||||
destination[1] = magic == NativeSaveMagic.S3SD ? (byte)'3' : (byte)'4';
|
||||
destination[2] = (byte)'S';
|
||||
destination[3] = (byte)'D';
|
||||
}
|
||||
|
||||
private static NativeSaveMagic ReadMagic(ReadOnlySpan<byte> source)
|
||||
{
|
||||
if (source[0] != 'S' || source[2] != 'S' || source[3] != 'D')
|
||||
throw new InvalidDataException("Native save magic is invalid.");
|
||||
return source[1] switch
|
||||
{
|
||||
(byte)'3' => NativeSaveMagic.S3SD,
|
||||
(byte)'4' => NativeSaveMagic.S4SD,
|
||||
_ => throw new InvalidDataException("Native save generation is unsupported."),
|
||||
};
|
||||
}
|
||||
|
||||
private static void ValidateMultiplier(ushort multiplier)
|
||||
{
|
||||
if (multiplier == 0 || (multiplier & 1) == 0)
|
||||
throw new InvalidDataException("Native save multiplier must be a nonzero odd value.");
|
||||
}
|
||||
|
||||
private static Encoding CreateShiftJis()
|
||||
{
|
||||
Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
|
||||
return Encoding.GetEncoding(932, EncoderFallback.ExceptionFallback, DecoderFallback.ExceptionFallback);
|
||||
}
|
||||
}
|
||||
118
engine/Age.Engine/Sys4/LzssEncoder.cs
Normal file
118
engine/Age.Engine/Sys4/LzssEncoder.cs
Normal file
@@ -0,0 +1,118 @@
|
||||
namespace Age.Engine.Sys4;
|
||||
|
||||
/// <summary>
|
||||
/// Encoder for Eushully's 4 KiB-ring LZSS stream. Tokens are grouped under an LSB-first flag byte;
|
||||
/// one bits are literals and zero bits are 12-bit ring offsets plus a four-bit length-minus-three.
|
||||
/// </summary>
|
||||
public static class LzssEncoder
|
||||
{
|
||||
private const int RingSize = 0x1000;
|
||||
private const int RingMask = RingSize - 1;
|
||||
private const int InitialRingPosition = 0xfee;
|
||||
private const int MaxMatchLength = 18;
|
||||
private const int CandidateLimit = 128;
|
||||
|
||||
/// <summary>
|
||||
/// Compress <paramref name="source"/>. When the encoded stream would not be smaller, return the
|
||||
/// original bytes; the native container marks that case by storing equal source/stored lengths.
|
||||
/// </summary>
|
||||
public static byte[] EncodeOrVerbatim(ReadOnlySpan<byte> source)
|
||||
{
|
||||
if (source.IsEmpty) return [];
|
||||
|
||||
var output = new List<byte>(source.Length);
|
||||
var positionsByPrefix = new Dictionary<int, LinkedList<int>>();
|
||||
int position = 0;
|
||||
|
||||
while (position < source.Length)
|
||||
{
|
||||
int flagIndex = output.Count;
|
||||
output.Add(0);
|
||||
byte flags = 0;
|
||||
|
||||
for (int bit = 0; bit < 8 && position < source.Length; bit++)
|
||||
{
|
||||
(int candidate, int length) = FindMatch(source, position, positionsByPrefix);
|
||||
if (length >= 3)
|
||||
{
|
||||
int ringOffset = (InitialRingPosition + candidate) & RingMask;
|
||||
output.Add((byte)ringOffset);
|
||||
output.Add((byte)(((ringOffset >> 4) & 0xf0) | (length - 3)));
|
||||
AddPositions(source, position, length, positionsByPrefix);
|
||||
position += length;
|
||||
}
|
||||
else
|
||||
{
|
||||
flags |= (byte)(1 << bit);
|
||||
output.Add(source[position]);
|
||||
AddPositions(source, position, 1, positionsByPrefix);
|
||||
position++;
|
||||
}
|
||||
}
|
||||
|
||||
output[flagIndex] = flags;
|
||||
}
|
||||
|
||||
return output.Count < source.Length ? output.ToArray() : source.ToArray();
|
||||
}
|
||||
|
||||
private static (int Candidate, int Length) FindMatch(
|
||||
ReadOnlySpan<byte> source,
|
||||
int position,
|
||||
Dictionary<int, LinkedList<int>> positionsByPrefix)
|
||||
{
|
||||
if (position > source.Length - 3) return (-1, 0);
|
||||
int key = PrefixKey(source, position);
|
||||
if (!positionsByPrefix.TryGetValue(key, out var candidates)) return (-1, 0);
|
||||
|
||||
int oldestAllowed = position - RingSize;
|
||||
while (candidates.First is { } first && first.Value < oldestAllowed)
|
||||
candidates.RemoveFirst();
|
||||
|
||||
int bestCandidate = -1, bestLength = 0, inspected = 0;
|
||||
for (LinkedListNode<int>? node = candidates.Last;
|
||||
node is not null && inspected < CandidateLimit;
|
||||
node = node.Previous, inspected++)
|
||||
{
|
||||
int candidate = node.Value;
|
||||
int distance = position - candidate;
|
||||
if (distance <= 0 || distance > RingSize) continue;
|
||||
|
||||
int limit = Math.Min(MaxMatchLength, source.Length - position);
|
||||
int length = 0;
|
||||
while (length < limit &&
|
||||
source[position + length] == source[candidate + (length % distance)])
|
||||
length++;
|
||||
|
||||
if (length > bestLength)
|
||||
{
|
||||
bestCandidate = candidate;
|
||||
bestLength = length;
|
||||
if (length == MaxMatchLength) break;
|
||||
}
|
||||
}
|
||||
return (bestCandidate, bestLength);
|
||||
}
|
||||
|
||||
private static void AddPositions(
|
||||
ReadOnlySpan<byte> source,
|
||||
int start,
|
||||
int count,
|
||||
Dictionary<int, LinkedList<int>> positionsByPrefix)
|
||||
{
|
||||
int end = Math.Min(start + count, source.Length - 2);
|
||||
for (int position = start; position < end; position++)
|
||||
{
|
||||
int key = PrefixKey(source, position);
|
||||
if (!positionsByPrefix.TryGetValue(key, out var positions))
|
||||
{
|
||||
positions = new LinkedList<int>();
|
||||
positionsByPrefix.Add(key, positions);
|
||||
}
|
||||
positions.AddLast(position);
|
||||
}
|
||||
}
|
||||
|
||||
private static int PrefixKey(ReadOnlySpan<byte> source, int position)
|
||||
=> source[position] | source[position + 1] << 8 | source[position + 2] << 16;
|
||||
}
|
||||
Reference in New Issue
Block a user