conco/OpenMaidEngine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Add synchronized MPEG movie audio

Browse Source
This commit is contained in:
gamer147
2026-07-25 11:49:50 -04:00
parent 1828701872
commit 843b282cc4
24 changed files with 1336 additions and 558 deletions
Download Patch File Download Diff File Expand all files Collapse all files

196
godot/FfmpegMovieDecoder.cs
View File

@@ -1,6 +1,7 @@
using System;
using System.Diagnostics;
using System.IO;
using System.Collections.Generic;
using System.Threading;
using Age.Engine.Sys4;
@@ -9,6 +10,11 @@ internal interface IMoviePacingClock
bool WaitUntil(long elapsedMilliseconds, WaitHandle cancellation);
}
internal interface IExternallyAdvancedMoviePacingClock : IMoviePacingClock
{
void AdvanceTo(long elapsedMilliseconds);
}
internal sealed class StopwatchMoviePacingClock : IMoviePacingClock
{
private readonly long _startedAt = Stopwatch.GetTimestamp();
@@ -26,9 +32,39 @@ internal sealed class StopwatchMoviePacingClock : IMoviePacingClock
}
}
internal sealed class ExternallyAdvancedMoviePacingClock : IExternallyAdvancedMoviePacingClock, IDisposable
{
private readonly AutoResetEvent _advanced = new(false);
private long _now;
public bool WaitUntil(long elapsedMilliseconds, WaitHandle cancellation)
{
while (Interlocked.Read(ref _now) < elapsedMilliseconds)
{
int signalled = WaitHandle.WaitAny([cancellation, _advanced]);
if (signalled == 0) return false;
}
return true;
}
public void AdvanceTo(long elapsedMilliseconds)
{
long current;
do
{
current = Interlocked.Read(ref _now);
if (elapsedMilliseconds <= current) return;
}
while (Interlocked.CompareExchange(ref _now, elapsedMilliseconds, current) != current);
_advanced.Set();
}
public void Dispose() => _advanced.Dispose();
}
/// <summary>
/// Timestamp-paced FFmpeg video delivery. The worker decodes no more than one frame ahead, publishes only
/// when its presentation timestamp is due, and reports completion after the final presentation interval.
/// Timestamp-paced FFmpeg delivery. Video-only streams retain monotonic stopwatch pacing. Audio-bearing
/// streams decode PCM into a bounded queue and advance video against the sound-device clock supplied by Godot.
/// </summary>
internal sealed class FfmpegMovieDecoder : IMovieDecoder
{
@@ -36,33 +72,70 @@ internal sealed class FfmpegMovieDecoder : IMovieDecoder
private readonly IMoviePacingClock _clock;
private readonly ManualResetEvent _cancel = new(false);
private readonly Thread _thread;
private readonly Thread? _audioThread;
private readonly object _frameLock = new();
private readonly object _audioLock = new();
private readonly Queue<MovieAudioChunk> _audioChunks = new();
private readonly AutoResetEvent _audioSpace = new(false);
private readonly int _maximumQueuedAudioFrames;
private RgbaImage? _latestFrame;
private volatile bool _completed;
private volatile bool _videoTimelineCompleted;
private volatile bool _audioDecodingCompleted;
private volatile bool _audioSubmitted;
private string? _failure;
private int _queuedAudioFrames;
private int _activeWorkers;
private int _disposed;
public long? StopTimeMs => _source.Info.StopTimeMs;
public bool IsCompleted => _completed;
public string? Failure => Volatile.Read(ref _failure);
public MovieAudioInfo? AudioInfo { get; }
public bool AudioDecodingCompleted => _audioDecodingCompleted;
public FfmpegMovieDecoder(MoviePayload movie)
: this(new FfmpegMovieSession(movie), new StopwatchMoviePacingClock()) { }
: this(new FfmpegMovieSession(movie), null) { }
internal FfmpegMovieDecoder(IFfmpegFrameSource source, IMoviePacingClock clock)
internal FfmpegMovieDecoder(IFfmpegFrameSource source, IMoviePacingClock? clock)
{
_source = source ?? throw new ArgumentNullException(nameof(source));
_clock = clock ?? throw new ArgumentNullException(nameof(clock));
_clock = clock ?? (source.Info.HasAudio
? new ExternallyAdvancedMoviePacingClock()
: new StopwatchMoviePacingClock());
if (source.Info.HasAudio)
{
AudioInfo = new MovieAudioInfo(source.Info.AudioSampleRate, source.Info.AudioChannels);
_maximumQueuedAudioFrames = Math.Max(source.Info.AudioSampleRate / 2,
source.Info.AudioFrameSamples * 2);
}
_thread = new Thread(DecodeThread)
{
IsBackground = true,
Name = "AGE FFmpeg movie",
Name = "AGE FFmpeg movie video",
};
try { _thread.Start(); }
_audioThread = source.Info.HasAudio
? new Thread(AudioDecodeThread)
{
IsBackground = true,
Name = "AGE FFmpeg movie audio",
}
: null;
_activeWorkers = _audioThread == null ? 1 : 2;
try
{
_thread.Start();
_audioThread?.Start();
}
catch
{
_cancel.Set();
if (_thread.IsAlive) _thread.Join();
if (_audioThread?.IsAlive == true) _audioThread.Join();
_source.Dispose();
_cancel.Dispose();
_audioSpace.Dispose();
if (_clock is IDisposable disposableClock) disposableClock.Dispose();
throw;
}
}
@@ -82,6 +155,34 @@ internal sealed class FfmpegMovieDecoder : IMovieDecoder
}
}
public bool TryTakeAudioChunk(out MovieAudioChunk chunk)
{
lock (_audioLock)
{
if (_audioChunks.Count == 0)
{
chunk = default!;
return false;
}
chunk = _audioChunks.Dequeue();
_queuedAudioFrames -= chunk.FrameCount;
}
_audioSpace.Set();
return true;
}
public void AdvancePlaybackClock(long elapsedMilliseconds)
{
if (_clock is IExternallyAdvancedMoviePacingClock external)
external.AdvanceTo(Math.Max(0, elapsedMilliseconds));
}
public void MarkAudioSubmitted()
{
_audioSubmitted = true;
UpdateCompletion();
}
private void DecodeThread()
{
try
@@ -96,7 +197,11 @@ internal sealed class FfmpegMovieDecoder : IMovieDecoder
throw new InvalidDataException("FFmpeg stream ended before producing a video frame");
long completionTime = Math.Max(_source.Info.StopTimeMs,
lastTimestamp + FrameIntervalMilliseconds(_source.Info));
if (_clock.WaitUntil(completionTime, _cancel)) _completed = true;
if (_clock.WaitUntil(completionTime, _cancel))
{
_videoTimelineCompleted = true;
UpdateCompletion();
}
return;
}
if (frame.PresentationTimeMs < 0 || frame.PresentationTimeMs < lastTimestamp)
@@ -110,15 +215,79 @@ internal sealed class FfmpegMovieDecoder : IMovieDecoder
}
catch (Exception error)
{
Volatile.Write(ref _failure, error.Message);
_completed = true; // decode failure must never strand an AGE movie wait
Fail(error);
}
finally
{
_source.Dispose();
WorkerCompleted();
}
}
private void AudioDecodeThread()
{
try
{
long priorTimestamp = -1;
while (!_cancel.WaitOne(0))
{
while (Volatile.Read(ref _queuedAudioFrames) >= _maximumQueuedAudioFrames)
{
int signalled = WaitHandle.WaitAny([_cancel, _audioSpace]);
if (signalled == 0) return;
}
if (!_source.TryDecodeNextAudioChunk(out FfmpegAudioChunk decoded))
{
_audioDecodingCompleted = true;
UpdateCompletion();
return;
}
if (decoded.FrameCount <= 0
|| decoded.InterleavedStereo.Length != checked(decoded.FrameCount * 2)
|| decoded.PresentationTimeMs < 0
|| decoded.PresentationTimeMs < priorTimestamp)
throw new InvalidDataException(
$"FFmpeg returned invalid audio block {decoded.FrameCount}f " +
$"at {decoded.PresentationTimeMs} ms after {priorTimestamp} ms");
var chunk = new MovieAudioChunk(decoded.InterleavedStereo, decoded.FrameCount,
decoded.PresentationTimeMs);
lock (_audioLock)
{
_audioChunks.Enqueue(chunk);
_queuedAudioFrames += chunk.FrameCount;
}
priorTimestamp = decoded.PresentationTimeMs;
}
}
catch (Exception error)
{
Fail(error);
}
finally
{
WorkerCompleted();
}
}
private void Fail(Exception error)
{
Interlocked.CompareExchange(ref _failure, error.Message, null);
_completed = true; // decode failure must never strand an AGE movie wait
_cancel.Set();
_audioSpace.Set();
}
private void UpdateCompletion()
{
if (_failure != null || !_videoTimelineCompleted) return;
if (AudioInfo != null && (!_audioDecodingCompleted || !_audioSubmitted)) return;
_completed = true;
}
private void WorkerCompleted()
{
if (Interlocked.Decrement(ref _activeWorkers) == 0) _source.Dispose();
}
private static long FrameIntervalMilliseconds(FfmpegMovieInfo info)
{
if (info.FrameRateNumerator <= 0 || info.FrameRateDenominator <= 0) return 0;
@@ -130,9 +299,14 @@ internal sealed class FfmpegMovieDecoder : IMovieDecoder
{
if (Interlocked.Exchange(ref _disposed, 1) != 0) return;
_cancel.Set();
_audioSpace.Set();
if (_thread.IsAlive && Thread.CurrentThread != _thread)
_thread.Join();
if (_audioThread?.IsAlive == true && Thread.CurrentThread != _audioThread)
_audioThread.Join();
_cancel.Dispose();
_audioSpace.Dispose();
if (_clock is IDisposable disposableClock) disposableClock.Dispose();
}
}