Move FFmpeg runtime to frontend
This commit is contained in:
gamer147
471
engine/Age.Engine.Frontend/FfmpegMovieDecoder.cs
Normal file
471
engine/Age.Engine.Frontend/FfmpegMovieDecoder.cs
Normal file
@@ -0,0 +1,471 @@
|
||||
using System;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using Age.Engine.Model;
|
||||
using Age.Engine.Sys4;
|
||||
|
||||
internal interface IMoviePacingClock
|
||||
{
|
||||
void StartPresentation() { }
|
||||
bool WaitUntil(long elapsedMilliseconds, WaitHandle cancellation);
|
||||
}
|
||||
|
||||
internal interface IExternallyAdvancedMoviePacingClock : IMoviePacingClock
|
||||
{
|
||||
void AdvanceTo(long elapsedMilliseconds);
|
||||
}
|
||||
|
||||
internal sealed class StopwatchMoviePacingClock : IMoviePacingClock
|
||||
{
|
||||
private long _startedAt;
|
||||
|
||||
public void StartPresentation()
|
||||
{
|
||||
long startedAt = Stopwatch.GetTimestamp();
|
||||
Interlocked.CompareExchange(ref _startedAt, startedAt, 0);
|
||||
}
|
||||
|
||||
public bool WaitUntil(long elapsedMilliseconds, WaitHandle cancellation)
|
||||
{
|
||||
StartPresentation();
|
||||
while (true)
|
||||
{
|
||||
long startedAt = Interlocked.Read(ref _startedAt);
|
||||
double remaining = elapsedMilliseconds
|
||||
- Stopwatch.GetElapsedTime(startedAt).TotalMilliseconds;
|
||||
if (remaining <= 0) return true;
|
||||
int waitMilliseconds = (int)Math.Clamp(Math.Ceiling(remaining), 1, 1000);
|
||||
if (cancellation.WaitOne(waitMilliseconds)) return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class ExternallyAdvancedMoviePacingClock : IExternallyAdvancedMoviePacingClock, IDisposable
|
||||
{
|
||||
private readonly AutoResetEvent _advanced = new(false);
|
||||
private long _now;
|
||||
|
||||
public bool WaitUntil(long elapsedMilliseconds, WaitHandle cancellation)
|
||||
{
|
||||
while (Interlocked.Read(ref _now) < elapsedMilliseconds)
|
||||
{
|
||||
int signalled = WaitHandle.WaitAny([cancellation, _advanced]);
|
||||
if (signalled == 0) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
public void AdvanceTo(long elapsedMilliseconds)
|
||||
{
|
||||
long current;
|
||||
do
|
||||
{
|
||||
current = Interlocked.Read(ref _now);
|
||||
if (elapsedMilliseconds <= current) return;
|
||||
}
|
||||
while (Interlocked.CompareExchange(ref _now, elapsedMilliseconds, current) != current);
|
||||
_advanced.Set();
|
||||
}
|
||||
|
||||
public void Dispose() => _advanced.Dispose();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Timestamp-paced FFmpeg delivery. Video-only streams retain monotonic stopwatch pacing. Audio-bearing
|
||||
/// streams decode PCM into a bounded queue and advance video against the sound-device clock supplied by Godot.
|
||||
/// </summary>
|
||||
internal sealed class FfmpegMovieDecoder : IMovieDecoder
|
||||
{
|
||||
private readonly IFfmpegFrameSource _source;
|
||||
private readonly IMoviePacingClock _clock;
|
||||
private readonly ManualResetEvent _cancel = new(false);
|
||||
private readonly ManualResetEvent _firstFramePresented = new(false);
|
||||
private readonly Thread _thread;
|
||||
private readonly Thread? _audioThread;
|
||||
private readonly object _frameLock = new();
|
||||
private readonly object _audioLock = new();
|
||||
private readonly Queue<MovieAudioChunk> _audioChunks = new();
|
||||
private readonly AutoResetEvent _audioSpace = new(false);
|
||||
private readonly int _maximumQueuedAudioFrames;
|
||||
private readonly long _initialPositionMs;
|
||||
private readonly long? _presentationDurationMs;
|
||||
private RgbaImage? _latestFrame;
|
||||
private volatile bool _completed;
|
||||
private volatile bool _videoTimelineCompleted;
|
||||
private volatile bool _audioDecodingCompleted;
|
||||
private volatile bool _audioSubmitted;
|
||||
private string? _failure;
|
||||
private int _queuedAudioFrames;
|
||||
private int _activeWorkers;
|
||||
private int _disposed;
|
||||
private int _firstFrameTaken;
|
||||
private long _firstFramePresentationTimeMs = -1;
|
||||
|
||||
public long? StopTimeMs => _source.Info.StopTimeMs;
|
||||
public long InitialPositionMs => _initialPositionMs;
|
||||
public bool IsCompleted => _completed;
|
||||
public string? Failure => Volatile.Read(ref _failure);
|
||||
public long? FirstFramePresentationTimeMs
|
||||
{
|
||||
get
|
||||
{
|
||||
long value = Interlocked.Read(ref _firstFramePresentationTimeMs);
|
||||
return value < 0 ? null : value;
|
||||
}
|
||||
}
|
||||
public MovieAudioInfo? AudioInfo { get; }
|
||||
public bool AudioDecodingCompleted => _audioDecodingCompleted;
|
||||
|
||||
public FfmpegMovieDecoder(
|
||||
MoviePayload movie, long initialPositionMs = 0, long? presentationDurationMs = null)
|
||||
: this(new FfmpegMovieSession(movie), null, initialPositionMs, presentationDurationMs) { }
|
||||
|
||||
internal FfmpegMovieDecoder(IFfmpegFrameSource source, IMoviePacingClock? clock,
|
||||
long initialPositionMs = 0, long? presentationDurationMs = null)
|
||||
{
|
||||
_source = source ?? throw new ArgumentNullException(nameof(source));
|
||||
_initialPositionMs = Math.Clamp(
|
||||
initialPositionMs,
|
||||
0,
|
||||
Math.Max(0, source.Info.StopTimeMs - 1));
|
||||
_presentationDurationMs = presentationDurationMs is >= 0
|
||||
? Math.Max(0, presentationDurationMs.Value)
|
||||
: null;
|
||||
try
|
||||
{
|
||||
if (_initialPositionMs > 0)
|
||||
source.Seek(_initialPositionMs);
|
||||
}
|
||||
catch
|
||||
{
|
||||
source.Dispose();
|
||||
throw;
|
||||
}
|
||||
_clock = clock ?? (source.Info.HasAudio
|
||||
? new ExternallyAdvancedMoviePacingClock()
|
||||
: new StopwatchMoviePacingClock());
|
||||
if (source.Info.HasAudio)
|
||||
{
|
||||
AudioInfo = new MovieAudioInfo(source.Info.AudioSampleRate, source.Info.AudioChannels);
|
||||
_maximumQueuedAudioFrames = Math.Max(source.Info.AudioSampleRate / 2,
|
||||
source.Info.AudioFrameSamples * 2);
|
||||
}
|
||||
_thread = new Thread(DecodeThread)
|
||||
{
|
||||
IsBackground = true,
|
||||
Name = "AGE FFmpeg movie video",
|
||||
};
|
||||
_audioThread = source.Info.HasAudio
|
||||
? new Thread(AudioDecodeThread)
|
||||
{
|
||||
IsBackground = true,
|
||||
Name = "AGE FFmpeg movie audio",
|
||||
}
|
||||
: null;
|
||||
_activeWorkers = _audioThread == null ? 1 : 2;
|
||||
try
|
||||
{
|
||||
_thread.Start();
|
||||
_audioThread?.Start();
|
||||
}
|
||||
catch
|
||||
{
|
||||
_cancel.Set();
|
||||
if (_thread.IsAlive) _thread.Join();
|
||||
if (_audioThread?.IsAlive == true) _audioThread.Join();
|
||||
_source.Dispose();
|
||||
_cancel.Dispose();
|
||||
_firstFramePresented.Dispose();
|
||||
_audioSpace.Dispose();
|
||||
if (_clock is IDisposable disposableClock) disposableClock.Dispose();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
public bool TryTakeFrame(out RgbaImage frame)
|
||||
{
|
||||
lock (_frameLock)
|
||||
{
|
||||
if (_latestFrame == null)
|
||||
{
|
||||
frame = default!;
|
||||
return false;
|
||||
}
|
||||
frame = _latestFrame;
|
||||
_latestFrame = null;
|
||||
}
|
||||
if (Interlocked.Exchange(ref _firstFrameTaken, 1) == 0)
|
||||
{
|
||||
_clock.StartPresentation();
|
||||
_firstFramePresented.Set();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
public bool TryTakeAudioChunk(out MovieAudioChunk chunk)
|
||||
{
|
||||
lock (_audioLock)
|
||||
{
|
||||
if (_audioChunks.Count == 0)
|
||||
{
|
||||
chunk = default!;
|
||||
return false;
|
||||
}
|
||||
chunk = _audioChunks.Dequeue();
|
||||
_queuedAudioFrames -= chunk.FrameCount;
|
||||
}
|
||||
_audioSpace.Set();
|
||||
return true;
|
||||
}
|
||||
|
||||
public void AdvancePlaybackClock(long elapsedMilliseconds)
|
||||
{
|
||||
if (_clock is IExternallyAdvancedMoviePacingClock external)
|
||||
external.AdvanceTo(Math.Max(0, elapsedMilliseconds));
|
||||
}
|
||||
|
||||
public void MarkAudioSubmitted()
|
||||
{
|
||||
_audioSubmitted = true;
|
||||
UpdateCompletion();
|
||||
}
|
||||
|
||||
private void DecodeThread()
|
||||
{
|
||||
try
|
||||
{
|
||||
long lastTimestamp = -1;
|
||||
long decodedFrames = 0;
|
||||
long firstTimestamp = -1;
|
||||
FfmpegVideoFrame? selectedSeekFrame = null;
|
||||
FfmpegVideoFrame? pendingAfterSeek = null;
|
||||
if (_initialPositionMs > 0)
|
||||
selectedSeekFrame = SelectFrameAtInitialPosition(out pendingAfterSeek);
|
||||
while (!_cancel.WaitOne(0))
|
||||
{
|
||||
FfmpegVideoFrame frame;
|
||||
if (selectedSeekFrame != null)
|
||||
{
|
||||
frame = selectedSeekFrame;
|
||||
selectedSeekFrame = null;
|
||||
}
|
||||
else if (pendingAfterSeek != null)
|
||||
{
|
||||
frame = pendingAfterSeek;
|
||||
pendingAfterSeek = null;
|
||||
}
|
||||
else if (!_source.TryDecodeNextVideoFrame(out frame))
|
||||
{
|
||||
if (decodedFrames == 0)
|
||||
throw new InvalidDataException("FFmpeg stream ended before producing a video frame");
|
||||
long completionTime = _presentationDurationMs
|
||||
?? PresentationDeadline(
|
||||
Math.Max(_source.Info.StopTimeMs,
|
||||
lastTimestamp + FrameIntervalMilliseconds(_source.Info)),
|
||||
firstTimestamp);
|
||||
if (_clock.WaitUntil(completionTime, _cancel))
|
||||
{
|
||||
_videoTimelineCompleted = true;
|
||||
UpdateCompletion();
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (frame.PresentationTimeMs < 0 || frame.PresentationTimeMs < lastTimestamp)
|
||||
throw new InvalidDataException(
|
||||
$"FFmpeg returned non-monotonic video timestamp {frame.PresentationTimeMs} after {lastTimestamp}");
|
||||
if (decodedFrames == 0)
|
||||
{
|
||||
firstTimestamp = frame.PresentationTimeMs;
|
||||
Interlocked.Exchange(ref _firstFramePresentationTimeMs, firstTimestamp);
|
||||
lock (_frameLock) _latestFrame = frame.Image;
|
||||
lastTimestamp = frame.PresentationTimeMs;
|
||||
decodedFrames++;
|
||||
int signalled = WaitHandle.WaitAny([_cancel, _firstFramePresented]);
|
||||
if (signalled == 0) return;
|
||||
continue;
|
||||
}
|
||||
if (!_clock.WaitUntil(PresentationDeadline(frame.PresentationTimeMs, firstTimestamp),
|
||||
_cancel))
|
||||
return;
|
||||
lock (_frameLock) _latestFrame = frame.Image;
|
||||
lastTimestamp = frame.PresentationTimeMs;
|
||||
decodedFrames++;
|
||||
}
|
||||
}
|
||||
catch (Exception error)
|
||||
{
|
||||
Fail(error);
|
||||
}
|
||||
finally
|
||||
{
|
||||
WorkerCompleted();
|
||||
}
|
||||
}
|
||||
|
||||
private FfmpegVideoFrame? SelectFrameAtInitialPosition(out FfmpegVideoFrame? pending)
|
||||
{
|
||||
pending = null;
|
||||
FfmpegVideoFrame? candidate = null;
|
||||
long priorTimestamp = -1;
|
||||
while (!_cancel.WaitOne(0) && _source.TryDecodeNextVideoFrame(out FfmpegVideoFrame frame))
|
||||
{
|
||||
if (frame.PresentationTimeMs < 0 || frame.PresentationTimeMs < priorTimestamp)
|
||||
throw new InvalidDataException(
|
||||
$"FFmpeg returned non-monotonic video timestamp {frame.PresentationTimeMs} " +
|
||||
$"after {priorTimestamp} during seek preroll");
|
||||
priorTimestamp = frame.PresentationTimeMs;
|
||||
if (frame.PresentationTimeMs <= _initialPositionMs)
|
||||
{
|
||||
candidate = frame;
|
||||
continue;
|
||||
}
|
||||
if (candidate == null)
|
||||
return frame;
|
||||
pending = frame;
|
||||
return candidate;
|
||||
}
|
||||
return candidate;
|
||||
}
|
||||
|
||||
private void AudioDecodeThread()
|
||||
{
|
||||
try
|
||||
{
|
||||
long priorSourceTimestamp = -1;
|
||||
long priorRebasedTimestamp = -1;
|
||||
while (!_cancel.WaitOne(0))
|
||||
{
|
||||
while (Volatile.Read(ref _queuedAudioFrames) >= _maximumQueuedAudioFrames)
|
||||
{
|
||||
int signalled = WaitHandle.WaitAny([_cancel, _audioSpace]);
|
||||
if (signalled == 0) return;
|
||||
}
|
||||
if (!_source.TryDecodeNextAudioChunk(out FfmpegAudioChunk decoded))
|
||||
{
|
||||
_audioDecodingCompleted = true;
|
||||
UpdateCompletion();
|
||||
return;
|
||||
}
|
||||
if (decoded.FrameCount <= 0
|
||||
|| decoded.InterleavedStereo.Length != checked(decoded.FrameCount * 2)
|
||||
|| decoded.PresentationTimeMs < 0
|
||||
|| decoded.PresentationTimeMs < priorSourceTimestamp)
|
||||
throw new InvalidDataException(
|
||||
$"FFmpeg returned invalid audio block {decoded.FrameCount}f " +
|
||||
$"at {decoded.PresentationTimeMs} ms after {priorSourceTimestamp} ms");
|
||||
priorSourceTimestamp = decoded.PresentationTimeMs;
|
||||
MovieAudioChunk? chunk = RebaseAudioChunk(decoded);
|
||||
if (chunk == null) continue;
|
||||
if (chunk.PresentationTimeMs < priorRebasedTimestamp)
|
||||
throw new InvalidDataException(
|
||||
$"FFmpeg seek produced non-monotonic rebased audio timestamp " +
|
||||
$"{chunk.PresentationTimeMs} after {priorRebasedTimestamp} ms");
|
||||
lock (_audioLock)
|
||||
{
|
||||
_audioChunks.Enqueue(chunk);
|
||||
_queuedAudioFrames += chunk.FrameCount;
|
||||
}
|
||||
priorRebasedTimestamp = chunk.PresentationTimeMs;
|
||||
}
|
||||
}
|
||||
catch (Exception error)
|
||||
{
|
||||
Fail(error);
|
||||
}
|
||||
finally
|
||||
{
|
||||
WorkerCompleted();
|
||||
}
|
||||
}
|
||||
|
||||
private MovieAudioChunk? RebaseAudioChunk(FfmpegAudioChunk decoded)
|
||||
{
|
||||
int trimFrames = 0;
|
||||
if (_initialPositionMs > decoded.PresentationTimeMs)
|
||||
{
|
||||
long deltaMs = _initialPositionMs - decoded.PresentationTimeMs;
|
||||
long required = checked(
|
||||
(deltaMs * (long)_source.Info.AudioSampleRate + 999) / 1000);
|
||||
trimFrames = checked((int)Math.Min(decoded.FrameCount, required));
|
||||
}
|
||||
if (trimFrames >= decoded.FrameCount) return null;
|
||||
|
||||
float[] samples = decoded.InterleavedStereo;
|
||||
int frameCount = decoded.FrameCount - trimFrames;
|
||||
if (trimFrames > 0)
|
||||
{
|
||||
var trimmed = new float[checked(frameCount * 2)];
|
||||
Array.Copy(samples, checked(trimFrames * 2), trimmed, 0, trimmed.Length);
|
||||
samples = trimmed;
|
||||
}
|
||||
long trimmedTimestamp = decoded.PresentationTimeMs
|
||||
+ trimFrames * 1000L / _source.Info.AudioSampleRate;
|
||||
long rebasedTimestamp = Math.Max(0, trimmedTimestamp - _initialPositionMs);
|
||||
return new MovieAudioChunk(samples, frameCount, rebasedTimestamp);
|
||||
}
|
||||
|
||||
private void Fail(Exception error)
|
||||
{
|
||||
Interlocked.CompareExchange(ref _failure, error.Message, null);
|
||||
_completed = true; // decode failure must never strand an AGE movie wait
|
||||
_cancel.Set();
|
||||
_audioSpace.Set();
|
||||
}
|
||||
|
||||
private void UpdateCompletion()
|
||||
{
|
||||
if (_failure != null || !_videoTimelineCompleted) return;
|
||||
if (AudioInfo != null && (!_audioDecodingCompleted || !_audioSubmitted)) return;
|
||||
_completed = true;
|
||||
}
|
||||
|
||||
private void WorkerCompleted()
|
||||
{
|
||||
if (Interlocked.Decrement(ref _activeWorkers) == 0) _source.Dispose();
|
||||
}
|
||||
|
||||
private static long FrameIntervalMilliseconds(FfmpegMovieInfo info)
|
||||
{
|
||||
if (info.FrameRateNumerator <= 0 || info.FrameRateDenominator <= 0) return 0;
|
||||
long scaledDenominator = checked((long)info.FrameRateDenominator * 1000);
|
||||
return Math.Max(1, scaledDenominator / info.FrameRateNumerator);
|
||||
}
|
||||
|
||||
private long PresentationDeadline(long sourceTimestamp, long firstVideoTimestamp)
|
||||
{
|
||||
// MPEG program streams may put the first video sample hundreds of milliseconds after the audio
|
||||
// stream's mux timestamp origin. DirectShow presents the video's first sample as video time zero;
|
||||
// retaining the absolute mux offset here would freeze that sample until the audio clock caught up.
|
||||
// Preserve every decoded frame and its cadence, but rebase the video stream to its first sample.
|
||||
long sourceElapsed = Math.Max(0, sourceTimestamp - firstVideoTimestamp);
|
||||
if (_presentationDurationMs is not { } duration
|
||||
|| _source.Info.StopTimeMs <= 0)
|
||||
return sourceElapsed;
|
||||
return checked(sourceElapsed * duration / _source.Info.StopTimeMs);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (Interlocked.Exchange(ref _disposed, 1) != 0) return;
|
||||
_cancel.Set();
|
||||
_firstFramePresented.Set();
|
||||
_audioSpace.Set();
|
||||
if (_thread.IsAlive && Thread.CurrentThread != _thread)
|
||||
_thread.Join();
|
||||
if (_audioThread?.IsAlive == true && Thread.CurrentThread != _audioThread)
|
||||
_audioThread.Join();
|
||||
_cancel.Dispose();
|
||||
_firstFramePresented.Dispose();
|
||||
_audioSpace.Dispose();
|
||||
if (_clock is IDisposable disposableClock) disposableClock.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class FfmpegMovieDecoderFactory : IMovieDecoderFactory
|
||||
{
|
||||
public IMovieDecoder Open(
|
||||
MoviePayload movie, long initialPositionMs = 0, long? presentationDurationMs = null)
|
||||
=> new FfmpegMovieDecoder(movie, initialPositionMs, presentationDurationMs);
|
||||
}
|
||||
Reference in New Issue
Block a user