Add synchronized MPEG movie audio

godot/FfmpegMovieNative.cs

@@ -10,26 +10,36 @@ internal readonly record struct FfmpegMovieInfo(
     long StopTimeMs,
     int FrameRateNumerator,
     int FrameRateDenominator,
-    bool HasAudio);
+    bool HasAudio,
+    int AudioSampleRate = 0,
+    int AudioChannels = 0,
+    int AudioFrameSamples = 0);
 
 internal sealed record FfmpegVideoFrame(RgbaImage Image, long PresentationTimeMs);
+internal sealed record FfmpegAudioChunk(float[] InterleavedStereo, int FrameCount, long PresentationTimeMs);
 
 internal interface IFfmpegFrameSource : IDisposable
 {
     FfmpegMovieInfo Info { get; }
     bool TryDecodeNextVideoFrame(out FfmpegVideoFrame frame);
+    bool TryDecodeNextAudioChunk(out FfmpegAudioChunk chunk)
+    {
+        chunk = default!;
+        return false;
+    }
 }
 
 /// <summary>Sequential, unpaced access to the project-owned FFmpeg C ABI for isolated probes and playback.</summary>
 internal sealed class FfmpegMovieSession : IFfmpegFrameSource
 {
     private FfmpegMovieHandle _handle;
+    private readonly object _decodeLock = new();
     public FfmpegMovieInfo Info { get; }
 
     public FfmpegMovieSession(MoviePayload movie)
     {
         ArgumentNullException.ThrowIfNull(movie);
-        if (FfmpegMovieNative.AbiVersion() != 1)
+        if (FfmpegMovieNative.AbiVersion() != 2)
             throw new InvalidOperationException("unsupported age_movie_ffmpeg ABI version");
 
         byte[] error = new byte[1024];
@@ -42,31 +52,97 @@ internal sealed class FfmpegMovieSession : IFfmpegFrameSource
                 $"FFmpeg movie open failed for {movie.Name}: {FfmpegMovieNative.DecodeUtf8(error)}");
         }
         Info = new FfmpegMovieInfo(nativeInfo.Width, nativeInfo.Height, nativeInfo.StopTimeMs,
-            nativeInfo.FrameRateNumerator, nativeInfo.FrameRateDenominator, nativeInfo.HasAudio != 0);
+            nativeInfo.FrameRateNumerator, nativeInfo.FrameRateDenominator, nativeInfo.HasAudio != 0,
+            nativeInfo.AudioSampleRate, nativeInfo.AudioChannels, nativeInfo.AudioFrameSamples);
         if (Info.Width <= 0 || Info.Height <= 0 || Info.StopTimeMs <= 0)
         {
             _handle.Dispose();
             throw new InvalidDataException(
                 $"FFmpeg returned invalid metadata for {movie.Name}: {Info.Width}x{Info.Height}, {Info.StopTimeMs} ms");
         }
+        if (Info.HasAudio && (Info.AudioSampleRate <= 0 || Info.AudioChannels != 2
+                              || Info.AudioFrameSamples <= 0))
+        {
+            _handle.Dispose();
+            throw new InvalidDataException(
+                $"FFmpeg returned invalid audio metadata for {movie.Name}: " +
+                $"{Info.AudioSampleRate} Hz, {Info.AudioChannels} channels, {Info.AudioFrameSamples} frames");
+        }
     }
 
     public bool TryDecodeNextVideoFrame(out FfmpegVideoFrame frame)
     {
         ObjectDisposedException.ThrowIf(_handle.IsClosed, this);
         byte[] pixels = new byte[checked(Info.Width * Info.Height * 4)];
-        int result = FfmpegMovieNative.DecodeVideo(_handle, pixels, (nuint)pixels.Length, out long ptsMs);
+        int result;
+        long ptsMs;
+        string? error = null;
+        lock (_decodeLock)
+        {
+            result = FfmpegMovieNative.DecodeVideo(
+                _handle, pixels, (nuint)pixels.Length, out ptsMs);
+            if (result != FfmpegMovieNative.EndOfFile && result != FfmpegMovieNative.Frame)
+                error = FfmpegMovieNative.LastError(_handle);
+        }
         if (result == FfmpegMovieNative.EndOfFile)
         {
             frame = default!;
             return false;
         }
         if (result != FfmpegMovieNative.Frame)
-            throw new InvalidDataException($"FFmpeg movie decode failed: {FfmpegMovieNative.LastError(_handle)}");
+            throw new InvalidDataException($"FFmpeg movie decode failed: {error}");
         frame = new FfmpegVideoFrame(new RgbaImage(Info.Width, Info.Height, pixels), ptsMs);
         return true;
     }
 
+    public bool TryDecodeNextAudioChunk(out FfmpegAudioChunk chunk)
+    {
+        ObjectDisposedException.ThrowIf(_handle.IsClosed, this);
+        if (!Info.HasAudio)
+        {
+            chunk = default!;
+            return false;
+        }
+
+        int capacity = Math.Max(Info.AudioFrameSamples, 1);
+        while (true)
+        {
+            float[] samples = new float[checked(capacity * 2)];
+            int result;
+            int frameCount;
+            long ptsMs;
+            string? error = null;
+            lock (_decodeLock)
+            {
+                result = FfmpegMovieNative.DecodeAudio(
+                    _handle, samples, (nuint)capacity, out frameCount, out ptsMs);
+                if (result != FfmpegMovieNative.EndOfFile
+                    && result != FfmpegMovieNative.Frame
+                    && result != FfmpegMovieNative.BufferTooSmall)
+                    error = FfmpegMovieNative.LastError(_handle);
+            }
+            if (result == FfmpegMovieNative.EndOfFile)
+            {
+                chunk = default!;
+                return false;
+            }
+            if (result == FfmpegMovieNative.BufferTooSmall)
+            {
+                if (frameCount <= capacity)
+                    throw new InvalidDataException("FFmpeg audio decode reported an invalid required buffer size");
+                capacity = frameCount;
+                continue;
+            }
+            if (result != FfmpegMovieNative.Frame)
+                throw new InvalidDataException($"FFmpeg movie audio decode failed: {error}");
+            if (frameCount <= 0 || frameCount > capacity)
+                throw new InvalidDataException($"FFmpeg returned invalid audio frame count {frameCount}");
+            if (frameCount != capacity) Array.Resize(ref samples, checked(frameCount * 2));
+            chunk = new FfmpegAudioChunk(samples, frameCount, ptsMs);
+            return true;
+        }
+    }
+
     public void Dispose() => _handle.Dispose();
 }
 
@@ -84,6 +160,7 @@ internal static class FfmpegMovieNative
 {
     internal const int EndOfFile = 0;
     internal const int Frame = 1;
+    internal const int BufferTooSmall = -3;
     private const string LibraryName = "age_movie_ffmpeg";
 
     [StructLayout(LayoutKind.Sequential)]
@@ -95,6 +172,9 @@ internal static class FfmpegMovieNative
         public int FrameRateNumerator;
         public int FrameRateDenominator;
         public int HasAudio;
+        public int AudioSampleRate;
+        public int AudioChannels;
+        public int AudioFrameSamples;
     }
 
     static FfmpegMovieNative()
@@ -160,6 +240,14 @@ internal static class FfmpegMovieNative
         nuint rgbaSize,
         out long presentationTimeMs);
 
+    [DllImport(LibraryName, EntryPoint = "age_movie_decode_audio", CallingConvention = CallingConvention.Cdecl)]
+    internal static extern int DecodeAudio(
+        FfmpegMovieHandle movie,
+        [Out] float[] stereo,
+        nuint frameCapacity,
+        out int frameCount,
+        out long presentationTimeMs);
+
     [DllImport(LibraryName, EntryPoint = "age_movie_last_error", CallingConvention = CallingConvention.Cdecl)]
     private static extern IntPtr LastErrorNative(FfmpegMovieHandle movie);