[video_core] clean up Android MediaCodec NVDEC support (#4154)

THIS MAY BE BLOBBED WITH TWO PREVIOUS ONES!
(git reset --mixed head~3, commit, profit)

Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/4154

src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/ui/SettingsFragmentPresenter.kt

@@ -269,7 +269,7 @@ class SettingsFragmentPresenter(
     // TODO(crueter): sub-submenus?
     private fun addGraphicsSettings(sl: ArrayList<SettingsItem>) {
         sl.apply {
-            // add(IntSetting.RENDERER_NVDEC_EMULATION.key)
+            add(IntSetting.RENDERER_NVDEC_EMULATION.key)
 
             add(IntSetting.RENDERER_RESOLUTION.key)
             add(IntSetting.RENDERER_VSYNC.key)

src/android/app/src/main/jni/CMakeLists.txt

@@ -27,7 +27,7 @@ if (ARCHITECTURE_arm64)
     target_link_libraries(yuzu-android PRIVATE adrenotools)
 endif()
 
-target_link_libraries(yuzu-android PRIVATE OpenSSL::SSL cpp-jwt::cpp-jwt)
+target_link_libraries(yuzu-android PRIVATE FFmpeg::FFmpeg OpenSSL::SSL cpp-jwt::cpp-jwt)
 if (ENABLE_UPDATE_CHECKER)
     target_compile_definitions(yuzu-android PUBLIC ENABLE_UPDATE_CHECKER)
 endif()

src/android/app/src/main/jni/native.cpp

@@ -36,6 +36,12 @@
 #include <frontend_common/content_manager.h>
 #include <jni.h>
 
+extern "C" {
+// Required for FFmpeg mediacodec
+#include <libavcodec/jni.h>
+#include <libavutil/log.h>
+}
+
 #include "common/android/multiplayer/multiplayer.h"
 #include "common/android/android_common.h"
 #include "common/android/id_cache.h"
@@ -680,6 +686,15 @@ const char* fallback_cpu_detection() {
 
 } // namespace
 
+// referenced by common
+extern "C" {
+jint InitFFmpegOnLoad(JavaVM *vm) {
+    av_jni_set_java_vm(vm, NULL);
+    av_log_set_level(AV_LOG_DEBUG);
+    return 0;
+}
+}
+
 extern "C" {
 
 void Java_org_yuzu_yuzu_1emu_NativeLibrary_surfaceChanged(JNIEnv* env, jobject instance,

src/common/android/id_cache.cpp

@@ -12,7 +12,6 @@
 #include "common/android/multiplayer/multiplayer.h"
 #include <network/network.h>
 
-
 static JavaVM *s_java_vm;
 static jclass s_native_library_class;
 static jclass s_disk_cache_progress_class;
@@ -427,8 +426,12 @@ namespace Common::Android {
     extern "C" {
 #endif
 
+    // see on src/android/app/src/main/jni/native.cpp
+    jint InitFFmpegOnLoad(JavaVM *vm);
+
     jint JNI_OnLoad(JavaVM *vm, void *reserved) {
         s_java_vm = vm;
+        InitFFmpegOnLoad(vm);
 
         JNIEnv *env;
         if (vm->GetEnv(reinterpret_cast<void **>(&env), JNI_VERSION) != JNI_OK)

src/video_core/host1x/codecs/decoder.cpp

@@ -20,58 +20,53 @@ Decoder::Decoder(Host1x::Host1x& host1x_, s32 id_, const Host1x::NvdecCommon::Nv
 
 Decoder::~Decoder() = default;
 
+void Decoder::SetFrameDimensions(s32 width, s32 height) {
+    if (width <= 0 || height <= 0) {
+        frame_dimensions.reset();
+        return;
+    }
+    frame_dimensions = FFmpeg::FrameDimensions{width, height};
+}
+
 void Decoder::Decode() {
     if (!initialized) {
         return;
     }
 
     const auto packet_data = ComposeFrame();
-    // Send assembled bitstream to decoder.
-    if (!decode_api.SendPacket(packet_data)) {
-        return;
-    }
-
-    // Only receive/store visible frames.
-    if (vp9_hidden_frame) {
-        return;
-    }
-
-    // Receive output frames from decoder.
-    auto frame = decode_api.ReceiveFrame();
-
-    if (!frame) {
-        return;
-    }
-
-    if (IsInterlaced()) {
-        auto [luma_top, luma_bottom, chroma_top, chroma_bottom] = GetInterlacedOffsets();
-        auto frame_copy = frame;
-
-        if (!frame.get()) {
-            LOG_ERROR(HW_GPU,
-                      "Nvdec {} failed to decode interlaced frame for top {:#X} bottom 0x{:X}", id,
-                      luma_top, luma_bottom);
-        }
-
-        if (UsingDecodeOrder()) {
-            host1x.frame_queue.PushDecodeOrder(id, luma_top, std::move(frame));
-            host1x.frame_queue.PushDecodeOrder(id, luma_bottom, std::move(frame_copy));
-        } else {
-            host1x.frame_queue.PushPresentOrder(id, luma_top, std::move(frame));
-            host1x.frame_queue.PushPresentOrder(id, luma_bottom, std::move(frame_copy));
-        }
+    FFmpeg::FrameOffsets offsets{};
+    offsets.hidden = vp9_hidden_frame;
+    offsets.interlaced = IsInterlaced();
+    if (offsets.interlaced) {
+        std::tie(offsets.luma, offsets.luma_bottom, std::ignore, std::ignore) =
+            GetInterlacedOffsets();
     } else {
-        auto [luma_offset, chroma_offset] = GetProgressiveOffsets();
+        std::tie(offsets.luma, std::ignore) = GetProgressiveOffsets();
+    }
 
-        if (!frame.get()) {
-            LOG_ERROR(HW_GPU, "Nvdec {} failed to decode progressive frame for luma {:#X}", id,
-                      luma_offset);
-        }
+    if (!decode_api.SendPacket(packet_data, offsets, GetFrameDimensions())) {
+        return;
+    }
 
+    auto push = [&](u64 luma, std::shared_ptr<FFmpeg::Frame> frame) {
         if (UsingDecodeOrder()) {
-            host1x.frame_queue.PushDecodeOrder(id, luma_offset, std::move(frame));
+            host1x.frame_queue.PushDecodeOrder(id, luma, std::move(frame));
         } else {
-            host1x.frame_queue.PushPresentOrder(id, luma_offset, std::move(frame));
+            host1x.frame_queue.PushPresentOrder(id, luma, std::move(frame));
+        }
+    };
+
+    while (auto result = decode_api.ReceiveFrame()) {
+        auto& [frame, o] = *result;
+        if (o.hidden || !frame) {
+            continue;
+        }
+        if (o.interlaced) {
+            auto frame_copy = frame;
+            push(o.luma, std::move(frame));
+            push(o.luma_bottom, std::move(frame_copy));
+        } else {
+            push(o.luma, std::move(frame));
         }
     }
 }

src/video_core/host1x/codecs/decoder.h

@@ -10,6 +10,7 @@
 #include <mutex>
 #include <optional>
 #include <string_view>
+#include <tuple>
 #include <ankerl/unordered_dense.h>
 #include <queue>
 
@@ -46,12 +47,19 @@ protected:
     virtual std::tuple<u64, u64, u64, u64> GetInterlacedOffsets() = 0;
     virtual bool IsInterlaced() = 0;
 
+    void SetFrameDimensions(s32 width, s32 height);
+
+    std::optional<FFmpeg::FrameDimensions> GetFrameDimensions() const {
+        return frame_dimensions;
+    }
+
     FFmpeg::DecodeApi decode_api;
     Host1x::Host1x& host1x;
     const Host1x::NvdecCommon::NvdecRegisters& regs;
     s32 id;
     bool initialized : 1 = false;
     bool vp9_hidden_frame : 1 = false;
+    std::optional<FFmpeg::FrameDimensions> frame_dimensions;
 };
 
 } // namespace Tegra

src/video_core/host1x/codecs/h264.cpp

@@ -52,6 +52,10 @@ bool H264::IsInterlaced() {
 
 std::span<const u8> H264::ComposeFrame() {
     host1x.gmmu_manager.ReadBlock(regs.picture_info_offset.Address(), &current_context, sizeof(H264DecoderContext));
+    const auto& params = current_context.h264_parameter_set;
+    SetFrameDimensions(static_cast<s32>(params.pic_width_in_mbs) * 16,
+                       static_cast<s32>(params.frame_height_in_mbs) * 16);
+
     const s64 frame_number = current_context.h264_parameter_set.frame_number.Value();
     if (!is_first_frame && frame_number != 0) {
         frame_scratch.resize_destructive(current_context.stream_len);

src/video_core/host1x/codecs/vp8.cpp

@@ -35,6 +35,7 @@ std::tuple<u64, u64, u64, u64> VP8::GetInterlacedOffsets() {
 
 std::span<const u8> VP8::ComposeFrame() {
     host1x.gmmu_manager.ReadBlock(regs.picture_info_offset.Address(), &current_context, sizeof(VP8PictureInfo));
+    SetFrameDimensions(current_context.frame_width, current_context.frame_height);
 
     const bool is_key_frame = current_context.key_frame == 1u;
     const auto bitstream_size = size_t(current_context.vld_buffer_size);

src/video_core/host1x/codecs/vp9.cpp

@@ -841,6 +841,7 @@ std::span<const u8> VP9::ComposeFrame() {
     {
         Vp9FrameContainer curr_frame = GetCurrentFrame();
         current_frame_info = curr_frame.info;
+        SetFrameDimensions(current_frame_info.frame_size.width, current_frame_info.frame_size.height);
         bitstream = std::move(curr_frame.bit_stream);
     }
     // The uncompressed header routine sets PrevProb parameters needed for the compressed header

src/video_core/host1x/ffmpeg.cpp

@@ -4,6 +4,11 @@
 // SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
+#include <cstring>
+#include <string>
+#include <string_view>
+#include <vector>
+
 #include "common/assert.h"
 #include "common/logging.h"
 #include "common/scope_exit.h"
@@ -19,15 +24,16 @@ extern "C" {
 #endif
 
 #include <libavutil/hwcontext.h>
+#include <libavutil/log.h>
 }
 
 namespace FFmpeg {
 
 namespace {
 
-constexpr AVPixelFormat PreferredGpuFormat = AV_PIX_FMT_NV12;
-constexpr AVPixelFormat PreferredCpuFormat = AV_PIX_FMT_YUV420P;
-constexpr std::array PreferredGpuDecoders = {
+constexpr AVPixelFormat PREFERRED_GPU_FORMAT = AV_PIX_FMT_NV12;
+constexpr AVPixelFormat PREFERRED_CPU_FORMAT = AV_PIX_FMT_YUV420P;
+constexpr std::array PREFERRED_GPU_DECODERS = {
 #if defined(_WIN32)
     AV_HWDEVICE_TYPE_CUDA,
     AV_HWDEVICE_TYPE_D3D11VA,
@@ -54,15 +60,14 @@ AVPixelFormat GetGpuFormat(AVCodecContext* codec_context, const AVPixelFormat* p
     if (desc && !(desc->flags & AV_PIX_FMT_FLAG_HWACCEL)) {
         for (int i = 0;; i++) {
             const AVCodecHWConfig* config = avcodec_get_hw_config(codec_context->codec, i);
-            if (!config) {
+            if (config) {
+                for (const auto type : PREFERRED_GPU_DECODERS)
+                    if (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX && config->device_type == type) {
+                        codec_context->pix_fmt = config->pix_fmt;
+                    }
+            } else {
                 break;
             }
-
-            for (const auto type : PreferredGpuDecoders) {
-                if (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX && config->device_type == type) {
-                    codec_context->pix_fmt = config->pix_fmt;
-                }
-            }
         }
     }
 
@@ -74,7 +79,7 @@ AVPixelFormat GetGpuFormat(AVCodecContext* codec_context, const AVPixelFormat* p
 
     LOG_INFO(HW_GPU, "Could not find supported GPU pixel format, falling back to CPU decoder");
     av_buffer_unref(&codec_context->hw_device_ctx);
-    codec_context->pix_fmt = PreferredCpuFormat;
+    codec_context->pix_fmt = PREFERRED_CPU_FORMAT;
     return codec_context->pix_fmt;
 }
 
@@ -84,6 +89,57 @@ std::string AVError(int errnum) {
     return errbuf;
 }
 
+#ifdef __ANDROID__
+// Match a 3- or 4-byte annex-B NAL start code at `i`. Returns its length, or 0.
+size_t FindNalStartCode(std::span<const u8> data, size_t i) {
+    const size_t n = data.size();
+    if (i + 3 < n && data[i] == 0 && data[i + 1] == 0 && data[i + 2] == 0 && data[i + 3] == 1) {
+        return 4;
+    }
+    if (i + 2 < n && data[i] == 0 && data[i + 1] == 0 && data[i + 2] == 1) {
+        return 3;
+    }
+    return 0;
+}
+
+// Pull SPS (NAL type 7) + PPS (NAL type 8) out of an annex-B frame into an
+// extradata buffer, each prefixed with a 4-byte start code. Eden synthesizes
+// these inline into the very first frame; h264_mediacodec wants them at open.
+std::vector<u8> ExtractH264ParameterSetExtradata(std::span<const u8> packet) {
+    std::vector<u8> extradata;
+    const size_t size = packet.size();
+    size_t i = 0;
+    while (i < size) {
+        const size_t sc = FindNalStartCode(packet, i);
+        if (sc == 0) {
+            ++i;
+            continue;
+        }
+        const size_t nal_start = i + sc;
+        if (nal_start >= size) {
+            break;
+        }
+        const u8 nal_type = packet[nal_start] & 0x1F;
+
+        size_t j = nal_start + 1;
+        while (j < size && FindNalStartCode(packet, j) == 0) {
+            ++j;
+        }
+
+        if (nal_type == 7 || nal_type == 8) {
+            constexpr u8 start[4] = {0, 0, 0, 1};
+            extradata.insert(extradata.end(), start, start + sizeof(start));
+            extradata.insert(extradata.end(), packet.begin() + nal_start, packet.begin() + j);
+        } else if (nal_type == 1 || nal_type == 5) {
+            break;
+        }
+        i = j;
+    }
+    return extradata;
+}
+
+#endif
+
 }
 
 Packet::Packet(std::span<const u8> data) {
@@ -118,7 +174,26 @@ Decoder::Decoder(Tegra::Host1x::NvdecCommon::VideoCodec codec) {
             return AV_CODEC_ID_NONE;
         }
     }();
-    m_codec = avcodec_find_decoder(av_codec);
+
+#ifdef __ANDROID__
+    // FFmpeg exposes MediaCodec via dedicated decoders rather than as a
+    // hw_config on the regular ones.
+    if (Settings::values.nvdec_emulation.GetValue() == Settings::NvdecEmulation::Gpu) {
+        const char* mc_name = nullptr;
+        switch (av_codec) {
+        case AV_CODEC_ID_H264: mc_name = "h264_mediacodec"; break;
+        case AV_CODEC_ID_VP8:  mc_name = "vp8_mediacodec";  break;
+        case AV_CODEC_ID_VP9:  mc_name = "vp9_mediacodec";  break;
+        default: break;
+        }
+        if (mc_name) {
+            m_codec = avcodec_find_decoder_by_name(mc_name);
+        }
+    }
+#endif
+    if (!m_codec) {
+        m_codec = avcodec_find_decoder(av_codec);
+    }
 }
 
 bool Decoder::SupportsDecodingOnDevice(AVPixelFormat* out_pix_fmt, AVHWDeviceType type) const {
@@ -142,13 +217,10 @@ bool Decoder::SupportsDecodingOnDevice(AVPixelFormat* out_pix_fmt, AVHWDeviceTyp
 std::vector<AVHWDeviceType> HardwareContext::GetSupportedDeviceTypes() {
     std::vector<AVHWDeviceType> types;
     AVHWDeviceType current_device_type = AV_HWDEVICE_TYPE_NONE;
-
     while (true) {
         current_device_type = av_hwdevice_iterate_types(current_device_type);
-        if (current_device_type == AV_HWDEVICE_TYPE_NONE) {
+        if (current_device_type == AV_HWDEVICE_TYPE_NONE)
             return types;
-        }
-
         types.push_back(current_device_type);
     }
 }
@@ -158,25 +230,20 @@ HardwareContext::~HardwareContext() {
 }
 
 bool HardwareContext::InitializeForDecoder(DecoderContext& decoder_context, const Decoder& decoder) {
-    const auto supported_types = GetSupportedDeviceTypes();
-    for (const auto type : PreferredGpuDecoders) {
-        AVPixelFormat hw_pix_fmt;
-
+    auto const supported_types = GetSupportedDeviceTypes();
+    for (auto const type : PREFERRED_GPU_DECODERS) {
         if (std::ranges::find(supported_types, type) == supported_types.end()) {
             LOG_DEBUG(HW_GPU, "{} explicitly unsupported", av_hwdevice_get_type_name(type));
             continue;
         }
-
-        if (!this->InitializeWithType(type)) {
-            continue;
-        }
-
-        if (decoder.SupportsDecodingOnDevice(&hw_pix_fmt, type)) {
-            decoder_context.InitializeHardwareDecoder(*this, hw_pix_fmt);
-            return true;
+        if (InitializeWithType(type)) {
+            AVPixelFormat hw_pix_fmt{};
+            if (decoder.SupportsDecodingOnDevice(&hw_pix_fmt, type)) {
+                decoder_context.InitializeHardwareDecoder(*this, hw_pix_fmt);
+                return true;
+            }
         }
     }
-
     return false;
 }
 
@@ -184,7 +251,7 @@ bool HardwareContext::InitializeWithType(AVHWDeviceType type) {
     av_buffer_unref(&m_gpu_decoder);
 
     if (const int ret = av_hwdevice_ctx_create(&m_gpu_decoder, type, nullptr, nullptr, 0); ret < 0) {
-        LOG_DEBUG(HW_GPU, "av_hwdevice_ctx_create({}) failed: {}", av_hwdevice_get_type_name(type), AVError(ret));
+        LOG_INFO(HW_GPU, "av_hwdevice_ctx_create({}) failed: {}", av_hwdevice_get_type_name(type), AVError(ret));
         return false;
     }
 
@@ -214,6 +281,9 @@ DecoderContext::DecoderContext(const Decoder& decoder) : m_decoder{decoder} {
     av_opt_set(m_codec_context->priv_data, "tune", "zerolatency", 0);
     m_codec_context->thread_count = 0;
     m_codec_context->thread_type &= ~FF_THREAD_FRAME;
+    // Forwarded into MediaCodec as KEY_LOW_LATENCY on Android.
+    m_codec_context->flags |= AV_CODEC_FLAG_LOW_DELAY;
+    m_codec_context->flags2 |= AV_CODEC_FLAG2_FAST;
 }
 
 DecoderContext::~DecoderContext() {
@@ -227,7 +297,19 @@ void DecoderContext::InitializeHardwareDecoder(const HardwareContext& context, A
     m_codec_context->pix_fmt = hw_pix_fmt;
 }
 
-bool DecoderContext::OpenContext(const Decoder& decoder) {
+bool DecoderContext::OpenContext(const Decoder& decoder, std::span<const u8> extradata) {
+    if (!extradata.empty()) {
+        av_freep(&m_codec_context->extradata);
+        m_codec_context->extradata = static_cast<u8*>(
+            av_mallocz(extradata.size() + AV_INPUT_BUFFER_PADDING_SIZE));
+        if (!m_codec_context->extradata) {
+            LOG_ERROR(HW_GPU, "Failed to allocate extradata");
+            return false;
+        }
+        std::memcpy(m_codec_context->extradata, extradata.data(), extradata.size());
+        m_codec_context->extradata_size = static_cast<int>(extradata.size());
+    }
+
     if (const int ret = avcodec_open2(m_codec_context, decoder.GetCodec(), nullptr); ret < 0) {
         LOG_ERROR(HW_GPU, "avcodec_open2 error: {}", AVError(ret));
         return false;
@@ -265,11 +347,17 @@ std::shared_ptr<Frame> DecoderContext::ReceiveFrame() {
 
     m_final_frame = std::make_shared<Frame>();
     if (m_codec_context->hw_device_ctx) {
-        m_final_frame->SetFormat(PreferredGpuFormat);
+#ifdef __ANDROID__
+        // c2.mtk.vp9.decoder, c2.mtk.vp89.decoder will be fine if we don't
+        // re-encode stuff twice :>
+        m_final_frame = std::move(intermediate_frame);
+#else
+        m_final_frame->SetFormat(PREFERRED_GPU_FORMAT);
         if (const int ret = av_hwframe_transfer_data(m_final_frame->GetFrame(), intermediate_frame->GetFrame(), 0); ret < 0) {
             LOG_ERROR(HW_GPU, "av_hwframe_transfer_data error: {}", AVError(ret));
             return {};
         }
+#endif
     } else {
         m_final_frame = std::move(intermediate_frame);
     }
@@ -281,9 +369,18 @@ void DecodeApi::Reset() {
     m_hardware_context.reset();
     m_decoder_context.reset();
     m_decoder.reset();
+    m_opened = false;
+    m_defer_android_mediacodec_open = false;
+    m_needs_h264_extradata = false;
+    m_next_pts = 0;
+    while (!m_pending_offsets.empty()) {
+        m_pending_offsets.pop();
+    }
 }
 
 bool DecodeApi::Initialize(Tegra::Host1x::NvdecCommon::VideoCodec codec) {
+    av_log_set_level(AV_LOG_DEBUG);
+
     this->Reset();
     m_decoder.emplace(codec);
     m_decoder_context.emplace(*m_decoder);
@@ -294,23 +391,80 @@ bool DecodeApi::Initialize(Tegra::Host1x::NvdecCommon::VideoCodec codec) {
         m_hardware_context->InitializeForDecoder(*m_decoder_context, *m_decoder);
     }
 
-    // Open the decoder context.
+#ifdef __ANDROID__
+    const std::string_view decoder_name = m_decoder->GetCodec() ? m_decoder->GetCodec()->name : "";
+
+    // MediaCodec decoders need the frame dimensions before avcodec_open2().
+    m_defer_android_mediacodec_open = decoder_name == "h264_mediacodec" ||
+                                      decoder_name == "vp8_mediacodec" ||
+                                      decoder_name == "vp9_mediacodec";
+
+    // h264_mediacodec also needs SPS/PPS in extradata at open. We pull them
+    // from the first frame's bitstream in SendPacket.
+    m_needs_h264_extradata = decoder_name == "h264_mediacodec";
+    if (m_defer_android_mediacodec_open) {
+        return true;
+    }
+#endif
+
     if (!m_decoder_context->OpenContext(*m_decoder)) {
         this->Reset();
         return false;
     }
+    m_opened = true;
 
     return true;
 }
 
-bool DecodeApi::SendPacket(std::span<const u8> packet_data) {
+bool DecodeApi::SendPacket(std::span<const u8> packet_data, const FrameOffsets& offsets,
+                           std::optional<FrameDimensions> dimensions) {
+    if (!m_opened) {
+        std::vector<u8> extradata;
+#ifdef __ANDROID__
+        if (m_defer_android_mediacodec_open) {
+            if (!dimensions) {
+                return true;
+            }
+
+            auto* ctx = m_decoder_context->GetCodecContext();
+            ctx->width = dimensions->width;
+            ctx->height = dimensions->height;
+            ctx->coded_width = dimensions->width;
+            ctx->coded_height = dimensions->height;
+        }
+
+        if (m_needs_h264_extradata) {
+            extradata = ExtractH264ParameterSetExtradata(packet_data);
+            if (extradata.empty()) {
+                return true;
+            }
+        }
+#endif
+        if (!m_decoder_context->OpenContext(*m_decoder, extradata)) {
+            this->Reset();
+            return false;
+        }
+        m_opened = true;
+    }
+    m_pending_offsets.push(offsets);
     FFmpeg::Packet packet(packet_data);
+    packet.GetPacket()->pts = m_next_pts;
+    packet.GetPacket()->dts = m_next_pts;
+    ++m_next_pts;
     return m_decoder_context->SendPacket(packet);
 }
 
-std::shared_ptr<Frame> DecodeApi::ReceiveFrame() {
-    // Receive raw frame from decoder.
-    return m_decoder_context->ReceiveFrame();
+std::optional<DecodeApi::DecodedFrame> DecodeApi::ReceiveFrame() {
+    auto frame = m_decoder_context->ReceiveFrame();
+    if (!frame) {
+        return std::nullopt;
+    }
+    FrameOffsets offsets{};
+    if (!m_pending_offsets.empty()) {
+        offsets = m_pending_offsets.front();
+        m_pending_offsets.pop();
+    }
+    return DecodedFrame{std::move(frame), offsets};
 }
 
 }

src/video_core/host1x/ffmpeg.h

@@ -179,7 +179,7 @@ public:
     ~DecoderContext();
 
     void InitializeHardwareDecoder(const HardwareContext& context, AVPixelFormat hw_pix_fmt);
-    bool OpenContext(const Decoder& decoder);
+    bool OpenContext(const Decoder& decoder, std::span<const u8> extradata = {});
     bool SendPacket(const Packet& packet);
     std::shared_ptr<Frame> ReceiveFrame();
 
@@ -198,6 +198,18 @@ private:
     bool m_decode_order{};
 };
 
+struct FrameOffsets {
+    bool interlaced{};
+    bool hidden{};
+    u64 luma{};
+    u64 luma_bottom{};
+};
+
+struct FrameDimensions {
+    s32 width{};
+    s32 height{};
+};
+
 class DecodeApi {
 public:
     YUZU_NON_COPYABLE(DecodeApi);
@@ -213,13 +225,24 @@ public:
         return m_decoder_context->UsingDecodeOrder();
     }
 
-    bool SendPacket(std::span<const u8> packet_data);
-    std::shared_ptr<Frame> ReceiveFrame();
+    bool SendPacket(std::span<const u8> packet_data, const FrameOffsets& offsets,
+                    std::optional<FrameDimensions> dimensions = std::nullopt);
+
+    struct DecodedFrame {
+        std::shared_ptr<Frame> frame;
+        FrameOffsets offsets;
+    };
+    std::optional<DecodedFrame> ReceiveFrame();
 
 private:
     std::optional<FFmpeg::Decoder> m_decoder;
     std::optional<FFmpeg::DecoderContext> m_decoder_context;
     std::optional<FFmpeg::HardwareContext> m_hardware_context;
+    bool m_opened{};
+    bool m_defer_android_mediacodec_open{};
+    bool m_needs_h264_extradata{};
+    s64 m_next_pts{};
+    std::queue<FrameOffsets> m_pending_offsets;
 };
 
 } // namespace FFmpeg

src/video_core/host1x/nvdec.cpp

@@ -31,6 +31,7 @@ Nvdec::Nvdec(Host1x& host1x_, s32 id_, u32 syncpt)
 
 Nvdec::~Nvdec() {
     LOG_INFO(HW_GPU, "Destroying nvdec {}", id);
+    host1x.frame_queue.Close(id);
 }
 
 void Nvdec::ProcessMethod(u32 method, u32 argument) {

src/video_core/host1x/vic.cpp

@@ -118,6 +118,7 @@ void Vic::Execute() noexcept {
     output_surface.resize(output_width * output_height);
 
     if (Settings::values.nvdec_emulation.GetValue() != Settings::NvdecEmulation::Off) {
+        bool decoded_frame = false;
         for (size_t i = 0; i < config.slot_structs.size(); i++) {
             if (auto& slot_config = config.slot_structs[i]; slot_config.config.slot_enable) {
                 auto const luma_offset = regs.surfaces[i][SurfaceIndex::Current].luma.Address();
@@ -136,11 +137,17 @@ void Vic::Execute() noexcept {
                         break;
                     }
                     Blend(config, slot_config, config.output_surface_config.out_pixel_format);
+                    decoded_frame = true;
                 } else {
-                    LOG_ERROR(HW_GPU, "Vic {} failed to get frame with offset {:#X}", id, luma_offset);
+                    LOG_TRACE(HW_GPU, "Vic {} failed to get frame with offset {:#X}", id, luma_offset);
                 }
             }
         }
+        if (decoded_frame) {
+            has_decoded_frame = true;
+        } else if (!has_decoded_frame) {
+            return;
+        }
     } else {
         // Fill the frame with black, as otherwise they can have random data and be very glitchy.
         std::fill(output_surface.begin(), output_surface.end(), Pixel{});

src/video_core/host1x/vic.h

@@ -628,6 +628,7 @@ private:
 
     s32 id;
     s32 nvdec_id{-1};
+    bool has_decoded_frame{};
     u32 syncpoint;
 };
 

src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp

@@ -37,8 +37,6 @@
 
 namespace Vulkan {
 namespace {
-using boost::container::small_vector;
-using boost::container::static_vector;
 using Shader::ImageBufferDescriptor;
 using Shader::Backend::SPIRV::RENDERAREA_LAYOUT_OFFSET;
 using Shader::Backend::SPIRV::RESCALING_LAYOUT_DOWN_FACTOR_OFFSET;
@@ -585,9 +583,9 @@ void GraphicsPipeline::MakePipeline(VkRenderPass render_pass) {
     } else {
         dynamic.raw1 = key.state.dynamic_state.raw1;
     }
-    static_vector<VkVertexInputBindingDescription, 32> vertex_bindings;
-    static_vector<VkVertexInputBindingDivisorDescriptionEXT, 32> vertex_binding_divisors;
-    static_vector<VkVertexInputAttributeDescription, 32> vertex_attributes;
+    boost::container::static_vector<VkVertexInputBindingDescription, 32> vertex_bindings;
+    boost::container::static_vector<VkVertexInputBindingDivisorDescriptionEXT, 32> vertex_binding_divisors;
+    boost::container::static_vector<VkVertexInputAttributeDescription, 32> vertex_attributes;
     if (!key.state.dynamic_vertex_input) {
         const size_t num_vertex_arrays = (std::min)(
             Maxwell::NumVertexArrays, static_cast<size_t>(device.GetMaxVertexInputBindings()));
@@ -812,7 +810,7 @@ void GraphicsPipeline::MakePipeline(VkRenderPass render_pass) {
     if (dynamic.depth_bounds_enable && !device.IsDepthBoundsSupported()) {
         LOG_WARNING(Render_Vulkan, "Depth bounds is enabled but not supported");
     }
-    static_vector<VkPipelineColorBlendAttachmentState, Maxwell::NumRenderTargets> cb_attachments;
+    boost::container::static_vector<VkPipelineColorBlendAttachmentState, Maxwell::NumRenderTargets> cb_attachments;
     const size_t num_attachments{NumAttachments(key.state)};
     for (size_t index = 0; index < num_attachments; ++index) {
         static constexpr std::array mask_table{
@@ -848,7 +846,7 @@ void GraphicsPipeline::MakePipeline(VkRenderPass render_pass) {
         .pAttachments = cb_attachments.data(),
         .blendConstants = {}
     };
-    static_vector<VkDynamicState, 34> dynamic_states{
+    boost::container::static_vector<VkDynamicState, 34> dynamic_states{
         VK_DYNAMIC_STATE_VIEWPORT,           VK_DYNAMIC_STATE_SCISSOR,
         VK_DYNAMIC_STATE_DEPTH_BIAS,         VK_DYNAMIC_STATE_BLEND_CONSTANTS,
         VK_DYNAMIC_STATE_DEPTH_BOUNDS,       VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK,
@@ -943,12 +941,9 @@ void GraphicsPipeline::MakePipeline(VkRenderPass render_pass) {
         .pNext = nullptr,
         .requiredSubgroupSize = GuestWarpSize,
     };
-    static_vector<VkPipelineShaderStageCreateInfo, 5> shader_stages;
+    boost::container::static_vector<VkPipelineShaderStageCreateInfo, 5> shader_stages;
     for (size_t stage = 0; stage < Maxwell::MaxShaderStage; ++stage) {
-        if (!spv_modules[stage]) {
-            continue;
-        }
-        [[maybe_unused]] auto& stage_ci =
+        if (spv_modules[stage]) {
             shader_stages.emplace_back(VkPipelineShaderStageCreateInfo{
                 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
                 .pNext = nullptr,
@@ -958,6 +953,7 @@ void GraphicsPipeline::MakePipeline(VkRenderPass render_pass) {
                 .pName = "main",
                 .pSpecializationInfo = nullptr,
             });
+        }
     }
     VkPipelineCreateFlags flags{};
     if (device.IsKhrPipelineExecutablePropertiesEnabled() && Settings::values.renderer_debug.GetValue()) {