[video_core] clean up Android MediaCodec NVDEC support (#4154)

THIS MAY BE BLOBBED WITH TWO PREVIOUS ONES!
(git reset --mixed head~3, commit, profit)

Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/4154

src/video_core/host1x/codecs/decoder.cpp

@@ -20,6 +20,14 @@ Decoder::Decoder(Host1x::Host1x& host1x_, s32 id_, const Host1x::NvdecCommon::Nv
 
 Decoder::~Decoder() = default;
 
+void Decoder::SetFrameDimensions(s32 width, s32 height) {
+    if (width <= 0 || height <= 0) {
+        frame_dimensions.reset();
+        return;
+    }
+    frame_dimensions = FFmpeg::FrameDimensions{width, height};
+}
+
 void Decoder::Decode() {
     if (!initialized) {
         return;
@@ -30,10 +38,10 @@ void Decoder::Decode() {
     offsets.hidden = vp9_hidden_frame;
     offsets.interlaced = IsInterlaced();
     if (offsets.interlaced) {
-        std::tie(offsets.luma, offsets.luma_bottom, offsets.chroma, offsets.chroma_bottom) =
+        std::tie(offsets.luma, offsets.luma_bottom, std::ignore, std::ignore) =
             GetInterlacedOffsets();
     } else {
-        std::tie(offsets.luma, offsets.chroma) = GetProgressiveOffsets();
+        std::tie(offsets.luma, std::ignore) = GetProgressiveOffsets();
     }
 
     if (!decode_api.SendPacket(packet_data, offsets, GetFrameDimensions())) {

src/video_core/host1x/codecs/decoder.h

@@ -10,6 +10,7 @@
 #include <mutex>
 #include <optional>
 #include <string_view>
+#include <tuple>
 #include <ankerl/unordered_dense.h>
 #include <queue>
 
@@ -45,8 +46,11 @@ protected:
     virtual std::tuple<u64, u64> GetProgressiveOffsets() = 0;
     virtual std::tuple<u64, u64, u64, u64> GetInterlacedOffsets() = 0;
     virtual bool IsInterlaced() = 0;
-    virtual std::optional<FFmpeg::FrameDimensions> GetFrameDimensions() {
-        return std::nullopt;
+
+    void SetFrameDimensions(s32 width, s32 height);
+
+    std::optional<FFmpeg::FrameDimensions> GetFrameDimensions() const {
+        return frame_dimensions;
     }
 
     FFmpeg::DecodeApi decode_api;
@@ -55,6 +59,7 @@ protected:
     s32 id;
     bool initialized : 1 = false;
     bool vp9_hidden_frame : 1 = false;
+    std::optional<FFmpeg::FrameDimensions> frame_dimensions;
 };
 
 } // namespace Tegra

src/video_core/host1x/codecs/h264.cpp

@@ -50,18 +50,12 @@ bool H264::IsInterlaced() {
            current_context.h264_parameter_set.luma_bot_offset.Address() != 0;
 }
 
-std::optional<FFmpeg::FrameDimensions> H264::GetFrameDimensions() {
-    const auto& params = current_context.h264_parameter_set;
-    const s32 width = static_cast<s32>(params.pic_width_in_mbs) * 16;
-    const s32 height = static_cast<s32>(params.frame_height_in_mbs) * 16;
-    if (width <= 0 || height <= 0) {
-        return std::nullopt;
-    }
-    return FFmpeg::FrameDimensions{width, height};
-}
-
 std::span<const u8> H264::ComposeFrame() {
     host1x.gmmu_manager.ReadBlock(regs.picture_info_offset.Address(), &current_context, sizeof(H264DecoderContext));
+    const auto& params = current_context.h264_parameter_set;
+    SetFrameDimensions(static_cast<s32>(params.pic_width_in_mbs) * 16,
+                       static_cast<s32>(params.frame_height_in_mbs) * 16);
+
     const s64 frame_number = current_context.h264_parameter_set.frame_number.Value();
     if (!is_first_frame && frame_number != 0) {
         frame_scratch.resize_destructive(current_context.stream_len);

src/video_core/host1x/codecs/h264.h

@@ -79,7 +79,6 @@ public:
     std::tuple<u64, u64> GetProgressiveOffsets() override;
     std::tuple<u64, u64, u64, u64> GetInterlacedOffsets() override;
     bool IsInterlaced() override;
-    std::optional<FFmpeg::FrameDimensions> GetFrameDimensions() override;
 
     std::string_view GetCurrentCodecName() const override {
         return "H264";

src/video_core/host1x/codecs/vp8.cpp

@@ -35,6 +35,7 @@ std::tuple<u64, u64, u64, u64> VP8::GetInterlacedOffsets() {
 
 std::span<const u8> VP8::ComposeFrame() {
     host1x.gmmu_manager.ReadBlock(regs.picture_info_offset.Address(), &current_context, sizeof(VP8PictureInfo));
+    SetFrameDimensions(current_context.frame_width, current_context.frame_height);
 
     const bool is_key_frame = current_context.key_frame == 1u;
     const auto bitstream_size = size_t(current_context.vld_buffer_size);

src/video_core/host1x/codecs/vp9.cpp

@@ -841,6 +841,7 @@ std::span<const u8> VP9::ComposeFrame() {
     {
         Vp9FrameContainer curr_frame = GetCurrentFrame();
         current_frame_info = curr_frame.info;
+        SetFrameDimensions(current_frame_info.frame_size.width, current_frame_info.frame_size.height);
         bitstream = std::move(curr_frame.bit_stream);
     }
     // The uncompressed header routine sets PrevProb parameters needed for the compressed header

src/video_core/host1x/ffmpeg.cpp

@@ -5,6 +5,7 @@
 // SPDX-License-Identifier: GPL-2.0-or-later
 
 #include <cstring>
+#include <string>
 #include <string_view>
 #include <vector>
 
@@ -23,6 +24,7 @@ extern "C" {
 #endif
 
 #include <libavutil/hwcontext.h>
+#include <libavutil/log.h>
 }
 
 namespace FFmpeg {
@@ -88,8 +90,8 @@ std::string AVError(int errnum) {
 }
 
 #ifdef __ANDROID__
-// Match a 3- or 4-byte annex-B start code at `i`. Returns its length, or 0.
-size_t MatchStartCode(std::span<const u8> data, size_t i) {
+// Match a 3- or 4-byte annex-B NAL start code at `i`. Returns its length, or 0.
+size_t FindNalStartCode(std::span<const u8> data, size_t i) {
     const size_t n = data.size();
     if (i + 3 < n && data[i] == 0 && data[i + 1] == 0 && data[i + 2] == 0 && data[i + 3] == 1) {
         return 4;
@@ -103,12 +105,12 @@ size_t MatchStartCode(std::span<const u8> data, size_t i) {
 // Pull SPS (NAL type 7) + PPS (NAL type 8) out of an annex-B frame into an
 // extradata buffer, each prefixed with a 4-byte start code. Eden synthesizes
 // these inline into the very first frame; h264_mediacodec wants them at open.
-std::vector<u8> ExtractH264AnnexBExtradata(std::span<const u8> packet) {
+std::vector<u8> ExtractH264ParameterSetExtradata(std::span<const u8> packet) {
     std::vector<u8> extradata;
     const size_t size = packet.size();
     size_t i = 0;
     while (i < size) {
-        const size_t sc = MatchStartCode(packet, i);
+        const size_t sc = FindNalStartCode(packet, i);
         if (sc == 0) {
             ++i;
             continue;
@@ -120,7 +122,7 @@ std::vector<u8> ExtractH264AnnexBExtradata(std::span<const u8> packet) {
         const u8 nal_type = packet[nal_start] & 0x1F;
 
         size_t j = nal_start + 1;
-        while (j < size && MatchStartCode(packet, j) == 0) {
+        while (j < size && FindNalStartCode(packet, j) == 0) {
             ++j;
         }
 
@@ -368,6 +370,7 @@ void DecodeApi::Reset() {
     m_decoder_context.reset();
     m_decoder.reset();
     m_opened = false;
+    m_defer_android_mediacodec_open = false;
     m_needs_h264_extradata = false;
     m_next_pts = 0;
     while (!m_pending_offsets.empty()) {
@@ -389,11 +392,17 @@ bool DecodeApi::Initialize(Tegra::Host1x::NvdecCommon::VideoCodec codec) {
     }
 
 #ifdef __ANDROID__
-    // h264_mediacodec needs SPS/PPS in extradata at open. We pull them from
-    // the first frame's bitstream in SendPacket.
-    m_needs_h264_extradata = m_decoder->GetCodec() &&
-                             std::string_view(m_decoder->GetCodec()->name) == "h264_mediacodec";
-    if (m_needs_h264_extradata) {
+    const std::string_view decoder_name = m_decoder->GetCodec() ? m_decoder->GetCodec()->name : "";
+
+    // MediaCodec decoders need the frame dimensions before avcodec_open2().
+    m_defer_android_mediacodec_open = decoder_name == "h264_mediacodec" ||
+                                      decoder_name == "vp8_mediacodec" ||
+                                      decoder_name == "vp9_mediacodec";
+
+    // h264_mediacodec also needs SPS/PPS in extradata at open. We pull them
+    // from the first frame's bitstream in SendPacket.
+    m_needs_h264_extradata = decoder_name == "h264_mediacodec";
+    if (m_defer_android_mediacodec_open) {
         return true;
     }
 #endif
@@ -412,17 +421,22 @@ bool DecodeApi::SendPacket(std::span<const u8> packet_data, const FrameOffsets&
     if (!m_opened) {
         std::vector<u8> extradata;
 #ifdef __ANDROID__
-        if (m_needs_h264_extradata) {
-            extradata = ExtractH264AnnexBExtradata(packet_data);
-            if (extradata.empty()) {
+        if (m_defer_android_mediacodec_open) {
+            if (!dimensions) {
                 return true;
             }
-            if (dimensions) {
-                auto* ctx = m_decoder_context->GetCodecContext();
-                ctx->width = dimensions->width;
-                ctx->height = dimensions->height;
-                ctx->coded_width = dimensions->width;
-                ctx->coded_height = dimensions->height;
+
+            auto* ctx = m_decoder_context->GetCodecContext();
+            ctx->width = dimensions->width;
+            ctx->height = dimensions->height;
+            ctx->coded_width = dimensions->width;
+            ctx->coded_height = dimensions->height;
+        }
+
+        if (m_needs_h264_extradata) {
+            extradata = ExtractH264ParameterSetExtradata(packet_data);
+            if (extradata.empty()) {
+                return true;
             }
         }
 #endif

src/video_core/host1x/ffmpeg.h

@@ -202,9 +202,7 @@ struct FrameOffsets {
     bool interlaced{};
     bool hidden{};
     u64 luma{};
-    u64 chroma{};
     u64 luma_bottom{};
-    u64 chroma_bottom{};
 };
 
 struct FrameDimensions {
@@ -241,6 +239,7 @@ private:
     std::optional<FFmpeg::DecoderContext> m_decoder_context;
     std::optional<FFmpeg::HardwareContext> m_hardware_context;
     bool m_opened{};
+    bool m_defer_android_mediacodec_open{};
     bool m_needs_h264_extradata{};
     s64 m_next_pts{};
     std::queue<FrameOffsets> m_pending_offsets;

src/video_core/host1x/vic.cpp

@@ -118,6 +118,7 @@ void Vic::Execute() noexcept {
     output_surface.resize(output_width * output_height);
 
     if (Settings::values.nvdec_emulation.GetValue() != Settings::NvdecEmulation::Off) {
+        bool decoded_frame = false;
         for (size_t i = 0; i < config.slot_structs.size(); i++) {
             if (auto& slot_config = config.slot_structs[i]; slot_config.config.slot_enable) {
                 auto const luma_offset = regs.surfaces[i][SurfaceIndex::Current].luma.Address();
@@ -136,11 +137,17 @@ void Vic::Execute() noexcept {
                         break;
                     }
                     Blend(config, slot_config, config.output_surface_config.out_pixel_format);
-                } else if (nvdec_id != -1) {
-                    LOG_ERROR(HW_GPU, "Vic {} failed to get frame with offset {:#X}", id, luma_offset);
+                    decoded_frame = true;
+                } else {
+                    LOG_TRACE(HW_GPU, "Vic {} failed to get frame with offset {:#X}", id, luma_offset);
                 }
             }
         }
+        if (decoded_frame) {
+            has_decoded_frame = true;
+        } else if (!has_decoded_frame) {
+            return;
+        }
     } else {
         // Fill the frame with black, as otherwise they can have random data and be very glitchy.
         std::fill(output_surface.begin(), output_surface.end(), Pixel{});

src/video_core/host1x/vic.h

@@ -628,6 +628,7 @@ private:
 
     s32 id;
     s32 nvdec_id{-1};
+    bool has_decoded_frame{};
     u32 syncpoint;
 };