debuggerx/eden
1
0
Fork 0
mirror of https://git.eden-emu.dev/eden-emu/eden.git synced 2026-08-25 08:49:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

[vulkan, android] Mediacodec implementation + Vulkan fixes (#4191)

Browse Source 
This is the first step into the conversion on full GPU video decoding for Android; currently due to the VIC structure, I couldn't set it on surface mode to prevent the latency when sending video decoded (which is processed by GPU now), because currently we convert YUV420 into the Nvidia's format each frame constantly on CPU, aside the requirements from other extensions to work + VIC rewrite, which is a work currently not planned to happen on this PR; the actual configuration for video decoding is ByteBuffer, using GPU to decode NVDEC data (all codecs supported, h264, vp8 and vp9) and send it to CPU for display purposes, which is more faster than relying purely on CPU for any drawing task, saving devices resources/ heating, the downside on this will be the slight latency when a new frame is displayed, which is gonna be a black frame for less than a second, nothing major to harm the experience rather than actually trying our best to take advantage on hardware accelerated.

Aside this, I also added a bunch of minor Vulkan fixes to grant drivers less thinkering when receiving spir-v instructions, meaning this has new bans for extensions on QCOM (following reported issues on other projects working around Adreno driver behavior), this more than providing performance aims to enhance the stability on the driver, performance it's gonna likely to be hit based on the UBO's (StorageBufferAccess) operations and SSBO (uniformStorageBufferAccess), there was an already existing path for the emulation which forces to wide them into 32bit packed operations. I also included some smaller changes/ bugs + VUID's fixes from earlier changes on my work.

Special Thanks:

-> Mr. Smoly Gidolard (@gidoly)

Sources:

1.- https://github.com/microsoft/DirectXShaderCompiler/issues/2842
2.- https://github.com/mlc-ai/web-llm/issues/836
3.- https://github.com/ggml-org/llama.cpp/issues/5186
4.- https://github.com/encounter/aurora/pull/202

Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/4191
This commit is contained in:
CamilleLaVey
2026-07-09 06:55:54 +02:00
committed by crueter
parent 3699795315
commit 372bdfccbc
33 changed files with 501 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/video_core/host1x/ffmpeg.cpp
+159 -10
View File
@@ -4,6 +4,10 @@
// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#include <cstring>
#include <string_view>
#include <vector>
#include "common/assert.h"
#include "common/logging.h"
#include "common/scope_exit.h"
@@ -84,6 +88,52 @@ std::string AVError(int errnum) {
return errbuf;
}
#if defined(__ANDROID__)
size_t FindNalStartCode(std::span<const u8> data, size_t i) {
const size_t n = data.size();
if (i + 3 < n && data[i] == 0 && data[i + 1] == 0 && data[i + 2] == 0 && data[i + 3] == 1) {
return 4;
}
if (i + 2 < n && data[i] == 0 && data[i + 1] == 0 && data[i + 2] == 1) {
return 3;
}
return 0;
}
std::vector<u8> ExtractH264ParameterSetExtradata(std::span<const u8> packet) {
std::vector<u8> extradata;
const size_t size = packet.size();
size_t i = 0;
while (i < size) {
const size_t sc = FindNalStartCode(packet, i);
if (sc == 0) {
++i;
continue;
}
const size_t nal_start = i + sc;
if (nal_start >= size) {
break;
}
const u8 nal_type = packet[nal_start] & 0x1F;
size_t j = nal_start + 1;
while (j < size && FindNalStartCode(packet, j) == 0) {
++j;
}
if (nal_type == 7 || nal_type == 8) {
constexpr u8 start[4] = {0, 0, 0, 1};
extradata.insert(extradata.end(), start, start + sizeof(start));
extradata.insert(extradata.end(), packet.begin() + nal_start, packet.begin() + j);
} else if (nal_type == 1 || nal_type == 5) {
break;
}
i = j;
}
return extradata;
}
#endif
}
Packet::Packet(std::span<const u8> data) {
@@ -118,7 +168,24 @@ Decoder::Decoder(Tegra::Host1x::NvdecCommon::VideoCodec codec) {
return AV_CODEC_ID_NONE;
}
}();
m_codec = avcodec_find_decoder(av_codec);
#if defined(__ANDROID__)
if (Settings::values.nvdec_emulation.GetValue() == Settings::NvdecEmulation::Gpu) {
const char* mc_name = nullptr;
switch (av_codec) {
case AV_CODEC_ID_H264: mc_name = "h264_mediacodec"; break;
case AV_CODEC_ID_VP8: mc_name = "vp8_mediacodec"; break;
case AV_CODEC_ID_VP9: mc_name = "vp9_mediacodec"; break;
default: break;
}
if (mc_name) {
m_codec = avcodec_find_decoder_by_name(mc_name);
}
}
#endif
if (!m_codec) {
m_codec = avcodec_find_decoder(av_codec);
}
}
bool Decoder::SupportsDecodingOnDevice(AVPixelFormat* out_pix_fmt, AVHWDeviceType type) const {
@@ -214,6 +281,10 @@ DecoderContext::DecoderContext(const Decoder& decoder) : m_decoder{decoder} {
av_opt_set(m_codec_context->priv_data, "tune", "zerolatency", 0);
m_codec_context->thread_count = 0;
m_codec_context->thread_type &= ~FF_THREAD_FRAME;
#if defined(__ANDROID__)
m_codec_context->flags |= AV_CODEC_FLAG_LOW_DELAY;
m_codec_context->flags2 |= AV_CODEC_FLAG2_FAST;
#endif
}
DecoderContext::~DecoderContext() {
@@ -227,15 +298,25 @@ void DecoderContext::InitializeHardwareDecoder(const HardwareContext& context, A
m_codec_context->pix_fmt = hw_pix_fmt;
}
bool DecoderContext::OpenContext(const Decoder& decoder) {
bool DecoderContext::OpenContext(const Decoder& decoder, std::span<const u8> extradata) {
if (!extradata.empty()) {
av_freep(&m_codec_context->extradata);
m_codec_context->extradata = static_cast<u8*>(
av_mallocz(extradata.size() + AV_INPUT_BUFFER_PADDING_SIZE));
if (!m_codec_context->extradata) {
LOG_ERROR(HW_GPU, "Failed to allocate extradata");
return false;
}
std::memcpy(m_codec_context->extradata, extradata.data(), extradata.size());
m_codec_context->extradata_size = static_cast<int>(extradata.size());
}
if (const int ret = avcodec_open2(m_codec_context, decoder.GetCodec(), nullptr); ret < 0) {
LOG_ERROR(HW_GPU, "avcodec_open2 error: {}", AVError(ret));
return false;
}
if (!m_codec_context->hw_device_ctx) {
LOG_INFO(HW_GPU, "Using FFmpeg CPU decoder");
}
LOG_INFO(HW_GPU, "Using decoder {}", decoder.GetCodec()->name);
return true;
}
@@ -281,6 +362,13 @@ void DecodeApi::Reset() {
m_hardware_context.reset();
m_decoder_context.reset();
m_decoder.reset();
m_opened = false;
m_defer_android_mediacodec_open = false;
m_needs_h264_extradata = false;
m_next_pts = 0;
while (!m_pending_offsets.empty()) {
m_pending_offsets.pop();
}
}
bool DecodeApi::Initialize(Tegra::Host1x::NvdecCommon::VideoCodec codec) {
@@ -288,29 +376,90 @@ bool DecodeApi::Initialize(Tegra::Host1x::NvdecCommon::VideoCodec codec) {
m_decoder.emplace(codec);
m_decoder_context.emplace(*m_decoder);
bool is_mediacodec = false;
#if defined(__ANDROID__)
const std::string_view decoder_name = m_decoder->GetCodec() ? m_decoder->GetCodec()->name : "";
is_mediacodec = decoder_name == "h264_mediacodec" ||
decoder_name == "vp8_mediacodec" ||
decoder_name == "vp9_mediacodec";
#endif
// Enable GPU decoding if requested.
if (Settings::values.nvdec_emulation.GetValue() == Settings::NvdecEmulation::Gpu) {
if (!is_mediacodec &&
Settings::values.nvdec_emulation.GetValue() == Settings::NvdecEmulation::Gpu) {
m_hardware_context.emplace();
m_hardware_context->InitializeForDecoder(*m_decoder_context, *m_decoder);
}
#if defined(__ANDROID__)
m_defer_android_mediacodec_open = is_mediacodec;
m_needs_h264_extradata = decoder_name == "h264_mediacodec";
if (m_defer_android_mediacodec_open) {
return true;
}
#endif
// Open the decoder context.
if (!m_decoder_context->OpenContext(*m_decoder)) {
this->Reset();
return false;
}
m_opened = true;
return true;
}
bool DecodeApi::SendPacket(std::span<const u8> packet_data) {
bool DecodeApi::SendPacket(std::span<const u8> packet_data, const FrameOffsets& offsets,
std::optional<FrameDimensions> dimensions) {
if (!m_opened) {
std::vector<u8> extradata;
#if defined(__ANDROID__)
if (m_defer_android_mediacodec_open) {
if (!dimensions) {
return true;
}
auto* ctx = m_decoder_context->GetCodecContext();
ctx->width = dimensions->width;
ctx->height = dimensions->height;
ctx->coded_width = dimensions->width;
ctx->coded_height = dimensions->height;
}
if (m_needs_h264_extradata) {
extradata = ExtractH264ParameterSetExtradata(packet_data);
if (extradata.empty()) {
return true;
}
}
#endif
if (!m_decoder_context->OpenContext(*m_decoder, extradata)) {
this->Reset();
return false;
}
m_opened = true;
}
if (!offsets.hidden) {
m_pending_offsets.push(offsets);
}
FFmpeg::Packet packet(packet_data);
packet.GetPacket()->pts = m_next_pts;
packet.GetPacket()->dts = m_next_pts;
++m_next_pts;
return m_decoder_context->SendPacket(packet);
}
std::shared_ptr<Frame> DecodeApi::ReceiveFrame() {
// Receive raw frame from decoder.
return m_decoder_context->ReceiveFrame();
std::optional<DecodeApi::DecodedFrame> DecodeApi::ReceiveFrame() {
auto frame = m_decoder_context->ReceiveFrame();
if (!frame) {
return std::nullopt;
}
FrameOffsets offsets{};
if (!m_pending_offsets.empty()) {
offsets = m_pending_offsets.front();
m_pending_offsets.pop();
}
return DecodedFrame{std::move(frame), offsets};
}
}