Compare commits

..

3 Commits

Author SHA1 Message Date
lizzie dfccf1980d fsfsf 2026-07-21 23:39:53 +02:00
lizzie 66b9649fe2 [core/hle/service/nvdrv] force std::unordered_map<> to retain pointer stability
Signed-off-by: lizzie <lizzie@eden-emu.dev>
2026-07-21 23:39:53 +02:00
lizzie 89004124a5 [video_core] use bool params for read/writes and cascade them thru the calltree (#4001)
should make codegen a tad bit better and reduce icache pressure for what is otherwise a glorified memcpy

Signed-off-by: lizzie <lizzie@eden-emu.dev>

Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/4001
Reviewed-by: Maufeat <sahyno1996@gmail.com>
Reviewed-by: CamilleLaVey <camillelavey99@gmail.com>
Reviewed-by: MaranBr <maranbr@eden-emu.dev>
2026-07-18 21:01:58 +02:00
100 changed files with 1235 additions and 5783 deletions
+1 -1
View File
@@ -65,7 +65,7 @@ android {
defaultConfig {
applicationId = "dev.eden.eden_emulator"
minSdk = 33
minSdk = 24
targetSdk = 36
versionName = getGitVersion()
versionCode = autoVersion
@@ -27,7 +27,6 @@ enum class BooleanSetting(override val key: String) : AbstractBooleanSetting {
RENDERER_ASYNCHRONOUS_GPU_EMULATION("use_asynchronous_gpu_emulation"),
RENDERER_ASYNC_PRESENTATION("async_presentation"),
RENDERER_ASYNCHRONOUS_SHADERS("use_asynchronous_shaders"),
RENDERER_UNIFIED_MEMORY("use_unified_memory"),
RENDERER_REACTIVE_FLUSHING("use_reactive_flushing"),
ENABLE_BUFFER_HISTORY("enable_buffer_history"),
USE_OPTIMIZED_VERTEX_BUFFERS("use_optimized_vertex_buffers"),
@@ -37,8 +36,6 @@ enum class BooleanSetting(override val key: String) : AbstractBooleanSetting {
RENDERER_DEBUG("debug"),
RENDERER_PATCH_OLD_QCOM_DRIVERS("patch_old_qcom_drivers"),
RENDERER_VERTEX_INPUT_DYNAMIC_STATE("vertex_input_dynamic_state"),
RENDERER_DYNAMIC_RENDERING("dynamic_rendering"),
RENDERER_WORKGROUP_MEMORY_EXPLICIT_LAYOUT("workgroup_memory_explicit_layout"),
RENDERER_SAMPLE_SHADING("sample_shading"),
GPU_UNSWIZZLE_ENABLED("gpu_unswizzle_enabled"),
PICTURE_IN_PICTURE("picture_in_picture"),
@@ -155,20 +155,6 @@ abstract class SettingsItem(
descriptionId = R.string.vertex_input_dynamic_state_description
)
)
put(
SwitchSetting(
BooleanSetting.RENDERER_DYNAMIC_RENDERING,
titleId = R.string.dynamic_rendering,
descriptionId = R.string.dynamic_rendering_description
)
)
put(
SwitchSetting(
BooleanSetting.RENDERER_WORKGROUP_MEMORY_EXPLICIT_LAYOUT,
titleId = R.string.workgroup_memory_explicit_layout,
descriptionId = R.string.workgroup_memory_explicit_layout_description
)
)
put(
SliderSetting(
IntSetting.RENDERER_SAMPLE_SHADING,
@@ -608,7 +594,7 @@ abstract class SettingsItem(
IntSetting.ANDROID_PIPELINE_WORKERS,
titleId = R.string.pipeline_worker_cores,
descriptionId = R.string.pipeline_worker_cores_description,
min = 2,
min = 4,
max = 8,
units = "cores"
)
@@ -699,13 +685,6 @@ abstract class SettingsItem(
descriptionId = R.string.renderer_asynchronous_shaders_description
)
)
put(
SwitchSetting(
BooleanSetting.RENDERER_UNIFIED_MEMORY,
titleId = R.string.renderer_unified_memory,
descriptionId = R.string.renderer_unified_memory_description
)
)
put(
SingleChoiceSetting(
IntSetting.FAST_GPU_TIME,
@@ -304,7 +304,6 @@ class SettingsFragmentPresenter(
add(BooleanSetting.EMULATE_BGR565.key)
add(BooleanSetting.RESCALE_HACK.key)
add(BooleanSetting.RENDERER_ASYNCHRONOUS_SHADERS.key)
add(BooleanSetting.RENDERER_UNIFIED_MEMORY.key)
add(IntSetting.ANDROID_PIPELINE_WORKERS.key)
add(BooleanSetting.RENDERER_ASYNCHRONOUS_GPU_EMULATION.key)
add(BooleanSetting.RENDERER_ASYNC_PRESENTATION.key)
@@ -314,8 +313,6 @@ class SettingsFragmentPresenter(
add(IntSetting.RENDERER_DYNA_STATE.key)
add(BooleanSetting.RENDERER_VERTEX_INPUT_DYNAMIC_STATE.key)
add(BooleanSetting.RENDERER_DYNAMIC_RENDERING.key)
add(BooleanSetting.RENDERER_WORKGROUP_MEMORY_EXPLICIT_LAYOUT.key)
add(IntSetting.RENDERER_SAMPLE_SHADING.key)
add(HeaderSetting(R.string.display))
@@ -147,6 +147,13 @@ namespace AndroidSettings {
&show_performance_overlay};
Settings::Setting<s32> pipeline_worker_count{linkage, 4, "pipeline_worker_count",
Settings::Category::Android,
Settings::Specialization::Default,
true,
true};
Settings::Setting<bool> show_input_overlay{linkage, true, "show_input_overlay",
Settings::Category::Overlay};
Settings::Setting<bool> overlay_snap_to_grid{linkage, false, "overlay_snap_to_grid",
@@ -524,8 +524,6 @@
<string name="rescale_hack_description">Enables a legacy handling for the rescale configuration pass for games by using a quick rescale path</string>
<string name="renderer_asynchronous_shaders">Use asynchronous shaders</string>
<string name="renderer_asynchronous_shaders_description">Compiles shaders asynchronously. This may reduce stutters but may also introduce glitches.</string>
<string name="renderer_unified_memory">Unified memory access (UMA)</string>
<string name="renderer_unified_memory_description">Allows GPU write buffer readbacks directly into guest memory, skipping the CPU staging copy.</string>
<string name="gpu_unswizzle_settings">GPU Unswizzle Settings</string>
<string name="gpu_unswizzle_settings_description">Configure GPU-based texture unswizzling parameters or disable it entirely. Adjust these settings to balance performance and texture loading quality.</string>
<string name="gpu_unswizzle_enable">Enable GPU Unswizzle</string>
@@ -546,10 +544,6 @@
<string name="disabled">Disabled</string>
<string name="vertex_input_dynamic_state">Vertex Input Dynamic State</string>
<string name="vertex_input_dynamic_state_description">Enabling this feature allows for more flexible vertex input handling, potentially reducing pipeline compilation time in vertex/buffer.</string>
<string name="dynamic_rendering">Dynamic Rendering</string>
<string name="dynamic_rendering_description">Render without render pass and framebuffer objects. Results vary by driver: some gain performance, others lose it.</string>
<string name="workgroup_memory_explicit_layout">Workgroup Memory Explicit Layout</string>
<string name="workgroup_memory_explicit_layout_description">Let shaders declare explicit layouts for workgroup memory. Disabled by default: some Qualcomm drivers are unstable with it.</string>
<string name="sample_shading_fraction">Sample Shading</string>
<string name="sample_shading_fraction_description">Allows the fragment shader to execute per sample in a multi-sampled fragment instead once per fragment. Improves graphics quality at the cost of some performance.</string>
+1 -290
View File
@@ -51,60 +51,14 @@
#endif // ^^^ POSIX ^^^
#include <atomic>
#include <mutex>
#include <random>
#include <vector>
#include "common/alignment.h"
#include "common/assert.h"
#include "common/free_region_manager.h"
#include "common/host_memory.h"
#include "common/logging.h"
#include "common/memory_detect.h"
#include "common/settings.h"
#ifdef __ANDROID__
#include <dlfcn.h>
#include <android/hardware_buffer.h>
namespace {
struct NativeHandle {
int version;
int numFds;
int numInts;
int data[1];
};
using PFN_AHardwareBuffer_getNativeHandle = const NativeHandle* (*)(const AHardwareBuffer*);
using PFN_AHardwareBuffer_isSupported = int (*)(const AHardwareBuffer_Desc*);
void* NativeWindowLibrary() {
static void* const lib = dlopen("libnativewindow.so", RTLD_NOW);
return lib;
}
PFN_AHardwareBuffer_getNativeHandle ResolveGetNativeHandle() {
void* const lib = NativeWindowLibrary();
if (lib == nullptr) {
return nullptr;
}
return reinterpret_cast<PFN_AHardwareBuffer_getNativeHandle>(
dlsym(lib, "AHardwareBuffer_getNativeHandle"));
}
PFN_AHardwareBuffer_isSupported ResolveIsSupported() {
void* const lib = NativeWindowLibrary();
if (lib == nullptr) {
return nullptr;
}
return reinterpret_cast<PFN_AHardwareBuffer_isSupported>(
dlsym(lib, "AHardwareBuffer_isSupported"));
}
} // namespace
#endif
#if defined(__ANDROID__) && __ANDROID_API__ < 30
#include <sys/syscall.h>
@@ -121,12 +75,6 @@ namespace Common {
[[maybe_unused]] constexpr size_t PageAlignment = 0x1000;
[[maybe_unused]] constexpr size_t HugePageSize = 0x200000;
static std::atomic<u64> committed_backing_size{};
u64 GetCommittedBackingSize() noexcept {
return committed_backing_size.load(std::memory_order_relaxed);
}
#ifdef _WIN32
// Manually imported for MinGW compatibility
@@ -561,11 +509,6 @@ public:
bool Init() {
long page_size = sysconf(_SC_PAGESIZE);
ASSERT_MSG(page_size == 0x1000, "page size {:#x} is incompatible with 4K paging", page_size);
#ifdef __ANDROID__
if (InitAhbBacking()) {
return InitVirtual();
}
#endif
// Backing memory initialization
#if defined(__sun__) || defined(__HAIKU__) || defined(__NetBSD__) || defined(__DragonFly__)
fd = shm_open_anon(O_RDWR | O_CREAT | O_EXCL | O_NOFOLLOW, 0600);
@@ -611,10 +554,7 @@ public:
return false;
}
return InitVirtual();
}
bool InitVirtual() {
// Virtual memory initialization
virtual_base = virtual_map_base = static_cast<u8*>(ChooseVirtualBase(virtual_size));
if (virtual_base == MAP_FAILED) {
LOG_CRITICAL(HW_Memory, "mmap failed: {}", strerror(errno));
@@ -627,180 +567,6 @@ public:
return true;
}
#ifdef __ANDROID__
static AHardwareBuffer_Desc MakeBlobDesc(size_t len) {
return AHardwareBuffer_Desc{
.width = static_cast<u32>(len),
.height = 1,
.layers = 1,
.format = AHARDWAREBUFFER_FORMAT_BLOB,
.usage = AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN |
AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN |
AHARDWAREBUFFER_USAGE_GPU_DATA_BUFFER,
.stride = 0,
.rfu0 = 0,
.rfu1 = 0,
};
}
static bool ProbeAhbBacking(PFN_AHardwareBuffer_getNativeHandle get_native_handle) {
const AHardwareBuffer_Desc desc = MakeBlobDesc(PageAlignment * 2);
AHardwareBuffer* buffer{};
if (AHardwareBuffer_allocate(&desc, &buffer) != 0 || buffer == nullptr) {
LOG_WARNING(HW_Memory, "Hardware buffer probe allocation failed");
return false;
}
const NativeHandle* const handle = get_native_handle(buffer);
if (handle == nullptr || handle->numFds < 1) {
LOG_WARNING(HW_Memory, "Hardware buffer has no mappable file descriptor");
AHardwareBuffer_release(buffer);
return false;
}
const int probe_fd = handle->data[0];
bool ok = true;
const auto try_map = [&](int prot, off_t offset, const char* what) {
if (!ok) {
return;
}
void* const ptr = mmap(nullptr, PageAlignment, prot, MAP_SHARED, probe_fd, offset);
if (ptr == MAP_FAILED) {
LOG_WARNING(HW_Memory, "Hardware buffer backing rejects {}: {}", what,
strerror(errno));
ok = false;
return;
}
munmap(ptr, PageAlignment);
};
try_map(PROT_READ | PROT_WRITE, 0, "shared mappings");
try_map(PROT_READ | PROT_WRITE, static_cast<off_t>(PageAlignment), "mappings at an offset");
#ifdef ARCHITECTURE_arm64
try_map(PROT_READ | PROT_EXEC, 0, "executable mappings");
#endif
AHardwareBuffer_release(buffer);
return ok;
}
bool InitAhbBacking() {
if (!Settings::values.use_unified_memory.GetValue()) {
return false;
}
static const PFN_AHardwareBuffer_getNativeHandle get_native_handle =
ResolveGetNativeHandle();
if (get_native_handle == nullptr) {
LOG_WARNING(HW_Memory, "AHardwareBuffer_getNativeHandle is not available");
return false;
}
static const PFN_AHardwareBuffer_isSupported is_supported = ResolveIsSupported();
if (is_supported == nullptr) {
LOG_WARNING(HW_Memory, "AHardwareBuffer_isSupported is not available");
return false;
}
const u64 total_physical = Common::GetMemInfo().TotalPhysicalMemory;
if (total_physical != 0 && backing_size > total_physical / 2) {
LOG_WARNING(HW_Memory,
"Hardware buffer backing would commit {} MiB on a {} MiB system, keeping "
"lazily committed memory",
backing_size >> 20, total_physical >> 20);
return false;
}
if (!ProbeAhbBacking(get_native_handle)) {
return false;
}
const auto try_window_size = [&](size_t window_size) -> bool {
const size_t num_windows = (backing_size + window_size - 1) / window_size;
std::vector<AHardwareBuffer*> buffers;
std::vector<int> buffer_fds;
const auto cleanup = [&] {
for (AHardwareBuffer* buffer : buffers) {
AHardwareBuffer_release(buffer);
}
buffers.clear();
buffer_fds.clear();
};
for (size_t i = 0; i < num_windows; ++i) {
const size_t len = (std::min)(window_size, backing_size - i * window_size);
const AHardwareBuffer_Desc desc = MakeBlobDesc(len);
AHardwareBuffer* buffer{};
if (AHardwareBuffer_allocate(&desc, &buffer) != 0 || buffer == nullptr) {
LOG_WARNING(HW_Memory, "Hardware buffer allocation failed for window {}", i);
cleanup();
return false;
}
buffers.push_back(buffer);
const NativeHandle* const handle = get_native_handle(buffer);
if (handle == nullptr || handle->numFds < 1) {
LOG_WARNING(HW_Memory, "Hardware buffer has no mappable file descriptor");
cleanup();
return false;
}
const int buffer_fd = handle->data[0];
const off_t buffer_len = lseek(buffer_fd, 0, SEEK_END);
if (buffer_len < static_cast<off_t>(len)) {
LOG_WARNING(HW_Memory, "Hardware buffer descriptor smaller than requested");
cleanup();
return false;
}
buffer_fds.push_back(buffer_fd);
}
u8* const base =
static_cast<u8*>(mmap(nullptr, backing_size, PROT_NONE,
MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0));
if (base == MAP_FAILED) {
cleanup();
return false;
}
for (size_t i = 0; i < num_windows; ++i) {
const size_t len = (std::min)(window_size, backing_size - i * window_size);
if (mmap(base + i * window_size, len, PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_FIXED, buffer_fds[i], 0) == MAP_FAILED) {
LOG_WARNING(HW_Memory, "Hardware buffer mmap failed: {}", strerror(errno));
munmap(base, backing_size);
cleanup();
return false;
}
}
backing_base = base;
ahb_windows = std::move(buffers);
ahb_fds = std::move(buffer_fds);
ahb_window_size = window_size;
ahb_backing = true;
committed_backing_size.store(backing_size, std::memory_order_relaxed);
LOG_INFO(HW_Memory,
"Guest memory backed by {} hardware buffer windows of {} MiB, {} MiB committed",
ahb_windows.size(), window_size >> 20, backing_size >> 20);
return true;
};
static constexpr size_t candidate_window_sizes[] = {
1024ULL << 20,
512ULL << 20,
256ULL << 20,
128ULL << 20,
};
for (const size_t candidate : candidate_window_sizes) {
const AHardwareBuffer_Desc window_desc = MakeBlobDesc(candidate);
if (is_supported(&window_desc) == 0) {
LOG_DEBUG(HW_Memory, "Allocator rejects {} MiB hardware buffer windows",
candidate >> 20);
continue;
}
if (try_window_size(candidate)) {
return true;
}
LOG_WARNING(HW_Memory, "Could not back guest memory with {} MiB windows",
candidate >> 20);
}
return false;
}
std::span<AHardwareBuffer* const> AhbWindows() const noexcept {
return ahb_windows;
}
size_t AhbWindowSize() const noexcept {
return ahb_backing ? ahb_window_size : 0;
}
#endif
~Impl() {
Release();
}
@@ -821,26 +587,6 @@ public:
#ifdef ARCHITECTURE_arm64
if (True(perms & MemoryPermission::Execute))
prot_flags |= PROT_EXEC;
#endif
#ifdef __ANDROID__
if (ahb_backing) {
size_t voff = virtual_offset;
size_t hoff = host_offset;
size_t remaining = length;
while (remaining > 0) {
const size_t window = hoff / ahb_window_size;
const size_t local = hoff % ahb_window_size;
const size_t chunk = (std::min)(remaining, ahb_window_size - local);
void* const ret =
mmap(virtual_base + voff, chunk, prot_flags, MAP_SHARED | MAP_FIXED,
ahb_fds[window], static_cast<off_t>(local));
ASSERT_MSG(ret != MAP_FAILED, "mmap: {}", strerror(errno));
voff += chunk;
hoff += chunk;
remaining -= chunk;
}
return;
}
#endif
int flags = (fd >= 0 ? MAP_SHARED : MAP_PRIVATE) | MAP_FIXED;
void* ret = mmap(virtual_base + virtual_offset, length, prot_flags, flags, fd, host_offset);
@@ -910,18 +656,6 @@ private:
int ret = close(fd);
ASSERT_MSG(ret == 0, "close failed: {}", strerror(errno));
}
#ifdef __ANDROID__
for (AHardwareBuffer* buffer : ahb_windows) {
AHardwareBuffer_release(buffer);
}
ahb_windows.clear();
ahb_fds.clear();
if (ahb_backing) {
committed_backing_size.store(0, std::memory_order_relaxed);
ahb_backing = false;
}
#endif
}
void AdjustMap(size_t* virtual_offset, size_t* length) {
@@ -947,13 +681,6 @@ private:
int fd{-1}; // memfd file descriptor, -1 is the error value of memfd_create
FreeRegionManager free_manager{};
#ifdef __ANDROID__
bool ahb_backing{};
std::vector<AHardwareBuffer*> ahb_windows;
std::vector<int> ahb_fds;
size_t ahb_window_size{};
#endif
};
#endif // ^^^ POSIX ^^^
@@ -1040,22 +767,6 @@ void HostMemory::ClearBackingRegion(size_t physical_offset, size_t length, u32 f
std::memset(backing_base + physical_offset, fill_value, length);
}
std::span<AHardwareBuffer* const> HostMemory::BackingHardwareBuffers() const noexcept {
#ifdef __ANDROID__
return impl ? impl->AhbWindows() : std::span<AHardwareBuffer* const>{};
#else
return {};
#endif
}
size_t HostMemory::BackingHardwareBufferWindowSize() const noexcept {
#ifdef __ANDROID__
return impl ? impl->AhbWindowSize() : 0;
#else
return 0;
#endif
}
void HostMemory::EnableDirectMappedAddress() {
#if !(defined(__OPENORBIS__) || defined(__managarm__))
if (impl) {
-13
View File
@@ -8,17 +8,12 @@
#include <memory>
#include <optional>
#include <span>
#include "common/common_funcs.h"
#include "common/common_types.h"
#include "common/virtual_buffer.h"
struct AHardwareBuffer;
namespace Common {
[[nodiscard]] u64 GetCommittedBackingSize() noexcept;
enum class MemoryPermission : u32 {
Read = 1 << 0,
Write = 1 << 1,
@@ -67,14 +62,6 @@ public:
return backing_base;
}
[[nodiscard]] size_t BackingSize() const noexcept {
return backing_size;
}
[[nodiscard]] std::span<AHardwareBuffer* const> BackingHardwareBuffers() const noexcept;
[[nodiscard]] size_t BackingHardwareBufferWindowSize() const noexcept;
[[nodiscard]] u8* VirtualBasePointer() noexcept {
return virtual_base;
}
-13
View File
@@ -587,9 +587,6 @@ struct Values {
SwitchableSetting<bool> use_asynchronous_shaders{linkage, false, "use_asynchronous_shaders",
Category::RendererHacks};
SwitchableSetting<bool> use_unified_memory{linkage, false, "use_unified_memory",
Category::RendererHacks};
SwitchableSetting<GpuUnswizzleSize> gpu_unswizzle_texture_size{linkage,
GpuUnswizzleSize::Large,
"gpu_unswizzle_texture_size",
@@ -638,16 +635,6 @@ struct Values {
#endif
"vertex_input_dynamic_state", Category::RendererExtensions};
SwitchableSetting<bool> dynamic_rendering{linkage, true, "dynamic_rendering",
Category::RendererExtensions};
SwitchableSetting<bool> workgroup_memory_explicit_layout{
linkage, false, "workgroup_memory_explicit_layout", Category::RendererExtensions};
SwitchableSetting<s32, true> pipeline_worker_count{
linkage, 2, 2, 8, "pipeline_worker_count", Category::RendererAdvanced,
Specialization::Scalar};
Setting<bool> renderer_debug{linkage, false, "debug", Category::RendererDebug};
Setting<bool> renderer_shader_feedback{linkage, false, "shader_feedback",
Category::RendererDebug};
+20 -139
View File
@@ -1,6 +1,5 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: 2013 Dolphin Emulator Project
// SPDX-FileCopyrightText: 2014 Citra Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
@@ -40,110 +39,6 @@
#include <unistd.h>
#endif
#ifdef __ANDROID__
#include <sys/resource.h>
#include <algorithm>
#include <fstream>
#include <utility>
#include <vector>
namespace {
[[maybe_unused]] constexpr int ANDROID_THREAD_PRIORITY_URGENT_AUDIO = -19;
[[maybe_unused]] constexpr int ANDROID_THREAD_PRIORITY_AUDIO = -16;
[[maybe_unused]] constexpr int ANDROID_THREAD_PRIORITY_URGENT_DISPLAY = -8;
[[maybe_unused]] constexpr int ANDROID_THREAD_PRIORITY_DISPLAY = -4;
[[maybe_unused]] constexpr int ANDROID_THREAD_PRIORITY_FOREGROUND = -2;
[[maybe_unused]] constexpr int ANDROID_THREAD_PRIORITY_MORE_FAVORABLE = -1;
[[maybe_unused]] constexpr int ANDROID_THREAD_PRIORITY_DEFAULT = 0;
[[maybe_unused]] constexpr int ANDROID_THREAD_PRIORITY_LESS_FAVORABLE = 1;
[[maybe_unused]] constexpr int ANDROID_THREAD_PRIORITY_BACKGROUND = 10;
[[maybe_unused]] constexpr int ANDROID_THREAD_PRIORITY_LOWEST = 19;
constexpr size_t ANDROID_MINIMUM_PERFORMANCE_CORES = 4;
cpu_set_t ComputePerformanceCoreMask() {
cpu_set_t mask;
CPU_ZERO(&mask);
cpu_set_t allowed;
CPU_ZERO(&allowed);
if (sched_getaffinity(gettid(), sizeof(allowed), &allowed) != 0) {
return mask;
}
std::vector<std::pair<long, int>> cores;
const int total = static_cast<int>(std::thread::hardware_concurrency());
for (int cpu = 0; cpu < total; ++cpu) {
if (!CPU_ISSET(cpu, &allowed)) {
continue;
}
long max_frequency = 0;
std::ifstream file("/sys/devices/system/cpu/cpu" + std::to_string(cpu) +
"/cpufreq/cpuinfo_max_freq");
if (!file || !(file >> max_frequency) || max_frequency <= 0) {
CPU_ZERO(&mask);
return mask;
}
cores.emplace_back(max_frequency, cpu);
}
if (cores.empty()) {
return mask;
}
std::sort(cores.begin(), cores.end(),
[](const auto& lhs, const auto& rhs) { return lhs.first > rhs.first; });
size_t taken = 0;
long cluster_frequency = cores.front().first;
for (const auto& [frequency, cpu] : cores) {
if (frequency != cluster_frequency) {
if (taken >= ANDROID_MINIMUM_PERFORMANCE_CORES) {
break;
}
cluster_frequency = frequency;
}
CPU_SET(cpu, &mask);
++taken;
}
return mask;
}
const cpu_set_t& PerformanceCoreMask() {
static const cpu_set_t mask = ComputePerformanceCoreMask();
return mask;
}
cpu_set_t ComputeEfficiencyCoreMask() {
cpu_set_t mask;
CPU_ZERO(&mask);
const cpu_set_t& performance = PerformanceCoreMask();
if (CPU_COUNT(&performance) == 0) {
return mask;
}
cpu_set_t allowed;
CPU_ZERO(&allowed);
if (sched_getaffinity(gettid(), sizeof(allowed), &allowed) != 0) {
return mask;
}
const int total = static_cast<int>(std::thread::hardware_concurrency());
for (int cpu = 0; cpu < total; ++cpu) {
if (CPU_ISSET(cpu, &allowed) && !CPU_ISSET(cpu, &performance)) {
CPU_SET(cpu, &mask);
}
}
return mask;
}
const cpu_set_t& EfficiencyCoreMask() {
static const cpu_set_t mask = ComputeEfficiencyCoreMask();
return mask;
}
} // Anonymous namespace
#endif
#include "common/cpu_features.h"
#ifdef ARCHITECTURE_x86_64
#ifdef _MSC_VER
@@ -183,21 +78,6 @@ void SetCurrentThreadPriority(ThreadPriority new_priority) {
}
}();
set_thread_priority(find_thread(NULL), priority);
#elif defined(__ANDROID__)
const int nice_value = [&]() {
switch (new_priority) {
case ThreadPriority::Low: return ANDROID_THREAD_PRIORITY_BACKGROUND;
case ThreadPriority::Normal: return ANDROID_THREAD_PRIORITY_DEFAULT;
case ThreadPriority::High: return ANDROID_THREAD_PRIORITY_DISPLAY;
case ThreadPriority::VeryHigh: return ANDROID_THREAD_PRIORITY_URGENT_DISPLAY;
case ThreadPriority::Critical: return ANDROID_THREAD_PRIORITY_AUDIO;
default: return ANDROID_THREAD_PRIORITY_DEFAULT;
}
}();
if (setpriority(PRIO_PROCESS, static_cast<id_t>(gettid()), nice_value) != 0) {
LOG_DEBUG(Common, "Could not set thread nice value to {}: {}", nice_value,
GetLastErrorMsg());
}
#else
pthread_t this_thread = pthread_self();
const auto scheduling_type = SCHED_OTHER;
@@ -252,28 +132,29 @@ void SetCurrentThreadName(const char* name) {
#endif
}
void SetCurrentThreadToPerformanceCores() {
void PinCurrentThreadToPerformanceCore(size_t core_id) {
ASSERT(core_id < 4);
// If we set a flag for a CPU that doesn't exist, the thread may not be allowed to
// run in ANY processor!
auto const total_cores = std::thread::hardware_concurrency();
if (core_id < total_cores) {
#if defined(__ANDROID__)
const cpu_set_t& mask = PerformanceCoreMask();
if (CPU_COUNT(&mask) == 0) {
return;
}
if (sched_setaffinity(gettid(), sizeof(mask), &mask) != 0) {
LOG_DEBUG(Common, "Could not restrict thread to performance cores: {}", GetLastErrorMsg());
}
cpu_set_t set;
CPU_ZERO(&set);
CPU_SET(core_id, &set);
sched_setaffinity(pthread_self(), sizeof(set), &set);
#elif defined(__linux__) || defined(__FreeBSD__)
cpu_set_t set;
CPU_ZERO(&set);
CPU_SET(core_id, &set);
pthread_setaffinity_np(pthread_self(), sizeof(set), &set);
#elif defined(_WIN32)
DWORD set = 1UL << core_id;
SetThreadAffinityMask(GetCurrentThread(), set);
#else
// No pin functionality implemented
#endif
}
void SetCurrentThreadToEfficiencyCores() {
#if defined(__ANDROID__)
const cpu_set_t& mask = EfficiencyCoreMask();
if (CPU_COUNT(&mask) == 0) {
return;
}
if (sched_setaffinity(gettid(), sizeof(mask), &mask) != 0) {
LOG_DEBUG(Common, "Could not restrict thread to efficiency cores: {}", GetLastErrorMsg());
}
#endif
}
#ifdef ARCHITECTURE_x86_64
+1 -7
View File
@@ -99,14 +99,8 @@ enum class ThreadPriority : u32 {
Critical = 4,
};
enum class ThreadPlacement : u32 {
Default = 0,
Background = 1,
};
void SetCurrentThreadPriority(ThreadPriority new_priority);
void SetCurrentThreadName(const char* name);
void SetCurrentThreadToPerformanceCores();
void SetCurrentThreadToEfficiencyCores();
void PinCurrentThreadToPerformanceCore(size_t core_id);
} // namespace Common
+3 -8
View File
@@ -1,4 +1,4 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
@@ -37,15 +37,10 @@ class StatefulThreadWorker {
using StateMaker = std::conditional_t<with_state, std::function<StateType()>, DummyCallable>;
public:
explicit StatefulThreadWorker(size_t num_workers, std::string name, StateMaker func = {},
ThreadPlacement placement = ThreadPlacement::Default)
explicit StatefulThreadWorker(size_t num_workers, std::string name, StateMaker func = {})
: workers_queued{num_workers}, thread_name{std::move(name)} {
const auto lambda = [this, func, placement](std::stop_token stop_token) {
const auto lambda = [this, func](std::stop_token stop_token) {
Common::SetCurrentThreadName(thread_name.c_str());
if (placement == ThreadPlacement::Background) {
Common::SetCurrentThreadPriority(ThreadPriority::Low);
Common::SetCurrentThreadToEfficiencyCores();
}
{
[[maybe_unused]] std::conditional_t<with_state, StateType, int> state{func()};
while (!stop_token.stop_requested()) {
+1 -3
View File
@@ -157,8 +157,6 @@ bool ArmNce::HandleGuestAlignmentFault(GuestContext* guest_ctx, void* raw_info,
return HandleFailedGuestFault(guest_ctx, raw_info, raw_context);
}
constexpr size_t NCE_WRITE_FAULT_CLUSTER_PAGES = 4;
bool ArmNce::HandleGuestAccessFault(GuestContext* guest_ctx, void* raw_info, void* raw_context) {
auto* info = static_cast<siginfo_t*>(raw_info);
@@ -167,7 +165,7 @@ bool ArmNce::HandleGuestAccessFault(GuestContext* guest_ctx, void* raw_info, voi
const Common::ProcessAddress addr =
(reinterpret_cast<u64>(info->si_addr) & ~Memory::YUZU_PAGEMASK);
auto& memory = guest_ctx->parent->m_running_thread->GetOwnerProcess()->GetMemory();
if (memory.InvalidateNCE(addr, Memory::YUZU_PAGESIZE * NCE_WRITE_FAULT_CLUSTER_PAGES)) {
if (memory.InvalidateNCE(addr, Memory::YUZU_PAGESIZE)) {
// We handled the access successfully and are returning to guest code.
return true;
}
+1 -5
View File
@@ -118,7 +118,6 @@ struct System::Impl {
is_multicore = Settings::values.use_multi_core.GetValue();
extended_memory_layout = Settings::values.memory_layout_mode.GetValue() != Settings::MemoryLayout::Memory_4Gb;
unified_memory = Settings::values.use_unified_memory.GetValue();
core_timing.SetMulticore(is_multicore);
core_timing.Initialize([&system]() { system.RegisterHostThread(); });
@@ -146,8 +145,7 @@ struct System::Impl {
!device_memory.has_value() ||
is_multicore != Settings::values.use_multi_core.GetValue() ||
extended_memory_layout != (Settings::values.memory_layout_mode.GetValue() !=
Settings::MemoryLayout::Memory_4Gb) ||
unified_memory != Settings::values.use_unified_memory.GetValue();
Settings::MemoryLayout::Memory_4Gb);
if (!must_reinitialize) {
return;
@@ -158,7 +156,6 @@ struct System::Impl {
is_multicore = Settings::values.use_multi_core.GetValue();
extended_memory_layout =
Settings::values.memory_layout_mode.GetValue() != Settings::MemoryLayout::Memory_4Gb;
unified_memory = Settings::values.use_unified_memory.GetValue();
Initialize(system);
}
@@ -506,7 +503,6 @@ struct System::Impl {
std::atomic_bool is_powered_on{};
bool is_multicore : 1 = false;
bool extended_memory_layout : 1 = false;
bool unified_memory : 1 = false;
bool exit_locked : 1 = false;
bool exit_requested : 1 = false;
bool nvdec_active : 1 = false;
+6 -1
View File
@@ -174,7 +174,12 @@ void CpuManager::RunThread(std::stop_token token, std::size_t core) {
std::string name = is_multicore ? ("CPUCore_" + std::to_string(core)) : std::string{"CPUThread"};
Common::SetCurrentThreadName(name.c_str());
Common::SetCurrentThreadPriority(Common::ThreadPriority::Critical);
Common::SetCurrentThreadToPerformanceCores();
#ifdef __ANDROID__
// Aimed specifically for Snapdragon 8 Elite devices
// This kills performance on desktop, but boosts perf for UMA devices
// like the S8E. Mediatek and Mali likely won't suffer.
Common::PinCurrentThreadToPerformanceCore(core);
#endif
auto& data = core_data[core];
data.host_context = Common::Fiber::ThreadToFiber();
-30
View File
@@ -20,8 +20,6 @@
#include "common/scratch_buffer.h"
#include "common/virtual_buffer.h"
struct AHardwareBuffer;
namespace Core {
constexpr size_t DEVICE_PAGEBITS = 12ULL;
@@ -97,26 +95,6 @@ public:
ApplyOpOnPAddr(address, buffer, operation);
}
u8* GetPhysicalBase() noexcept {
return reinterpret_cast<u8*>(physical_base);
}
const u8* GetPhysicalBase() const noexcept {
return reinterpret_cast<const u8*>(physical_base);
}
size_t GetPhysicalSize() const noexcept {
return physical_size;
}
std::span<AHardwareBuffer* const> GetBackingHardwareBuffers() const noexcept {
return ahb_windows;
}
size_t GetBackingHardwareBufferWindowSize() const noexcept {
return ahb_window_size;
}
PAddr GetPhysicalRawAddressFromDAddr(DAddr address) const {
PAddr subbits = PAddr(address & page_mask);
auto paddr = tracked_entries[(address >> page_bits)].compressed_physical_ptr;
@@ -148,10 +126,6 @@ public:
// New batch API to update multiple ranges with a single lock acquisition.
void UpdatePagesCachedBatch(std::span<const std::pair<DAddr, size_t>> ranges, s32 delta);
void UpdateTexturePagesCount(DAddr addr, size_t size, s32 delta);
[[nodiscard]] bool IsRegionTextureCached(DAddr addr, size_t size) const noexcept;
private:
struct TranslationEntry {
DAddr guest_page{};
@@ -197,9 +171,6 @@ private:
std::unique_ptr<DeviceMemoryManagerAllocator<Traits>> impl;
const uintptr_t physical_base;
const size_t physical_size;
const std::span<AHardwareBuffer* const> ahb_windows;
const size_t ahb_window_size;
DeviceInterface* device_inter;
struct TrackedEntry {
@@ -263,7 +234,6 @@ private:
(1ULL << (device_virtual_bits - page_bits)) / subentries;
using CachedPages = std::array<CounterEntry, num_counter_entries>;
std::unique_ptr<CachedPages> cached_pages;
std::unique_ptr<CachedPages> texture_cached_pages;
Common::RangeMutex counter_guard;
std::mutex mapping_guard;
-26
View File
@@ -171,16 +171,12 @@ struct DeviceMemoryManagerAllocator {
template <typename Traits>
DeviceMemoryManager<Traits>::DeviceMemoryManager(const DeviceMemory& device_memory_)
: physical_base{uintptr_t(device_memory_.buffer.BackingBasePointer())}
, physical_size{device_memory_.buffer.BackingSize()}
, ahb_windows{device_memory_.buffer.BackingHardwareBuffers()}
, ahb_window_size{device_memory_.buffer.BackingHardwareBufferWindowSize()}
, device_inter{nullptr}
, compressed_device_addr(1ULL << ((Settings::values.memory_layout_mode.GetValue() == Settings::MemoryLayout::Memory_4Gb ? physical_min_bits : physical_max_bits) - Memory::YUZU_PAGEBITS))
, tracked_entries(device_as_size >> Memory::YUZU_PAGEBITS)
{
impl = std::make_unique<DeviceMemoryManagerAllocator<Traits>>();
cached_pages = std::make_unique<CachedPages>();
texture_cached_pages = std::make_unique<CachedPages>();
const size_t total_virtual = device_as_size >> Memory::YUZU_PAGEBITS;
for (size_t i = 0; i < total_virtual; i++) {
@@ -629,28 +625,6 @@ void DeviceMemoryManager<Traits>::UpdatePagesCachedCount(DAddr addr, size_t size
UpdatePagesCachedCountNoLock(addr, size, delta);
}
template <typename Traits>
void DeviceMemoryManager<Traits>::UpdateTexturePagesCount(DAddr addr, size_t size, s32 delta) {
Common::ScopedRangeLock lk(counter_guard, addr, size);
const size_t page_end = Common::DivCeil(addr + size, Memory::YUZU_PAGESIZE);
for (size_t page = addr >> Memory::YUZU_PAGEBITS; page != page_end; ++page) {
CounterAtomicType& count = texture_cached_pages->at(page >> subentries_shift).Count(page);
count.fetch_add(static_cast<CounterType>(delta), std::memory_order_release);
}
}
template <typename Traits>
bool DeviceMemoryManager<Traits>::IsRegionTextureCached(DAddr addr, size_t size) const noexcept {
const size_t page_end = Common::DivCeil(addr + size, Memory::YUZU_PAGESIZE);
for (size_t page = addr >> Memory::YUZU_PAGEBITS; page != page_end; ++page) {
if (texture_cached_pages->at(page >> subentries_shift).Count(page).load(
std::memory_order_acquire) != 0) {
return true;
}
}
return false;
}
template <typename Traits>
void DeviceMemoryManager<Traits>::UpdatePagesCachedBatch(std::span<const std::pair<DAddr, size_t>> ranges, s32 delta) {
if (ranges.empty()) {
+2 -2
View File
@@ -12,6 +12,7 @@
#include <memory>
#include <mutex>
#include <optional>
#include <unordered_map>
#include <ankerl/unordered_dense.h>
#include <assert.h>
@@ -161,8 +162,7 @@ private:
std::list<std::shared_ptr<Handle>> unmap_queue{};
std::mutex unmap_queue_lock{}; //!< Protects access to `unmap_queue`
ankerl::unordered_dense::map<Handle::Id, std::shared_ptr<Handle>>
handles{}; //!< Main owning map of handles
std::unordered_map<Handle::Id, std::shared_ptr<Handle>> handles{}; //!< Main owning map of handles
std::mutex handles_lock; //!< Protects access to `handles`
static constexpr u32 HandleIdIncrement{
@@ -230,11 +230,6 @@ std::unique_ptr<TranslationMap> InitializeTranslations(QObject* parent) {
tr("Preserves GPU-modified data by reading it back before uploading.\nSome games require this to render certain effects properly."));
INSERT(Settings, use_asynchronous_shaders, tr("Enable asynchronous shader compilation"),
tr("May reduce shader stutter."));
INSERT(Settings, use_unified_memory, tr("Enable unified memory access (UMA)"),
tr("Lets the GPU write buffer readbacks directly into guest memory."));
INSERT(Settings, pipeline_worker_count, tr("Pipeline Worker Threads"),
tr("Number of threads used to build Vulkan pipelines.\n"
"Higher values speed up compilation at the cost of heat and power."));
INSERT(Settings, fast_gpu_time, tr("Fast GPU Time"),
tr("Overclocks the emulated GPU to increase dynamic resolution and render "
"distance.\nUse 256 for maximal performance and 512 for maximal graphics fidelity."));
@@ -292,12 +287,6 @@ std::unique_ptr<TranslationMap> InitializeTranslations(QObject* parent) {
INSERT(Settings, vertex_input_dynamic_state, tr("Vertex Input Dynamic State"),
tr("Enables vertex input dynamic state feature for better quality and performance."));
INSERT(Settings, dynamic_rendering, tr("Dynamic Rendering"),
tr("Renders without render pass and framebuffer objects.\n"
"Results vary by driver: some gain performance, others lose it."));
INSERT(Settings, workgroup_memory_explicit_layout, QString(), QString());
INSERT(
Settings, sample_shading, tr("Sample Shading"),
tr("Allows the fragment shader to execute per sample in a multi-sampled fragment "
@@ -1,4 +1,4 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
@@ -665,8 +665,6 @@ void EmitShuffleDown(EmitContext& ctx, IR::Inst& inst, ScalarU32 value, ScalarU3
const IR::Value& clamp, const IR::Value& segmentation_mask);
void EmitShuffleButterfly(EmitContext& ctx, IR::Inst& inst, ScalarU32 value, ScalarU32 index,
const IR::Value& clamp, const IR::Value& segmentation_mask);
void EmitQuadBroadcast(EmitContext& ctx, IR::Inst& inst, ScalarU32 value, ScalarU32 lane);
void EmitQuadSwap(EmitContext& ctx, IR::Inst& inst, ScalarU32 value, ScalarU32 direction);
void EmitFSwizzleAdd(EmitContext& ctx, IR::Inst& inst, ScalarF32 op_a, ScalarF32 op_b,
ScalarU32 swizzle);
void EmitDPdxFine(EmitContext& ctx, IR::Inst& inst, ScalarF32 op_a);
@@ -1,6 +1,3 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
@@ -100,24 +97,6 @@ void EmitShuffleButterfly(EmitContext& ctx, IR::Inst& inst, ScalarU32 value, Sca
Shuffle(ctx, inst, value, index, clamp, segmentation_mask, "XOR");
}
void EmitQuadBroadcast(EmitContext& ctx, IR::Inst& inst, ScalarU32 value, ScalarU32 lane) {
const Register ret{ctx.reg_alloc.Define(inst)};
ctx.Add("AND.U RC.x,{}.threadid,~3;"
"AND.U RC.y,{},3;"
"OR.U RC.x,RC.x,RC.y;"
"SHFIDX.U {},{},RC.x,0x1C03;"
"MOV.U {}.x,{}.y;",
ctx.stage_name, lane, ret, value, ret, ret);
}
void EmitQuadSwap(EmitContext& ctx, IR::Inst& inst, ScalarU32 value, ScalarU32 direction) {
const Register ret{ctx.reg_alloc.Define(inst)};
ctx.Add("ADD.U RC.x,{},1;"
"SHFXOR.U {},{},RC.x,0x1C03;"
"MOV.U {}.x,{}.y;",
direction, ret, value, ret, ret);
}
void EmitFSwizzleAdd(EmitContext& ctx, IR::Inst& inst, ScalarF32 op_a, ScalarF32 op_b,
ScalarU32 swizzle) {
const auto ret{ctx.reg_alloc.Define(inst)};
@@ -1,4 +1,4 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
@@ -743,10 +743,6 @@ void EmitShuffleDown(EmitContext& ctx, IR::Inst& inst, std::string_view value,
void EmitShuffleButterfly(EmitContext& ctx, IR::Inst& inst, std::string_view value,
std::string_view index, std::string_view clamp,
std::string_view segmentation_mask);
void EmitQuadBroadcast(EmitContext& ctx, IR::Inst& inst, std::string_view value,
std::string_view lane);
void EmitQuadSwap(EmitContext& ctx, IR::Inst& inst, std::string_view value,
std::string_view direction);
void EmitFSwizzleAdd(EmitContext& ctx, IR::Inst& inst, std::string_view op_a, std::string_view op_b,
std::string_view swizzle);
void EmitDPdxFine(EmitContext& ctx, IR::Inst& inst, std::string_view op_a);
@@ -1,6 +1,3 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
@@ -203,18 +200,6 @@ void EmitShuffleButterfly(EmitContext& ctx, IR::Inst& inst, std::string_view val
ctx.AddU32("{}=shfl_in_bounds?shfl_result:{};", inst, value);
}
void EmitQuadBroadcast(EmitContext& ctx, IR::Inst& inst, std::string_view value,
std::string_view lane) {
const auto src_thread_id{fmt::format("(({}&~3)|({}& 3))", THREAD_ID, lane)};
ctx.AddU32("{}=readInvocationARB({},{});", inst, value, src_thread_id);
}
void EmitQuadSwap(EmitContext& ctx, IR::Inst& inst, std::string_view value,
std::string_view direction) {
const auto src_thread_id{fmt::format("({}^({}+1))", THREAD_ID, direction)};
ctx.AddU32("{}=readInvocationARB({},{});", inst, value, src_thread_id);
}
void EmitFSwizzleAdd(EmitContext& ctx, IR::Inst& inst, std::string_view op_a, std::string_view op_b,
std::string_view swizzle) {
const auto mask{fmt::format("({}>>((gl_SubGroupInvocationARB&3)<<1))&3", swizzle)};
@@ -322,11 +322,6 @@ void DefineEntryPoint(const IR::Program& program, EmitContext& ctx, Id main) {
if (ctx.runtime_info.force_early_z) {
ctx.AddExecutionMode(main, spv::ExecutionMode::EarlyFragmentTests);
}
if (ctx.profile.support_shader_quad_control && program.info.uses_quad_shuffles) {
ctx.AddExtension("SPV_KHR_quad_control");
ctx.AddCapability(spv::Capability::QuadControlKHR);
ctx.AddExecutionMode(main, spv::ExecutionMode::RequireFullQuadsKHR);
}
break;
default:
throw NotImplementedException("Stage {}", program.stage);
@@ -448,12 +443,6 @@ void SetupCapabilities(const Profile& profile, const Info& info, EmitContext& ct
ctx.AddCapability(spv::Capability::GroupNonUniformVote);
}
}
if (info.uses_quad_shuffles) {
if (profile.support_quad_shuffles) {
ctx.AddCapability(spv::Capability::GroupNonUniformQuad);
}
ctx.AddCapability(spv::Capability::GroupNonUniformShuffle);
}
if (info.uses_int64_bit_atomics && profile.support_int64_atomics) {
ctx.AddCapability(spv::Capability::Int64Atomics);
}
@@ -1,4 +1,4 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
@@ -622,8 +622,6 @@ Id EmitShuffleDown(EmitContext& ctx, IR::Inst* inst, Id value, Id index, Id clam
Id segmentation_mask);
Id EmitShuffleButterfly(EmitContext& ctx, IR::Inst* inst, Id value, Id index, Id clamp,
Id segmentation_mask);
Id EmitQuadBroadcast(EmitContext& ctx, Id value, Id lane);
Id EmitQuadSwap(EmitContext& ctx, Id value, Id direction);
Id EmitFSwizzleAdd(EmitContext& ctx, Id op_a, Id op_b, Id swizzle);
Id EmitDPdxFine(EmitContext& ctx, Id op_a);
Id EmitDPdyFine(EmitContext& ctx, Id op_a);
@@ -260,21 +260,6 @@ Id EmitShuffleButterfly(EmitContext& ctx, IR::Inst* inst, Id value, Id index, Id
return SelectValue(ctx, in_range, value, src_thread_id);
}
Id EmitQuadBroadcast(EmitContext& ctx, Id value, Id lane) {
if (ctx.profile.support_quad_shuffles) {
return ctx.OpGroupNonUniformQuadBroadcast(ctx.U32[1], SubgroupScope(ctx), value, lane);
}
const Id base{ctx.OpBitwiseAnd(ctx.U32[1], GetThreadId(ctx), ctx.Const(~3u))};
const Id local_lane{ctx.OpBitwiseAnd(ctx.U32[1], lane, ctx.Const(3u))};
const Id src_thread_id{ctx.OpBitwiseOr(ctx.U32[1], base, local_lane)};
return ctx.OpGroupNonUniformShuffle(ctx.U32[1], SubgroupScope(ctx), value, src_thread_id);
}
Id EmitQuadSwap(EmitContext& ctx, Id value, Id direction) {
const Id xor_mask{ctx.OpIAdd(ctx.U32[1], direction, ctx.Const(1u))};
return ctx.OpGroupNonUniformShuffleXor(ctx.U32[1], SubgroupScope(ctx), value, xor_mask);
}
Id EmitFSwizzleAdd(EmitContext& ctx, Id op_a, Id op_b, Id swizzle) {
const Id three{ctx.Const(3U)};
Id mask{GetThreadId(ctx)};
@@ -1,4 +1,4 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
@@ -2100,14 +2100,6 @@ U32 IREmitter::ShuffleButterfly(const IR::U32& value, const IR::U32& index, cons
return Inst<U32>(Opcode::ShuffleButterfly, value, index, clamp, seg_mask);
}
U32 IREmitter::QuadBroadcast(const IR::U32& value, const IR::U32& lane) {
return Inst<U32>(Opcode::QuadBroadcast, value, lane);
}
U32 IREmitter::QuadSwap(const IR::U32& value, const IR::U32& direction) {
return Inst<U32>(Opcode::QuadSwap, value, direction);
}
F32 IREmitter::FSwizzleAdd(const F32& a, const F32& b, const U32& swizzle, FpControl control) {
return Inst<F32>(Opcode::FSwizzleAdd, Flags{control}, a, b, swizzle);
}
@@ -1,4 +1,4 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
@@ -394,8 +394,6 @@ public:
const IR::U32& seg_mask);
[[nodiscard]] U32 ShuffleButterfly(const IR::U32& value, const IR::U32& index,
const IR::U32& clamp, const IR::U32& seg_mask);
[[nodiscard]] U32 QuadBroadcast(const IR::U32& value, const IR::U32& lane);
[[nodiscard]] U32 QuadSwap(const IR::U32& value, const IR::U32& direction);
[[nodiscard]] F32 FSwizzleAdd(const F32& a, const F32& b, const U32& swizzle,
FpControl control = {});
@@ -10,7 +10,7 @@ namespace Shader::IR {
namespace Detail {
OpcodeMeta META_TABLE[] = {
OpcodeMeta META_TABLE[532] = {
#define OPCODE(name_token, type_token, ...) \
{ \
.name{#name_token}, \
@@ -21,7 +21,7 @@ OpcodeMeta META_TABLE[] = {
#undef OPCODE
};
u8 NUM_ARGS[] = {
u8 NUM_ARGS[532] = {
#define OPCODE(name_token, type_token, ...) u8(CalculateNumArgsOf(Opcode::name_token)),
#include "opcodes.inc"
#undef OPCODE
+2 -2
View File
@@ -57,12 +57,12 @@ static constexpr Type F64x2{Type::F64x2};
static constexpr Type F64x3{Type::F64x3};
static constexpr Type F64x4{Type::F64x4};
extern OpcodeMeta META_TABLE[];
extern OpcodeMeta META_TABLE[532];
constexpr size_t CalculateNumArgsOf(Opcode op) noexcept {
const auto& arg_types = META_TABLE[size_t(op)].arg_types;
return size_t(std::distance(arg_types.begin(), std::ranges::find(arg_types, Type::Void)));
}
extern u8 NUM_ARGS[];
extern u8 NUM_ARGS[532];
} // namespace Detail
/// Get return type of an opcode
@@ -1,6 +1,3 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
@@ -582,8 +579,6 @@ OPCODE(ShuffleIndex, U32, U32,
OPCODE(ShuffleUp, U32, U32, U32, U32, U32, )
OPCODE(ShuffleDown, U32, U32, U32, U32, U32, )
OPCODE(ShuffleButterfly, U32, U32, U32, U32, U32, )
OPCODE(QuadBroadcast, U32, U32, U32, )
OPCODE(QuadSwap, U32, U32, U32, )
OPCODE(FSwizzleAdd, F32, F32, F32, U32, )
OPCODE(DPdxFine, F32, F32, )
OPCODE(DPdyFine, F32, F32, )
@@ -1,4 +1,4 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
@@ -39,6 +39,17 @@ IR::U32 scaleIndex(IR::IREmitter& ir, IR::U32 index, Shift shift) {
}
}
IR::U32 skewBytes(IR::IREmitter& ir, SZ sizeRead) {
const IR::U32 lane = ir.LaneId();
switch (sizeRead) {
case SZ::U8: return lane;
case SZ::U16: return ir.ShiftLeftLogical(lane, ir.Imm32(1));
case SZ::U32:
case SZ::F32: return ir.ShiftLeftLogical(lane, ir.Imm32(2));
default: UNREACHABLE();
}
}
} // Anonymous namespace
void TranslatorVisitor::ISBERD(u64 insn) {
@@ -57,41 +68,61 @@ void TranslatorVisitor::ISBERD(u64 insn) {
BitField<47, 2, Shift> shift;
} const isberd{insn};
if (isberd.skew != 0) {
throw NotImplementedException("ISBERD SKEW");
}
if (isberd.o != 0) {
throw NotImplementedException("ISBERD O");
}
if (isberd.sz.Value() > SZ::F32) {
throw NotImplementedException("ISBERD SZ {}",
static_cast<u64>(isberd.sz.Value()));
}
if (isberd.shift.Value() > Shift::B32) {
throw NotImplementedException("ISBERD Shift {}",
static_cast<u64>(isberd.shift.Value()));
IR::U32 index{};
if (isberd.src_reg_num.Value() == 0xFF) {
index = ir.Imm32(isberd.imm.Value());
} else {
const IR::U32 scaledIndex = scaleIndex(ir, X(isberd.src_reg.Value()), isberd.shift.Value());
index = ir.IAdd(scaledIndex, ir.Imm32(isberd.imm.Value()));
}
switch (isberd.mode.Value()) {
case Mode::Default:
X(isberd.dest_reg.Value(), X(isberd.src_reg.Value()));
return;
case Mode::Attr: {
IR::U32 offset{};
if (isberd.src_reg_num.Value() == 0xFF) {
offset = ir.Imm32(isberd.imm.Value());
} else {
const IR::U32 index{
scaleIndex(ir, X(isberd.src_reg.Value()), isberd.shift.Value())};
offset = ir.IAdd(index, ir.Imm32(isberd.imm.Value()));
if (isberd.o.Value()) {
if (isberd.skew.Value()) {
index = ir.IAdd(index, skewBytes(ir, isberd.sz.Value()));
}
X(isberd.dest_reg.Value(), ir.BitCast<IR::U32>(ir.GetAttributeIndexed(offset)));
const IR::U64 index64 = ir.UConvert(64, index);
IR::U32 globalLoaded{};
switch (isberd.sz.Value()) {
case SZ::U8: globalLoaded = ir.LoadGlobalU8 (index64); break;
case SZ::U16: globalLoaded = ir.LoadGlobalU16(index64); break;
case SZ::U32:
case SZ::F32: globalLoaded = ir.LoadGlobal32(index64); break;
default: UNREACHABLE();
}
X(isberd.dest_reg.Value(), globalLoaded);
return;
}
default:
throw NotImplementedException("ISBERD Mode {}",
static_cast<u64>(isberd.mode.Value()));
if (isberd.mode.Value() != Mode::Default) {
if (isberd.skew.Value()) {
index = ir.IAdd(index, skewBytes(ir, SZ::U32));
}
IR::F32 float_index{};
switch (isberd.mode.Value()) {
case Mode::Patch: float_index = ir.GetPatch(index.Patch());
break;
case Mode::Prim: float_index = ir.GetAttribute(index.Attribute());
break;
case Mode::Attr: float_index = ir.GetAttributeIndexed(index);
break;
default: UNREACHABLE();
}
X(isberd.dest_reg.Value(), ir.BitCast<IR::U32>(float_index));
return;
}
if (isberd.skew.Value()) {
X(isberd.dest_reg.Value(), ir.IAdd(X(isberd.src_reg.Value()), ir.LaneId()));
return;
}
// Fallback copy
X(isberd.dest_reg.Value(), X(isberd.src_reg.Value()));
}
} // namespace Shader::Maxwell
@@ -1,4 +1,4 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
@@ -36,10 +36,7 @@ enum class ShuffleMode : u64 {
}
}
constexpr u32 QUAD_MASK = (28u << 8) | 3u;
void Shuffle(TranslatorVisitor& v, u64 insn, const IR::U32& index, const IR::U32& mask,
bool index_is_imm, u32 index_imm, bool mask_is_imm, u32 mask_imm) {
void Shuffle(TranslatorVisitor& v, u64 insn, const IR::U32& index, const IR::U32& mask) {
union {
u64 insn;
BitField<0, 8, IR::Reg> dest_reg;
@@ -48,21 +45,6 @@ void Shuffle(TranslatorVisitor& v, u64 insn, const IR::U32& index, const IR::U32
BitField<48, 3, IR::Pred> pred;
} const shfl{insn};
const bool is_quad_candidate{mask_is_imm && mask_imm == QUAD_MASK && index_is_imm &&
v.env.ShaderStage() == Stage::Fragment};
if (is_quad_candidate) {
if (shfl.mode == ShuffleMode::IDX && index_imm <= 3) {
v.X(shfl.dest_reg, v.ir.QuadBroadcast(v.X(shfl.src_reg), v.ir.Imm32(index_imm)));
v.ir.SetPred(shfl.pred, v.ir.Imm1(true));
return;
}
if (shfl.mode == ShuffleMode::BFLY && index_imm >= 1 && index_imm <= 3) {
v.X(shfl.dest_reg, v.ir.QuadSwap(v.X(shfl.src_reg), v.ir.Imm32(index_imm - 1)));
v.ir.SetPred(shfl.pred, v.ir.Imm1(true));
return;
}
}
const IR::U32 result{ShuffleOperation(v.ir, v.X(shfl.src_reg), index, mask, shfl.mode)};
v.ir.SetPred(shfl.pred, v.ir.GetInBoundsFromOp(result));
v.X(shfl.dest_reg, result);
@@ -77,14 +59,11 @@ void TranslatorVisitor::SHFL(u64 insn) {
BitField<29, 1, u64> src_b_flag;
BitField<34, 13, u64> src_b_imm;
} const flags{insn};
const bool index_is_imm{flags.src_a_flag != 0};
const bool mask_is_imm{flags.src_b_flag != 0};
const IR::U32 src_a{index_is_imm ? ir.Imm32(static_cast<u32>(flags.src_a_imm))
: GetReg20(insn)};
const IR::U32 src_b{mask_is_imm ? ir.Imm32(static_cast<u32>(flags.src_b_imm))
: GetReg39(insn)};
Shuffle(*this, insn, src_a, src_b, index_is_imm, static_cast<u32>(flags.src_a_imm),
mask_is_imm, static_cast<u32>(flags.src_b_imm));
const IR::U32 src_a{flags.src_a_flag != 0 ? ir.Imm32(static_cast<u32>(flags.src_a_imm))
: GetReg20(insn)};
const IR::U32 src_b{flags.src_b_flag != 0 ? ir.Imm32(static_cast<u32>(flags.src_b_imm))
: GetReg39(insn)};
Shuffle(*this, insn, src_a, src_b);
}
} // namespace Shader::Maxwell
@@ -498,10 +498,6 @@ void VisitUsages(Info& info, IR::Inst& inst) {
case IR::Opcode::ShuffleButterfly:
info.uses_subgroup_shuffles = true;
break;
case IR::Opcode::QuadBroadcast:
case IR::Opcode::QuadSwap:
info.uses_quad_shuffles = true;
break;
case IR::Opcode::GetCbufU8:
case IR::Opcode::GetCbufS8:
case IR::Opcode::GetCbufU16:
-2
View File
@@ -37,8 +37,6 @@ struct Profile {
bool support_explicit_workgroup_layout{};
bool support_workgroup_layout_8bit_access{};
bool support_workgroup_layout_16bit_access{};
bool support_shader_quad_control{};
bool support_quad_shuffles{};
bool support_vote{};
u32 supported_subgroup_stages{0x7F};
bool support_viewport_index_layer_non_geometry{};
-1
View File
@@ -252,7 +252,6 @@ struct Info {
bool uses_is_helper_invocation{};
bool uses_subgroup_invocation_id{};
bool uses_subgroup_shuffles{};
bool uses_quad_shuffles{};
std::array<bool, 30> uses_patches{};
std::array<Interpolation, 32> interpolation{};
+1 -3
View File
@@ -33,7 +33,7 @@ add_library(video_core STATIC
control/channel_state_cache.h
control/scheduler.cpp
control/scheduler.h
deferred_destruction_queue.h
delayed_destruction_ring.h
dirty_flags.cpp
dirty_flags.h
dma_pusher.cpp
@@ -158,8 +158,6 @@ add_library(video_core STATIC
renderer_vulkan/vk_compute_pass.h
renderer_vulkan/vk_compute_pipeline.cpp
renderer_vulkan/vk_compute_pipeline.h
renderer_vulkan/vk_descriptor_buffer.cpp
renderer_vulkan/vk_descriptor_buffer.h
renderer_vulkan/vk_descriptor_pool.cpp
renderer_vulkan/vk_descriptor_pool.h
renderer_vulkan/vk_fence_manager.cpp
+88 -283
View File
@@ -7,7 +7,6 @@
#pragma once
#include <algorithm>
#include <bit>
#include <memory>
#include <numeric>
@@ -32,84 +31,44 @@ BufferCache<P>::BufferCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, R
immediately_free = (Settings::values.vram_usage_mode.GetValue() == Settings::VramUsageMode::Aggressive);
#endif
if (!runtime.CanReportMemoryUsage()) {
memory_budget = FALLBACK_MEMORY_BUDGET;
minimum_memory = DEFAULT_EXPECTED_MEMORY;
critical_memory = DEFAULT_CRITICAL_MEMORY;
return;
}
memory_budget = runtime.GetDeviceLocalMemory();
const s64 device_local_memory = static_cast<s64>(runtime.GetDeviceLocalMemory());
const s64 min_spacing_expected = device_local_memory - 1_GiB;
const s64 min_spacing_critical = device_local_memory - 512_MiB;
const s64 mem_threshold = (std::min)(device_local_memory, TARGET_THRESHOLD);
const s64 min_vacancy_expected = (6 * mem_threshold) / 10;
const s64 min_vacancy_critical = (2 * mem_threshold) / 10;
minimum_memory = static_cast<u64>(
(std::max)((std::min)(device_local_memory - min_vacancy_expected, min_spacing_expected),
DEFAULT_EXPECTED_MEMORY));
critical_memory = static_cast<u64>(
(std::max)((std::min)(device_local_memory - min_vacancy_critical, min_spacing_critical),
DEFAULT_CRITICAL_MEMORY));
}
template <class P>
BufferCache<P>::~BufferCache() = default;
template <class P>
u64 BufferCache<P>::DeviceUsage(bool force_refresh) {
if (!runtime.CanReportMemoryUsage()) {
return total_used_memory;
}
if (force_refresh || usage_refresh_countdown == 0) {
cached_device_usage = runtime.GetDeviceAllocationUsage();
usage_refresh_countdown = USAGE_REFRESH_INTERVAL;
} else {
--usage_refresh_countdown;
}
return cached_device_usage;
}
template <class P>
u64 BufferCache<P>::ReclaimMemory(u64 target_bytes, bool allow_download) {
if (target_bytes == 0 || in_reclaim) {
return 0;
}
in_reclaim = true;
u64 freed = 0;
const auto clean_up = [&](BufferId buffer_id) {
if (freed >= target_bytes) {
void BufferCache<P>::RunGarbageCollector() {
const bool aggressive_gc = total_used_memory >= critical_memory;
const u64 ticks_to_destroy = aggressive_gc ? 60 : 120;
int num_iterations = aggressive_gc ? 64 : 32;
const auto clean_up = [this, &num_iterations](BufferId buffer_id) {
if (num_iterations == 0) {
return true;
}
--num_iterations;
auto& buffer = slot_buffers[buffer_id];
if (!allow_download && IsRegionGpuModified(buffer.CpuAddr(), buffer.SizeBytes())) {
return false;
}
const u64 buffer_bytes = Common::AlignUp(buffer.SizeBytes(), 1024);
DownloadBufferMemory(buffer);
DeleteBuffer(buffer_id);
freed += buffer_bytes;
return false;
};
const u64 cold_tick =
frame_tick > RECLAIM_GUARD_FRAMES ? frame_tick - RECLAIM_GUARD_FRAMES : 0;
lru_cache.ForEachItemBelow(cold_tick, clean_up);
if (freed == 0) {
lru_cache.ForEachItemBelow(frame_tick > 0 ? frame_tick - 1 : 0, clean_up);
}
in_reclaim = false;
usage_refresh_countdown = 0;
reclaim_stalled = freed == 0;
if (freed > 0) {
reclaim_wait_sync_point = runtime.CurrentSyncPoint();
}
return freed;
}
template <class P>
void BufferCache<P>::EnsureHeadroom(bool allow_download) {
if (reclaim_stalled) {
return;
}
if (runtime.CompletedSyncPoint() < reclaim_wait_sync_point) {
return;
}
const u64 limit = memory_budget > RECLAIM_HEADROOM ? memory_budget - RECLAIM_HEADROOM : 0;
const u64 usage = DeviceUsage(false);
if (usage <= limit) {
return;
}
const u64 target = (limit / 100) * RECLAIM_TARGET_PERCENT;
const u64 excess = usage - target;
const u64 usage_mib = (std::max)(usage >> 20, u64{1});
const u64 share = (((excess >> 20) * (total_used_memory >> 20)) / usage_mib) << 20;
ReclaimMemory((std::min)(share, total_used_memory), allow_download);
lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, clean_up);
}
template <class P>
@@ -137,11 +96,15 @@ void BufferCache<P>::TickFrame() {
const bool skip_preferred = hits * 256 < shots * 251;
channel_state->uniform_buffer_skip_cache_size = skip_preferred ? DEFAULT_SKIP_CACHE_SIZE : 0;
usage_refresh_countdown = 0;
reclaim_stalled = false;
EnsureHeadroom(true);
// If we can obtain the memory info, use it instead of the estimate.
if (runtime.CanReportMemoryUsage()) {
total_used_memory = runtime.GetDeviceMemoryUsage();
}
if (total_used_memory >= minimum_memory) {
RunGarbageCollector();
}
++frame_tick;
sentenced_buffers.Reclaim(runtime.CompletedSyncPoint());
delayed_destruction_ring.Tick();
for (auto& buffer : async_buffers_death_ring) {
runtime.FreeDeferredStagingBuffer(buffer);
@@ -212,71 +175,9 @@ std::optional<VideoCore::RasterizerDownloadArea> BufferCache<P>::GetFlushArea(DA
template <class P>
void BufferCache<P>::DownloadMemory(DAddr device_addr, u64 size) {
if constexpr (!USE_MEMORY_MAPS) {
std::scoped_lock lock{mutex};
ForEachBufferInRange(device_addr, size, [&](BufferId, Buffer& buffer) {
DownloadBufferMemory(buffer, device_addr, size);
});
return;
}
boost::container::small_vector<std::pair<BufferCopy, BufferId>, 8> downloads;
u64 total_size_bytes = 0;
u64 largest_copy = 0;
std::unique_lock lock{mutex};
ForEachBufferInRange(device_addr, size, [&](BufferId buffer_id, Buffer& buffer) {
memory_tracker.ForEachDownloadRangeAndClear(
device_addr, size, [&](u64 device_addr_out, u64 range_size) {
const DAddr buffer_addr = buffer.CpuAddr();
const auto add_download = [&](DAddr start, DAddr end) {
const u64 new_offset = start - buffer_addr;
const u64 new_size = end - start;
downloads.push_back({
BufferCopy{
.src_offset = new_offset,
.dst_offset = total_size_bytes,
.size = new_size,
},
buffer_id,
});
constexpr u64 align = 64ULL;
constexpr u64 mask = ~(align - 1ULL);
total_size_bytes += (new_size + align - 1) & mask;
largest_copy = (std::max)(largest_copy, new_size);
};
gpu_modified_ranges.ForEachInRange(device_addr_out, range_size, add_download);
ClearDownload(device_addr_out, range_size);
gpu_modified_ranges.Subtract(device_addr_out, range_size);
});
ForEachBufferInRange(device_addr, size, [&](BufferId, Buffer& buffer) {
DownloadBufferMemory(buffer, device_addr, size);
});
if (total_size_bytes == 0) {
return;
}
auto download_staging = runtime.DownloadStagingBuffer(total_size_bytes);
boost::container::small_vector<BufferCopy, 8> writebacks;
runtime.PreCopyBarrier();
for (auto& [copy, buffer_id] : downloads) {
copy.dst_offset += download_staging.offset;
Buffer& buffer = slot_buffers[buffer_id];
buffer.MarkUsage(copy.src_offset, copy.size);
const std::array copies{copy};
runtime.CopyBuffer(download_staging.buffer, buffer, copies, false);
BufferCopy writeback{copy};
writeback.src_offset = static_cast<u64>(buffer.CpuAddr()) + copy.src_offset;
writebacks.push_back(writeback);
}
runtime.PostCopyBarrier();
lock.unlock();
runtime.Finish();
const u8* const base = download_staging.mapped_span.data();
for (const BufferCopy& writeback : writebacks) {
const u64 staging_offset = writeback.dst_offset - download_staging.offset;
device_memory.WriteBlockUnsafe(static_cast<DAddr>(writeback.src_offset),
base + staging_offset, writeback.size);
}
}
template <class P>
@@ -313,7 +214,7 @@ bool BufferCache<P>::DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 am
auto& src_buffer = slot_buffers[buffer_a];
auto& dest_buffer = slot_buffers[buffer_b];
SynchronizeBuffer(src_buffer, *cpu_src_address, static_cast<u32>(amount));
memory_tracker.UnmarkRegionAsCpuModified(*cpu_dest_address, static_cast<u32>(amount));
SynchronizeBuffer(dest_buffer, *cpu_dest_address, static_cast<u32>(amount));
std::array copies{BufferCopy{
.src_offset = src_buffer.Offset(*cpu_src_address),
.dst_offset = dest_buffer.Offset(*cpu_dest_address),
@@ -772,44 +673,32 @@ void BufferCache<P>::PopAsyncFlushes() {
template <class P>
void BufferCache<P>::PopAsyncBuffers() {
struct Writeback {
DAddr addr;
const u8* src;
u64 size;
};
boost::container::small_vector<Writeback, 8> writebacks;
{
std::scoped_lock lock{mutex};
if (async_buffers.empty()) {
return;
}
if (!async_buffers.front().has_value()) {
async_buffers.pop_front();
return;
}
auto& downloads = pending_downloads.front();
auto& async_buffer = async_buffers.front();
const u8* base = async_buffer->mapped_span.data();
const size_t base_offset = async_buffer->offset;
for (const auto& copy : downloads) {
const DAddr device_addr = static_cast<DAddr>(copy.src_offset);
const u64 dst_offset = copy.dst_offset - base_offset;
const u8* read_mapped_memory = base + dst_offset;
async_downloads.ForEachInRange(device_addr, copy.size, [&](DAddr start, DAddr end, s32) {
writebacks.push_back(
{start, &read_mapped_memory[start - device_addr], end - start});
});
async_downloads.Subtract(device_addr, copy.size, [&](DAddr start, DAddr end) {
gpu_modified_ranges.Subtract(start, end - start);
});
}
async_buffers_death_ring.emplace_back(*async_buffer);
if (async_buffers.empty()) {
return;
}
if (!async_buffers.front().has_value()) {
async_buffers.pop_front();
pending_downloads.pop_front();
return;
}
for (const auto& wb : writebacks) {
device_memory.WriteBlockUnsafe(wb.addr, wb.src, wb.size);
auto& downloads = pending_downloads.front();
auto& async_buffer = async_buffers.front();
u8* base = async_buffer->mapped_span.data();
const size_t base_offset = async_buffer->offset;
for (const auto& copy : downloads) {
const DAddr device_addr = static_cast<DAddr>(copy.src_offset);
const u64 dst_offset = copy.dst_offset - base_offset;
const u8* read_mapped_memory = base + dst_offset;
async_downloads.ForEachInRange(device_addr, copy.size, [&](DAddr start, DAddr end, s32) {
device_memory.WriteBlockUnsafe(start, &read_mapped_memory[start - device_addr],
end - start);
});
async_downloads.Subtract(device_addr, copy.size, [&](DAddr start, DAddr end) {
gpu_modified_ranges.Subtract(start, end - start);
});
}
async_buffers_death_ring.emplace_back(*async_buffer);
async_buffers.pop_front();
pending_downloads.pop_front();
}
template <class P>
@@ -920,46 +809,46 @@ void BufferCache<P>::BindHostVertexBuffers() {
if (use_optimized_vertex_buffers) {
auto& flags = maxwell3d->dirty.flags;
const u32 enabled_mask = enabled_vertex_buffers_mask;
bool any_dirty = false;
u32 pending_mask = enabled_mask;
while (pending_mask != 0) {
const u32 index = std::countr_zero(pending_mask);
pending_mask &= (pending_mask - 1);
u32 enabled_mask = enabled_vertex_buffers_mask;
HostBindings<Buffer> bindings{};
u32 last_index = (std::numeric_limits<u32>::max)();
const auto flush_bindings = [&]() {
if (bindings.buffers.empty()) {
return;
}
bindings.max_index = bindings.min_index + static_cast<u32>(bindings.buffers.size());
runtime.BindVertexBuffers(bindings);
bindings = HostBindings<Buffer>{};
last_index = (std::numeric_limits<u32>::max)();
};
while (enabled_mask != 0) {
const u32 index = std::countr_zero(enabled_mask);
enabled_mask &= (enabled_mask - 1);
const Binding& binding = VertexBufferSlot(index);
Buffer& buffer = slot_buffers[binding.buffer_id];
TouchBuffer(buffer, binding.buffer_id);
SynchronizeBuffer(buffer, binding.device_addr, binding.size);
any_dirty |= flags[Dirty::VertexBuffer0 + index];
}
if (enabled_mask == 0 || !any_dirty) {
return;
}
const u32 min_index = static_cast<u32>(std::countr_zero(enabled_mask));
const u32 max_index = 32u - static_cast<u32>(std::countl_zero(enabled_mask));
HostBindings<Buffer> bindings{};
bindings.min_index = min_index;
bindings.max_index = max_index;
for (u32 index = min_index; index < max_index; ++index) {
flags[Dirty::VertexBuffer0 + index] = false;
const u32 stride = maxwell3d->regs.vertex_streams[index].stride;
if ((enabled_mask & (1u << index)) == 0) {
bindings.buffers.push_back(&slot_buffers[NULL_BUFFER_ID]);
bindings.offsets.push_back(0);
bindings.sizes.push_back(0);
bindings.strides.push_back(stride);
if (!flags[Dirty::VertexBuffer0 + index]) {
flush_bindings();
continue;
}
const Binding& binding = VertexBufferSlot(index);
Buffer& buffer = slot_buffers[binding.buffer_id];
flags[Dirty::VertexBuffer0 + index] = false;
const u32 stride = maxwell3d->regs.vertex_streams[index].stride;
const u32 offset = buffer.Offset(binding.device_addr);
buffer.MarkUsage(offset, binding.size);
if (!bindings.buffers.empty() && index != last_index + 1) {
flush_bindings();
}
if (bindings.buffers.empty()) {
bindings.min_index = index;
}
bindings.buffers.push_back(&buffer);
bindings.offsets.push_back(offset);
bindings.sizes.push_back(binding.size);
bindings.strides.push_back(stride);
last_index = index;
}
runtime.BindVertexBuffers(bindings);
flush_bindings();
} else {
HostBindings<typename P::Buffer> host_bindings;
bool any_valid{false};
@@ -1032,6 +921,7 @@ void BufferCache<P>::BindHostGraphicsUniformBuffers(size_t stage) {
template <class P>
void BufferCache<P>::BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32 binding_index, bool needs_bind) {
++channel_state->uniform_cache_shots[0];
const Binding& binding = channel_state->uniform_buffers[stage][index];
const DAddr device_addr = binding.device_addr;
const u32 size = (std::min)(binding.size, (*channel_state->uniform_buffer_sizes)[stage][index]);
@@ -1050,12 +940,8 @@ void BufferCache<P>::BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32
return alignment > 1 && (offset % alignment) != 0;
}
}();
const bool cached_buffer_is_current =
has_host_buffer && !memory_tracker.IsRegionCpuModified(device_addr, size);
const bool use_fast_buffer = needs_alignment_stream
|| (has_host_buffer && !cached_buffer_is_current
&& size <= channel_state->uniform_buffer_skip_cache_size
|| (has_host_buffer && size <= channel_state->uniform_buffer_skip_cache_size
&& !memory_tracker.IsRegionGpuModified(device_addr, size));
if (use_fast_buffer) {
if constexpr (IS_OPENGL) {
@@ -1082,7 +968,7 @@ void BufferCache<P>::BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32
device_memory.ReadBlockUnsafe(device_addr, span.data(), size);
return;
}
++channel_state->uniform_cache_shots[0];
// Classic cached path
if (SynchronizeBuffer(buffer, device_addr, size)) {
++channel_state->uniform_cache_hits[0];
}
@@ -1690,7 +1576,6 @@ void BufferCache<P>::JoinOverlap(BufferId new_buffer_id, BufferId overlap_id,
template <class P>
BufferId BufferCache<P>::CreateBuffer(DAddr device_addr, u32 wanted_size) {
EnsureHeadroom(false);
DAddr device_addr_end = Common::AlignUp(device_addr + wanted_size, CACHING_PAGESIZE);
device_addr = Common::AlignDown(device_addr, CACHING_PAGESIZE);
wanted_size = static_cast<u32>(device_addr_end - device_addr);
@@ -1728,7 +1613,7 @@ void BufferCache<P>::ChangeRegister(BufferId buffer_id) {
total_used_memory += Common::AlignUp(size, 1024);
buffer.setLRUID(lru_cache.Insert(buffer_id, frame_tick));
} else {
total_used_memory -= std::min<u64>(total_used_memory, Common::AlignUp(size, 1024));
total_used_memory -= Common::AlignUp(size, 1024);
lru_cache.Free(buffer.getLRUID());
}
const DAddr device_addr_begin = buffer.CpuAddr();
@@ -1814,80 +1699,6 @@ void BufferCache<P>::ImmediateUploadMemory([[maybe_unused]] Buffer& buffer,
}
}
template <class P>
bool BufferCache<P>::TryUnifiedDownloadMemory([[maybe_unused]] Buffer& buffer,
[[maybe_unused]] std::span<BufferCopy> copies) {
if constexpr (USE_UNIFIED_MEMORY) {
const u8* const physical_base = device_memory.GetPhysicalBase();
const u64 unified_size = runtime.UnifiedMemorySize();
const u64 window_size = runtime.UnifiedMemoryWindowSize();
if (window_size == 0) {
return false;
}
boost::container::small_vector<u64, 4> window_ids;
boost::container::small_vector<boost::container::small_vector<BufferCopy, 16>, 4> groups;
const auto group_for = [&](u64 window) -> boost::container::small_vector<BufferCopy, 16>& {
for (size_t i = 0; i < window_ids.size(); ++i) {
if (window_ids[i] == window) {
return groups[i];
}
}
window_ids.push_back(window);
groups.emplace_back();
return groups.back();
};
for (const BufferCopy& copy : copies) {
const DAddr device_addr = buffer.CpuAddr() + copy.src_offset;
u64 downloaded = 0;
while (downloaded < copy.size) {
const DAddr page_addr = device_addr + downloaded;
const u8* const ptr = device_memory.GetPointer<u8>(page_addr);
if (ptr == nullptr) {
return false;
}
const u64 page_offset = page_addr & Core::DEVICE_PAGEMASK;
u64 chunk = (std::min)(copy.size - downloaded,
static_cast<u64>(Core::DEVICE_PAGESIZE) - page_offset);
const u64 phys_offset = static_cast<u64>(ptr - physical_base);
if (phys_offset + chunk > unified_size) {
return false;
}
const u64 window = phys_offset / window_size;
const u64 local_offset = phys_offset % window_size;
chunk = (std::min)(chunk, window_size - local_offset);
auto& group = group_for(window);
if (!group.empty()) {
BufferCopy& last = group.back();
if (last.src_offset + last.size == copy.src_offset + downloaded &&
last.dst_offset + last.size == local_offset) {
last.size += chunk;
downloaded += chunk;
continue;
}
}
group.push_back(BufferCopy{
.src_offset = copy.src_offset + downloaded,
.dst_offset = local_offset,
.size = chunk,
});
downloaded += chunk;
}
}
for (const BufferCopy& copy : copies) {
buffer.MarkUsage(copy.src_offset, copy.size);
}
for (size_t i = 0; i < window_ids.size(); ++i) {
const std::span<BufferCopy> group_span(groups[i].data(), groups[i].size());
runtime.CopyBuffer(runtime.UnifiedMemoryWindowBuffer(window_ids[i]), buffer,
group_span, true);
}
runtime.Finish();
return true;
} else {
return false;
}
}
template <class P>
void BufferCache<P>::MappedUploadMemory([[maybe_unused]] Buffer& buffer,
[[maybe_unused]] u64 total_size_bytes,
@@ -1991,12 +1802,6 @@ void BufferCache<P>::DownloadBufferMemory(Buffer& buffer, DAddr device_addr, u64
}
if constexpr (USE_MEMORY_MAPS) {
if constexpr (USE_UNIFIED_MEMORY) {
if (runtime.HasUnifiedMemory() &&
TryUnifiedDownloadMemory(buffer, std::span(copies.data(), copies.size()))) {
return;
}
}
auto download_staging = runtime.DownloadStagingBuffer(total_size_bytes);
const u8* const mapped_memory = download_staging.mapped_span.data();
const std::span<BufferCopy> copies_span(copies.data(), copies.data() + copies.size());
@@ -2067,7 +1872,7 @@ void BufferCache<P>::DeleteBuffer(BufferId buffer_id, bool do_not_mark) {
#ifdef YUZU_LEGACY
if (!do_not_mark || !immediately_free)
#endif
sentenced_buffers.Push(std::move(slot_buffers[buffer_id]), runtime.CurrentSyncPoint());
delayed_destruction_ring.Push(std::move(slot_buffers[buffer_id]));
slot_buffers.erase(buffer_id);
+14 -25
View File
@@ -9,7 +9,6 @@
#include <algorithm>
#include <array>
#include <bit>
#include <deque>
#include <functional>
#include <memory>
#include <mutex>
@@ -31,7 +30,7 @@
#include "common/slot_vector.h"
#include "video_core/buffer_cache/buffer_base.h"
#include "video_core/control/channel_state_cache.h"
#include "video_core/deferred_destruction_queue.h"
#include "video_core/delayed_destruction_ring.h"
#include "video_core/dirty_flags.h"
#include "video_core/engines/maxwell_3d.h"
#include "video_core/engines/kepler_compute.h"
@@ -181,18 +180,15 @@ class BufferCache : public VideoCommon::ChannelSetupCaches<BufferCacheChannelInf
static constexpr bool USE_MEMORY_MAPS = P::USE_MEMORY_MAPS;
static constexpr bool SEPARATE_IMAGE_BUFFERS_BINDINGS = P::SEPARATE_IMAGE_BUFFER_BINDINGS;
static constexpr bool USE_MEMORY_MAPS_FOR_UPLOADS = P::USE_MEMORY_MAPS_FOR_UPLOADS;
static constexpr bool USE_UNIFIED_MEMORY = P::USE_UNIFIED_MEMORY;
#ifdef YUZU_LEGACY
static constexpr u64 RECLAIM_HEADROOM = 384_MiB;
static constexpr s64 TARGET_THRESHOLD = 3_GiB;
#else
static constexpr u64 RECLAIM_HEADROOM = 512_MiB;
static constexpr s64 TARGET_THRESHOLD = 4_GiB;
#endif
static constexpr u64 FALLBACK_MEMORY_BUDGET = 2_GiB;
static constexpr u32 USAGE_REFRESH_INTERVAL = 16;
static constexpr u64 RECLAIM_GUARD_FRAMES = 8;
static constexpr u64 RECLAIM_TARGET_PERCENT = 95;
static constexpr s64 DEFAULT_EXPECTED_MEMORY = 512_MiB;
static constexpr s64 DEFAULT_CRITICAL_MEMORY = 1_GiB;
// Debug Flags.
@@ -219,8 +215,6 @@ public:
void TickFrame();
u64 ReclaimMemory(u64 target_bytes, bool allow_download);
void WriteMemory(DAddr device_addr, u64 size);
void CachedWriteMemory(DAddr device_addr, u64 size);
@@ -364,9 +358,7 @@ private:
((device_addr + size) & ~Core::DEVICE_PAGEMASK);
}
u64 DeviceUsage(bool force_refresh);
void EnsureHeadroom(bool allow_download);
void RunGarbageCollector();
void BindHostIndexBuffer();
@@ -451,8 +443,6 @@ private:
void MappedUploadMemory(Buffer& buffer, u64 total_size_bytes, std::span<BufferCopy> copies);
bool TryUnifiedDownloadMemory(Buffer& buffer, std::span<BufferCopy> copies);
void DownloadBufferMemory(Buffer& buffer_id);
void DownloadBufferMemory(Buffer& buffer_id, DAddr device_addr, u64 size);
@@ -485,7 +475,12 @@ private:
Tegra::MaxwellDeviceMemoryManager& device_memory;
Common::SlotVector<Buffer> slot_buffers;
DeferredDestructionQueue<Buffer> sentenced_buffers;
#ifdef YUZU_LEGACY
static constexpr size_t TICKS_TO_DESTROY = 6;
#else
static constexpr size_t TICKS_TO_DESTROY = 8;
#endif
DelayedDestructionRing<Buffer, TICKS_TO_DESTROY> delayed_destruction_ring;
const Tegra::Engines::Maxwell3D::DrawManager::IndirectParams* current_draw_indirect{};
@@ -520,14 +515,8 @@ private:
Common::LeastRecentlyUsedCache<LRUItemParams> lru_cache;
u64 frame_tick = 0;
u64 total_used_memory = 0;
u64 memory_budget = 0;
u64 cached_device_usage = 0;
/// Sync point the last reclaim's evictions were queued at. Their memory is not back with the
/// device until this completes, so reclaiming again before then measures stale usage.
u64 reclaim_wait_sync_point = 0;
u32 usage_refresh_countdown = 0;
bool in_reclaim = false;
bool reclaim_stalled = false;
u64 minimum_memory = 0;
u64 critical_memory = 0;
BufferId inline_buffer_id;
#ifdef YUZU_LEGACY
bool immediately_free = false;
@@ -1,56 +0,0 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
#pragma once
#include <cstddef>
#include <utility>
#include <boost/container/deque.hpp>
#include <boost/container/options.hpp>
#include "common/common_types.h"
namespace VideoCommon {
template <typename T>
class DeferredDestructionQueue {
public:
void Push(T&& object, u64 sync_point) {
entries.emplace_back(std::move(object), sync_point);
}
void Reclaim(u64 completed_sync_point) {
while (!entries.empty() && entries.front().sync_point <= completed_sync_point) {
entries.pop_front();
}
}
void Clear() {
entries.clear();
}
[[nodiscard]] size_t Size() const noexcept {
return entries.size();
}
[[nodiscard]] bool Empty() const noexcept {
return entries.empty();
}
private:
struct Entry {
Entry(T&& object_, u64 sync_point_) noexcept
: object{std::move(object_)}, sync_point{sync_point_} {}
T object;
u64 sync_point;
};
using EntryDequeOptions =
boost::container::deque_options<boost::container::block_size<8u>>::type;
boost::container::deque<Entry, void, EntryDequeOptions> entries;
};
} // namespace VideoCommon
+34
View File
@@ -0,0 +1,34 @@
// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#pragma once
#include <array>
#include <cstddef>
#include <utility>
#include <vector>
namespace VideoCommon {
/// Container to push objects to be destroyed a few ticks in the future
template <typename T, size_t TICKS_TO_DESTROY>
class DelayedDestructionRing {
public:
void Tick() {
index = (index + 1) % TICKS_TO_DESTROY;
elements[index].clear();
}
void Push(T&& object) {
elements[index].push_back(std::move(object));
}
private:
size_t index = 0;
std::array<std::vector<T>, TICKS_TO_DESTROY> elements;
};
} // namespace VideoCommon
+1 -2
View File
@@ -71,8 +71,7 @@ void Fermi2D::Blit() {
constexpr s64 null_derivative = 1ULL << 32;
Surface src = regs.src;
const auto bytes_per_pixel = BytesPerBlock(PixelFormatFromRenderTargetFormat(src.format));
const u64 src_area = static_cast<u64>(src.width) * static_cast<u64>(src.height);
const bool delegate_to_gpu = src_area > 512ULL * 512ULL && bytes_per_pixel <= 8 &&
const bool delegate_to_gpu = src.width > 512 && src.height > 512 && bytes_per_pixel <= 8 &&
src.format != regs.dst.format;
auto srcX = args.src_x0;
+10 -10
View File
@@ -18,7 +18,7 @@
#include "common/common_types.h"
#include "common/settings.h"
#include "common/thread.h"
#include "video_core/deferred_destruction_queue.h"
#include "video_core/delayed_destruction_ring.h"
#include "video_core/gpu.h"
#include "video_core/host1x/host1x.h"
#include "video_core/host1x/syncpoint_manager.h"
@@ -50,8 +50,7 @@ public:
/// Notify the fence manager about a new frame
void TickFrame() {
std::unique_lock lock(ring_guard);
++retire_tick;
sentenced_fences.Reclaim(retire_tick > RETIRE_DELAY ? retire_tick - RETIRE_DELAY : 0);
delayed_destruction_ring.Tick();
}
// Unlike other fences, this one doesn't
@@ -92,6 +91,9 @@ public:
func();
}
fences.push(std::move(new_fence));
if (should_flush) {
rasterizer.FlushCommands();
}
if constexpr (can_async_check) {
guard.unlock();
cv.notify_all();
@@ -184,7 +186,7 @@ private:
}
{
std::unique_lock lock(ring_guard);
sentenced_fences.Push(std::move(current_fence), retire_tick);
delayed_destruction_ring.Push(std::move(current_fence));
}
fences.pop();
}
@@ -217,7 +219,7 @@ private:
}
{
std::unique_lock lock(ring_guard);
sentenced_fences.Push(std::move(current_fence), retire_tick);
delayed_destruction_ring.Push(std::move(current_fence));
}
}
}
@@ -236,10 +238,10 @@ private:
void PopAsyncFlushes() {
{
std::scoped_lock lock{texture_cache.mutex};
std::scoped_lock lock{buffer_cache.mutex, texture_cache.mutex};
texture_cache.PopAsyncFlushes();
buffer_cache.PopAsyncFlushes();
}
buffer_cache.PopAsyncFlushes();
query_cache.PopAsyncFlushes();
}
@@ -262,9 +264,7 @@ private:
std::jthread fence_thread;
static constexpr u64 RETIRE_DELAY = 8;
u64 retire_tick = 1;
DeferredDestructionQueue<TFence> sentenced_fences;
DelayedDestructionRing<TFence, 8> delayed_destruction_ring;
};
} // namespace VideoCommon
-1
View File
@@ -30,7 +30,6 @@ void ThreadManager::StartThread(VideoCore::RendererBase& renderer, Core::Fronten
thread = std::jthread([&](std::stop_token stop_token) {
Common::SetCurrentThreadName("GPU");
Common::SetCurrentThreadPriority(Common::ThreadPriority::Critical);
Common::SetCurrentThreadToPerformanceCores();
system.RegisterHostThread();
auto current_context = context.Acquire();
@@ -17,13 +17,11 @@ set(SHADER_FILES
${CMAKE_CURRENT_SOURCE_DIR}/astc_decoder.comp
${CMAKE_CURRENT_SOURCE_DIR}/blit_color_float.frag
${CMAKE_CURRENT_SOURCE_DIR}/block_linear_unswizzle_2d.comp
${CMAKE_CURRENT_SOURCE_DIR}/block_linear_unswizzle_2d_buffer.comp
${CMAKE_CURRENT_SOURCE_DIR}/blit_color_msaa.frag
${CMAKE_CURRENT_SOURCE_DIR}/blit_depth_msaa.frag
${CMAKE_CURRENT_SOURCE_DIR}/blit_depth_stencil_msaa.frag
${CMAKE_CURRENT_SOURCE_DIR}/block_linear_unswizzle_3d.comp
${CMAKE_CURRENT_SOURCE_DIR}/block_linear_unswizzle_3d_bcn.comp
${CMAKE_CURRENT_SOURCE_DIR}/block_linear_unswizzle_3d_buffer.comp
${CMAKE_CURRENT_SOURCE_DIR}/convert_abgr8_to_d24s8.frag
${CMAKE_CURRENT_SOURCE_DIR}/convert_abgr8_to_d32f.frag
${CMAKE_CURRENT_SOURCE_DIR}/convert_d32f_to_abgr8.frag
@@ -34,7 +32,6 @@ set(SHADER_FILES
${CMAKE_CURRENT_SOURCE_DIR}/convert_msaa_to_non_msaa.frag
${CMAKE_CURRENT_SOURCE_DIR}/convert_non_msaa_to_msaa.comp
${CMAKE_CURRENT_SOURCE_DIR}/convert_non_msaa_to_msaa.frag
${CMAKE_CURRENT_SOURCE_DIR}/convert_non_msaa_to_msaa_depth.frag
${CMAKE_CURRENT_SOURCE_DIR}/convert_s8d24_to_abgr8.frag
${CMAKE_CURRENT_SOURCE_DIR}/full_screen_triangle.vert
${CMAKE_CURRENT_SOURCE_DIR}/fxaa.frag
+16 -10
View File
@@ -77,8 +77,14 @@ uvec4 local_buff;
uvec4 color_endpoint_data;
int color_bitsread = 0;
#define MAX_WEIGHT_VALUES 64
uint result_vector[MAX_WEIGHT_VALUES];
// Global "vector" to be pushed into when decoding
// At most will require BLOCK_WIDTH x BLOCK_HEIGHT in single plane mode
// At most will require BLOCK_WIDTH x BLOCK_HEIGHT x 2 in dual plane mode
// So the maximum would be 144 (12 x 12) elements, x 2 for two planes
#define DIVCEIL(number, divisor) (number + divisor - 1) / divisor
#define ARRAY_NUM_ELEMENTS 144
#define VECTOR_ARRAY_SIZE DIVCEIL(ARRAY_NUM_ELEMENTS * 2, 4)
uint result_vector[ARRAY_NUM_ELEMENTS * 2];
int result_index = 0;
uint result_vector_max_index;
@@ -486,7 +492,7 @@ void DecodeColorValues(uvec4 modes, uint num_partitions, uint color_data_bits, o
A = ReplicateBitTo9((bitval & 1));
switch (encoding) {
case JUST_BITS:
color_values[out_index++] = FastReplicateTo8(bitval, bitlen);
color_values[++out_index] = FastReplicateTo8(bitval, bitlen);
break;
case TRIT: {
D = QuintTritValue(val);
@@ -565,7 +571,7 @@ void DecodeColorValues(uvec4 modes, uint num_partitions, uint color_data_bits, o
uint T = (D * C) + B;
T ^= A;
T = (A & 0x80) | (T >> 2);
color_values[out_index++] = T;
color_values[++out_index] = T;
}
}
}
@@ -747,12 +753,12 @@ void ComputeEndpoints(out uvec4 ep1, out uvec4 ep2, uint color_endpoint_mode, ui
#define READ_UINT_VALUES(N) \
uvec4 V[2]; \
for (uint i = 0; i < N; i++) { \
V[i / 4][i % 4] = color_values[colvals_index++]; \
V[i / 4][i % 4] = color_values[++colvals_index]; \
}
#define READ_INT_VALUES(N) \
ivec4 V[2]; \
for (uint i = 0; i < N; i++) { \
V[i / 4][i % 4] = int(color_values[colvals_index++]); \
V[i / 4][i % 4] = int(color_values[++colvals_index]); \
}
switch (color_endpoint_mode) {
@@ -1219,10 +1225,6 @@ void DecompressBlock(ivec3 coord) {
FillError(coord);
return;
}
if (GetNumWeightValues(size_params, dual_plane) > MAX_WEIGHT_VALUES) {
FillError(coord);
return;
}
uint partition_index = 1;
uvec4 color_endpoint_mode = uvec4(0);
uint ced_pointer = 0;
@@ -1382,7 +1384,11 @@ void DecompressBlock(ivec3 coord) {
p = Cf / 65535.0f;
}
#ifdef VULKAN
imageStore(dest_image, coord + ivec3(i, j, 0), p.gbar);
#else
imageStore(dest_image, coord + ivec3(i, j, 0), clamp(p, 0.0f, 1.0f).gbar);
#endif
}
}
}
@@ -1,104 +0,0 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
#version 430
#extension GL_EXT_shader_16bit_storage : require
#extension GL_EXT_shader_8bit_storage : require
#define BINDING_INPUT_BUFFER 0
#define BINDING_OUTPUT_BUFFER 1
layout(push_constant) uniform PushConstants {
uvec3 dim;
uint bytes_per_block_log2;
uvec3 origin;
uint layer_stride;
uint block_size;
uint x_shift;
uint block_height;
uint block_height_mask;
} pc;
layout(binding = BINDING_INPUT_BUFFER, std430) buffer InputBufferU32 { uint u32data[]; };
layout(binding = BINDING_INPUT_BUFFER, std430) buffer InputBufferU64 { uvec2 u64data[]; };
layout(binding = BINDING_INPUT_BUFFER, std430) buffer InputBufferU128 { uvec4 u128data[]; };
layout(binding = BINDING_OUTPUT_BUFFER, std430) writeonly buffer OutputBuffer {
uint out_u32[];
};
layout(local_size_x = 16, local_size_y = 8, local_size_z = 1) in;
const uint GOB_SIZE_X = 64;
const uint GOB_SIZE_Y = 8;
const uint GOB_SIZE_X_SHIFT = 6;
const uint GOB_SIZE_Y_SHIFT = 3;
const uint GOB_SIZE_SHIFT = GOB_SIZE_X_SHIFT + GOB_SIZE_Y_SHIFT;
const uvec2 SWIZZLE_MASK = uvec2(GOB_SIZE_X - 1u, GOB_SIZE_Y - 1u);
uint SwizzleTable(uint pos) {
const uint t[8] = uint[](
0x12100200, 0x13110301, 0x16140604, 0x17150705,
0x1a180a08, 0x1b190b09, 0x1e1c0e0c, 0x1f1d0f0d
);
const uint i = pos >> 4;
const uint h = (t[i / 4] >> ((i % 4) * 8)) & 0xff;
return (h << 4) | (pos & 0xf);
}
uint SwizzleOffset(uvec2 pos) {
pos = pos & SWIZZLE_MASK;
return SwizzleTable(pos.y * 64u + pos.x);
}
uvec4 ReadTexel(uint offset) {
switch (pc.bytes_per_block_log2) {
case 2u:
return uvec4(u32data[offset / 4u], 0u, 0u, 0u);
case 3u:
return uvec4(u64data[offset / 8u], 0u, 0u);
case 4u:
return u128data[offset / 16u];
}
return uvec4(0u);
}
void main() {
uvec3 coord = gl_GlobalInvocationID;
if (coord.x >= pc.dim.x || coord.y >= pc.dim.y || coord.z >= pc.dim.z) {
return;
}
uvec3 pos = coord + pc.origin;
pos.x <<= pc.bytes_per_block_log2;
uint swizzle = SwizzleOffset(pos.xy);
uint block_y = pos.y >> GOB_SIZE_Y_SHIFT;
uint offset = 0u;
offset += pos.z * pc.layer_stride;
offset += (block_y >> pc.block_height) * pc.block_size;
offset += (block_y & pc.block_height_mask) << GOB_SIZE_SHIFT;
offset += (pos.x >> GOB_SIZE_X_SHIFT) << pc.x_shift;
offset += swizzle;
uvec4 texel = ReadTexel(offset);
uint words = 1u << (pc.bytes_per_block_log2 - 2u);
uint linear_index = coord.x + coord.y * pc.dim.x + coord.z * pc.dim.x * pc.dim.y;
uint out_idx = linear_index * words;
out_u32[out_idx] = texel.x;
if (words > 1u) {
out_u32[out_idx + 1u] = texel.y;
}
if (words > 2u) {
out_u32[out_idx + 2u] = texel.z;
out_u32[out_idx + 3u] = texel.w;
}
}
@@ -1,105 +0,0 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
#version 430
#define BINDING_INPUT_BUFFER 0
#define BINDING_OUTPUT_BUFFER 1
layout(push_constant) uniform PushConstants {
uvec3 dim;
uint bytes_per_block_log2;
uvec3 origin;
uint slice_size;
uint block_size;
uint x_shift;
uint block_height;
uint block_height_mask;
uint block_depth;
uint block_depth_mask;
} pc;
layout(binding = BINDING_INPUT_BUFFER, std430) buffer InputBufferU32 { uint u32data[]; };
layout(binding = BINDING_INPUT_BUFFER, std430) buffer InputBufferU64 { uvec2 u64data[]; };
layout(binding = BINDING_INPUT_BUFFER, std430) buffer InputBufferU128 { uvec4 u128data[]; };
layout(binding = BINDING_OUTPUT_BUFFER, std430) writeonly buffer OutputBuffer {
uint out_u32[];
};
layout(local_size_x = 8, local_size_y = 8, local_size_z = 4) in;
const uint GOB_SIZE_X = 64;
const uint GOB_SIZE_Y = 8;
const uint GOB_SIZE_X_SHIFT = 6;
const uint GOB_SIZE_Y_SHIFT = 3;
const uint GOB_SIZE_SHIFT = GOB_SIZE_X_SHIFT + GOB_SIZE_Y_SHIFT;
const uvec2 SWIZZLE_MASK = uvec2(GOB_SIZE_X - 1u, GOB_SIZE_Y - 1u);
uint SwizzleTable(uint pos) {
const uint t[8] = uint[](
0x12100200, 0x13110301, 0x16140604, 0x17150705,
0x1a180a08, 0x1b190b09, 0x1e1c0e0c, 0x1f1d0f0d
);
const uint i = pos >> 4;
const uint h = (t[i / 4] >> ((i % 4) * 8)) & 0xff;
return (h << 4) | (pos & 0xf);
}
uint SwizzleOffset(uvec2 pos) {
pos = pos & SWIZZLE_MASK;
return SwizzleTable(pos.y * 64u + pos.x);
}
uvec4 ReadTexel(uint offset) {
switch (pc.bytes_per_block_log2) {
case 2u:
return uvec4(u32data[offset / 4u], 0u, 0u, 0u);
case 3u:
return uvec4(u64data[offset / 8u], 0u, 0u);
case 4u:
return u128data[offset / 16u];
}
return uvec4(0u);
}
void main() {
uvec3 coord = gl_GlobalInvocationID;
if (coord.x >= pc.dim.x || coord.y >= pc.dim.y || coord.z >= pc.dim.z) {
return;
}
uvec3 pos = coord + pc.origin;
pos.x <<= pc.bytes_per_block_log2;
uint swizzle = SwizzleOffset(pos.xy);
uint block_y = pos.y >> GOB_SIZE_Y_SHIFT;
uint offset = 0u;
offset += (pos.z >> pc.block_depth) * pc.slice_size;
offset += (pos.z & pc.block_depth_mask) << (GOB_SIZE_SHIFT + pc.block_height);
offset += (block_y >> pc.block_height) * pc.block_size;
offset += (block_y & pc.block_height_mask) << GOB_SIZE_SHIFT;
offset += (pos.x >> GOB_SIZE_X_SHIFT) << pc.x_shift;
offset += swizzle;
uvec4 texel = ReadTexel(offset);
uint words = 1u << (pc.bytes_per_block_log2 - 2u);
uint linear_index = coord.x + coord.y * pc.dim.x + coord.z * pc.dim.x * pc.dim.y;
uint out_idx = linear_index * words;
out_u32[out_idx] = texel.x;
if (words > 1u) {
out_u32[out_idx + 1u] = texel.y;
}
if (words > 2u) {
out_u32[out_idx + 2u] = texel.z;
out_u32[out_idx + 3u] = texel.w;
}
}
@@ -1,19 +0,0 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
#version 450 core
layout(binding = 0) uniform sampler2D img_in;
layout(push_constant) uniform PushConstants {
ivec2 dst_offset;
ivec2 src_offset;
ivec2 scale;
};
void main() {
const ivec2 msaa_coord = ivec2(gl_FragCoord.xy) - dst_offset;
const ivec2 sample_offset = ivec2(gl_SampleID % scale.x, gl_SampleID / scale.x);
const ivec2 coord = msaa_coord * scale + sample_offset + src_offset;
gl_FragDepth = texelFetch(img_in, coord, 0).r;
}
+93 -193
View File
@@ -58,9 +58,8 @@ MemoryManager::MemoryManager(Core::System& system_, u64 address_space_bits_, GPU
MemoryManager::~MemoryManager() = default;
template <bool is_big_page>
MemoryManager::EntryType MemoryManager::GetEntry(size_t position) const {
if constexpr (is_big_page) {
MemoryManager::EntryType MemoryManager::GetEntry(size_t position, bool is_big_page) const {
if (is_big_page) {
position = position >> big_page_bits;
const u64 entry_mask = big_entries[position / 32];
const size_t sub_index = position % 32;
@@ -73,9 +72,8 @@ MemoryManager::EntryType MemoryManager::GetEntry(size_t position) const {
}
}
template <bool is_big_page>
void MemoryManager::SetEntry(size_t position, MemoryManager::EntryType entry) {
if constexpr (is_big_page) {
void MemoryManager::SetEntry(size_t position, MemoryManager::EntryType entry, bool is_big_page) {
if (is_big_page) {
position = position >> big_page_bits;
const u64 entry_mask = big_entries[position / 32];
const size_t sub_index = position % 32;
@@ -108,23 +106,21 @@ inline void MemoryManager::SetBigPageContinuous(size_t big_page_index, bool valu
(~(1ULL << sub_index) & continuous_mask) | (value ? 1ULL << sub_index : 0);
}
template <MemoryManager::EntryType entry_type>
GPUVAddr MemoryManager::PageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr dev_addr, size_t size,
PTEKind kind) {
GPUVAddr MemoryManager::PageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr dev_addr, size_t size, PTEKind kind, MemoryManager::EntryType entry_type) {
[[maybe_unused]] u64 remaining_size{size};
if constexpr (entry_type == EntryType::Mapped) {
if (entry_type == EntryType::Mapped) {
page_table.ReserveRange(gpu_addr, size);
}
for (u64 offset{}; offset < size; offset += page_size) {
const GPUVAddr current_gpu_addr = gpu_addr + offset;
[[maybe_unused]] const auto current_entry_type = GetEntry<false>(current_gpu_addr);
SetEntry<false>(current_gpu_addr, entry_type);
[[maybe_unused]] const auto current_entry_type = GetEntry(current_gpu_addr, false);
SetEntry(current_gpu_addr, entry_type, false);
if (current_entry_type != entry_type) {
rasterizer->ModifyGPUMemory(unique_identifier, current_gpu_addr, page_size);
}
if constexpr (entry_type == EntryType::Mapped) {
if (entry_type == EntryType::Mapped) {
const DAddr current_dev_addr = dev_addr + offset;
const auto index = PageEntryIndex<false>(current_gpu_addr);
const auto index = PageEntryIndex(current_gpu_addr, false);
const u32 sub_value = static_cast<u32>(current_dev_addr >> cpu_page_bits);
page_table[index] = sub_value;
}
@@ -134,20 +130,18 @@ GPUVAddr MemoryManager::PageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr de
return gpu_addr;
}
template <MemoryManager::EntryType entry_type>
GPUVAddr MemoryManager::BigPageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr dev_addr,
size_t size, PTEKind kind) {
GPUVAddr MemoryManager::BigPageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr dev_addr, size_t size, PTEKind kind, MemoryManager::EntryType entry_type) {
[[maybe_unused]] u64 remaining_size{size};
for (u64 offset{}; offset < size; offset += big_page_size) {
const GPUVAddr current_gpu_addr = gpu_addr + offset;
[[maybe_unused]] const auto current_entry_type = GetEntry<true>(current_gpu_addr);
SetEntry<true>(current_gpu_addr, entry_type);
[[maybe_unused]] const auto current_entry_type = GetEntry(current_gpu_addr, true);
SetEntry(current_gpu_addr, entry_type, true);
if (current_entry_type != entry_type) {
rasterizer->ModifyGPUMemory(unique_identifier, current_gpu_addr, big_page_size);
}
if constexpr (entry_type == EntryType::Mapped) {
if (entry_type == EntryType::Mapped) {
const DAddr current_dev_addr = dev_addr + offset;
const auto index = PageEntryIndex<true>(current_gpu_addr);
const auto index = PageEntryIndex(current_gpu_addr, true);
const u32 sub_value = static_cast<u32>(current_dev_addr >> cpu_page_bits);
big_page_table_dev[index] = sub_value;
const bool is_continuous = ([&] {
@@ -181,19 +175,16 @@ void MemoryManager::BindRasterizer(VideoCore::RasterizerInterface* rasterizer_)
rasterizer = rasterizer_;
}
GPUVAddr MemoryManager::Map(GPUVAddr gpu_addr, DAddr dev_addr, std::size_t size, PTEKind kind,
bool is_big_pages) {
if (is_big_pages) [[likely]] {
return BigPageTableOp<EntryType::Mapped>(gpu_addr, dev_addr, size, kind);
}
return PageTableOp<EntryType::Mapped>(gpu_addr, dev_addr, size, kind);
GPUVAddr MemoryManager::Map(GPUVAddr gpu_addr, DAddr dev_addr, std::size_t size, PTEKind kind, bool is_big_pages) {
if (is_big_pages)
return BigPageTableOp(gpu_addr, dev_addr, size, kind, EntryType::Mapped);
return PageTableOp(gpu_addr, dev_addr, size, kind, EntryType::Mapped);
}
GPUVAddr MemoryManager::MapSparse(GPUVAddr gpu_addr, std::size_t size, bool is_big_pages) {
if (is_big_pages) [[likely]] {
return BigPageTableOp<EntryType::Reserved>(gpu_addr, 0, size, PTEKind::INVALID);
}
return PageTableOp<EntryType::Reserved>(gpu_addr, 0, size, PTEKind::INVALID);
if (is_big_pages)
return BigPageTableOp(gpu_addr, 0, size, PTEKind::INVALID, EntryType::Reserved);
return PageTableOp(gpu_addr, 0, size, PTEKind::INVALID, EntryType::Reserved);
}
void MemoryManager::Unmap(GPUVAddr gpu_addr, std::size_t size) {
@@ -207,26 +198,21 @@ void MemoryManager::Unmap(GPUVAddr gpu_addr, std::size_t size) {
}
page_stash.clear();
BigPageTableOp<EntryType::Free>(gpu_addr, 0, size, PTEKind::INVALID);
PageTableOp<EntryType::Free>(gpu_addr, 0, size, PTEKind::INVALID);
BigPageTableOp(gpu_addr, 0, size, PTEKind::INVALID, EntryType::Free);
PageTableOp(gpu_addr, 0, size, PTEKind::INVALID, EntryType::Free);
}
std::optional<DAddr> MemoryManager::GpuToCpuAddress(GPUVAddr gpu_addr) const {
if (!IsWithinGPUAddressRange(gpu_addr)) [[unlikely]] {
return std::nullopt;
}
if (GetEntry<true>(gpu_addr) != EntryType::Mapped) [[unlikely]] {
if (GetEntry<false>(gpu_addr) != EntryType::Mapped) {
if (GetEntry(gpu_addr, true) != EntryType::Mapped) [[unlikely]] {
if (GetEntry(gpu_addr, false) != EntryType::Mapped)
return std::nullopt;
}
const DAddr dev_addr_base = static_cast<DAddr>(page_table[PageEntryIndex<false>(gpu_addr)])
<< cpu_page_bits;
const DAddr dev_addr_base = DAddr(page_table[PageEntryIndex(gpu_addr, false)]) << cpu_page_bits;
return dev_addr_base + (gpu_addr & page_mask);
}
const DAddr dev_addr_base =
static_cast<DAddr>(big_page_table_dev[PageEntryIndex<true>(gpu_addr)]) << cpu_page_bits;
const DAddr dev_addr_base = DAddr(big_page_table_dev[PageEntryIndex(gpu_addr, true)]) << cpu_page_bits;
return dev_addr_base + (gpu_addr & big_page_mask);
}
@@ -299,10 +285,8 @@ const u8* MemoryManager::GetPointer(GPUVAddr gpu_addr) const {
#pragma inline_recursion(on)
#endif
template <bool is_big_pages, typename FuncMapped, typename FuncReserved, typename FuncUnmapped>
inline void MemoryManager::MemoryOperation(GPUVAddr gpu_src_addr, std::size_t size,
FuncMapped&& func_mapped, FuncReserved&& func_reserved,
FuncUnmapped&& func_unmapped) const {
template <typename FuncMapped, typename FuncReserved, typename FuncUnmapped>
inline void MemoryManager::MemoryOperation(GPUVAddr gpu_src_addr, std::size_t size, bool is_big_page, FuncMapped&& func_mapped, FuncReserved&& func_reserved, FuncUnmapped&& func_unmapped) const {
using FuncMappedReturn =
typename std::invoke_result<FuncMapped, std::size_t, std::size_t, std::size_t>::type;
using FuncReservedReturn =
@@ -315,7 +299,7 @@ inline void MemoryManager::MemoryOperation(GPUVAddr gpu_src_addr, std::size_t si
u64 used_page_size;
u64 used_page_mask;
u64 used_page_bits;
if constexpr (is_big_pages) {
if (is_big_page) {
used_page_size = big_page_size;
used_page_mask = big_page_mask;
used_page_bits = big_page_bits;
@@ -332,7 +316,7 @@ inline void MemoryManager::MemoryOperation(GPUVAddr gpu_src_addr, std::size_t si
while (remaining_size > 0) {
const std::size_t copy_amount{
(std::min)(static_cast<std::size_t>(used_page_size) - page_offset, remaining_size)};
auto entry = GetEntry<is_big_pages>(current_address);
auto entry = GetEntry(current_address, is_big_page);
if (entry == EntryType::Mapped) [[likely]] {
if constexpr (BOOL_BREAK_MAPPED) {
if (func_mapped(page_index, page_offset, copy_amount)) {
@@ -367,164 +351,91 @@ inline void MemoryManager::MemoryOperation(GPUVAddr gpu_src_addr, std::size_t si
}
}
template <bool is_safe>
void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std::size_t size,
[[maybe_unused]] VideoCommon::CacheType which) const {
const u8* run_src{nullptr};
u8* run_dst{nullptr};
std::size_t run_size{0};
auto flush_run = [&] {
if (run_size == 0) {
return;
}
std::memcpy(run_dst, run_src, run_size);
run_src = nullptr;
run_dst = nullptr;
run_size = 0;
};
auto append_run = [&](const u8* physical, std::size_t copy_amount) {
if (physical == nullptr) [[unlikely]] {
flush_run();
std::memset(dest_buffer, 0, copy_amount);
return;
}
if (run_size != 0 && run_src + run_size == physical &&
run_dst + run_size == static_cast<u8*>(dest_buffer)) {
run_size += copy_amount;
return;
}
flush_run();
run_src = physical;
run_dst = static_cast<u8*>(dest_buffer);
run_size = copy_amount;
};
auto set_to_zero = [&]([[maybe_unused]] std::size_t page_index,
[[maybe_unused]] std::size_t offset, std::size_t copy_amount) {
flush_run();
void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std::size_t size, [[maybe_unused]] VideoCommon::CacheType which, bool unsafe) const {
auto set_to_zero = [&]([[maybe_unused]] std::size_t page_index, [[maybe_unused]] std::size_t offset, std::size_t copy_amount) {
std::memset(dest_buffer, 0, copy_amount);
dest_buffer = static_cast<u8*>(dest_buffer) + copy_amount;
};
auto mapped_normal = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
const DAddr dev_addr_base =
(static_cast<DAddr>(page_table[page_index]) << cpu_page_bits) + offset;
if constexpr (is_safe) {
const DAddr dev_addr_base = (DAddr(page_table[page_index]) << cpu_page_bits) + offset;
if (!unsafe) {
rasterizer->FlushRegion(dev_addr_base, copy_amount, which);
}
append_run(memory.GetPointer<u8>(dev_addr_base), copy_amount);
u8* physical = memory.GetPointer<u8>(dev_addr_base);
std::memcpy(dest_buffer, physical, copy_amount);
dest_buffer = static_cast<u8*>(dest_buffer) + copy_amount;
};
auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
const DAddr dev_addr_base =
(static_cast<DAddr>(big_page_table_dev[page_index]) << cpu_page_bits) + offset;
if constexpr (is_safe) {
const DAddr dev_addr_base = (DAddr(big_page_table_dev[page_index]) << cpu_page_bits) + offset;
if (!unsafe) {
rasterizer->FlushRegion(dev_addr_base, copy_amount, which);
}
if (!IsBigPageContinuous(page_index)) [[unlikely]] {
flush_run();
memory.ReadBlockUnsafe(dev_addr_base, dest_buffer, copy_amount);
} else {
append_run(memory.GetPointer<u8>(dev_addr_base), copy_amount);
u8* physical = memory.GetPointer<u8>(dev_addr_base);
std::memcpy(dest_buffer, physical, copy_amount);
}
dest_buffer = static_cast<u8*>(dest_buffer) + copy_amount;
};
auto read_short_pages = [&](std::size_t page_index, std::size_t offset,
std::size_t copy_amount) {
auto read_short_pages = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
GPUVAddr base = (page_index << big_page_bits) + offset;
MemoryOperation<false>(base, copy_amount, mapped_normal, set_to_zero, set_to_zero);
MemoryOperation(base, copy_amount, false, mapped_normal, set_to_zero, set_to_zero);
};
MemoryOperation<true>(gpu_src_addr, size, mapped_big, set_to_zero, read_short_pages);
flush_run();
MemoryOperation(gpu_src_addr, size, true, mapped_big, set_to_zero, read_short_pages);
}
void MemoryManager::ReadBlock(GPUVAddr gpu_src_addr, void* dest_buffer, std::size_t size,
VideoCommon::CacheType which) const {
ReadBlockImpl<true>(gpu_src_addr, dest_buffer, size, which);
void MemoryManager::ReadBlock(GPUVAddr gpu_src_addr, void* dest_buffer, std::size_t size, VideoCommon::CacheType which) const {
ReadBlockImpl(gpu_src_addr, dest_buffer, size, which, false);
}
void MemoryManager::ReadBlockUnsafe(GPUVAddr gpu_src_addr, void* dest_buffer,
const std::size_t size) const {
ReadBlockImpl<false>(gpu_src_addr, dest_buffer, size, VideoCommon::CacheType::None);
void MemoryManager::ReadBlockUnsafe(GPUVAddr gpu_src_addr, void* dest_buffer, const std::size_t size) const {
ReadBlockImpl(gpu_src_addr, dest_buffer, size, VideoCommon::CacheType::None, true);
}
template <bool is_safe>
void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size,
[[maybe_unused]] VideoCommon::CacheType which) {
const u8* run_src{nullptr};
u8* run_dst{nullptr};
std::size_t run_size{0};
auto flush_run = [&] {
if (run_size == 0) {
return;
}
std::memcpy(run_dst, run_src, run_size);
run_src = nullptr;
run_dst = nullptr;
run_size = 0;
};
auto append_run = [&](u8* physical, std::size_t copy_amount) {
if (physical == nullptr) [[unlikely]] {
flush_run();
return;
}
if (run_size != 0 && run_dst + run_size == physical &&
run_src + run_size == static_cast<const u8*>(src_buffer)) {
run_size += copy_amount;
return;
}
flush_run();
run_src = static_cast<const u8*>(src_buffer);
run_dst = physical;
run_size = copy_amount;
};
auto just_advance = [&]([[maybe_unused]] std::size_t page_index,
[[maybe_unused]] std::size_t offset, std::size_t copy_amount) {
flush_run();
void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size, [[maybe_unused]] VideoCommon::CacheType which, bool unsafe) {
auto just_advance = [&]([[maybe_unused]] std::size_t page_index, [[maybe_unused]] std::size_t offset, std::size_t copy_amount) {
src_buffer = static_cast<const u8*>(src_buffer) + copy_amount;
};
auto mapped_normal = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
const DAddr dev_addr_base =
(static_cast<DAddr>(page_table[page_index]) << cpu_page_bits) + offset;
if constexpr (is_safe) {
const DAddr dev_addr_base = (DAddr(page_table[page_index]) << cpu_page_bits) + offset;
if (!unsafe) {
rasterizer->InvalidateRegion(dev_addr_base, copy_amount, which);
}
append_run(memory.GetPointer<u8>(dev_addr_base), copy_amount);
u8* physical = memory.GetPointer<u8>(dev_addr_base);
std::memcpy(physical, src_buffer, copy_amount);
src_buffer = static_cast<const u8*>(src_buffer) + copy_amount;
};
auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
const DAddr dev_addr_base =
(static_cast<DAddr>(big_page_table_dev[page_index]) << cpu_page_bits) + offset;
if constexpr (is_safe) {
const DAddr dev_addr_base = (DAddr(big_page_table_dev[page_index]) << cpu_page_bits) + offset;
if (!unsafe) {
rasterizer->InvalidateRegion(dev_addr_base, copy_amount, which);
}
if (!IsBigPageContinuous(page_index)) [[unlikely]] {
flush_run();
memory.WriteBlockUnsafe(dev_addr_base, src_buffer, copy_amount);
} else {
append_run(memory.GetPointer<u8>(dev_addr_base), copy_amount);
u8* physical = memory.GetPointer<u8>(dev_addr_base);
std::memcpy(physical, src_buffer, copy_amount);
}
src_buffer = static_cast<const u8*>(src_buffer) + copy_amount;
};
auto write_short_pages = [&](std::size_t page_index, std::size_t offset,
std::size_t copy_amount) {
auto write_short_pages = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
GPUVAddr base = (page_index << big_page_bits) + offset;
MemoryOperation<false>(base, copy_amount, mapped_normal, just_advance, just_advance);
MemoryOperation(base, copy_amount, false, mapped_normal, just_advance, just_advance);
};
MemoryOperation<true>(gpu_dest_addr, size, mapped_big, just_advance, write_short_pages);
flush_run();
MemoryOperation(gpu_dest_addr, size, true, mapped_big, just_advance, write_short_pages);
}
void MemoryManager::WriteBlock(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size,
VideoCommon::CacheType which) {
WriteBlockImpl<true>(gpu_dest_addr, src_buffer, size, which);
void MemoryManager::WriteBlock(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size, VideoCommon::CacheType which) {
WriteBlockImpl(gpu_dest_addr, src_buffer, size, which, false);
}
void MemoryManager::WriteBlockUnsafe(GPUVAddr gpu_dest_addr, const void* src_buffer,
std::size_t size) {
WriteBlockImpl<false>(gpu_dest_addr, src_buffer, size, VideoCommon::CacheType::None);
void MemoryManager::WriteBlockUnsafe(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size) {
WriteBlockImpl(gpu_dest_addr, src_buffer, size, VideoCommon::CacheType::None, true);
}
void MemoryManager::WriteBlockCached(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size) {
WriteBlockImpl<false>(gpu_dest_addr, src_buffer, size, VideoCommon::CacheType::None);
WriteBlockImpl(gpu_dest_addr, src_buffer, size, VideoCommon::CacheType::None, true);
accumulator.Add(gpu_dest_addr, size);
}
@@ -535,21 +446,18 @@ void MemoryManager::FlushRegion(GPUVAddr gpu_addr, size_t size,
[[maybe_unused]] std::size_t copy_amount) {};
auto mapped_normal = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
const DAddr dev_addr_base =
(static_cast<DAddr>(page_table[page_index]) << cpu_page_bits) + offset;
const DAddr dev_addr_base = (DAddr(page_table[page_index]) << cpu_page_bits) + offset;
rasterizer->FlushRegion(dev_addr_base, copy_amount, which);
};
auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
const DAddr dev_addr_base =
(static_cast<DAddr>(big_page_table_dev[page_index]) << cpu_page_bits) + offset;
const DAddr dev_addr_base = (DAddr(big_page_table_dev[page_index]) << cpu_page_bits) + offset;
rasterizer->FlushRegion(dev_addr_base, copy_amount, which);
};
auto flush_short_pages = [&](std::size_t page_index, std::size_t offset,
std::size_t copy_amount) {
auto flush_short_pages = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
GPUVAddr base = (page_index << big_page_bits) + offset;
MemoryOperation<false>(base, copy_amount, mapped_normal, do_nothing, do_nothing);
MemoryOperation(base, copy_amount, false, mapped_normal, do_nothing, do_nothing);
};
MemoryOperation<true>(gpu_addr, size, mapped_big, do_nothing, flush_short_pages);
MemoryOperation(gpu_addr, size, true, mapped_big, do_nothing, flush_short_pages);
}
bool MemoryManager::IsMemoryDirty(GPUVAddr gpu_addr, size_t size,
@@ -574,10 +482,10 @@ bool MemoryManager::IsMemoryDirty(GPUVAddr gpu_addr, size_t size,
auto check_short_pages = [&](std::size_t page_index, std::size_t offset,
std::size_t copy_amount) {
GPUVAddr base = (page_index << big_page_bits) + offset;
MemoryOperation<false>(base, copy_amount, mapped_normal, do_nothing, do_nothing);
MemoryOperation(base, copy_amount, false, mapped_normal, do_nothing, do_nothing);
return result;
};
MemoryOperation<true>(gpu_addr, size, mapped_big, do_nothing, check_short_pages);
MemoryOperation(gpu_addr, size, true, mapped_big, do_nothing, check_short_pages);
return result;
}
@@ -614,10 +522,10 @@ size_t MemoryManager::MaxContinuousRange(GPUVAddr gpu_addr, size_t size) const {
auto check_short_pages = [&](std::size_t page_index, std::size_t offset,
std::size_t copy_amount) {
GPUVAddr base = (page_index << big_page_bits) + offset;
MemoryOperation<false>(base, copy_amount, short_check, fail, fail);
MemoryOperation(base, copy_amount, false, short_check, fail, fail);
return result;
};
MemoryOperation<true>(gpu_addr, size, big_check, fail, check_short_pages);
MemoryOperation(gpu_addr, size, true, big_check, fail, check_short_pages);
return range_so_far;
}
@@ -633,21 +541,18 @@ void MemoryManager::InvalidateRegion(GPUVAddr gpu_addr, size_t size,
[[maybe_unused]] std::size_t copy_amount) {};
auto mapped_normal = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
const DAddr dev_addr_base =
(static_cast<DAddr>(page_table[page_index]) << cpu_page_bits) + offset;
const DAddr dev_addr_base = (DAddr(page_table[page_index]) << cpu_page_bits) + offset;
rasterizer->InvalidateRegion(dev_addr_base, copy_amount, which);
};
auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
const DAddr dev_addr_base =
(static_cast<DAddr>(big_page_table_dev[page_index]) << cpu_page_bits) + offset;
const DAddr dev_addr_base = (DAddr(big_page_table_dev[page_index]) << cpu_page_bits) + offset;
rasterizer->InvalidateRegion(dev_addr_base, copy_amount, which);
};
auto invalidate_short_pages = [&](std::size_t page_index, std::size_t offset,
std::size_t copy_amount) {
auto invalidate_short_pages = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
GPUVAddr base = (page_index << big_page_bits) + offset;
MemoryOperation<false>(base, copy_amount, mapped_normal, do_nothing, do_nothing);
MemoryOperation(base, copy_amount, false, mapped_normal, do_nothing, do_nothing);
};
MemoryOperation<true>(gpu_addr, size, mapped_big, do_nothing, invalidate_short_pages);
MemoryOperation(gpu_addr, size, true, mapped_big, do_nothing, invalidate_short_pages);
}
void MemoryManager::CopyBlock(GPUVAddr gpu_dest_addr, GPUVAddr gpu_src_addr, std::size_t size,
@@ -659,16 +564,16 @@ void MemoryManager::CopyBlock(GPUVAddr gpu_dest_addr, GPUVAddr gpu_src_addr, std
}
bool MemoryManager::IsGranularRange(GPUVAddr gpu_addr, std::size_t size) const {
if (GetEntry<true>(gpu_addr) == EntryType::Mapped) [[likely]] {
if (GetEntry(gpu_addr, true) == EntryType::Mapped) [[likely]] {
size_t page_index = gpu_addr >> big_page_bits;
if (IsBigPageContinuous(page_index)) [[likely]] {
const std::size_t page{(gpu_addr & big_page_mask) + size};
const std::size_t page{(page_index & big_page_mask) + size};
return page <= big_page_size;
}
const std::size_t page{(gpu_addr & Core::DEVICE_PAGEMASK) + size};
return page <= Core::DEVICE_PAGESIZE;
}
if (GetEntry<false>(gpu_addr) != EntryType::Mapped) {
if (GetEntry(gpu_addr, false) != EntryType::Mapped) {
return false;
}
const std::size_t page{(gpu_addr & Core::DEVICE_PAGEMASK) + size};
@@ -706,10 +611,10 @@ bool MemoryManager::IsContinuousRange(GPUVAddr gpu_addr, std::size_t size) const
auto check_short_pages = [&](std::size_t page_index, std::size_t offset,
std::size_t copy_amount) {
GPUVAddr base = (page_index << big_page_bits) + offset;
MemoryOperation<false>(base, copy_amount, short_check, fail, fail);
MemoryOperation(base, copy_amount, false, short_check, fail, fail);
return !result;
};
MemoryOperation<true>(gpu_addr, size, big_check, fail, check_short_pages);
MemoryOperation(gpu_addr, size, true, big_check, fail, check_short_pages);
return result;
}
@@ -722,13 +627,12 @@ bool MemoryManager::IsFullyMappedRange(GPUVAddr gpu_addr, std::size_t size) cons
};
auto pass = [&]([[maybe_unused]] std::size_t page_index, [[maybe_unused]] std::size_t offset,
[[maybe_unused]] std::size_t copy_amount) { return false; };
auto check_short_pages = [&](std::size_t page_index, std::size_t offset,
std::size_t copy_amount) {
auto check_short_pages = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
GPUVAddr base = (page_index << big_page_bits) + offset;
MemoryOperation<false>(base, copy_amount, pass, pass, fail);
MemoryOperation(base, copy_amount, false, pass, pass, fail);
return !result;
};
MemoryOperation<true>(gpu_addr, size, pass, fail, check_short_pages);
MemoryOperation(gpu_addr, size, true, pass, fail, check_short_pages);
return result;
}
@@ -740,13 +644,9 @@ MemoryManager::GetSubmappedRange(GPUVAddr gpu_addr, std::size_t size) const {
}
template <bool is_gpu_address>
void MemoryManager::GetSubmappedRangeImpl(
GPUVAddr gpu_addr, std::size_t size,
boost::container::small_vector<
std::pair<std::conditional_t<is_gpu_address, GPUVAddr, DAddr>, std::size_t>, 32>& result)
void MemoryManager::GetSubmappedRangeImpl(GPUVAddr gpu_addr, std::size_t size, boost::container::small_vector<std::pair<std::conditional_t<is_gpu_address, GPUVAddr, DAddr>, std::size_t>, 32>& result)
const {
std::optional<std::pair<std::conditional_t<is_gpu_address, GPUVAddr, DAddr>, std::size_t>>
last_segment{};
std::optional<std::pair<std::conditional_t<is_gpu_address, GPUVAddr, DAddr>, std::size_t>> last_segment{};
std::optional<DAddr> old_page_addr{};
const auto split = [&last_segment, &result]([[maybe_unused]] std::size_t page_index,
[[maybe_unused]] std::size_t offset,
@@ -802,9 +702,9 @@ void MemoryManager::GetSubmappedRangeImpl(
};
auto do_short_pages = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
GPUVAddr base = (page_index << big_page_bits) + offset;
MemoryOperation<false>(base, copy_amount, extend_size_short, split, split);
MemoryOperation(base, copy_amount, false, extend_size_short, split, split);
};
MemoryOperation<true>(gpu_addr, size, extend_size_big, split, do_short_pages);
MemoryOperation(gpu_addr, size, true, extend_size_big, split, do_short_pages);
split(0, 0, 0);
}
+19 -40
View File
@@ -45,7 +45,7 @@ public:
static constexpr bool HAS_FLUSH_INVALIDATION = true;
size_t GetID() const {
inline size_t GetID() const noexcept {
return unique_identifier;
}
@@ -66,16 +66,15 @@ public:
[[nodiscard]] const u8* GetPointer(GPUVAddr addr) const;
template <typename T>
[[nodiscard]] T* GetPointer(GPUVAddr addr) {
const auto address{GpuToCpuAddress(addr)};
if (!address) {
[[nodiscard]] inline T* GetPointer(GPUVAddr addr) noexcept {
const auto address = GpuToCpuAddress(addr);
if (!address)
return {};
}
return memory.GetPointer<T>(*address);
}
template <typename T>
[[nodiscard]] const T* GetPointer(GPUVAddr addr) const {
[[nodiscard]] inline const T* GetPointer(GPUVAddr addr) const noexcept {
return GetPointer<T*>(addr);
}
@@ -85,12 +84,9 @@ public:
* in the Host Memory counterpart. Note: This functions cause Host GPU Memory
* Flushes and Invalidations, respectively to each operation.
*/
void ReadBlock(GPUVAddr gpu_src_addr, void* dest_buffer, std::size_t size,
VideoCommon::CacheType which = VideoCommon::CacheType::All) const;
void WriteBlock(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size,
VideoCommon::CacheType which = VideoCommon::CacheType::All);
void CopyBlock(GPUVAddr gpu_dest_addr, GPUVAddr gpu_src_addr, std::size_t size,
VideoCommon::CacheType which = VideoCommon::CacheType::All);
void ReadBlock(GPUVAddr gpu_src_addr, void* dest_buffer, std::size_t size, VideoCommon::CacheType which = VideoCommon::CacheType::All) const;
void WriteBlock(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size, VideoCommon::CacheType which = VideoCommon::CacheType::All);
void CopyBlock(GPUVAddr gpu_dest_addr, GPUVAddr gpu_src_addr, std::size_t size, VideoCommon::CacheType which = VideoCommon::CacheType::All);
/**
* ReadBlockUnsafe and WriteBlockUnsafe are special versions of ReadBlock and
@@ -160,21 +156,14 @@ public:
u8* GetSpan(const GPUVAddr src_addr, const std::size_t size);
private:
template <bool is_big_pages, typename FuncMapped, typename FuncReserved, typename FuncUnmapped>
inline void MemoryOperation(GPUVAddr gpu_src_addr, std::size_t size, FuncMapped&& func_mapped,
FuncReserved&& func_reserved, FuncUnmapped&& func_unmapped) const;
template <typename FuncMapped, typename FuncReserved, typename FuncUnmapped>
inline void MemoryOperation(GPUVAddr gpu_src_addr, std::size_t size, bool is_big_page, FuncMapped&& func_mapped, FuncReserved&& func_reserved, FuncUnmapped&& func_unmapped) const;
template <bool is_safe>
void ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std::size_t size,
VideoCommon::CacheType which) const;
void ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std::size_t size, VideoCommon::CacheType which, bool unsafe) const;
void WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size, VideoCommon::CacheType which, bool unsafe);
template <bool is_safe>
void WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size,
VideoCommon::CacheType which);
template <bool is_big_page>
[[nodiscard]] std::size_t PageEntryIndex(GPUVAddr gpu_addr) const {
if constexpr (is_big_page) {
[[nodiscard]] std::size_t PageEntryIndex(GPUVAddr gpu_addr, bool is_big_page) const {
if (is_big_page) {
return (gpu_addr >> big_page_bits) & big_page_table_mask;
} else {
return (gpu_addr >> page_bits) & page_table_mask;
@@ -187,9 +176,7 @@ private:
template <bool is_gpu_address>
void GetSubmappedRangeImpl(
GPUVAddr gpu_addr, std::size_t size,
boost::container::small_vector<
std::pair<std::conditional_t<is_gpu_address, GPUVAddr, DAddr>, std::size_t>, 32>&
result) const;
boost::container::small_vector<std::pair<std::conditional_t<is_gpu_address, GPUVAddr, DAddr>, std::size_t>, 32>& result) const;
Core::System& system;
MaxwellDeviceMemoryManager& memory;
@@ -219,19 +206,11 @@ private:
std::vector<u64> entries;
std::vector<u64> big_entries;
template <EntryType entry_type>
GPUVAddr PageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr dev_addr, size_t size,
PTEKind kind);
GPUVAddr PageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr dev_addr, size_t size, PTEKind kind, EntryType entry_type);
GPUVAddr BigPageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr dev_addr, size_t size, PTEKind kind, EntryType entry_type);
template <EntryType entry_type>
GPUVAddr BigPageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr dev_addr, size_t size,
PTEKind kind);
template <bool is_big_page>
inline EntryType GetEntry(size_t position) const;
template <bool is_big_page>
inline void SetEntry(size_t position, EntryType entry);
inline EntryType GetEntry(size_t position, bool is_big_page) const;
inline void SetEntry(size_t position, EntryType entry, bool is_big_page);
Common::MultiLevelPageTable<u32> page_table;
Common::RangeMap<GPUVAddr, PTEKind> kind_map;
@@ -93,17 +93,7 @@ public:
void PostCopyBarrier();
void Finish();
void TickFrame(Common::SlotVector<Buffer>&) noexcept {
++sync_point;
}
u64 CurrentSyncPoint() const noexcept {
return sync_point;
}
u64 CompletedSyncPoint() const noexcept {
return sync_point > SYNC_POINT_DELAY ? sync_point - SYNC_POINT_DELAY : 0;
}
void TickFrame(Common::SlotVector<Buffer>&) noexcept {}
void ClearBuffer(Buffer& dest_buffer, u32 offset, size_t size, u32 value);
@@ -138,10 +128,6 @@ public:
u64 GetDeviceMemoryUsage() const;
u64 GetDeviceAllocationUsage() const {
return GetDeviceMemoryUsage();
}
void BindFastUniformBuffer(size_t stage, u32 binding_index, u32 size) {
const GLuint handle = fast_uniforms[stage][binding_index].handle;
const GLsizeiptr gl_size = static_cast<GLsizeiptr>(size);
@@ -227,13 +213,9 @@ private:
GL_FRAGMENT_PROGRAM_PARAMETER_BUFFER_NV,
};
static constexpr u64 SYNC_POINT_DELAY = 8;
const Device& device;
StagingBufferPool& staging_buffer_pool;
u64 sync_point = 1;
bool has_fast_buffer_sub_data = false;
bool use_assembly_shaders = false;
bool has_unified_vertex_buffers = false;
@@ -279,7 +261,6 @@ struct BufferCacheParams {
// TODO: Investigate why OpenGL seems to perform worse with persistently mapped buffer uploads
static constexpr bool USE_MEMORY_MAPS_FOR_UPLOADS = false;
static constexpr bool USE_UNIFIED_MEMORY = false;
};
using BufferCache = VideoCommon::BufferCache<BufferCacheParams>;
@@ -485,6 +485,7 @@ void RasterizerOpenGL::FlushRegion(DAddr addr, u64 size, VideoCommon::CacheType
texture_cache.DownloadMemory(addr, size);
}
if ((True(which & VideoCommon::CacheType::BufferCache))) {
std::scoped_lock lock{buffer_cache.mutex};
buffer_cache.DownloadMemory(addr, size);
}
if ((True(which & VideoCommon::CacheType::QueryCache))) {
@@ -87,10 +87,6 @@ public:
u64 GetDeviceMemoryUsage() const;
u64 GetDeviceAllocationUsage() const {
return GetDeviceMemoryUsage();
}
bool CanReportMemoryUsage() const {
return device.CanReportMemoryUsage();
}
@@ -143,19 +139,7 @@ public:
bool HasNativeASTC() const noexcept;
void TickFrame() {
++sync_point;
}
u64 CurrentSyncPoint() const noexcept {
return sync_point;
}
u64 CompletedSyncPoint() const noexcept {
return sync_point > SYNC_POINT_DELAY ? sync_point - SYNC_POINT_DELAY : 0;
}
void WaitSyncPoint(u64) {}
void TickFrame() {}
StateTracker& GetStateTracker() {
return state_tracker;
@@ -190,9 +174,6 @@ private:
std::array<OGLFramebuffer, 4> rescale_read_fbos;
const Settings::ResolutionScalingInfo& resolution;
u64 device_access_memory;
static constexpr u64 SYNC_POINT_DELAY = 8;
u64 sync_point = 1;
};
class Image : public VideoCommon::ImageBase {
@@ -389,7 +370,6 @@ struct TextureCacheParams {
static constexpr bool HAS_EMULATED_COPIES = true;
static constexpr bool HAS_DEVICE_MEMORY_INFO = true;
static constexpr bool IMPLEMENTS_ASYNC_DOWNLOADS = true;
static constexpr bool HAS_TIMELINE_SYNC_POINTS = false;
using Runtime = OpenGL::TextureCacheRuntime;
using Image = OpenGL::Image;
+94 -240
View File
@@ -21,7 +21,6 @@
#include "video_core/host_shaders/convert_depth_to_float_frag_spv.h"
#include "video_core/host_shaders/convert_float_to_depth_frag_spv.h"
#include "video_core/host_shaders/convert_msaa_to_non_msaa_frag_spv.h"
#include "video_core/host_shaders/convert_non_msaa_to_msaa_depth_frag_spv.h"
#include "video_core/host_shaders/convert_non_msaa_to_msaa_frag_spv.h"
#include "video_core/host_shaders/convert_s8d24_to_abgr8_frag_spv.h"
#include "video_core/host_shaders/full_screen_triangle_vert_spv.h"
@@ -520,8 +519,7 @@ void RecordShaderReadBarrier(Scheduler& scheduler, const ImageView& image_view)
}
[[nodiscard]] vk::ImageView MakeMSAACopyView(const vk::Device& device, VkImage image,
VkFormat format, u32 base_level,
VkImageAspectFlags aspect_mask) {
VkFormat format, u32 base_level) {
return device.CreateImageView(VkImageViewCreateInfo{
.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
.pNext = nullptr,
@@ -536,7 +534,7 @@ void RecordShaderReadBarrier(Scheduler& scheduler, const ImageView& image_view)
.a = VK_COMPONENT_SWIZZLE_IDENTITY,
},
.subresourceRange{
.aspectMask = aspect_mask,
.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
.baseMipLevel = base_level,
.levelCount = 1,
.baseArrayLayer = 0,
@@ -547,10 +545,6 @@ void RecordShaderReadBarrier(Scheduler& scheduler, const ImageView& image_view)
void BeginRenderPass(vk::CommandBuffer& cmdbuf, const Framebuffer* framebuffer) {
const VkRenderPass render_pass = framebuffer->RenderPass();
if (!render_pass) {
framebuffer->BeginRendering(cmdbuf);
return;
}
const VkFramebuffer framebuffer_handle = framebuffer->Handle();
const VkExtent2D render_area = framebuffer->RenderArea();
const VkRenderPassBeginInfo renderpass_bi{
@@ -567,31 +561,6 @@ void BeginRenderPass(vk::CommandBuffer& cmdbuf, const Framebuffer* framebuffer)
};
cmdbuf.BeginRenderPass(renderpass_bi, VK_SUBPASS_CONTENTS_INLINE);
}
void EndRenderPass(vk::CommandBuffer& cmdbuf, const Framebuffer* framebuffer) {
if (framebuffer->RenderPass()) {
cmdbuf.EndRenderPass();
} else {
cmdbuf.EndRendering();
}
}
[[nodiscard]] VkPipelineRenderingCreateInfo MakePipelineRenderingCreateInfo(
const Framebuffer* framebuffer) {
return VkPipelineRenderingCreateInfo{
.sType = VK_STRUCTURE_TYPE_PIPELINE_RENDERING_CREATE_INFO,
.pNext = nullptr,
.viewMask = 0,
.colorAttachmentCount = framebuffer->NumColorAttachments(),
.pColorAttachmentFormats = framebuffer->ColorAttachmentFormats().data(),
.depthAttachmentFormat = framebuffer->HasAspectDepthBit()
? framebuffer->DepthAttachmentFormat()
: VK_FORMAT_UNDEFINED,
.stencilAttachmentFormat = framebuffer->HasAspectStencilBit()
? framebuffer->DepthAttachmentFormat()
: VK_FORMAT_UNDEFINED,
};
}
} // Anonymous namespace
BlitImageHelper::BlitImageHelper(const Device& device_, Scheduler& scheduler_,
@@ -641,8 +610,6 @@ BlitImageHelper::BlitImageHelper(const Device& device_, Scheduler& scheduler_,
convert_s8d24_to_abgr8_frag(BuildShader(device, CONVERT_S8D24_TO_ABGR8_FRAG_SPV)),
convert_msaa_to_non_msaa_frag(BuildShader(device, CONVERT_MSAA_TO_NON_MSAA_FRAG_SPV)),
convert_non_msaa_to_msaa_frag(BuildShader(device, CONVERT_NON_MSAA_TO_MSAA_FRAG_SPV)),
convert_non_msaa_to_msaa_depth_frag(
BuildShader(device, CONVERT_NON_MSAA_TO_MSAA_DEPTH_FRAG_SPV)),
linear_sampler(device.GetLogical().CreateSampler(SAMPLER_CREATE_INFO<VK_FILTER_LINEAR>)),
nearest_sampler(device.GetLogical().CreateSampler(SAMPLER_CREATE_INFO<VK_FILTER_NEAREST>)) {}
@@ -656,12 +623,10 @@ void BlitImageHelper::BlitColor(const Framebuffer* dst_framebuffer, const ImageV
const BlitImagePipelineKey key{
.renderpass = dst_framebuffer->RenderPass(),
.operation = operation,
.color_formats = dst_framebuffer->ColorAttachmentFormats(),
.depth_format = dst_framebuffer->DepthAttachmentFormat(),
};
const VkPipelineLayout layout = *one_texture_pipeline_layout;
const VkSampler sampler = is_linear ? *linear_sampler : *nearest_sampler;
const VkPipeline pipeline = FindOrEmplaceColorPipeline(key, dst_framebuffer);
const VkPipeline pipeline = FindOrEmplaceColorPipeline(key);
const VkImageView src_view = src_image_view.Handle(Shader::TextureType::Color2D);
RecordShaderReadBarrier(scheduler, src_image_view);
@@ -686,11 +651,9 @@ void BlitImageHelper::BlitColor(const Framebuffer* dst_framebuffer, VkImageView
const BlitImagePipelineKey key{
.renderpass = dst_framebuffer->RenderPass(),
.operation = Tegra::Engines::Fermi2D::Operation::SrcCopy,
.color_formats = dst_framebuffer->ColorAttachmentFormats(),
.depth_format = dst_framebuffer->DepthAttachmentFormat(),
};
const VkPipelineLayout layout = *one_texture_pipeline_layout;
const VkPipeline pipeline = FindOrEmplaceColorPipeline(key, dst_framebuffer);
const VkPipeline pipeline = FindOrEmplaceColorPipeline(key);
scheduler.RequestOutsideRenderPassOperationContext();
scheduler.Record([this, dst_framebuffer, src_image_view, src_image, src_sampler, dst_region,
src_region, src_size, pipeline, layout](vk::CommandBuffer cmdbuf) {
@@ -703,7 +666,7 @@ void BlitImageHelper::BlitColor(const Framebuffer* dst_framebuffer, VkImageView
nullptr);
BindBlitState(cmdbuf, layout, dst_region, src_region, src_size);
cmdbuf.Draw(3, 1, 0, 0);
EndRenderPass(cmdbuf, dst_framebuffer);
cmdbuf.EndRenderPass();
});
}
@@ -713,12 +676,10 @@ void BlitImageHelper::BlitColorMSAA(const Framebuffer* dst_framebuffer,
const BlitMSAAPipelineKey key{
.renderpass = dst_framebuffer->RenderPass(),
.samples = dst_framebuffer->Samples(),
.color_formats = dst_framebuffer->ColorAttachmentFormats(),
.depth_format = dst_framebuffer->DepthAttachmentFormat(),
};
const VkPipelineLayout layout = *one_texture_pipeline_layout;
const VkSampler sampler = *nearest_sampler;
const VkPipeline pipeline = FindOrEmplaceBlitColorMSAAPipeline(key, dst_framebuffer);
const VkPipeline pipeline = FindOrEmplaceBlitColorMSAAPipeline(key);
const VkImageView src_view = src_image_view.Handle(Shader::TextureType::Color2D);
RecordShaderReadBarrier(scheduler, src_image_view);
@@ -742,7 +703,7 @@ void BlitImageHelper::ResolveDepthStencil(const Framebuffer* dst_framebuffer,
const bool resolve_stencil =
dst_framebuffer->HasAspectStencilBit() && device.IsExtShaderStencilExportSupported();
const VkPipeline pipeline =
FindOrEmplaceResolveDepthStencilPipeline(dst_framebuffer, resolve_stencil);
FindOrEmplaceResolveDepthStencilPipeline(dst_framebuffer->RenderPass(), resolve_stencil);
const VkPipelineLayout layout =
resolve_stencil ? *two_textures_pipeline_layout : *one_texture_pipeline_layout;
const VkSampler sampler = *nearest_sampler;
@@ -786,12 +747,10 @@ void BlitImageHelper::BlitDepthStencil(const Framebuffer* dst_framebuffer,
const BlitImagePipelineKey key{
.renderpass = dst_framebuffer->RenderPass(),
.operation = operation,
.color_formats = dst_framebuffer->ColorAttachmentFormats(),
.depth_format = dst_framebuffer->DepthAttachmentFormat(),
};
const VkPipelineLayout layout = *two_textures_pipeline_layout;
const VkSampler sampler = *nearest_sampler;
const VkPipeline pipeline = FindOrEmplaceDepthStencilPipeline(key, dst_framebuffer);
const VkPipeline pipeline = FindOrEmplaceDepthStencilPipeline(key);
const VkImageView src_depth_view = src_image_view.DepthView();
const VkImageView src_stencil_view = src_image_view.StencilView();
@@ -813,25 +772,25 @@ void BlitImageHelper::BlitDepthStencil(const Framebuffer* dst_framebuffer,
void BlitImageHelper::ConvertD32ToR32(const Framebuffer* dst_framebuffer,
const ImageView& src_image_view) {
ConvertDepthToColorPipeline(convert_d32_to_r32_pipeline, dst_framebuffer);
ConvertDepthToColorPipeline(convert_d32_to_r32_pipeline, dst_framebuffer->RenderPass());
Convert(*convert_d32_to_r32_pipeline, dst_framebuffer, src_image_view);
}
void BlitImageHelper::ConvertR32ToD32(const Framebuffer* dst_framebuffer,
const ImageView& src_image_view) {
ConvertColorToDepthPipeline(convert_r32_to_d32_pipeline, dst_framebuffer);
ConvertColorToDepthPipeline(convert_r32_to_d32_pipeline, dst_framebuffer->RenderPass());
Convert(*convert_r32_to_d32_pipeline, dst_framebuffer, src_image_view);
}
void BlitImageHelper::ConvertD16ToR16(const Framebuffer* dst_framebuffer,
const ImageView& src_image_view) {
ConvertDepthToColorPipeline(convert_d16_to_r16_pipeline, dst_framebuffer);
ConvertDepthToColorPipeline(convert_d16_to_r16_pipeline, dst_framebuffer->RenderPass());
Convert(*convert_d16_to_r16_pipeline, dst_framebuffer, src_image_view);
}
void BlitImageHelper::ConvertR16ToD16(const Framebuffer* dst_framebuffer,
const ImageView& src_image_view) {
ConvertColorToDepthPipeline(convert_r16_to_d16_pipeline, dst_framebuffer);
ConvertColorToDepthPipeline(convert_r16_to_d16_pipeline, dst_framebuffer->RenderPass());
Convert(*convert_r16_to_d16_pipeline, dst_framebuffer, src_image_view);
}
@@ -842,35 +801,35 @@ void BlitImageHelper::ConvertABGR8ToD24S8(const Framebuffer* dst_framebuffer,
LOG_WARNING(Render_Vulkan, "ConvertABGR8ToD24S8 requires shader_stencil_export, skipping");
return;
}
ConvertPipelineDepthTargetEx(convert_abgr8_to_d24s8_pipeline, dst_framebuffer,
ConvertPipelineDepthTargetEx(convert_abgr8_to_d24s8_pipeline, dst_framebuffer->RenderPass(),
convert_abgr8_to_d24s8_frag);
Convert(*convert_abgr8_to_d24s8_pipeline, dst_framebuffer, src_image_view);
}
void BlitImageHelper::ConvertABGR8ToD32F(const Framebuffer* dst_framebuffer,
const ImageView& src_image_view) {
ConvertPipelineDepthTargetEx(convert_abgr8_to_d32f_pipeline, dst_framebuffer,
ConvertPipelineDepthTargetEx(convert_abgr8_to_d32f_pipeline, dst_framebuffer->RenderPass(),
convert_abgr8_to_d32f_frag);
Convert(*convert_abgr8_to_d32f_pipeline, dst_framebuffer, src_image_view);
}
void BlitImageHelper::ConvertD32FToABGR8(const Framebuffer* dst_framebuffer,
ImageView& src_image_view) {
ConvertPipelineColorTargetEx(convert_d32f_to_abgr8_pipeline, dst_framebuffer,
ConvertPipelineColorTargetEx(convert_d32f_to_abgr8_pipeline, dst_framebuffer->RenderPass(),
convert_d32f_to_abgr8_frag);
ConvertDepthStencil(*convert_d32f_to_abgr8_pipeline, dst_framebuffer, src_image_view);
}
void BlitImageHelper::ConvertD24S8ToABGR8(const Framebuffer* dst_framebuffer,
ImageView& src_image_view) {
ConvertPipelineColorTargetEx(convert_d24s8_to_abgr8_pipeline, dst_framebuffer,
ConvertPipelineColorTargetEx(convert_d24s8_to_abgr8_pipeline, dst_framebuffer->RenderPass(),
convert_d24s8_to_abgr8_frag);
ConvertDepthStencil(*convert_d24s8_to_abgr8_pipeline, dst_framebuffer, src_image_view);
}
void BlitImageHelper::ConvertS8D24ToABGR8(const Framebuffer* dst_framebuffer,
ImageView& src_image_view) {
ConvertPipelineColorTargetEx(convert_s8d24_to_abgr8_pipeline, dst_framebuffer,
ConvertPipelineColorTargetEx(convert_s8d24_to_abgr8_pipeline, dst_framebuffer->RenderPass(),
convert_s8d24_to_abgr8_frag);
ConvertDepthStencil(*convert_s8d24_to_abgr8_pipeline, dst_framebuffer, src_image_view);
}
@@ -881,10 +840,8 @@ void BlitImageHelper::ClearColor(const Framebuffer* dst_framebuffer, u8 color_ma
const BlitImagePipelineKey key{
.renderpass = dst_framebuffer->RenderPass(),
.operation = Tegra::Engines::Fermi2D::Operation::BlendPremult,
.color_formats = dst_framebuffer->ColorAttachmentFormats(),
.depth_format = dst_framebuffer->DepthAttachmentFormat(),
};
const VkPipeline pipeline = FindOrEmplaceClearColorPipeline(key, dst_framebuffer);
const VkPipeline pipeline = FindOrEmplaceClearColorPipeline(key);
const VkPipelineLayout layout = *clear_color_pipeline_layout;
scheduler.RequestRenderpass(dst_framebuffer);
scheduler.Record(
@@ -910,10 +867,8 @@ void BlitImageHelper::ClearDepthStencil(const Framebuffer* dst_framebuffer, bool
.stencil_mask = stencil_mask,
.stencil_compare_mask = stencil_compare_mask,
.stencil_ref = stencil_ref,
.color_formats = dst_framebuffer->ColorAttachmentFormats(),
.depth_format = dst_framebuffer->DepthAttachmentFormat(),
};
const VkPipeline pipeline = FindOrEmplaceClearStencilPipeline(key, dst_framebuffer);
const VkPipeline pipeline = FindOrEmplaceClearStencilPipeline(key);
const VkPipelineLayout layout = *clear_color_pipeline_layout;
scheduler.RequestRenderpass(dst_framebuffer);
scheduler.Record([pipeline, layout, clear_depth, dst_region](vk::CommandBuffer cmdbuf) {
@@ -940,34 +895,16 @@ void BlitImageHelper::CopyMSAA(RenderPassCache& render_pass_cache, VkImage dst_i
const s32 scale_y = 1 << samples_y;
const VkSampleCountFlagBits samples =
msaa_to_non_msaa ? VK_SAMPLE_COUNT_1_BIT : SampleCountFlag(num_samples);
const auto dst_surface_type = VideoCore::Surface::GetFormatType(dst_format);
const bool is_depth = dst_surface_type == VideoCore::Surface::SurfaceType::Depth ||
dst_surface_type == VideoCore::Surface::SurfaceType::DepthStencil;
const bool has_stencil = dst_surface_type == VideoCore::Surface::SurfaceType::DepthStencil;
const VkImageAspectFlags view_aspect =
is_depth ? VK_IMAGE_ASPECT_DEPTH_BIT : VK_IMAGE_ASPECT_COLOR_BIT;
VkImageAspectFlags barrier_aspect = VK_IMAGE_ASPECT_COLOR_BIT;
if (is_depth) {
barrier_aspect = VK_IMAGE_ASPECT_DEPTH_BIT;
if (has_stencil) {
barrier_aspect |= VK_IMAGE_ASPECT_STENCIL_BIT;
}
}
RenderPassKey renderpass_key{};
renderpass_key.color_formats.fill(VideoCore::Surface::PixelFormat::Invalid);
if (is_depth) {
renderpass_key.depth_format = dst_format;
} else {
renderpass_key.color_formats[0] = dst_format;
renderpass_key.depth_format = VideoCore::Surface::PixelFormat::Invalid;
}
renderpass_key.color_formats[0] = dst_format;
renderpass_key.depth_format = VideoCore::Surface::PixelFormat::Invalid;
renderpass_key.samples = samples;
const VkRenderPass renderpass = render_pass_cache.Get(renderpass_key);
const MSAACopyPipelineKey key{
.renderpass = renderpass,
.samples = samples,
.msaa_to_non_msaa = msaa_to_non_msaa,
.is_depth = is_depth,
};
const VkPipeline pipeline = FindOrEmplaceMSAACopyPipeline(key);
const VkPipelineLayout layout = *msaa_copy_pipeline_layout;
@@ -983,10 +920,10 @@ void BlitImageHelper::CopyMSAA(RenderPassCache& render_pass_cache, VkImage dst_i
ASSERT(copy.dst_subresource.num_layers == 1);
vk::ImageView src_view =
MakeMSAACopyView(device.GetLogical(), src_image, src_vk_format,
static_cast<u32>(copy.src_subresource.base_level), view_aspect);
static_cast<u32>(copy.src_subresource.base_level));
vk::ImageView dst_view =
MakeMSAACopyView(device.GetLogical(), dst_image, dst_vk_format,
static_cast<u32>(copy.dst_subresource.base_level), view_aspect);
static_cast<u32>(copy.dst_subresource.base_level));
const VkOffset2D dst_offset{copy.dst_offset.x, copy.dst_offset.y};
const VkExtent2D dst_extent{copy.extent.width, copy.extent.height};
const VkRect2D render_area{
@@ -1012,64 +949,50 @@ void BlitImageHelper::CopyMSAA(RenderPassCache& render_pass_cache, VkImage dst_i
scheduler.RequestOutsideRenderPassOperationContext();
scheduler.Record([this, pipeline, layout, sampler, renderpass,
framebuffer_handle = *framebuffer, src_view_handle = *src_view,
src = src_image, dst = dst_image, render_area, is_depth, barrier_aspect,
src = src_image, dst = dst_image, render_area,
push_constants](vk::CommandBuffer cmdbuf) {
const VkImageSubresourceRange src_range{
.aspectMask = barrier_aspect,
constexpr VkImageSubresourceRange color_range{
.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
.baseMipLevel = 0,
.levelCount = VK_REMAINING_MIP_LEVELS,
.baseArrayLayer = 0,
.layerCount = VK_REMAINING_ARRAY_LAYERS,
};
const VkImageSubresourceRange dst_range = src_range;
const VkAccessFlags attachment_read =
is_depth ? VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT
: VK_ACCESS_COLOR_ATTACHMENT_READ_BIT;
const VkAccessFlags attachment_write =
is_depth ? VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT
: VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
const VkPipelineStageFlags depth_stage =
VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT |
VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT;
const VkPipelineStageFlags attachment_stage =
is_depth ? depth_stage
: static_cast<VkPipelineStageFlags>(
VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT);
const std::array pre_barriers{
VkImageMemoryBarrier{
.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
.pNext = nullptr,
.srcAccessMask = attachment_write | VK_ACCESS_SHADER_WRITE_BIT |
VK_ACCESS_TRANSFER_WRITE_BIT,
.srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT |
VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT,
.dstAccessMask = VK_ACCESS_SHADER_READ_BIT,
.oldLayout = VK_IMAGE_LAYOUT_GENERAL,
.newLayout = VK_IMAGE_LAYOUT_GENERAL,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.image = src,
.subresourceRange = src_range,
.subresourceRange = color_range,
},
VkImageMemoryBarrier{
.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
.pNext = nullptr,
.srcAccessMask = attachment_write | VK_ACCESS_SHADER_WRITE_BIT |
VK_ACCESS_TRANSFER_WRITE_BIT,
.dstAccessMask = attachment_read | attachment_write,
.srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT |
VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT,
.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_READ_BIT |
VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT,
.oldLayout = VK_IMAGE_LAYOUT_GENERAL,
.newLayout = VK_IMAGE_LAYOUT_GENERAL,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.image = dst,
.subresourceRange = dst_range,
.subresourceRange = color_range,
},
};
cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT |
VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT |
VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT |
VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT |
VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT |
VK_PIPELINE_STAGE_TRANSFER_BIT,
VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | attachment_stage,
VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT |
VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT,
0, nullptr, nullptr, pre_barriers);
const VkRenderPassBeginInfo renderpass_bi{
.sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO,
@@ -1102,16 +1025,16 @@ void BlitImageHelper::CopyMSAA(RenderPassCache& render_pass_cache, VkImage dst_i
const VkImageMemoryBarrier post_barrier{
.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
.pNext = nullptr,
.srcAccessMask = attachment_write,
.srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT,
.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_TRANSFER_READ_BIT,
.oldLayout = VK_IMAGE_LAYOUT_GENERAL,
.newLayout = VK_IMAGE_LAYOUT_GENERAL,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.image = dst,
.subresourceRange = dst_range,
.subresourceRange = color_range,
};
cmdbuf.PipelineBarrier(attachment_stage,
cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT,
VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT |
VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT |
VK_PIPELINE_STAGE_TRANSFER_BIT,
@@ -1217,14 +1140,12 @@ void BlitImageHelper::ConvertDepthStencil(VkPipeline pipeline, const Framebuffer
scheduler.InvalidateState();
}
VkPipeline BlitImageHelper::FindOrEmplaceColorPipeline(const BlitImagePipelineKey& key,
const Framebuffer* framebuffer) {
VkPipeline BlitImageHelper::FindOrEmplaceColorPipeline(const BlitImagePipelineKey& key) {
const auto it = std::ranges::find(blit_color_keys, key);
if (it != blit_color_keys.end()) {
return *blit_color_pipelines[std::distance(blit_color_keys.begin(), it)];
}
blit_color_keys.push_back(key);
const VkPipelineRenderingCreateInfo rendering_ci = MakePipelineRenderingCreateInfo(framebuffer);
const std::array stages = MakeStages(*full_screen_vert, *blit_color_to_color_frag);
const VkPipelineColorBlendAttachmentState blend_attachment{
@@ -1252,7 +1173,7 @@ VkPipeline BlitImageHelper::FindOrEmplaceColorPipeline(const BlitImagePipelineKe
const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device);
blit_color_pipelines.push_back(device.GetLogical().CreateGraphicsPipeline({
.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
.pNext = key.renderpass ? nullptr : &rendering_ci,
.pNext = nullptr,
.flags = 0,
.stageCount = static_cast<u32>(stages.size()),
.pStages = stages.data(),
@@ -1270,23 +1191,21 @@ VkPipeline BlitImageHelper::FindOrEmplaceColorPipeline(const BlitImagePipelineKe
.subpass = 0,
.basePipelineHandle = VK_NULL_HANDLE,
.basePipelineIndex = 0,
}, device.StaticPipelineCache()));
}));
return *blit_color_pipelines.back();
}
VkPipeline BlitImageHelper::FindOrEmplaceDepthStencilPipeline(const BlitImagePipelineKey& key,
const Framebuffer* framebuffer) {
VkPipeline BlitImageHelper::FindOrEmplaceDepthStencilPipeline(const BlitImagePipelineKey& key) {
const auto it = std::ranges::find(blit_depth_stencil_keys, key);
if (it != blit_depth_stencil_keys.end()) {
return *blit_depth_stencil_pipelines[std::distance(blit_depth_stencil_keys.begin(), it)];
}
blit_depth_stencil_keys.push_back(key);
const VkPipelineRenderingCreateInfo rendering_ci = MakePipelineRenderingCreateInfo(framebuffer);
const std::array stages = MakeStages(*full_screen_vert, *blit_depth_stencil_frag);
const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device);
blit_depth_stencil_pipelines.push_back(device.GetLogical().CreateGraphicsPipeline({
.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
.pNext = key.renderpass ? nullptr : &rendering_ci,
.pNext = nullptr,
.flags = 0,
.stageCount = static_cast<u32>(stages.size()),
.pStages = stages.data(),
@@ -1304,50 +1223,42 @@ VkPipeline BlitImageHelper::FindOrEmplaceDepthStencilPipeline(const BlitImagePip
.subpass = 0,
.basePipelineHandle = VK_NULL_HANDLE,
.basePipelineIndex = 0,
}, device.StaticPipelineCache()));
}));
return *blit_depth_stencil_pipelines.back();
}
VkPipeline BlitImageHelper::FindOrEmplaceClearColorPipeline(const BlitImagePipelineKey& key,
const Framebuffer* framebuffer) {
VkPipeline BlitImageHelper::FindOrEmplaceClearColorPipeline(const BlitImagePipelineKey& key) {
const auto it = std::ranges::find(clear_color_keys, key);
if (it != clear_color_keys.end()) {
return *clear_color_pipelines[std::distance(clear_color_keys.begin(), it)];
}
clear_color_keys.push_back(key);
const VkPipelineRenderingCreateInfo rendering_ci = MakePipelineRenderingCreateInfo(framebuffer);
const std::array stages = MakeStages(*clear_color_vert, *clear_color_frag);
const u32 num_color = framebuffer->NumColorAttachments();
constexpr VkColorComponentFlags full_write_mask =
VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT |
VK_COLOR_COMPONENT_A_BIT;
std::array<VkPipelineColorBlendAttachmentState, VideoCommon::NUM_RT> blend_attachments{};
for (u32 index = 0; index < num_color; ++index) {
blend_attachments[index] = VkPipelineColorBlendAttachmentState{
.blendEnable = index == 0 ? VK_TRUE : VK_FALSE,
.srcColorBlendFactor = VK_BLEND_FACTOR_CONSTANT_COLOR,
.dstColorBlendFactor = VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_COLOR,
.colorBlendOp = VK_BLEND_OP_ADD,
.srcAlphaBlendFactor = VK_BLEND_FACTOR_CONSTANT_ALPHA,
.dstAlphaBlendFactor = VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_ALPHA,
.alphaBlendOp = VK_BLEND_OP_ADD,
.colorWriteMask = index == 0 ? full_write_mask : VkColorComponentFlags{0},
};
}
const VkPipelineColorBlendAttachmentState color_blend_attachment_state{
.blendEnable = VK_TRUE,
.srcColorBlendFactor = VK_BLEND_FACTOR_CONSTANT_COLOR,
.dstColorBlendFactor = VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_COLOR,
.colorBlendOp = VK_BLEND_OP_ADD,
.srcAlphaBlendFactor = VK_BLEND_FACTOR_CONSTANT_ALPHA,
.dstAlphaBlendFactor = VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_ALPHA,
.alphaBlendOp = VK_BLEND_OP_ADD,
.colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT |
VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT,
};
const VkPipelineColorBlendStateCreateInfo color_blend_state_generic_create_info{
.sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
.pNext = nullptr,
.flags = 0,
.logicOpEnable = VK_FALSE,
.logicOp = VK_LOGIC_OP_CLEAR,
.attachmentCount = num_color,
.pAttachments = blend_attachments.data(),
.attachmentCount = 1,
.pAttachments = &color_blend_attachment_state,
.blendConstants = {0.0f, 0.0f, 0.0f, 0.0f},
};
const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device);
clear_color_pipelines.push_back(device.GetLogical().CreateGraphicsPipeline({
.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
.pNext = key.renderpass ? nullptr : &rendering_ci,
.pNext = nullptr,
.flags = 0,
.stageCount = static_cast<u32>(stages.size()),
.pStages = stages.data(),
@@ -1365,31 +1276,18 @@ VkPipeline BlitImageHelper::FindOrEmplaceClearColorPipeline(const BlitImagePipel
.subpass = 0,
.basePipelineHandle = VK_NULL_HANDLE,
.basePipelineIndex = 0,
}, device.StaticPipelineCache()));
}));
return *clear_color_pipelines.back();
}
VkPipeline BlitImageHelper::FindOrEmplaceClearStencilPipeline(
const BlitDepthStencilPipelineKey& key, const Framebuffer* framebuffer) {
const BlitDepthStencilPipelineKey& key) {
const auto it = std::ranges::find(clear_stencil_keys, key);
if (it != clear_stencil_keys.end()) {
return *clear_stencil_pipelines[std::distance(clear_stencil_keys.begin(), it)];
}
clear_stencil_keys.push_back(key);
const VkPipelineRenderingCreateInfo rendering_ci = MakePipelineRenderingCreateInfo(framebuffer);
const std::array stages = MakeStages(*clear_color_vert, *clear_stencil_frag);
const u32 num_color = framebuffer->NumColorAttachments();
std::array<VkPipelineColorBlendAttachmentState, VideoCommon::NUM_RT> blend_attachments{};
const VkPipelineColorBlendStateCreateInfo color_blend_ci{
.sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
.pNext = nullptr,
.flags = 0,
.logicOpEnable = VK_FALSE,
.logicOp = VK_LOGIC_OP_CLEAR,
.attachmentCount = num_color,
.pAttachments = blend_attachments.data(),
.blendConstants = {0.0f, 0.0f, 0.0f, 0.0f},
};
const auto stencil = VkStencilOpState{
.failOp = VK_STENCIL_OP_KEEP,
.passOp = VK_STENCIL_OP_REPLACE,
@@ -1416,7 +1314,7 @@ VkPipeline BlitImageHelper::FindOrEmplaceClearStencilPipeline(
const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device);
clear_stencil_pipelines.push_back(device.GetLogical().CreateGraphicsPipeline({
.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
.pNext = key.renderpass ? nullptr : &rendering_ci,
.pNext = nullptr,
.flags = 0,
.stageCount = static_cast<u32>(stages.size()),
.pStages = stages.data(),
@@ -1427,25 +1325,23 @@ VkPipeline BlitImageHelper::FindOrEmplaceClearStencilPipeline(
.pRasterizationState = &PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
.pMultisampleState = &PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
.pDepthStencilState = &depth_stencil_ci,
.pColorBlendState = &color_blend_ci,
.pColorBlendState = &PIPELINE_COLOR_BLEND_STATE_GENERIC_CREATE_INFO,
.pDynamicState = &PIPELINE_DYNAMIC_STATE_CREATE_INFO,
.layout = *clear_color_pipeline_layout,
.renderPass = key.renderpass,
.subpass = 0,
.basePipelineHandle = VK_NULL_HANDLE,
.basePipelineIndex = 0,
}, device.StaticPipelineCache()));
}));
return *clear_stencil_pipelines.back();
}
VkPipeline BlitImageHelper::FindOrEmplaceBlitColorMSAAPipeline(const BlitMSAAPipelineKey& key,
const Framebuffer* framebuffer) {
VkPipeline BlitImageHelper::FindOrEmplaceBlitColorMSAAPipeline(const BlitMSAAPipelineKey& key) {
const auto it = std::ranges::find(blit_msaa_color_keys, key);
if (it != blit_msaa_color_keys.end()) {
return *blit_msaa_color_pipelines[std::distance(blit_msaa_color_keys.begin(), it)];
}
blit_msaa_color_keys.push_back(key);
const VkPipelineRenderingCreateInfo rendering_ci = MakePipelineRenderingCreateInfo(framebuffer);
const std::array stages = MakeStages(*full_screen_vert, *blit_color_msaa_frag);
const VkPipelineMultisampleStateCreateInfo multisample_ci{
.sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
@@ -1461,7 +1357,7 @@ VkPipeline BlitImageHelper::FindOrEmplaceBlitColorMSAAPipeline(const BlitMSAAPip
const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device);
blit_msaa_color_pipelines.push_back(device.GetLogical().CreateGraphicsPipeline({
.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
.pNext = key.renderpass ? nullptr : &rendering_ci,
.pNext = nullptr,
.flags = 0,
.stageCount = static_cast<u32>(stages.size()),
.pStages = stages.data(),
@@ -1479,32 +1375,26 @@ VkPipeline BlitImageHelper::FindOrEmplaceBlitColorMSAAPipeline(const BlitMSAAPip
.subpass = 0,
.basePipelineHandle = VK_NULL_HANDLE,
.basePipelineIndex = 0,
}, device.StaticPipelineCache()));
}));
return *blit_msaa_color_pipelines.back();
}
VkPipeline BlitImageHelper::FindOrEmplaceResolveDepthStencilPipeline(
const Framebuffer* framebuffer, bool resolve_stencil) {
const VkRenderPass renderpass = framebuffer->RenderPass();
const ResolveDepthStencilPipelineKey key{
.renderpass = renderpass,
.depth_format = framebuffer->DepthAttachmentFormat(),
};
VkPipeline BlitImageHelper::FindOrEmplaceResolveDepthStencilPipeline(VkRenderPass renderpass,
bool resolve_stencil) {
auto& keys = resolve_stencil ? resolve_depth_stencil_keys : resolve_depth_keys;
auto& pipelines = resolve_stencil ? resolve_depth_stencil_pipelines : resolve_depth_pipelines;
const auto it = std::ranges::find(keys, key);
const auto it = std::ranges::find(keys, renderpass);
if (it != keys.end()) {
return *pipelines[std::distance(keys.begin(), it)];
}
keys.push_back(key);
const VkPipelineRenderingCreateInfo rendering_ci = MakePipelineRenderingCreateInfo(framebuffer);
keys.push_back(renderpass);
const std::array stages =
MakeStages(*full_screen_vert,
resolve_stencil ? *blit_depth_stencil_msaa_frag : *blit_depth_msaa_frag);
const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device);
pipelines.push_back(device.GetLogical().CreateGraphicsPipeline({
.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
.pNext = renderpass ? nullptr : &rendering_ci,
.pNext = nullptr,
.flags = 0,
.stageCount = static_cast<u32>(stages.size()),
.pStages = stages.data(),
@@ -1523,7 +1413,7 @@ VkPipeline BlitImageHelper::FindOrEmplaceResolveDepthStencilPipeline(
.subpass = 0,
.basePipelineHandle = VK_NULL_HANDLE,
.basePipelineIndex = 0,
}, device.StaticPipelineCache()));
}));
return *pipelines.back();
}
@@ -1533,36 +1423,9 @@ VkPipeline BlitImageHelper::FindOrEmplaceMSAACopyPipeline(const MSAACopyPipeline
return *msaa_copy_pipelines[std::distance(msaa_copy_keys.begin(), it)];
}
msaa_copy_keys.push_back(key);
const VkShaderModule frag_module =
key.msaa_to_non_msaa
? *convert_msaa_to_non_msaa_frag
: (key.is_depth ? *convert_non_msaa_to_msaa_depth_frag
: *convert_non_msaa_to_msaa_frag);
const std::array stages = MakeStages(*clear_color_vert, frag_module);
const VkPipelineDepthStencilStateCreateInfo depth_stencil_ci{
.sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO,
.pNext = nullptr,
.flags = 0,
.depthTestEnable = VK_TRUE,
.depthWriteEnable = VK_TRUE,
.depthCompareOp = VK_COMPARE_OP_ALWAYS,
.depthBoundsTestEnable = VK_FALSE,
.stencilTestEnable = VK_FALSE,
.front = {},
.back = {},
.minDepthBounds = 0.0f,
.maxDepthBounds = 0.0f,
};
static constexpr VkPipelineColorBlendStateCreateInfo no_color_blend_ci{
.sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
.pNext = nullptr,
.flags = 0,
.logicOpEnable = VK_FALSE,
.logicOp = VK_LOGIC_OP_CLEAR,
.attachmentCount = 0,
.pAttachments = nullptr,
.blendConstants = {0.0f, 0.0f, 0.0f, 0.0f},
};
const std::array stages = MakeStages(*clear_color_vert, key.msaa_to_non_msaa
? *convert_msaa_to_non_msaa_frag
: *convert_non_msaa_to_msaa_frag);
const VkPipelineMultisampleStateCreateInfo multisample_ci{
.sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
.pNext = nullptr,
@@ -1587,42 +1450,37 @@ VkPipeline BlitImageHelper::FindOrEmplaceMSAACopyPipeline(const MSAACopyPipeline
.pViewportState = &PIPELINE_VIEWPORT_STATE_CREATE_INFO,
.pRasterizationState = &PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
.pMultisampleState = &multisample_ci,
.pDepthStencilState = key.is_depth ? &depth_stencil_ci : nullptr,
.pColorBlendState = key.is_depth ? &no_color_blend_ci
: &PIPELINE_COLOR_BLEND_STATE_GENERIC_CREATE_INFO,
.pDepthStencilState = nullptr,
.pColorBlendState = &PIPELINE_COLOR_BLEND_STATE_GENERIC_CREATE_INFO,
.pDynamicState = &PIPELINE_DYNAMIC_STATE_CREATE_INFO,
.layout = *msaa_copy_pipeline_layout,
.renderPass = key.renderpass,
.subpass = 0,
.basePipelineHandle = VK_NULL_HANDLE,
.basePipelineIndex = 0,
}, device.StaticPipelineCache()));
}));
return *msaa_copy_pipelines.back();
}
void BlitImageHelper::ConvertDepthToColorPipeline(vk::Pipeline& pipeline,
const Framebuffer* framebuffer) {
ConvertPipeline(pipeline, framebuffer, false);
void BlitImageHelper::ConvertDepthToColorPipeline(vk::Pipeline& pipeline, VkRenderPass renderpass) {
ConvertPipeline(pipeline, renderpass, false);
}
void BlitImageHelper::ConvertColorToDepthPipeline(vk::Pipeline& pipeline,
const Framebuffer* framebuffer) {
ConvertPipeline(pipeline, framebuffer, true);
void BlitImageHelper::ConvertColorToDepthPipeline(vk::Pipeline& pipeline, VkRenderPass renderpass) {
ConvertPipeline(pipeline, renderpass, true);
}
void BlitImageHelper::ConvertPipelineEx(vk::Pipeline& pipeline, const Framebuffer* framebuffer,
void BlitImageHelper::ConvertPipelineEx(vk::Pipeline& pipeline, VkRenderPass renderpass,
vk::ShaderModule& module, bool single_texture,
bool is_target_depth) {
if (pipeline) {
return;
}
const VkRenderPass renderpass = framebuffer->RenderPass();
const VkPipelineRenderingCreateInfo rendering_ci = MakePipelineRenderingCreateInfo(framebuffer);
const std::array stages = MakeStages(*full_screen_vert, *module);
const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device);
pipeline = device.GetLogical().CreateGraphicsPipeline(VkGraphicsPipelineCreateInfo{
.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
.pNext = renderpass ? nullptr : &rendering_ci,
.pNext = nullptr,
.flags = 0,
.stageCount = static_cast<u32>(stages.size()),
.pStages = stages.data(),
@@ -1641,35 +1499,31 @@ void BlitImageHelper::ConvertPipelineEx(vk::Pipeline& pipeline, const Framebuffe
.subpass = 0,
.basePipelineHandle = VK_NULL_HANDLE,
.basePipelineIndex = 0,
}, device.StaticPipelineCache());
});
}
void BlitImageHelper::ConvertPipelineColorTargetEx(vk::Pipeline& pipeline,
const Framebuffer* framebuffer,
void BlitImageHelper::ConvertPipelineColorTargetEx(vk::Pipeline& pipeline, VkRenderPass renderpass,
vk::ShaderModule& module) {
ConvertPipelineEx(pipeline, framebuffer, module, false, false);
ConvertPipelineEx(pipeline, renderpass, module, false, false);
}
void BlitImageHelper::ConvertPipelineDepthTargetEx(vk::Pipeline& pipeline,
const Framebuffer* framebuffer,
void BlitImageHelper::ConvertPipelineDepthTargetEx(vk::Pipeline& pipeline, VkRenderPass renderpass,
vk::ShaderModule& module) {
ConvertPipelineEx(pipeline, framebuffer, module, true, true);
ConvertPipelineEx(pipeline, renderpass, module, true, true);
}
void BlitImageHelper::ConvertPipeline(vk::Pipeline& pipeline, const Framebuffer* framebuffer,
void BlitImageHelper::ConvertPipeline(vk::Pipeline& pipeline, VkRenderPass renderpass,
bool is_target_depth) {
if (pipeline) {
return;
}
const VkRenderPass renderpass = framebuffer->RenderPass();
const VkPipelineRenderingCreateInfo rendering_ci = MakePipelineRenderingCreateInfo(framebuffer);
VkShaderModule frag_shader =
is_target_depth ? *convert_float_to_depth_frag : *convert_depth_to_float_frag;
const std::array stages = MakeStages(*full_screen_vert, frag_shader);
const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device);
pipeline = device.GetLogical().CreateGraphicsPipeline(VkGraphicsPipelineCreateInfo{
.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
.pNext = renderpass ? nullptr : &rendering_ci,
.pNext = nullptr,
.flags = 0,
.stageCount = static_cast<u32>(stages.size()),
.pStages = stages.data(),
@@ -1688,7 +1542,7 @@ void BlitImageHelper::ConvertPipeline(vk::Pipeline& pipeline, const Framebuffer*
.subpass = 0,
.basePipelineHandle = VK_NULL_HANDLE,
.basePipelineIndex = 0,
}, device.StaticPipelineCache());
});
}
} // namespace Vulkan
+14 -34
View File
@@ -33,8 +33,6 @@ struct BlitImagePipelineKey {
VkRenderPass renderpass;
Tegra::Engines::Fermi2D::Operation operation;
std::array<VkFormat, VideoCommon::NUM_RT> color_formats;
VkFormat depth_format;
};
struct BlitDepthStencilPipelineKey {
@@ -45,8 +43,6 @@ struct BlitDepthStencilPipelineKey {
u8 stencil_mask;
u32 stencil_compare_mask;
u32 stencil_ref;
std::array<VkFormat, VideoCommon::NUM_RT> color_formats;
VkFormat depth_format;
};
struct MSAACopyPipelineKey {
@@ -55,7 +51,6 @@ struct MSAACopyPipelineKey {
VkRenderPass renderpass;
VkSampleCountFlagBits samples;
bool msaa_to_non_msaa;
bool is_depth;
};
struct BlitMSAAPipelineKey {
@@ -63,15 +58,6 @@ struct BlitMSAAPipelineKey {
VkRenderPass renderpass;
VkSampleCountFlagBits samples;
std::array<VkFormat, VideoCommon::NUM_RT> color_formats;
VkFormat depth_format;
};
struct ResolveDepthStencilPipelineKey {
constexpr auto operator<=>(const ResolveDepthStencilPipelineKey&) const noexcept = default;
VkRenderPass renderpass;
VkFormat depth_format;
};
class BlitImageHelper {
@@ -137,36 +123,31 @@ private:
void ConvertDepthStencil(VkPipeline pipeline, const Framebuffer* dst_framebuffer,
ImageView& src_image_view);
[[nodiscard]] VkPipeline FindOrEmplaceColorPipeline(const BlitImagePipelineKey& key,
const Framebuffer* framebuffer);
[[nodiscard]] VkPipeline FindOrEmplaceColorPipeline(const BlitImagePipelineKey& key);
[[nodiscard]] VkPipeline FindOrEmplaceDepthStencilPipeline(const BlitImagePipelineKey& key,
const Framebuffer* framebuffer);
[[nodiscard]] VkPipeline FindOrEmplaceDepthStencilPipeline(const BlitImagePipelineKey& key);
[[nodiscard]] VkPipeline FindOrEmplaceClearColorPipeline(const BlitImagePipelineKey& key,
const Framebuffer* framebuffer);
[[nodiscard]] VkPipeline FindOrEmplaceClearColorPipeline(const BlitImagePipelineKey& key);
[[nodiscard]] VkPipeline FindOrEmplaceClearStencilPipeline(
const BlitDepthStencilPipelineKey& key, const Framebuffer* framebuffer);
const BlitDepthStencilPipelineKey& key);
[[nodiscard]] VkPipeline FindOrEmplaceMSAACopyPipeline(const MSAACopyPipelineKey& key);
[[nodiscard]] VkPipeline FindOrEmplaceBlitColorMSAAPipeline(const BlitMSAAPipelineKey& key,
const Framebuffer* framebuffer);
[[nodiscard]] VkPipeline FindOrEmplaceResolveDepthStencilPipeline(const Framebuffer* framebuffer,
[[nodiscard]] VkPipeline FindOrEmplaceBlitColorMSAAPipeline(const BlitMSAAPipelineKey& key);
[[nodiscard]] VkPipeline FindOrEmplaceResolveDepthStencilPipeline(VkRenderPass renderpass,
bool resolve_stencil);
void ConvertPipeline(vk::Pipeline& pipeline, const Framebuffer* framebuffer,
bool is_target_depth);
void ConvertPipeline(vk::Pipeline& pipeline, VkRenderPass renderpass, bool is_target_depth);
void ConvertDepthToColorPipeline(vk::Pipeline& pipeline, const Framebuffer* framebuffer);
void ConvertDepthToColorPipeline(vk::Pipeline& pipeline, VkRenderPass renderpass);
void ConvertColorToDepthPipeline(vk::Pipeline& pipeline, const Framebuffer* framebuffer);
void ConvertColorToDepthPipeline(vk::Pipeline& pipeline, VkRenderPass renderpass);
void ConvertPipelineEx(vk::Pipeline& pipeline, const Framebuffer* framebuffer,
void ConvertPipelineEx(vk::Pipeline& pipeline, VkRenderPass renderpass,
vk::ShaderModule& module, bool single_texture, bool is_target_depth);
void ConvertPipelineColorTargetEx(vk::Pipeline& pipeline, const Framebuffer* framebuffer,
void ConvertPipelineColorTargetEx(vk::Pipeline& pipeline, VkRenderPass renderpass,
vk::ShaderModule& module);
void ConvertPipelineDepthTargetEx(vk::Pipeline& pipeline, const Framebuffer* framebuffer,
void ConvertPipelineDepthTargetEx(vk::Pipeline& pipeline, VkRenderPass renderpass,
vk::ShaderModule& module);
const Device& device;
@@ -199,7 +180,6 @@ private:
vk::ShaderModule convert_s8d24_to_abgr8_frag;
vk::ShaderModule convert_msaa_to_non_msaa_frag;
vk::ShaderModule convert_non_msaa_to_msaa_frag;
vk::ShaderModule convert_non_msaa_to_msaa_depth_frag;
vk::Sampler linear_sampler;
vk::Sampler nearest_sampler;
@@ -215,9 +195,9 @@ private:
std::vector<vk::Pipeline> msaa_copy_pipelines;
std::vector<BlitMSAAPipelineKey> blit_msaa_color_keys;
std::vector<vk::Pipeline> blit_msaa_color_pipelines;
std::vector<ResolveDepthStencilPipelineKey> resolve_depth_keys;
std::vector<VkRenderPass> resolve_depth_keys;
std::vector<vk::Pipeline> resolve_depth_pipelines;
std::vector<ResolveDepthStencilPipelineKey> resolve_depth_stencil_keys;
std::vector<VkRenderPass> resolve_depth_stencil_keys;
std::vector<vk::Pipeline> resolve_depth_stencil_pipelines;
struct MSAACopyResources {
u64 tick;
@@ -164,9 +164,7 @@ void FixedPipelineState::Refresh(Tegra::Engines::Maxwell3D& maxwell3d, DynamicFe
}
provoking_vertex_last.Assign(use_last_provoking_vertex ? 1 : 0);
if (!features.has_dynamic_state3_conservative_raster_mode) {
conservative_raster_enable.Assign(regs.conservative_raster_enable != 0 ? 1 : 0);
}
conservative_raster_enable.Assign(regs.conservative_raster_enable != 0 ? 1 : 0);
smooth_lines.Assign(regs.line_anti_alias_enable != 0 ? 1 : 0);
alpha_to_coverage_enabled.Assign(regs.anti_alias_alpha_control.alpha_to_coverage != 0 ? 1 : 0);
alpha_to_one_enabled.Assign(regs.anti_alias_alpha_control.alpha_to_one != 0 ? 1 : 0);
@@ -362,35 +360,18 @@ void FixedPipelineState::DynamicState::Refresh2(const Maxwell& regs,
depth_bias_enable.Assign(enabled_lut[POLYGON_OFFSET_ENABLE_LUT[topology_index]] != 0 ? 1 : 0);
}
bool IsDepthClipEnabled(const Maxwell& regs) {
const auto clip = regs.viewport_clip_control.geometry_clip.Value();
return clip == Maxwell::ViewportClipControl::GeometryClip::Passthrough ||
clip == Maxwell::ViewportClipControl::GeometryClip::FrustumXYZ ||
clip == Maxwell::ViewportClipControl::GeometryClip::FrustumZ;
}
bool IsDepthClampEnabled(const Maxwell& regs, bool has_depth_clip_enable) {
if (!IsDepthClipEnabled(regs)) {
return true;
}
if (!has_depth_clip_enable) {
return false;
}
return regs.viewport_clip_control.pixel_min_z.Value() != 0 ||
regs.viewport_clip_control.pixel_max_z.Value() != 0;
}
void FixedPipelineState::DynamicState::Refresh3(const Maxwell& regs,
const DynamicFeatures& features) {
if (!features.has_dynamic_state3_logic_op_enable) {
logic_op_enable.Assign(regs.logic_op.enable != 0 ? 1 : 0);
}
if (features.has_depth_clip_enable) {
depth_clip_disabled.Assign(IsDepthClipEnabled(regs) ? 0 : 1);
}
if (!features.has_dynamic_state3_depth_clamp_enable) {
depth_clamp_disabled.Assign(
IsDepthClampEnabled(regs, features.has_depth_clip_enable) ? 0 : 1);
depth_clamp_disabled.Assign(regs.viewport_clip_control.geometry_clip ==
Maxwell::ViewportClipControl::GeometryClip::Passthrough ||
regs.viewport_clip_control.geometry_clip ==
Maxwell::ViewportClipControl::GeometryClip::FrustumXYZ ||
regs.viewport_clip_control.geometry_clip ==
Maxwell::ViewportClipControl::GeometryClip::FrustumZ);
}
if (!features.has_dynamic_state3_line_stipple_enable) {
line_stipple_enable.Assign(regs.line_stipple_enable);
@@ -30,8 +30,6 @@ struct DynamicFeatures {
bool has_extended_dynamic_state_3_blend;
bool has_extended_dynamic_state_3_enables;
bool has_dynamic_state3_depth_clamp_enable;
bool has_dynamic_state3_conservative_raster_mode;
bool has_depth_clip_enable;
bool has_dynamic_state3_logic_op_enable;
bool has_dynamic_state3_line_stipple_enable;
bool has_dynamic_vertex_input;
@@ -167,7 +165,6 @@ struct FixedPipelineState {
BitField<10, 1, u32> logic_op_enable;
BitField<11, 1, u32> depth_clamp_disabled;
BitField<12, 1, u32> line_stipple_enable;
BitField<13, 1, u32> depth_clip_disabled;
};
union {
u32 raw2;
@@ -301,9 +298,6 @@ static_assert(std::has_unique_object_representations_v<FixedPipelineState>);
static_assert(std::is_trivially_copyable_v<FixedPipelineState>);
static_assert(std::is_trivially_constructible_v<FixedPipelineState>);
bool IsDepthClipEnabled(const Maxwell& regs);
bool IsDepthClampEnabled(const Maxwell& regs, bool has_depth_clip_enable);
} // namespace Vulkan
namespace std {
@@ -47,93 +47,6 @@ using Shader::Backend::SPIRV::NUM_TEXTURE_AND_IMAGE_SCALING_WORDS;
return std::nullopt;
}
[[nodiscard]] inline VkDeviceSize DescriptorSizeForType(const Device& device,
VkDescriptorType type) {
const auto& props = device.DescriptorBufferProperties();
const bool robust = device.IsRobustBufferAccessEnabled();
switch (type) {
case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER:
return robust ? props.robustUniformBufferDescriptorSize : props.uniformBufferDescriptorSize;
case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
return robust ? props.robustStorageBufferDescriptorSize : props.storageBufferDescriptorSize;
case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
return robust ? props.robustUniformTexelBufferDescriptorSize
: props.uniformTexelBufferDescriptorSize;
case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER:
return robust ? props.robustStorageTexelBufferDescriptorSize
: props.storageTexelBufferDescriptorSize;
case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
return props.combinedImageSamplerDescriptorSize;
case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:
return props.storageImageDescriptorSize;
default:
return 0;
}
}
struct DescriptorBufferBinding {
VkDescriptorType type;
u32 count;
VkDeviceSize offset;
VkDeviceSize stride;
};
struct DescriptorBufferLayout {
VkDeviceSize size{};
boost::container::small_vector<DescriptorBufferBinding, 32> bindings;
[[nodiscard]] bool Empty() const noexcept {
return bindings.empty();
}
};
inline void WriteDescriptorBuffer(const Device& device, const DescriptorBufferLayout& layout,
const DescriptorUpdateEntry* payload, u8* host) {
const vk::Device& dev = device.GetLogical();
for (const DescriptorBufferBinding& binding : layout.bindings) {
for (u32 index = 0; index < binding.count; ++index) {
const DescriptorUpdateEntry& entry = *(payload++);
const VkDescriptorAddressInfoEXT address_info{
.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_ADDRESS_INFO_EXT,
.pNext = nullptr,
.address = entry.address.address,
.range = entry.address.range,
.format = entry.address.format,
};
VkDescriptorGetInfoEXT get_info{
.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_GET_INFO_EXT,
.pNext = nullptr,
.type = binding.type,
.data{},
};
switch (binding.type) {
case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER:
get_info.data.pUniformBuffer = &address_info;
break;
case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
get_info.data.pStorageBuffer = &address_info;
break;
case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
get_info.data.pUniformTexelBuffer = &address_info;
break;
case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER:
get_info.data.pStorageTexelBuffer = &address_info;
break;
case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
get_info.data.pCombinedImageSampler = &entry.image;
break;
case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:
get_info.data.pStorageImage = &entry.image;
break;
default:
continue;
}
dev.GetDescriptorEXT(get_info, binding.stride,
host + binding.offset + index * binding.stride);
}
}
}
[[nodiscard]] inline u32 NumDescriptorEntries(const Shader::Info& info) {
return Shader::NumDescriptors(info.constant_buffer_descriptors) +
Shader::NumDescriptors(info.storage_buffers_descriptors) +
@@ -152,59 +65,16 @@ public:
num_descriptors <= device->MaxPushDescriptors();
}
bool CanUseDescriptorBuffer() const noexcept {
return device->IsExtDescriptorBufferSupported() && !bindings.empty() &&
!CanUsePushDescriptor() &&
device->DescriptorBufferProperties().combinedImageSamplerDescriptorSingleArray;
}
DescriptorBufferLayout MakeDescriptorBufferLayout(VkDescriptorSetLayout layout) const {
DescriptorBufferLayout result;
if (!layout) {
return result;
}
const vk::Device& dev = device->GetLogical();
result.size = dev.GetDescriptorSetLayoutSizeEXT(layout);
result.bindings.reserve(bindings.size());
for (const VkDescriptorSetLayoutBinding& binding : bindings) {
result.bindings.push_back(DescriptorBufferBinding{
.type = binding.descriptorType,
.count = binding.descriptorCount,
.offset = dev.GetDescriptorSetLayoutBindingOffsetEXT(layout, binding.binding),
.stride = DescriptorSizeForType(*device, binding.descriptorType),
});
}
return result;
}
vk::DescriptorSetLayout CreateDescriptorSetLayout(bool use_push_descriptor,
bool use_descriptor_buffer = false) const {
// TODO(crueter): utilize layout binding flags
vk::DescriptorSetLayout CreateDescriptorSetLayout(bool use_push_descriptor) const {
if (bindings.empty()) {
return nullptr;
}
VkDescriptorSetLayoutCreateFlags flags = 0;
if (use_push_descriptor) {
flags |= VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR;
}
if (use_descriptor_buffer) {
flags |= VK_DESCRIPTOR_SET_LAYOUT_CREATE_DESCRIPTOR_BUFFER_BIT_EXT;
}
boost::container::small_vector<VkDescriptorBindingFlags, 32> binding_flags;
VkDescriptorSetLayoutBindingFlagsCreateInfo binding_flags_ci{};
const void* pnext = nullptr;
if (!use_push_descriptor && device->IsDescriptorBindingPartiallyBoundSupported()) {
binding_flags.assign(bindings.size(), VK_DESCRIPTOR_BINDING_PARTIALLY_BOUND_BIT);
binding_flags_ci = {
.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_BINDING_FLAGS_CREATE_INFO,
.pNext = nullptr,
.bindingCount = static_cast<u32>(binding_flags.size()),
.pBindingFlags = binding_flags.data(),
};
pnext = &binding_flags_ci;
}
const VkDescriptorSetLayoutCreateFlags flags =
use_push_descriptor ? VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR : 0;
return device->GetLogical().CreateDescriptorSetLayout({
.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
.pNext = pnext,
.pNext = nullptr,
.flags = flags,
.bindingCount = static_cast<u32>(bindings.size()),
.pBindings = bindings.data(),
@@ -491,7 +491,7 @@ static vk::Pipeline CreateWrappedPipelineImpl(
.subpass = 0,
.basePipelineHandle = 0,
.basePipelineIndex = 0,
}, device.StaticPipelineCache());
});
}
vk::Pipeline CreateWrappedPipeline(const Device& device, vk::RenderPass& renderpass,
@@ -69,9 +69,6 @@ vk::Buffer CreateBuffer(const Device& device, const MemoryAllocator& memory_allo
if (device.IsExtConditionalRendering()) {
flags |= VK_BUFFER_USAGE_CONDITIONAL_RENDERING_BIT_EXT;
}
if (device.IsBufferDeviceAddressSupported()) {
flags |= VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT;
}
const VkBufferCreateInfo buffer_ci = {
.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,
.pNext = nullptr,
@@ -94,9 +91,6 @@ Buffer::Buffer(BufferCacheRuntime& runtime, VideoCommon::NullBufferParams null_p
device = &runtime.device;
buffer = runtime.CreateNullBuffer();
is_null = true;
if (device->IsBufferDeviceAddressSupported()) {
device_address = device->GetLogical().GetBufferDeviceAddress(*buffer);
}
}
Buffer::Buffer(BufferCacheRuntime& runtime, DAddr cpu_addr_, u64 size_bytes_)
@@ -106,9 +100,6 @@ Buffer::Buffer(BufferCacheRuntime& runtime, DAddr cpu_addr_, u64 size_bytes_)
if (runtime.device.HasDebuggingToolAttached()) {
buffer.SetObjectNameEXT(fmt::format("Buffer 0x{:x}", CpuAddr()).c_str());
}
if (device->IsBufferDeviceAddressSupported()) {
device_address = device->GetLogical().GetBufferDeviceAddress(*buffer);
}
}
void Buffer::MarkUsage(u64 offset, u64 size) noexcept {
@@ -255,6 +246,7 @@ protected:
StagingBufferPool& staging_pool;
vk::Buffer buffer{};
MemoryCommit memory_commit{};
VkIndexType index_type{};
u32 num_indices = 0;
};
@@ -364,16 +356,6 @@ BufferCacheRuntime::BufferCacheRuntime(const Device& device_, MemoryAllocator& m
scheduler_, staging_pool_);
}
void BufferCacheRuntime::TryEnableUnifiedMemory(void* base, size_t size,
std::span<AHardwareBuffer* const> hardware_buffers,
size_t hardware_buffer_window) {
unified_memory = std::make_unique<HostMemoryImport>(device, base, size, hardware_buffers,
hardware_buffer_window);
if (!unified_memory->IsValid()) {
unified_memory.reset();
}
}
StagingBufferRef BufferCacheRuntime::UploadStagingBuffer(size_t size) {
return staging_pool.Request(size, MemoryUsage::Upload);
}
@@ -382,10 +364,6 @@ StagingBufferRef BufferCacheRuntime::DownloadStagingBuffer(size_t size, bool def
return staging_pool.Request(size, MemoryUsage::Download, deferred);
}
VkFormat BufferCacheRuntime::TexelBufferFormat(VideoCore::Surface::PixelFormat format) const {
return MaxwellToVK::SurfaceFormat(device, FormatType::Buffer, false, format).format;
}
void BufferCacheRuntime::FreeDeferredStagingBuffer(StagingBufferRef& ref) {
staging_pool.FreeDeferred(ref);
}
@@ -398,10 +376,6 @@ u64 BufferCacheRuntime::GetDeviceMemoryUsage() const {
return device.GetDeviceMemoryUsage();
}
u64 BufferCacheRuntime::GetDeviceAllocationUsage() const {
return device.GetMemoryBudgetInfo().allocation_bytes;
}
bool BufferCacheRuntime::CanReportMemoryUsage() const {
return device.CanReportMemoryUsage();
}
@@ -430,16 +404,6 @@ u64 BufferCacheRuntime::KnownGpuTick() {
return scheduler.GetMasterSemaphore().KnownGpuTick();
}
u64 BufferCacheRuntime::CurrentSyncPoint() const noexcept {
return scheduler.GetMasterSemaphore().CurrentTick();
}
u64 BufferCacheRuntime::CompletedSyncPoint() const {
auto& master_semaphore = scheduler.GetMasterSemaphore();
master_semaphore.Refresh();
return master_semaphore.KnownGpuTick();
}
void BufferCacheRuntime::Wait(u64 buffer_tick) {
scheduler.Wait(buffer_tick);
}
@@ -677,7 +641,6 @@ void BufferCacheRuntime::BindTransformFeedbackBuffer(u32 index, VkBuffer buffer,
offset = 0;
size = 0;
}
scheduler.MarkTransformFeedbackUsed();
scheduler.Record([index, buffer, offset, size](vk::CommandBuffer cmdbuf) {
const VkDeviceSize vk_offset = offset;
const VkDeviceSize vk_size = size;
@@ -690,26 +653,19 @@ void BufferCacheRuntime::BindTransformFeedbackBuffers(VideoCommon::HostBindings<
// Already logged in the rasterizer
return;
}
const u32 count = std::min<u32>(static_cast<u32>(bindings.buffers.size()),
VideoCommon::NUM_TRANSFORM_FEEDBACK_BUFFERS);
std::array<VkBuffer, VideoCommon::NUM_TRANSFORM_FEEDBACK_BUFFERS> handles{};
std::array<VkDeviceSize, VideoCommon::NUM_TRANSFORM_FEEDBACK_BUFFERS> offsets{};
std::array<VkDeviceSize, VideoCommon::NUM_TRANSFORM_FEEDBACK_BUFFERS> sizes{};
for (u32 i = 0; i < count; ++i) {
boost::container::static_vector<VkBuffer, VideoCommon::NUM_VERTEX_BUFFERS> buffer_handles(bindings.buffers.size());
for (u32 i = 0; i < bindings.buffers.size(); ++i) {
auto handle = bindings.buffers[i]->Handle();
if (handle == VK_NULL_HANDLE) {
ReserveNullBuffer();
handle = *null_buffer;
} else {
offsets[i] = bindings.offsets[i];
sizes[i] = bindings.sizes[i];
bindings.offsets[i] = 0;
bindings.sizes[i] = 0;
}
handles[i] = handle;
buffer_handles[i] = handle;
}
scheduler.MarkTransformFeedbackUsed();
scheduler.Record([count, handles, offsets, sizes](vk::CommandBuffer cmdbuf) {
cmdbuf.BindTransformFeedbackBuffersEXT(0, count, handles.data(), offsets.data(),
sizes.data());
scheduler.Record([bindings_ = std::move(bindings), buffer_handles_ = std::move(buffer_handles)](vk::CommandBuffer cmdbuf) {
cmdbuf.BindTransformFeedbackBuffersEXT(0, u32(buffer_handles_.size()), buffer_handles_.data(), bindings_.offsets.data(), bindings_.sizes.data());
});
}
@@ -734,9 +690,6 @@ vk::Buffer BufferCacheRuntime::CreateNullBuffer() {
if (device.IsExtTransformFeedbackSupported()) {
create_info.usage |= VK_BUFFER_USAGE_TRANSFORM_FEEDBACK_BUFFER_BIT_EXT;
}
if (device.IsBufferDeviceAddressSupported()) {
create_info.usage |= VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT;
}
vk::Buffer ret = memory_allocator.CreateBuffer(create_info, MemoryUsage::DeviceLocal);
if (device.HasDebuggingToolAttached()) {
ret.SetObjectNameEXT("Null buffer");
@@ -7,8 +7,6 @@
#pragma once
#include <limits>
#include <memory>
#include <span>
#include "video_core/buffer_cache/buffer_cache_base.h"
#include "video_core/buffer_cache/memory_tracker_base.h"
@@ -41,10 +39,6 @@ public:
return *buffer;
}
[[nodiscard]] VkDeviceAddress DeviceAddress() const noexcept {
return device_address;
}
[[nodiscard]] bool IsRegionUsed(u64 offset, u64 size) const noexcept {
return tracker.IsUsed(offset, size);
}
@@ -76,7 +70,6 @@ private:
vk::Buffer buffer;
std::vector<BufferView> views;
VideoCommon::UsageTracker tracker;
VkDeviceAddress device_address{};
u64 last_usage_tick{};
bool is_null{};
};
@@ -99,26 +92,6 @@ public:
void TickFrame(Common::SlotVector<Buffer>& slot_buffers) noexcept;
void TryEnableUnifiedMemory(void* base, size_t size,
std::span<AHardwareBuffer* const> hardware_buffers,
size_t hardware_buffer_window);
[[nodiscard]] bool HasUnifiedMemory() const noexcept {
return unified_memory != nullptr && unified_memory->IsValid();
}
[[nodiscard]] u64 UnifiedMemorySize() const noexcept {
return unified_memory ? unified_memory->GetSize() : 0;
}
[[nodiscard]] u64 UnifiedMemoryWindowSize() const noexcept {
return unified_memory ? unified_memory->GetWindowSize() : 0;
}
[[nodiscard]] VkBuffer UnifiedMemoryWindowBuffer(size_t index) const noexcept {
return unified_memory ? unified_memory->GetWindowBuffer(index) : VK_NULL_HANDLE;
}
u64 CurrentTick();
u64 KnownGpuTick();
@@ -127,16 +100,10 @@ public:
void Finish();
u64 CurrentSyncPoint() const noexcept;
u64 CompletedSyncPoint() const;
u64 GetDeviceLocalMemory() const;
u64 GetDeviceMemoryUsage() const;
u64 GetDeviceAllocationUsage() const;
bool CanReportMemoryUsage() const;
u32 GetUniformBufferAlignment() const;
@@ -178,25 +145,22 @@ public:
[[maybe_unused]] u32 binding_index,
u32 size) {
const StagingBufferRef ref = staging_pool.Request(size, MemoryUsage::Upload);
guest_descriptor_queue.AddBuffer(ref.buffer, ref.device_address,
static_cast<u32>(ref.offset), size);
BindBuffer(ref.buffer, static_cast<u32>(ref.offset), size);
return ref.mapped_span;
}
void BindUniformBuffer(const Buffer& buffer, u32 offset, u32 size) {
void BindUniformBuffer(VkBuffer buffer, u32 offset, u32 size) {
BindBuffer(buffer, offset, size);
}
void BindStorageBuffer(const Buffer& buffer, u32 offset, u32 size,
void BindStorageBuffer(VkBuffer buffer, u32 offset, u32 size,
[[maybe_unused]] bool is_written) {
BindBuffer(buffer, offset, size);
}
void BindTextureBuffer(Buffer& buffer, u32 offset, u32 size,
VideoCore::Surface::PixelFormat format) {
guest_descriptor_queue.AddTexelBuffer(buffer.View(offset, size, format),
buffer.DeviceAddress(), offset, size,
TexelBufferFormat(format));
guest_descriptor_queue.AddTexelBuffer(buffer.View(offset, size, format));
}
bool ShouldLimitDynamicStorageBuffers() const {
@@ -208,17 +172,14 @@ public:
}
private:
void BindBuffer(const Buffer& buffer, u32 offset, u32 size) {
const VkBuffer handle = buffer.Handle();
if (handle == VK_NULL_HANDLE) {
guest_descriptor_queue.AddBuffer(handle, 0, 0, VK_WHOLE_SIZE);
void BindBuffer(VkBuffer buffer, u32 offset, u32 size) {
if (buffer == VK_NULL_HANDLE) {
guest_descriptor_queue.AddBuffer(buffer, 0, VK_WHOLE_SIZE);
} else {
guest_descriptor_queue.AddBuffer(handle, buffer.DeviceAddress(), offset, size);
guest_descriptor_queue.AddBuffer(buffer, offset, size);
}
}
VkFormat TexelBufferFormat(VideoCore::Surface::PixelFormat format) const;
void ReserveNullBuffer();
vk::Buffer CreateNullBuffer();
@@ -232,7 +193,6 @@ private:
std::shared_ptr<QuadStripIndexBuffer> quad_strip_index_buffer;
vk::Buffer null_buffer;
std::unique_ptr<HostMemoryImport> unified_memory;
std::unique_ptr<Uint8Pass> uint8_pass;
QuadIndexedPass quad_index_pass;
@@ -255,7 +215,6 @@ struct BufferCacheParams {
static constexpr bool USE_MEMORY_MAPS = true;
static constexpr bool SEPARATE_IMAGE_BUFFER_BINDINGS = false;
static constexpr bool USE_MEMORY_MAPS_FOR_UPLOADS = true;
static constexpr bool USE_UNIFIED_MEMORY = true;
};
using BufferCache = VideoCommon::BufferCache<BufferCacheParams>;
@@ -1,13 +1,9 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#include <cstddef>
#include "video_core/renderer_vulkan/vk_command_pool.h"
#include "video_core/renderer_vulkan/vk_master_semaphore.h"
#include "video_core/vulkan_common/vulkan_device.h"
#include "video_core/vulkan_common/vulkan_wrapper.h"
@@ -18,52 +14,32 @@ constexpr size_t COMMAND_BUFFER_POOL_SIZE = 4;
struct CommandPool::Pool {
vk::CommandPool handle;
vk::CommandBuffers cmdbufs;
u64 tick;
};
CommandPool::CommandPool(MasterSemaphore& master_semaphore_, const Device& device_)
: master_semaphore{master_semaphore_}, device{device_} {}
: ResourcePool(master_semaphore_, COMMAND_BUFFER_POOL_SIZE), device{device_} {}
CommandPool::~CommandPool() = default;
void CommandPool::AllocatePool() {
void CommandPool::Allocate(size_t begin, size_t end) {
// Command buffers are going to be committed, recorded, executed every single usage cycle.
// They are also going to be reset when committed.
Pool& pool = pools.emplace_back();
pool.handle = device.GetLogical().CreateCommandPool({
.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO,
.pNext = nullptr,
.flags = VK_COMMAND_POOL_CREATE_TRANSIENT_BIT,
.flags =
VK_COMMAND_POOL_CREATE_TRANSIENT_BIT | VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT,
.queueFamilyIndex = device.GetGraphicsFamily(),
});
pool.cmdbufs = pool.handle.Allocate(COMMAND_BUFFER_POOL_SIZE);
pool.tick = 0;
}
void CommandPool::AcquirePool() {
if (!pools.empty()) {
master_semaphore.Refresh();
const u64 gpu_tick = master_semaphore.KnownGpuTick();
for (size_t i = 0; i < pools.size(); ++i) {
const size_t candidate = (current_pool + 1 + i) % pools.size();
if (gpu_tick >= pools[candidate].tick) {
current_pool = candidate;
current_index = 0;
pools[current_pool].handle.Reset();
return;
}
}
}
AllocatePool();
current_pool = pools.size() - 1;
current_index = 0;
}
VkCommandBuffer CommandPool::Commit() {
if (pools.empty() || current_index >= COMMAND_BUFFER_POOL_SIZE) {
AcquirePool();
}
Pool& pool = pools[current_pool];
pool.tick = master_semaphore.CurrentTick();
return pool.cmdbufs[current_index++];
const size_t index = CommitResource();
const auto pool_index = index / COMMAND_BUFFER_POOL_SIZE;
const auto sub_index = index % COMMAND_BUFFER_POOL_SIZE;
return pools[pool_index].cmdbufs[sub_index];
}
} // namespace Vulkan
@@ -1,6 +1,3 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
@@ -9,7 +6,7 @@
#include <cstddef>
#include <vector>
#include "common/common_types.h"
#include "video_core/renderer_vulkan/vk_resource_pool.h"
#include "video_core/vulkan_common/vulkan_wrapper.h"
namespace Vulkan {
@@ -17,24 +14,20 @@ namespace Vulkan {
class Device;
class MasterSemaphore;
class CommandPool final {
class CommandPool final : public ResourcePool {
public:
explicit CommandPool(MasterSemaphore& master_semaphore_, const Device& device_);
~CommandPool();
~CommandPool() override;
void Allocate(size_t begin, size_t end) override;
VkCommandBuffer Commit();
private:
struct Pool;
void AllocatePool();
void AcquirePool();
MasterSemaphore& master_semaphore;
const Device& device;
std::vector<Pool> pools;
size_t current_pool = 0;
size_t current_index = 0;
};
} // namespace Vulkan
@@ -5,12 +5,10 @@
// SPDX-License-Identifier: GPL-2.0-or-later
#include <array>
#include <cstring>
#include <memory>
#include <numeric>
#include <optional>
#include <utility>
#include <vector>
#include "video_core/renderer_vulkan/vk_texture_cache.h"
@@ -24,9 +22,7 @@
#include "video_core/host_shaders/resolve_conditional_render_comp_spv.h"
#include "video_core/host_shaders/vulkan_quad_indexed_comp_spv.h"
#include "video_core/host_shaders/vulkan_uint8_comp_spv.h"
#include "video_core/host_shaders/block_linear_unswizzle_2d_buffer_comp_spv.h"
#include "video_core/host_shaders/block_linear_unswizzle_3d_bcn_comp_spv.h"
#include "video_core/host_shaders/block_linear_unswizzle_3d_buffer_comp_spv.h"
#include "video_core/renderer_vulkan/vk_compute_pass.h"
#include "video_core/surface.h"
#include "video_core/renderer_vulkan/vk_descriptor_pool.h"
@@ -35,7 +31,6 @@
#include "video_core/renderer_vulkan/vk_update_descriptor.h"
#include "video_core/texture_cache/accelerated_swizzle.h"
#include "video_core/texture_cache/types.h"
#include "video_core/texture_cache/util.h"
#include "video_core/textures/decoders.h"
#include "video_core/vulkan_common/vulkan_device.h"
#include "video_core/vulkan_common/vulkan_wrapper.h"
@@ -273,7 +268,7 @@ ComputePass::ComputePass(const Device& device_, Scheduler& scheduler, Descriptor
.layout = *layout,
.basePipelineHandle = {},
.basePipelineIndex = 0,
}, device.StaticPipelineCache());
});
}
ComputePass::~ComputePass() = default;
@@ -575,7 +570,7 @@ void ASTCDecoderPass::Assemble(Image& image, const StagingBufferRef& map,
const void* const descriptor_data{compute_pass_descriptor_queue.UpdateData()};
// To unswizzle the ASTC data
const auto params = VideoCommon::Accelerated::MakeBlockLinearSwizzle2DParams(swizzle, image.info);
const auto params = MakeBlockLinearSwizzle2DParams(swizzle, image.info);
ASSERT(params.origin == (std::array<u32, 3>{0, 0, 0}));
ASSERT(params.destination == (std::array<s32, 3>{0, 0, 0}));
ASSERT(params.bytes_per_block_log2 == 4);
@@ -718,7 +713,7 @@ void BlockLinearUnswizzle3DPass::Unswizzle(
ASSERT(swizzles.size() == 1);
const auto& sw = swizzles[0];
const auto params = VideoCommon::Accelerated::MakeBlockLinearSwizzle3DParams(sw, image.info);
const auto params = MakeBlockLinearSwizzle3DParams(sw, image.info);
const u32 blocks_x = (image.info.size.width + 3) / 4;
const u32 blocks_y = (image.info.size.height + 3) / 4;
@@ -877,626 +872,4 @@ void BlockLinearUnswizzle3DPass::UnswizzleChunk(
});
}
namespace {
constexpr u32 BL2D_BINDING_INPUT_BUFFER = 0;
constexpr u32 BL2D_BINDING_OUTPUT_BUFFER = 1;
struct alignas(16) BlockLinearUnswizzle2DPushConstants {
std::array<u32, 3> dim;
u32 bytes_per_block_log2;
std::array<u32, 3> origin;
u32 layer_stride;
u32 block_size;
u32 x_shift;
u32 block_height;
u32 block_height_mask;
};
static_assert(sizeof(BlockLinearUnswizzle2DPushConstants) <= 128);
constexpr std::array<VkDescriptorSetLayoutBinding, 2> BL2D_BINDINGS{{
{
.binding = BL2D_BINDING_INPUT_BUFFER,
.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
.descriptorCount = 1,
.stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
.pImmutableSamplers = nullptr,
},
{
.binding = BL2D_BINDING_OUTPUT_BUFFER,
.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
.descriptorCount = 1,
.stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
.pImmutableSamplers = nullptr,
},
}};
constexpr std::array<VkDescriptorUpdateTemplateEntry, 2> BL2D_TEMPLATE{{
{
.dstBinding = BL2D_BINDING_INPUT_BUFFER,
.dstArrayElement = 0,
.descriptorCount = 1,
.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
.offset = BL2D_BINDING_INPUT_BUFFER * sizeof(DescriptorUpdateEntry),
.stride = sizeof(DescriptorUpdateEntry),
},
{
.dstBinding = BL2D_BINDING_OUTPUT_BUFFER,
.dstArrayElement = 0,
.descriptorCount = 1,
.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
.offset = BL2D_BINDING_OUTPUT_BUFFER * sizeof(DescriptorUpdateEntry),
.stride = sizeof(DescriptorUpdateEntry),
},
}};
constexpr DescriptorBankInfo BL2D_BANK_INFO{
.uniform_buffers = 0,
.storage_buffers = 2,
.texture_buffers = 0,
.image_buffers = 0,
.textures = 0,
.images = 0,
.score = 2,
};
constexpr bool BL2D_VERIFY_AGAINST_CPU = false;
} // Anonymous namespace
BlockLinearUnswizzle2DPass::BlockLinearUnswizzle2DPass(
const Device& device_, Scheduler& scheduler_, DescriptorPool& descriptor_pool_,
StagingBufferPool& staging_buffer_pool_,
ComputePassDescriptorQueue& compute_pass_descriptor_queue_)
: ComputePass(device_, scheduler_, descriptor_pool_, BL2D_BINDINGS, BL2D_TEMPLATE,
BL2D_BANK_INFO,
COMPUTE_PUSH_CONSTANT_RANGE<sizeof(BlockLinearUnswizzle2DPushConstants)>,
BLOCK_LINEAR_UNSWIZZLE_2D_BUFFER_COMP_SPV),
scheduler{scheduler_}, staging_buffer_pool{staging_buffer_pool_},
compute_pass_descriptor_queue{compute_pass_descriptor_queue_} {}
BlockLinearUnswizzle2DPass::~BlockLinearUnswizzle2DPass() = default;
bool BlockLinearUnswizzle2DPass::IsSupported(const VideoCommon::ImageInfo& info) {
if (info.type != VideoCommon::ImageType::e2D) {
return false;
}
if (info.resources.levels != 1) {
return false;
}
if (info.num_samples > 1) {
return false;
}
if (VideoCore::Surface::IsPixelFormatASTC(info.format) ||
VideoCore::Surface::IsPixelFormatBCn(info.format)) {
return false;
}
const u32 bytes_per_block = VideoCore::Surface::BytesPerBlock(info.format);
if (bytes_per_block != 4 && bytes_per_block != 8 && bytes_per_block != 16) {
return false;
}
return VideoCore::Surface::DefaultBlockWidth(info.format) == 1 &&
VideoCore::Surface::DefaultBlockHeight(info.format) == 1;
}
void BlockLinearUnswizzle2DPass::Unswizzle(
Image& image, const StagingBufferRef& swizzled,
std::span<const VideoCommon::SwizzleParameters> swizzles) {
using namespace VideoCommon::Accelerated;
if (swizzles.empty()) {
return;
}
const VideoCommon::SwizzleParameters& sw = swizzles.front();
const auto params = VideoCommon::Accelerated::MakeBlockLinearSwizzle2DParams(sw, image.info);
const u32 width = sw.num_tiles.width;
const u32 height = sw.num_tiles.height;
const u32 depth = image.info.resources.layers;
const u32 bytes_per_block = 1u << params.bytes_per_block_log2;
const VkDeviceSize output_size =
static_cast<VkDeviceSize>(width) * height * depth * bytes_per_block;
const StagingBufferRef output = staging_buffer_pool.Request(
static_cast<size_t>(output_size), MemoryUsage::DeviceLocal);
BlockLinearUnswizzle2DPushConstants pc{};
pc.dim = {width, height, depth};
pc.bytes_per_block_log2 = params.bytes_per_block_log2;
pc.origin = params.origin;
pc.layer_stride = params.layer_stride;
pc.block_size = params.block_size;
pc.x_shift = params.x_shift;
pc.block_height = params.block_height;
pc.block_height_mask = params.block_height_mask;
scheduler.RequestOutsideRenderPassOperationContext();
compute_pass_descriptor_queue.Acquire(scheduler, 2);
compute_pass_descriptor_queue.AddBuffer(swizzled.buffer,
sw.buffer_offset + swizzled.offset,
image.guest_size_bytes - sw.buffer_offset);
compute_pass_descriptor_queue.AddBuffer(output.buffer, output.offset, output_size);
const void* descriptor_data = compute_pass_descriptor_queue.UpdateData();
const VkDescriptorSet set = descriptor_allocator.Commit();
const u32 gx = Common::DivCeil(width, 16u);
const u32 gy = Common::DivCeil(height, 8u);
const bool is_initialized = image.ExchangeInitialization();
const VkBuffer out_buffer = output.buffer;
const VkDeviceSize out_offset = output.offset;
const VkImage dst_image = image.Handle();
const VkImageAspectFlags aspect = image.AspectMask();
scheduler.Record([this, set, descriptor_data, pc, gx, gy, depth, output_size, out_buffer,
out_offset, dst_image, aspect, width, height,
is_initialized](vk::CommandBuffer cmdbuf) {
if (dst_image == VK_NULL_HANDLE || out_buffer == VK_NULL_HANDLE) {
return;
}
device.GetLogical().UpdateDescriptorSet(set, *descriptor_template, descriptor_data);
cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_COMPUTE, *pipeline);
cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_COMPUTE, *layout, 0, set, {});
cmdbuf.PushConstants(*layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc);
cmdbuf.Dispatch(gx, gy, depth);
const VkBufferMemoryBarrier buffer_barrier{
.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER,
.pNext = nullptr,
.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT,
.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.buffer = out_buffer,
.offset = out_offset,
.size = output_size,
};
const VkImageMemoryBarrier pre_copy{
.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
.pNext = nullptr,
.srcAccessMask = static_cast<VkAccessFlags>(
is_initialized ? VK_ACCESS_SHADER_READ_BIT : VK_ACCESS_NONE),
.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,
.oldLayout = is_initialized ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_UNDEFINED,
.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.image = dst_image,
.subresourceRange{
.aspectMask = aspect,
.baseMipLevel = 0,
.levelCount = VK_REMAINING_MIP_LEVELS,
.baseArrayLayer = 0,
.layerCount = VK_REMAINING_ARRAY_LAYERS,
},
};
cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT |
(is_initialized ? vk::PIPELINE_STAGE_GRAPHICS_COMPUTE
: VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT),
VK_PIPELINE_STAGE_TRANSFER_BIT, 0, {}, buffer_barrier, pre_copy);
const VkBufferImageCopy copy{
.bufferOffset = out_offset,
.bufferRowLength = 0,
.bufferImageHeight = 0,
.imageSubresource{
.aspectMask = aspect,
.mipLevel = 0,
.baseArrayLayer = 0,
.layerCount = depth,
},
.imageOffset = {0, 0, 0},
.imageExtent = {width, height, 1},
};
cmdbuf.CopyBufferToImage(out_buffer, dst_image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, copy);
const VkImageMemoryBarrier post_copy{
.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
.pNext = nullptr,
.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,
.dstAccessMask = VK_ACCESS_SHADER_READ_BIT,
.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
.newLayout = VK_IMAGE_LAYOUT_GENERAL,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.image = dst_image,
.subresourceRange{
.aspectMask = aspect,
.baseMipLevel = 0,
.levelCount = VK_REMAINING_MIP_LEVELS,
.baseArrayLayer = 0,
.layerCount = VK_REMAINING_ARRAY_LAYERS,
},
};
cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT,
vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, 0, {}, {}, post_copy);
});
if constexpr (BL2D_VERIFY_AGAINST_CPU) {
VerifyAgainstCpu(swizzled, sw, output, output_size, width, height, depth,
bytes_per_block);
}
}
void BlockLinearUnswizzle2DPass::VerifyAgainstCpu(const StagingBufferRef& swizzled,
const VideoCommon::SwizzleParameters& sw,
const StagingBufferRef& gpu_output,
VkDeviceSize output_size, u32 width, u32 height,
u32 depth, u32 bytes_per_block) {
const StagingBufferRef readback =
staging_buffer_pool.Request(static_cast<size_t>(output_size), MemoryUsage::Download);
const VkBuffer src = gpu_output.buffer;
const VkDeviceSize src_offset = gpu_output.offset;
const VkBuffer dst = readback.buffer;
const VkDeviceSize dst_offset = readback.offset;
scheduler.Record([src, src_offset, dst, dst_offset, output_size](vk::CommandBuffer cmdbuf) {
const VkBufferMemoryBarrier barrier{
.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER,
.pNext = nullptr,
.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT,
.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.buffer = src,
.offset = src_offset,
.size = output_size,
};
cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT,
0, {}, barrier, {});
const VkBufferCopy copy{
.srcOffset = src_offset,
.dstOffset = dst_offset,
.size = output_size,
};
cmdbuf.CopyBuffer(src, dst, copy);
});
scheduler.Finish();
const size_t size = static_cast<size_t>(output_size);
std::vector<u8> reference(size);
const std::span<const u8> input{swizzled.mapped_span.data() + sw.buffer_offset,
swizzled.mapped_span.size() - sw.buffer_offset};
Tegra::Texture::UnswizzleTexture(reference, input, bytes_per_block, width, height, depth,
sw.block.height, sw.block.depth);
const u8* gpu_data = readback.mapped_span.data();
if (std::memcmp(reference.data(), gpu_data, size) == 0) {
LOG_INFO(Render_Vulkan, "BL2D verify OK: {}x{}x{} bpb={} ({} bytes)", width, height, depth,
bytes_per_block, size);
return;
}
size_t first_diff = size;
size_t num_diff = 0;
for (size_t i = 0; i < size; ++i) {
if (reference[i] != gpu_data[i]) {
if (first_diff == size) {
first_diff = i;
}
++num_diff;
}
}
LOG_CRITICAL(Render_Vulkan,
"BL2D verify FAILED: {}x{}x{} bpb={} block_height={} first_diff={} "
"num_diff={}/{} cpu=0x{:02x} gpu=0x{:02x}",
width, height, depth, bytes_per_block, sw.block.height, first_diff,
num_diff, size, reference[first_diff], gpu_data[first_diff]);
}
namespace {
constexpr u32 BL3DB_BINDING_INPUT_BUFFER = 0;
constexpr u32 BL3DB_BINDING_OUTPUT_BUFFER = 1;
struct alignas(16) BlockLinearUnswizzle3DBufferPushConstants {
std::array<u32, 3> dim;
u32 bytes_per_block_log2;
std::array<u32, 3> origin;
u32 slice_size;
u32 block_size;
u32 x_shift;
u32 block_height;
u32 block_height_mask;
u32 block_depth;
u32 block_depth_mask;
};
static_assert(sizeof(BlockLinearUnswizzle3DBufferPushConstants) <= 128);
constexpr std::array<VkDescriptorSetLayoutBinding, 2> BL3DB_BINDINGS{{
{
.binding = BL3DB_BINDING_INPUT_BUFFER,
.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
.descriptorCount = 1,
.stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
.pImmutableSamplers = nullptr,
},
{
.binding = BL3DB_BINDING_OUTPUT_BUFFER,
.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
.descriptorCount = 1,
.stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
.pImmutableSamplers = nullptr,
},
}};
constexpr std::array<VkDescriptorUpdateTemplateEntry, 2> BL3DB_TEMPLATE{{
{
.dstBinding = BL3DB_BINDING_INPUT_BUFFER,
.dstArrayElement = 0,
.descriptorCount = 1,
.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
.offset = BL3DB_BINDING_INPUT_BUFFER * sizeof(DescriptorUpdateEntry),
.stride = sizeof(DescriptorUpdateEntry),
},
{
.dstBinding = BL3DB_BINDING_OUTPUT_BUFFER,
.dstArrayElement = 0,
.descriptorCount = 1,
.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
.offset = BL3DB_BINDING_OUTPUT_BUFFER * sizeof(DescriptorUpdateEntry),
.stride = sizeof(DescriptorUpdateEntry),
},
}};
constexpr DescriptorBankInfo BL3DB_BANK_INFO{
.uniform_buffers = 0,
.storage_buffers = 2,
.texture_buffers = 0,
.image_buffers = 0,
.textures = 0,
.images = 0,
.score = 2,
};
constexpr bool BL3DB_VERIFY_AGAINST_CPU = false;
} // Anonymous namespace
BlockLinearUnswizzle3DBufferPass::BlockLinearUnswizzle3DBufferPass(
const Device& device_, Scheduler& scheduler_, DescriptorPool& descriptor_pool_,
StagingBufferPool& staging_buffer_pool_,
ComputePassDescriptorQueue& compute_pass_descriptor_queue_)
: ComputePass(device_, scheduler_, descriptor_pool_, BL3DB_BINDINGS, BL3DB_TEMPLATE,
BL3DB_BANK_INFO,
COMPUTE_PUSH_CONSTANT_RANGE<sizeof(BlockLinearUnswizzle3DBufferPushConstants)>,
BLOCK_LINEAR_UNSWIZZLE_3D_BUFFER_COMP_SPV),
scheduler{scheduler_}, staging_buffer_pool{staging_buffer_pool_},
compute_pass_descriptor_queue{compute_pass_descriptor_queue_} {}
BlockLinearUnswizzle3DBufferPass::~BlockLinearUnswizzle3DBufferPass() = default;
bool BlockLinearUnswizzle3DBufferPass::IsSupported(const Device& device,
const VideoCommon::ImageInfo& info) {
if (info.type != VideoCommon::ImageType::e3D) {
return false;
}
if (info.resources.levels != 1 || info.resources.layers != 1) {
return false;
}
if (info.num_samples > 1) {
return false;
}
if (info.size.depth <= 1) {
return false;
}
if (VideoCore::Surface::IsPixelFormatASTC(info.format)) {
return false;
}
if (VideoCore::Surface::IsPixelFormatBCn(info.format) && !device.IsOptimalBcnSupported()) {
return false;
}
const u32 bytes_per_block = VideoCore::Surface::BytesPerBlock(info.format);
return bytes_per_block == 4 || bytes_per_block == 8 || bytes_per_block == 16;
}
void BlockLinearUnswizzle3DBufferPass::Unswizzle(
Image& image, const StagingBufferRef& swizzled,
std::span<const VideoCommon::SwizzleParameters> swizzles) {
if (swizzles.empty()) {
return;
}
const VideoCommon::SwizzleParameters& sw = swizzles.front();
const auto params = VideoCommon::Accelerated::MakeBlockLinearSwizzle3DParams(sw, image.info);
const u32 blocks_x = sw.num_tiles.width;
const u32 blocks_y = sw.num_tiles.height;
const u32 blocks_z = sw.num_tiles.depth;
const u32 bytes_per_block = 1u << params.bytes_per_block_log2;
const VkDeviceSize output_size =
static_cast<VkDeviceSize>(blocks_x) * blocks_y * blocks_z * bytes_per_block;
const StagingBufferRef output =
staging_buffer_pool.Request(static_cast<size_t>(output_size), MemoryUsage::DeviceLocal);
BlockLinearUnswizzle3DBufferPushConstants pc{};
pc.dim = {blocks_x, blocks_y, blocks_z};
pc.bytes_per_block_log2 = params.bytes_per_block_log2;
pc.origin = params.origin;
pc.slice_size = params.slice_size;
pc.block_size = params.block_size;
pc.x_shift = params.x_shift;
pc.block_height = params.block_height;
pc.block_height_mask = params.block_height_mask;
pc.block_depth = params.block_depth;
pc.block_depth_mask = params.block_depth_mask;
scheduler.RequestOutsideRenderPassOperationContext();
compute_pass_descriptor_queue.Acquire(scheduler, 2);
compute_pass_descriptor_queue.AddBuffer(swizzled.buffer, sw.buffer_offset + swizzled.offset,
image.guest_size_bytes - sw.buffer_offset);
compute_pass_descriptor_queue.AddBuffer(output.buffer, output.offset, output_size);
const void* descriptor_data = compute_pass_descriptor_queue.UpdateData();
const VkDescriptorSet set = descriptor_allocator.Commit();
const u32 gx = Common::DivCeil(blocks_x, 8u);
const u32 gy = Common::DivCeil(blocks_y, 8u);
const u32 gz = Common::DivCeil(blocks_z, 4u);
const bool is_initialized = image.ExchangeInitialization();
const VkBuffer out_buffer = output.buffer;
const VkDeviceSize out_offset = output.offset;
const VkImage dst_image = image.Handle();
const VkImageAspectFlags aspect = image.AspectMask();
const VkExtent3D extent{
.width = image.info.size.width,
.height = image.info.size.height,
.depth = image.info.size.depth,
};
scheduler.Record([this, set, descriptor_data, pc, gx, gy, gz, output_size, out_buffer,
out_offset, dst_image, aspect, extent,
is_initialized](vk::CommandBuffer cmdbuf) {
if (dst_image == VK_NULL_HANDLE || out_buffer == VK_NULL_HANDLE) {
return;
}
device.GetLogical().UpdateDescriptorSet(set, *descriptor_template, descriptor_data);
cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_COMPUTE, *pipeline);
cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_COMPUTE, *layout, 0, set, {});
cmdbuf.PushConstants(*layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc);
cmdbuf.Dispatch(gx, gy, gz);
const VkBufferMemoryBarrier buffer_barrier{
.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER,
.pNext = nullptr,
.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT,
.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.buffer = out_buffer,
.offset = out_offset,
.size = output_size,
};
const VkImageMemoryBarrier pre_copy{
.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
.pNext = nullptr,
.srcAccessMask = static_cast<VkAccessFlags>(
is_initialized ? VK_ACCESS_SHADER_READ_BIT : VK_ACCESS_NONE),
.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,
.oldLayout = is_initialized ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_UNDEFINED,
.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.image = dst_image,
.subresourceRange{
.aspectMask = aspect,
.baseMipLevel = 0,
.levelCount = VK_REMAINING_MIP_LEVELS,
.baseArrayLayer = 0,
.layerCount = VK_REMAINING_ARRAY_LAYERS,
},
};
cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT |
(is_initialized ? vk::PIPELINE_STAGE_GRAPHICS_COMPUTE
: VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT),
VK_PIPELINE_STAGE_TRANSFER_BIT, 0, {}, buffer_barrier, pre_copy);
const VkBufferImageCopy copy{
.bufferOffset = out_offset,
.bufferRowLength = 0,
.bufferImageHeight = 0,
.imageSubresource{
.aspectMask = aspect,
.mipLevel = 0,
.baseArrayLayer = 0,
.layerCount = 1,
},
.imageOffset = {0, 0, 0},
.imageExtent = extent,
};
cmdbuf.CopyBufferToImage(out_buffer, dst_image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, copy);
const VkImageMemoryBarrier post_copy{
.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
.pNext = nullptr,
.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,
.dstAccessMask = VK_ACCESS_SHADER_READ_BIT,
.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
.newLayout = VK_IMAGE_LAYOUT_GENERAL,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.image = dst_image,
.subresourceRange{
.aspectMask = aspect,
.baseMipLevel = 0,
.levelCount = VK_REMAINING_MIP_LEVELS,
.baseArrayLayer = 0,
.layerCount = VK_REMAINING_ARRAY_LAYERS,
},
};
cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT,
vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, 0, {}, {}, post_copy);
});
if constexpr (BL3DB_VERIFY_AGAINST_CPU) {
VerifyAgainstCpu(swizzled, sw, image.info, output, output_size, blocks_x, blocks_y,
blocks_z, bytes_per_block);
}
}
void BlockLinearUnswizzle3DBufferPass::VerifyAgainstCpu(
const StagingBufferRef& swizzled, const VideoCommon::SwizzleParameters& sw,
const VideoCommon::ImageInfo& info, const StagingBufferRef& gpu_output,
VkDeviceSize output_size, u32 blocks_x, u32 blocks_y, u32 blocks_z, u32 bytes_per_block) {
const StagingBufferRef readback =
staging_buffer_pool.Request(static_cast<size_t>(output_size), MemoryUsage::Download);
const VkBuffer src = gpu_output.buffer;
const VkDeviceSize src_offset = gpu_output.offset;
const VkBuffer dst = readback.buffer;
const VkDeviceSize dst_offset = readback.offset;
scheduler.Record([src, src_offset, dst, dst_offset, output_size](vk::CommandBuffer cmdbuf) {
const VkBufferMemoryBarrier barrier{
.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER,
.pNext = nullptr,
.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT,
.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.buffer = src,
.offset = src_offset,
.size = output_size,
};
cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT,
0, {}, barrier, {});
const VkBufferCopy copy{
.srcOffset = src_offset,
.dstOffset = dst_offset,
.size = output_size,
};
cmdbuf.CopyBuffer(src, dst, copy);
});
scheduler.Finish();
const size_t size = static_cast<size_t>(output_size);
std::vector<u8> reference(size);
const std::span<const u8> input{swizzled.mapped_span.data() + sw.buffer_offset,
swizzled.mapped_span.size() - sw.buffer_offset};
const u32 stride_alignment = VideoCommon::CalculateLevelStrideAlignment(info, sw.level);
Tegra::Texture::UnswizzleTexture(reference, input, bytes_per_block, blocks_x, blocks_y, blocks_z,
sw.block.height, sw.block.depth, stride_alignment);
const u8* gpu_data = readback.mapped_span.data();
if (std::memcmp(reference.data(), gpu_data, size) == 0) {
LOG_INFO(Render_Vulkan, "BL3D verify OK: {}x{}x{} bpb={} ({} bytes)", blocks_x, blocks_y,
blocks_z, bytes_per_block, size);
return;
}
size_t first_diff = size;
size_t num_diff = 0;
for (size_t i = 0; i < size; ++i) {
if (reference[i] != gpu_data[i]) {
if (first_diff == size) {
first_diff = i;
}
++num_diff;
}
}
LOG_CRITICAL(Render_Vulkan,
"BL3D verify FAILED: {}x{}x{} bpb={} block_height={} block_depth={} "
"first_diff={} num_diff={}/{} cpu=0x{:02x} gpu=0x{:02x}",
blocks_x, blocks_y, blocks_z, bytes_per_block, sw.block.height, sw.block.depth,
first_diff, num_diff, size, reference[first_diff], gpu_data[first_diff]);
}
} // namespace Vulkan
@@ -164,54 +164,4 @@ private:
ComputePassDescriptorQueue& compute_pass_descriptor_queue;
};
class BlockLinearUnswizzle2DPass final : public ComputePass {
public:
explicit BlockLinearUnswizzle2DPass(const Device& device_, Scheduler& scheduler_,
DescriptorPool& descriptor_pool_,
StagingBufferPool& staging_buffer_pool_,
ComputePassDescriptorQueue& compute_pass_descriptor_queue_);
~BlockLinearUnswizzle2DPass();
[[nodiscard]] static bool IsSupported(const VideoCommon::ImageInfo& info);
void Unswizzle(Image& image, const StagingBufferRef& swizzled,
std::span<const VideoCommon::SwizzleParameters> swizzles);
private:
void VerifyAgainstCpu(const StagingBufferRef& swizzled,
const VideoCommon::SwizzleParameters& sw,
const StagingBufferRef& gpu_output, VkDeviceSize output_size, u32 width,
u32 height, u32 depth, u32 bytes_per_block);
Scheduler& scheduler;
StagingBufferPool& staging_buffer_pool;
ComputePassDescriptorQueue& compute_pass_descriptor_queue;
};
class BlockLinearUnswizzle3DBufferPass final : public ComputePass {
public:
explicit BlockLinearUnswizzle3DBufferPass(
const Device& device_, Scheduler& scheduler_, DescriptorPool& descriptor_pool_,
StagingBufferPool& staging_buffer_pool_,
ComputePassDescriptorQueue& compute_pass_descriptor_queue_);
~BlockLinearUnswizzle3DBufferPass();
[[nodiscard]] static bool IsSupported(const Device& device,
const VideoCommon::ImageInfo& info);
void Unswizzle(Image& image, const StagingBufferRef& swizzled,
std::span<const VideoCommon::SwizzleParameters> swizzles);
private:
void VerifyAgainstCpu(const StagingBufferRef& swizzled,
const VideoCommon::SwizzleParameters& sw,
const VideoCommon::ImageInfo& info,
const StagingBufferRef& gpu_output, VkDeviceSize output_size,
u32 blocks_x, u32 blocks_y, u32 blocks_z, u32 bytes_per_block);
Scheduler& scheduler;
StagingBufferPool& staging_buffer_pool;
ComputePassDescriptorQueue& compute_pass_descriptor_queue;
};
} // namespace Vulkan
@@ -34,14 +34,12 @@ using Tegra::Texture::TexturePair;
ComputePipeline::ComputePipeline(const Device& device_, Scheduler& scheduler, vk::PipelineCache& pipeline_cache_,
DescriptorPool& descriptor_pool,
GuestDescriptorQueue& guest_descriptor_queue_,
DescriptorBufferRing& descriptor_buffer_ring_,
Common::ThreadWorker* thread_worker,
PipelineStatistics* pipeline_statistics,
VideoCore::ShaderNotify* shader_notify, const Shader::Info& info_,
vk::ShaderModule spv_module_, u64 shader_hash_)
: device{device_},
pipeline_cache(pipeline_cache_), guest_descriptor_queue{guest_descriptor_queue_},
descriptor_buffer_ring{descriptor_buffer_ring_}, info{info_},
pipeline_cache(pipeline_cache_), guest_descriptor_queue{guest_descriptor_queue_}, info{info_},
shader_hash{shader_hash_}, spv_module(std::move(spv_module_)) {
if (shader_notify) {
shader_notify->MarkShaderBuilding();
@@ -50,35 +48,18 @@ ComputePipeline::ComputePipeline(const Device& device_, Scheduler& scheduler, vk
uniform_buffer_sizes.begin());
num_descriptor_entries = NumDescriptorEntries(info);
DescriptorLayoutBuilder builder{device};
builder.Add(info, VK_SHADER_STAGE_COMPUTE_BIT);
auto func{[this, &scheduler, &descriptor_pool, shader_notify, pipeline_statistics] {
DescriptorLayoutBuilder builder{device};
builder.Add(info, VK_SHADER_STAGE_COMPUTE_BIT);
uses_push_descriptor = builder.CanUsePushDescriptor();
uses_descriptor_buffer = builder.CanUseDescriptorBuffer() && descriptor_buffer_ring.IsValid();
descriptor_set_layout =
builder.CreateDescriptorSetLayout(uses_push_descriptor, uses_descriptor_buffer);
if (uses_descriptor_buffer) {
descriptor_buffer_layout = builder.MakeDescriptorBufferLayout(*descriptor_set_layout);
if (descriptor_buffer_layout.size > DescriptorBufferRing::MaxAllocationSize()) {
LOG_WARNING(Render_Vulkan,
"Compute shader {:016X} needs {} descriptor bytes, falling back to sets",
shader_hash, descriptor_buffer_layout.size);
uses_descriptor_buffer = false;
descriptor_buffer_layout = {};
descriptor_set_layout = builder.CreateDescriptorSetLayout(false);
}
}
pipeline_layout = builder.CreatePipelineLayout(*descriptor_set_layout);
if (!uses_descriptor_buffer) {
uses_push_descriptor = builder.CanUsePushDescriptor();
descriptor_set_layout = builder.CreateDescriptorSetLayout(uses_push_descriptor);
pipeline_layout = builder.CreatePipelineLayout(*descriptor_set_layout);
descriptor_update_template =
builder.CreateTemplate(*descriptor_set_layout, *pipeline_layout, uses_push_descriptor);
if (!uses_push_descriptor) {
descriptor_allocator =
descriptor_pool.Allocator(device, scheduler, *descriptor_set_layout, info);
descriptor_allocator = descriptor_pool.Allocator(device, scheduler, *descriptor_set_layout, info);
}
}
auto func{[this, shader_notify, pipeline_statistics] {
const VkPipelineShaderStageRequiredSubgroupSizeCreateInfoEXT subgroup_size_ci{
.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_REQUIRED_SUBGROUP_SIZE_CREATE_INFO_EXT,
.pNext = nullptr,
@@ -88,20 +69,9 @@ ComputePipeline::ComputePipeline(const Device& device_, Scheduler& scheduler, vk
if (device.IsKhrPipelineExecutablePropertiesEnabled() && Settings::values.renderer_debug.GetValue()) {
flags |= VK_PIPELINE_CREATE_CAPTURE_STATISTICS_BIT_KHR;
}
if (uses_descriptor_buffer) {
flags |= VK_PIPELINE_CREATE_DESCRIPTOR_BUFFER_BIT_EXT;
}
VkPipelineCreationFeedback creation_feedback{};
const VkPipelineCreationFeedbackCreateInfo feedback_ci{
.sType = VK_STRUCTURE_TYPE_PIPELINE_CREATION_FEEDBACK_CREATE_INFO,
.pNext = nullptr,
.pPipelineCreationFeedback = &creation_feedback,
.pipelineStageCreationFeedbackCount = 0,
.pPipelineStageCreationFeedbacks = nullptr,
};
const VkComputePipelineCreateInfo compute_ci{
.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
.pNext = device.IsExtPipelineCreationFeedbackSupported() ? &feedback_ci : nullptr,
.pNext = nullptr,
.flags = flags,
.stage{
.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
@@ -131,14 +101,6 @@ ComputePipeline::ComputePipeline(const Device& device_, Scheduler& scheduler, vk
return;
}
if ((creation_feedback.flags & VK_PIPELINE_CREATION_FEEDBACK_VALID_BIT) != 0) {
const bool cache_hit =
(creation_feedback.flags &
VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT) != 0;
LOG_DEBUG(Render_Vulkan, "Compute pipeline {:016X} cache_hit={} duration={}us",
shader_hash, cache_hit, creation_feedback.duration / 1000);
}
// Log compute pipeline creation
if (GPU::Logging::IsActive()) {
GPU::Logging::GPULogger::GetInstance().LogPipelineStateChange(
@@ -166,7 +128,7 @@ ComputePipeline::ComputePipeline(const Device& device_, Scheduler& scheduler, vk
void ComputePipeline::Configure(Tegra::Engines::KeplerCompute& kepler_compute,
Tegra::MemoryManager& gpu_memory, Scheduler& scheduler,
BufferCache& buffer_cache, TextureCache& texture_cache) {
guest_descriptor_queue.Acquire(scheduler, num_descriptor_entries, uses_descriptor_buffer);
guest_descriptor_queue.Acquire(scheduler, num_descriptor_entries);
buffer_cache.SetComputeUniformBufferState(info.constant_buffer_mask, &uniform_buffer_sizes);
buffer_cache.UnbindComputeStorageBuffers();
@@ -287,22 +249,9 @@ void ComputePipeline::Configure(Tegra::Engines::KeplerCompute& kepler_compute,
GPU::Logging::GPULogger::GetInstance().LogPipelineBind(true, "compute pipeline");
}
const DescriptorUpdateEntry* const descriptor_data{guest_descriptor_queue.UpdateData()};
VkDeviceSize descriptor_buffer_offset{};
bool descriptor_buffer_ready{false};
if (uses_descriptor_buffer) {
const DescriptorBufferRing::Allocation alloc{
descriptor_buffer_ring.Allocate(scheduler, descriptor_buffer_layout.size)};
if (alloc.host) {
WriteDescriptorBuffer(device, descriptor_buffer_layout, descriptor_data, alloc.host);
descriptor_buffer_offset = alloc.offset;
descriptor_buffer_ready = true;
}
}
const void* const descriptor_data{guest_descriptor_queue.UpdateData()};
const bool is_rescaling = !info.texture_descriptors.empty() || !info.image_descriptors.empty();
scheduler.Record([this, descriptor_data, is_rescaling, descriptor_buffer_offset,
descriptor_buffer_ready,
scheduler.Record([this, descriptor_data, is_rescaling,
rescaling_data = rescaling.Data()](vk::CommandBuffer cmdbuf) {
if (!pipeline) {
return;
@@ -316,17 +265,7 @@ void ComputePipeline::Configure(Tegra::Engines::KeplerCompute& kepler_compute,
RESCALING_LAYOUT_WORDS_OFFSET, sizeof(rescaling_data),
rescaling_data.data());
}
if (uses_descriptor_buffer) {
if (!descriptor_buffer_ready) {
return;
}
const VkDescriptorBufferBindingInfoEXT binding_info{
descriptor_buffer_ring.BindingInfo()};
cmdbuf.BindDescriptorBuffersEXT(binding_info);
const u32 buffer_index{};
cmdbuf.SetDescriptorBufferOffsetsEXT(VK_PIPELINE_BIND_POINT_COMPUTE, *pipeline_layout,
0, buffer_index, descriptor_buffer_offset);
} else if (uses_push_descriptor) {
if (uses_push_descriptor) {
cmdbuf.PushDescriptorSetWithTemplateKHR(*descriptor_update_template, *pipeline_layout,
0, descriptor_data);
} else {
@@ -13,9 +13,7 @@
#include "common/common_types.h"
#include "common/thread_worker.h"
#include "shader_recompiler/shader_info.h"
#include "video_core/renderer_vulkan/pipeline_helper.h"
#include "video_core/renderer_vulkan/vk_buffer_cache.h"
#include "video_core/renderer_vulkan/vk_descriptor_buffer.h"
#include "video_core/renderer_vulkan/vk_descriptor_pool.h"
#include "video_core/renderer_vulkan/vk_texture_cache.h"
#include "video_core/renderer_vulkan/vk_update_descriptor.h"
@@ -36,7 +34,6 @@ public:
explicit ComputePipeline(const Device& device, Scheduler& scheduler, vk::PipelineCache& pipeline_cache,
DescriptorPool& descriptor_pool,
GuestDescriptorQueue& guest_descriptor_queue,
DescriptorBufferRing& descriptor_buffer_ring,
Common::ThreadWorker* thread_worker,
PipelineStatistics* pipeline_statistics,
VideoCore::ShaderNotify* shader_notify, const Shader::Info& info,
@@ -59,7 +56,6 @@ private:
const Device& device;
vk::PipelineCache& pipeline_cache;
GuestDescriptorQueue& guest_descriptor_queue;
DescriptorBufferRing& descriptor_buffer_ring;
Shader::Info info;
u64 shader_hash{};
u32 num_descriptor_entries{};
@@ -69,8 +65,6 @@ private:
vk::ShaderModule spv_module;
vk::DescriptorSetLayout descriptor_set_layout;
bool uses_push_descriptor{false};
bool uses_descriptor_buffer{false};
DescriptorBufferLayout descriptor_buffer_layout;
DescriptorAllocator descriptor_allocator;
vk::PipelineLayout pipeline_layout;
vk::DescriptorUpdateTemplate descriptor_update_template;
@@ -1,98 +0,0 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
#include <algorithm>
#include "common/alignment.h"
#include "common/assert.h"
#include "common/logging.h"
#include "video_core/renderer_vulkan/vk_descriptor_buffer.h"
#include "video_core/renderer_vulkan/vk_scheduler.h"
#include "video_core/vulkan_common/vulkan_device.h"
namespace Vulkan {
DescriptorBufferRing::DescriptorBufferRing(const Device& device_,
MemoryAllocator& memory_allocator)
: device{device_} {
if (!device.IsExtDescriptorBufferSupported() || !device.IsBufferDeviceAddressSupported()) {
return;
}
alignment = std::max<VkDeviceSize>(
device.DescriptorBufferProperties().descriptorBufferOffsetAlignment, 1);
const VkDeviceSize total = FRAME_SIZE * FRAMES_IN_FLIGHT + alignment;
const VkBufferCreateInfo buffer_ci{
.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,
.pNext = nullptr,
.flags = 0,
.size = total,
.usage = VK_BUFFER_USAGE_RESOURCE_DESCRIPTOR_BUFFER_BIT_EXT |
VK_BUFFER_USAGE_SAMPLER_DESCRIPTOR_BUFFER_BIT_EXT |
VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT,
.sharingMode = VK_SHARING_MODE_EXCLUSIVE,
.queueFamilyIndexCount = 0,
.pQueueFamilyIndices = nullptr,
};
buffer = memory_allocator.CreateBuffer(buffer_ci, MemoryUsage::Upload);
if (!buffer.IsHostVisible()) {
LOG_WARNING(Render_Vulkan, "Descriptor buffer is not host visible, disabling");
buffer.reset();
return;
}
if (device.HasDebuggingToolAttached()) {
buffer.SetObjectNameEXT("Descriptor buffer");
}
const VkDeviceAddress raw_address = device.GetLogical().GetBufferDeviceAddress(*buffer);
base_address = Common::AlignUp(raw_address, alignment);
base_host = buffer.Mapped().data() + (base_address - raw_address);
}
DescriptorBufferRing::~DescriptorBufferRing() = default;
void DescriptorBufferRing::TickFrame() {
if (++frame_index >= FRAMES_IN_FLIGHT) {
frame_index = 0;
}
frame_start = static_cast<VkDeviceSize>(frame_index) * FRAME_SIZE;
cursor = 0;
frame_reused = true;
}
DescriptorBufferRing::Allocation DescriptorBufferRing::Allocate(Scheduler& scheduler,
VkDeviceSize size) {
ASSERT(buffer);
const VkDeviceSize needed = Common::AlignUp(size, alignment);
if (needed > FRAME_SIZE) {
LOG_ERROR(Render_Vulkan, "Descriptor set of {} bytes exceeds frame capacity {}", needed,
FRAME_SIZE);
return Allocation{};
}
if (frame_reused) {
frame_reused = false;
scheduler.Wait(frame_ticks[frame_index]);
}
if (cursor + needed > FRAME_SIZE) {
LOG_WARNING(Render_Vulkan, "Descriptor buffer frame exhausted, stalling on the GPU");
scheduler.Finish();
cursor = 0;
}
const VkDeviceSize offset = frame_start + cursor;
cursor += needed;
frame_ticks[frame_index] = scheduler.CurrentTick();
return Allocation{
.host = base_host + offset,
.offset = offset,
};
}
VkDescriptorBufferBindingInfoEXT DescriptorBufferRing::BindingInfo() const noexcept {
return VkDescriptorBufferBindingInfoEXT{
.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_BUFFER_BINDING_INFO_EXT,
.pNext = nullptr,
.address = base_address,
.usage = VK_BUFFER_USAGE_RESOURCE_DESCRIPTOR_BUFFER_BIT_EXT |
VK_BUFFER_USAGE_SAMPLER_DESCRIPTOR_BUFFER_BIT_EXT,
};
}
} // namespace Vulkan
@@ -1,57 +0,0 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
#pragma once
#include <array>
#include "common/common_types.h"
#include "video_core/vulkan_common/vulkan_memory_allocator.h"
#include "video_core/vulkan_common/vulkan_wrapper.h"
namespace Vulkan {
class Device;
class Scheduler;
class DescriptorBufferRing final {
static constexpr size_t FRAMES_IN_FLIGHT = 8;
static constexpr VkDeviceSize FRAME_SIZE = 512 * 1024;
public:
explicit DescriptorBufferRing(const Device& device_, MemoryAllocator& memory_allocator);
~DescriptorBufferRing();
struct Allocation {
u8* host{};
VkDeviceSize offset{};
};
[[nodiscard]] static constexpr VkDeviceSize MaxAllocationSize() noexcept {
return FRAME_SIZE;
}
void TickFrame();
[[nodiscard]] Allocation Allocate(Scheduler& scheduler, VkDeviceSize size);
[[nodiscard]] VkDescriptorBufferBindingInfoEXT BindingInfo() const noexcept;
[[nodiscard]] bool IsValid() const noexcept {
return static_cast<bool>(buffer);
}
private:
const Device& device;
vk::Buffer buffer;
VkDeviceAddress base_address{};
u8* base_host{};
VkDeviceSize alignment{1};
size_t frame_index{};
VkDeviceSize frame_start{};
VkDeviceSize cursor{};
std::array<u64, FRAMES_IN_FLIGHT> frame_ticks{};
bool frame_reused{};
};
} // namespace Vulkan
@@ -37,7 +37,7 @@ void InnerFence::Wait() {
if (is_stubbed) {
return;
}
scheduler.WaitSubmitted(wait_tick);
scheduler.Wait(wait_tick);
}
FenceManager::FenceManager(VideoCore::RasterizerInterface& rasterizer_, Tegra::GPU& gpu_,
@@ -5,7 +5,6 @@
// SPDX-License-Identifier: GPL-2.0-or-later
#include <algorithm>
#include <cstring>
#include <iostream>
#include <span>
@@ -251,15 +250,13 @@ GraphicsPipeline::GraphicsPipeline(
Scheduler& scheduler_, BufferCache& buffer_cache_, TextureCache& texture_cache_,
vk::PipelineCache& pipeline_cache_, VideoCore::ShaderNotify* shader_notify,
const Device& device_, DescriptorPool& descriptor_pool,
GuestDescriptorQueue& guest_descriptor_queue_, DescriptorBufferRing& descriptor_buffer_ring_,
Common::ThreadWorker* worker_thread,
GuestDescriptorQueue& guest_descriptor_queue_, Common::ThreadWorker* worker_thread,
PipelineStatistics* pipeline_statistics, RenderPassCache& render_pass_cache,
const GraphicsPipelineCacheKey& key_, std::array<vk::ShaderModule, NUM_STAGES> stages,
const std::array<const Shader::Info*, NUM_STAGES>& infos)
: key{key_}, device{device_}, texture_cache{texture_cache_}, buffer_cache{buffer_cache_},
pipeline_cache(pipeline_cache_), scheduler{scheduler_},
guest_descriptor_queue{guest_descriptor_queue_},
descriptor_buffer_ring{descriptor_buffer_ring_}, spv_modules{std::move(stages)} {
guest_descriptor_queue{guest_descriptor_queue_}, spv_modules{std::move(stages)} {
if (shader_notify) {
shader_notify->MarkShaderBuilding();
}
@@ -279,40 +276,21 @@ GraphicsPipeline::GraphicsPipeline(
num_descriptor_entries += NumDescriptorEntries(*info);
}
fragment_has_color0_output = stage_infos[NUM_STAGES - 1].stores_frag_color[0];
auto func{[this, shader_notify, &render_pass_cache, &descriptor_pool, pipeline_statistics] {
DescriptorLayoutBuilder builder{MakeBuilder(device, stage_infos)};
uses_push_descriptor = builder.CanUsePushDescriptor();
descriptor_set_layout = builder.CreateDescriptorSetLayout(uses_push_descriptor);
DescriptorLayoutBuilder builder{MakeBuilder(device, stage_infos)};
uses_push_descriptor = builder.CanUsePushDescriptor();
uses_descriptor_buffer = builder.CanUseDescriptorBuffer() && descriptor_buffer_ring.IsValid();
descriptor_set_layout =
builder.CreateDescriptorSetLayout(uses_push_descriptor, uses_descriptor_buffer);
if (uses_descriptor_buffer) {
descriptor_buffer_layout = builder.MakeDescriptorBufferLayout(*descriptor_set_layout);
if (descriptor_buffer_layout.size > DescriptorBufferRing::MaxAllocationSize()) {
LOG_WARNING(Render_Vulkan,
"Graphics pipeline {:016X} needs {} descriptor bytes, falling back to sets",
key.Hash(), descriptor_buffer_layout.size);
uses_descriptor_buffer = false;
descriptor_buffer_layout = {};
descriptor_set_layout = builder.CreateDescriptorSetLayout(uses_push_descriptor);
if (!uses_push_descriptor) {
descriptor_allocator = descriptor_pool.Allocator(device, scheduler, *descriptor_set_layout, stage_infos);
}
}
const VkDescriptorSetLayout set_layout{*descriptor_set_layout};
pipeline_layout = builder.CreatePipelineLayout(set_layout);
if (!uses_descriptor_buffer) {
const VkDescriptorSetLayout set_layout{*descriptor_set_layout};
pipeline_layout = builder.CreatePipelineLayout(set_layout);
descriptor_update_template =
builder.CreateTemplate(set_layout, *pipeline_layout, uses_push_descriptor);
if (!uses_push_descriptor) {
descriptor_allocator =
descriptor_pool.Allocator(device, scheduler, set_layout, stage_infos);
}
}
auto func{[this, shader_notify, &render_pass_cache, pipeline_statistics] {
VkRenderPass render_pass{};
if (!device.IsKhrDynamicRenderingSupported()) {
render_pass = render_pass_cache.Get(MakeRenderPassKey(key.state, device));
}
const VkRenderPass render_pass{render_pass_cache.Get(MakeRenderPassKey(key.state, device))};
Validate();
try {
MakePipeline(render_pass);
@@ -518,7 +496,7 @@ bool GraphicsPipeline::ConfigureImpl(bool is_indexed) {
buffer_cache.UpdateGraphicsBuffers(is_indexed);
buffer_cache.BindHostGeometryBuffers(is_indexed);
guest_descriptor_queue.Acquire(scheduler, num_descriptor_entries, uses_descriptor_buffer);
guest_descriptor_queue.Acquire(scheduler, num_descriptor_entries);
RescalingPushConstant rescaling;
RenderAreaPushConstant render_area;
@@ -587,33 +565,8 @@ void GraphicsPipeline::ConfigureDraw(const RescalingPushConstant& rescaling,
}
const void* const descriptor_data{guest_descriptor_queue.UpdateData()};
bool update_descriptors = true;
if (descriptor_set_layout && !uses_push_descriptor && !uses_descriptor_buffer) {
const auto* const entries = static_cast<const DescriptorUpdateEntry*>(descriptor_data);
update_descriptors =
bind_pipeline || last_descriptor_payload.size() != num_descriptor_entries ||
std::memcmp(last_descriptor_payload.data(), entries,
num_descriptor_entries * sizeof(DescriptorUpdateEntry)) != 0;
if (update_descriptors) {
last_descriptor_payload.assign(entries, entries + num_descriptor_entries);
}
}
VkDeviceSize descriptor_buffer_offset{};
bool descriptor_buffer_ready{false};
if (descriptor_set_layout && uses_descriptor_buffer) {
const DescriptorBufferRing::Allocation alloc{
descriptor_buffer_ring.Allocate(scheduler, descriptor_buffer_layout.size)};
if (alloc.host) {
WriteDescriptorBuffer(device, descriptor_buffer_layout,
static_cast<const DescriptorUpdateEntry*>(descriptor_data),
alloc.host);
descriptor_buffer_offset = alloc.offset;
descriptor_buffer_ready = true;
}
}
scheduler.Record([this, descriptor_data, bind_pipeline, update_descriptors,
descriptor_buffer_offset, descriptor_buffer_ready,
rescaling_data = rescaling.Data(), is_rescaling, update_rescaling,
scheduler.Record([this, descriptor_data, bind_pipeline, rescaling_data = rescaling.Data(),
is_rescaling, update_rescaling,
uses_render_area = render_area.uses_render_area,
render_area_data = render_area.words](vk::CommandBuffer cmdbuf) {
if (bind_pipeline) {
@@ -640,20 +593,10 @@ void GraphicsPipeline::ConfigureDraw(const RescalingPushConstant& rescaling,
if (!descriptor_set_layout) {
return;
}
if (uses_descriptor_buffer) {
if (!descriptor_buffer_ready) {
return;
}
const VkDescriptorBufferBindingInfoEXT binding_info{
descriptor_buffer_ring.BindingInfo()};
cmdbuf.BindDescriptorBuffersEXT(binding_info);
const u32 buffer_index{};
cmdbuf.SetDescriptorBufferOffsetsEXT(VK_PIPELINE_BIND_POINT_GRAPHICS, *pipeline_layout,
0, buffer_index, descriptor_buffer_offset);
} else if (uses_push_descriptor) {
if (uses_push_descriptor) {
cmdbuf.PushDescriptorSetWithTemplateKHR(*descriptor_update_template, *pipeline_layout,
0, descriptor_data);
} else if (update_descriptors) {
} else {
const VkDescriptorSet descriptor_set{descriptor_allocator.Commit()};
const vk::Device& dev{device.GetLogical()};
dev.UpdateDescriptorSet(descriptor_set, *descriptor_update_template, descriptor_data);
@@ -814,13 +757,16 @@ void GraphicsPipeline::MakePipeline(VkRenderPass render_pass) {
.lineWidth = 1.0f,
// TODO(alekpop): Transfer from regs
};
const VkLineRasterizationModeEXT line_raster_mode =
device.GetLineRasterizationMode(key.state.smooth_lines != 0);
const bool stippled_lines_supported = device.SupportsStippleForMode(line_raster_mode);
const bool smooth_lines_supported =
device.IsExtLineRasterizationSupported() && device.SupportsSmoothLines();
const bool stippled_lines_supported =
device.IsExtLineRasterizationSupported() && device.SupportsStippledRectangularLines();
VkPipelineRasterizationLineStateCreateInfoEXT line_state{
.sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_LINE_STATE_CREATE_INFO_EXT,
.pNext = nullptr,
.lineRasterizationMode = line_raster_mode,
.lineRasterizationMode = key.state.smooth_lines != 0 && smooth_lines_supported
? VK_LINE_RASTERIZATION_MODE_RECTANGULAR_SMOOTH_EXT
: VK_LINE_RASTERIZATION_MODE_RECTANGULAR_EXT,
.stippledLineEnable =
(dynamic.line_stipple_enable && stippled_lines_supported) ? VK_TRUE : VK_FALSE,
.lineStippleFactor = key.state.line_stipple_factor,
@@ -859,16 +805,6 @@ void GraphicsPipeline::MakePipeline(VkRenderPass render_pass) {
if (device.IsExtProvokingVertexSupported()) {
provoking_vertex.pNext = std::exchange(rasterization_ci.pNext, &provoking_vertex);
}
VkPipelineRasterizationDepthClipStateCreateInfoEXT depth_clip_state{
.sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_DEPTH_CLIP_STATE_CREATE_INFO_EXT,
.pNext = nullptr,
.flags = 0,
.depthClipEnable = static_cast<VkBool32>(dynamic.depth_clip_disabled == 0 ? VK_TRUE
: VK_FALSE),
};
if (device.IsExtDepthClipEnableSupported()) {
depth_clip_state.pNext = std::exchange(rasterization_ci.pNext, &depth_clip_state);
}
const bool supports_alpha_output = fragment_has_color0_output;
const bool alpha_to_one_supported = device.SupportsAlphaToOne();
@@ -1059,65 +995,10 @@ void GraphicsPipeline::MakePipeline(VkRenderPass render_pass) {
if (device.IsKhrPipelineExecutablePropertiesEnabled() && Settings::values.renderer_debug.GetValue()) {
flags |= VK_PIPELINE_CREATE_CAPTURE_STATISTICS_BIT_KHR;
}
if (uses_descriptor_buffer) {
flags |= VK_PIPELINE_CREATE_DESCRIPTOR_BUFFER_BIT_EXT;
}
const RenderPassKey renderpass_key{MakeRenderPassKey(key.state, device)};
std::array<VkFormat, Maxwell::NumRenderTargets> color_attachment_formats{};
for (size_t index = 0; index < renderpass_key.color_formats.size(); ++index) {
const PixelFormat pixel_format{renderpass_key.color_formats[index]};
if (pixel_format == PixelFormat::Invalid) {
color_attachment_formats[index] = VK_FORMAT_UNDEFINED;
continue;
}
color_attachment_formats[index] =
MaxwellToVK::SurfaceFormat(device, FormatType::Optimal, true, pixel_format).format;
}
VkFormat depth_attachment_format{VK_FORMAT_UNDEFINED};
VkFormat stencil_attachment_format{VK_FORMAT_UNDEFINED};
if (renderpass_key.depth_format != PixelFormat::Invalid) {
const VkFormat format{
MaxwellToVK::SurfaceFormat(device, FormatType::Optimal, true,
renderpass_key.depth_format)
.format};
const auto surface_type{VideoCore::Surface::GetFormatType(renderpass_key.depth_format)};
if (surface_type == VideoCore::Surface::SurfaceType::Depth ||
surface_type == VideoCore::Surface::SurfaceType::DepthStencil) {
depth_attachment_format = format;
}
if (surface_type == VideoCore::Surface::SurfaceType::Stencil ||
surface_type == VideoCore::Surface::SurfaceType::DepthStencil) {
stencil_attachment_format = format;
}
}
const VkPipelineRenderingCreateInfo rendering_ci{
.sType = VK_STRUCTURE_TYPE_PIPELINE_RENDERING_CREATE_INFO,
.pNext = nullptr,
.viewMask = 0,
.colorAttachmentCount = static_cast<u32>(NumAttachments(key.state)),
.pColorAttachmentFormats = color_attachment_formats.data(),
.depthAttachmentFormat = depth_attachment_format,
.stencilAttachmentFormat = stencil_attachment_format,
};
VkPipelineCreationFeedback creation_feedback{};
const VkPipelineCreationFeedbackCreateInfo feedback_ci{
.sType = VK_STRUCTURE_TYPE_PIPELINE_CREATION_FEEDBACK_CREATE_INFO,
.pNext = device.IsKhrDynamicRenderingSupported() ? &rendering_ci : nullptr,
.pPipelineCreationFeedback = &creation_feedback,
.pipelineStageCreationFeedbackCount = 0,
.pPipelineStageCreationFeedbacks = nullptr,
};
const void* const create_next =
device.IsExtPipelineCreationFeedbackSupported()
? static_cast<const void*>(&feedback_ci)
: (device.IsKhrDynamicRenderingSupported() ? static_cast<const void*>(&rendering_ci)
: nullptr);
pipeline = device.GetLogical().CreateGraphicsPipeline({
.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
.pNext = create_next,
.pNext = nullptr,
.flags = flags,
.stageCount = static_cast<u32>(shader_stages.size()),
.pStages = shader_stages.data(),
@@ -1137,14 +1018,6 @@ void GraphicsPipeline::MakePipeline(VkRenderPass render_pass) {
.basePipelineIndex = 0,
}, *pipeline_cache);
if ((creation_feedback.flags & VK_PIPELINE_CREATION_FEEDBACK_VALID_BIT) != 0) {
const bool cache_hit =
(creation_feedback.flags &
VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT) != 0;
LOG_DEBUG(Render_Vulkan, "Graphics pipeline {:016X} cache_hit={} duration={}us",
key.Hash(), cache_hit, creation_feedback.duration / 1000);
}
// Log graphics pipeline creation
if (GPU::Logging::IsActive()) {
const std::string pipeline_info = fmt::format(
@@ -12,18 +12,14 @@
#include <condition_variable>
#include <mutex>
#include <type_traits>
#include <vector>
#include "common/thread_worker.h"
#include "shader_recompiler/shader_info.h"
#include "video_core/engines/maxwell_3d.h"
#include "video_core/renderer_vulkan/fixed_pipeline_state.h"
#include "video_core/renderer_vulkan/pipeline_helper.h"
#include "video_core/renderer_vulkan/vk_buffer_cache.h"
#include "video_core/renderer_vulkan/vk_descriptor_buffer.h"
#include "video_core/renderer_vulkan/vk_descriptor_pool.h"
#include "video_core/renderer_vulkan/vk_texture_cache.h"
#include "video_core/renderer_vulkan/vk_update_descriptor.h"
#include "video_core/vulkan_common/vulkan_wrapper.h"
namespace VideoCore {
@@ -80,8 +76,7 @@ public:
Scheduler& scheduler, BufferCache& buffer_cache, TextureCache& texture_cache,
vk::PipelineCache& pipeline_cache, VideoCore::ShaderNotify* shader_notify,
const Device& device, DescriptorPool& descriptor_pool,
GuestDescriptorQueue& guest_descriptor_queue,
DescriptorBufferRing& descriptor_buffer_ring, Common::ThreadWorker* worker_thread,
GuestDescriptorQueue& guest_descriptor_queue, Common::ThreadWorker* worker_thread,
PipelineStatistics* pipeline_statistics, RenderPassCache& render_pass_cache,
const GraphicsPipelineCacheKey& key, std::array<vk::ShaderModule, NUM_STAGES> stages,
const std::array<const Shader::Info*, NUM_STAGES>& infos);
@@ -153,7 +148,6 @@ private:
vk::PipelineCache& pipeline_cache;
Scheduler& scheduler;
GuestDescriptorQueue& guest_descriptor_queue;
DescriptorBufferRing& descriptor_buffer_ring;
bool (*configure_func)(GraphicsPipeline*, bool){};
@@ -176,14 +170,10 @@ private:
vk::DescriptorUpdateTemplate descriptor_update_template;
vk::Pipeline pipeline;
DescriptorBufferLayout descriptor_buffer_layout;
std::vector<DescriptorUpdateEntry> last_descriptor_payload;
std::condition_variable build_condvar;
std::mutex build_mutex;
std::atomic_bool is_built{false};
bool uses_push_descriptor{false};
bool uses_descriptor_buffer{false};
};
} // namespace Vulkan
@@ -17,7 +17,6 @@
#include "common/cityhash.h"
#include "common/fs/fs.h"
#include "common/fs/path_util.h"
#include "common/settings.h"
#include "common/thread_worker.h"
#include "core/core.h"
#include "shader_recompiler/backend/spirv/emit_spirv.h"
@@ -46,6 +45,10 @@
#include "video_core/vulkan_common/vulkan_wrapper.h"
#include "video_core/gpu_logging/gpu_logging.h"
#ifdef __ANDROID__
#include "../../android/app/src/main/jni/android_settings.h"
#endif
namespace Vulkan {
namespace {
@@ -60,8 +63,6 @@ using VideoCommon::GenericEnvironment;
using VideoCommon::GraphicsEnvironment;
constexpr u32 CACHE_VERSION = 18;
constexpr size_t VULKAN_CACHE_FLUSH_PIPELINES = 128;
constexpr size_t VULKAN_CACHE_FLUSH_MIN_SECONDS = 30;
constexpr std::array<char, 8> VULKAN_CACHE_MAGIC_NUMBER{'y', 'u', 'z', 'u', 'v', 'k', 'c', 'h'};
template <typename Container>
@@ -303,8 +304,12 @@ size_t GetTotalPipelineWorkers() {
const size_t max_core_threads =
std::max<size_t>(static_cast<size_t>(std::thread::hardware_concurrency()), 2ULL) - 1ULL;
#ifdef __ANDROID__
const s32 configured = Settings::values.pipeline_worker_count.GetValue();
const size_t desired = static_cast<size_t>(std::clamp(configured, 2, 8));
const int configured = AndroidSettings::values.pipeline_worker_count.GetValue();
const int clamped = std::clamp(configured, 4, 8);
const size_t desired = static_cast<size_t>(clamped);
if (desired == 0) {
return 1ULL;
}
return std::min(max_core_threads, desired);
#else
return max_core_threads;
@@ -335,20 +340,17 @@ PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_,
const Device& device_, Scheduler& scheduler_,
DescriptorPool& descriptor_pool_,
GuestDescriptorQueue& guest_descriptor_queue_,
DescriptorBufferRing& descriptor_buffer_ring_,
RenderPassCache& render_pass_cache_, BufferCache& buffer_cache_,
TextureCache& texture_cache_, VideoCore::ShaderNotify& shader_notify_)
: VideoCommon::ShaderCache{device_memory_}, device{device_}, scheduler{scheduler_},
descriptor_pool{descriptor_pool_}, guest_descriptor_queue{guest_descriptor_queue_},
descriptor_buffer_ring{descriptor_buffer_ring_},
render_pass_cache{render_pass_cache_}, buffer_cache{buffer_cache_},
texture_cache{texture_cache_}, shader_notify{shader_notify_},
use_asynchronous_shaders{Settings::values.use_asynchronous_shaders.GetValue()},
use_vulkan_pipeline_cache{Settings::values.use_vulkan_driver_pipeline_cache.GetValue()},
workers(device.HasBrokenParallelShaderCompiling() ? 1ULL : GetTotalPipelineWorkers(),
"VkPipelineBuilder"),
serialization_thread(1, "VkPipelineSerialization", {},
Common::ThreadPlacement::Background) {
serialization_thread(1, "VkPipelineSerialization") {
const auto& float_control{device.FloatControlProperties()};
const VkDriverId driver_id{device.GetDriverID()};
const VkShaderStageFlags subgroup_stages{device.GetSubgroupSupportedStages()};
@@ -399,8 +401,6 @@ PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_,
device.IsWorkgroupMemoryExplicitLayout8BitAccessSupported(),
.support_workgroup_layout_16bit_access =
device.IsWorkgroupMemoryExplicitLayout16BitAccessSupported(),
.support_shader_quad_control = device.IsKhrShaderQuadControlSupported(),
.support_quad_shuffles = device.IsSubgroupFeatureSupported(VK_SUBGROUP_FEATURE_QUAD_BIT),
.support_vote = device.IsSubgroupFeatureSupported(VK_SUBGROUP_FEATURE_VOTE_BIT),
.supported_subgroup_stages = supported_subgroup_stages,
.support_viewport_index_layer_non_geometry =
@@ -514,11 +514,6 @@ PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_,
dynamic_features.has_dynamic_state3_depth_clamp_enable =
dynamic_features.has_extended_dynamic_state_3_enables &&
device.SupportsDynamicState3DepthClampEnable();
dynamic_features.has_dynamic_state3_conservative_raster_mode =
dynamic_features.has_extended_dynamic_state_3_enables &&
device.SupportsDynamicState3ConservativeRasterizationMode();
dynamic_features.has_depth_clip_enable =
device.IsExtDepthClipEnableSupported();
dynamic_features.has_dynamic_state3_logic_op_enable =
dynamic_features.has_extended_dynamic_state_3_enables &&
device.SupportsDynamicState3LogicOpEnable();
@@ -531,8 +526,7 @@ PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_,
device.IsExtVertexInputDynamicStateSupported() &&
Settings::values.vertex_input_dynamic_state.GetValue();
dynamic_features.has_provoking_vertex =
device.IsExtProvokingVertexSupported();
dynamic_features.has_provoking_vertex = device.IsExtProvokingVertexSupported();
dynamic_features.has_provoking_vertex_first_mode =
device.SupportsProvokingVertexFirstMode();
dynamic_features.has_provoking_vertex_last_mode =
@@ -702,10 +696,6 @@ void PipelineCache::LoadDiskResources(u64 title_id, std::stop_token stop_loading
if (use_vulkan_pipeline_cache) {
SerializeVulkanPipelineCache(vulkan_pipeline_cache_filename, vulkan_pipeline_cache,
CACHE_VERSION);
size_t size = 0;
vulkan_pipeline_cache.Read(&size, nullptr);
last_cache_size.store(size, std::memory_order_relaxed);
last_flush = std::chrono::steady_clock::now();
}
if (state.statistics) {
@@ -713,35 +703,6 @@ void PipelineCache::LoadDiskResources(u64 title_id, std::stop_token stop_loading
}
}
void PipelineCache::QueueVulkanPipelineCacheFlush() {
if (!use_vulkan_pipeline_cache || vulkan_pipeline_cache_filename.empty()) {
return;
}
if (++pipelines_since_flush < VULKAN_CACHE_FLUSH_PIPELINES) {
return;
}
const auto now = std::chrono::steady_clock::now();
const auto megabytes = last_cache_size.load(std::memory_order_relaxed) / (1024 * 1024);
const std::chrono::seconds interval{
std::max<size_t>(VULKAN_CACHE_FLUSH_MIN_SECONDS, megabytes)};
if (last_flush.time_since_epoch().count() != 0 && now - last_flush < interval) {
return;
}
if (flush_in_flight.exchange(true, std::memory_order_acq_rel)) {
return;
}
pipelines_since_flush = 0;
last_flush = now;
serialization_thread.QueueWork([this] {
SerializeVulkanPipelineCache(vulkan_pipeline_cache_filename, vulkan_pipeline_cache,
CACHE_VERSION);
size_t size = 0;
vulkan_pipeline_cache.Read(&size, nullptr);
last_cache_size.store(size, std::memory_order_relaxed);
flush_in_flight.store(false, std::memory_order_release);
});
}
GraphicsPipeline* PipelineCache::CurrentGraphicsPipelineSlowPath() {
const auto [pair, is_new]{graphics_cache.try_emplace(graphics_key)};
auto& pipeline{pair->second};
@@ -780,7 +741,7 @@ std::unique_ptr<GraphicsPipeline> PipelineCache::CreateGraphicsPipeline(
std::span<Shader::Environment* const> envs, PipelineStatistics* statistics,
bool build_in_parallel) try {
auto hash = key.Hash();
LOG_DEBUG(Render_Vulkan, "{:#016x}", hash);
LOG_INFO(Render_Vulkan, "0x{:016x}", hash);
size_t env_index{0};
std::array<Shader::IR::Program, Maxwell::MaxShaderProgram> programs;
const bool uses_vertex_a{key.unique_hashes[0] != 0};
@@ -875,8 +836,8 @@ std::unique_ptr<GraphicsPipeline> PipelineCache::CreateGraphicsPipeline(
Common::ThreadWorker* const thread_worker{build_in_parallel ? &workers : nullptr};
return std::make_unique<GraphicsPipeline>(
scheduler, buffer_cache, texture_cache, vulkan_pipeline_cache, &shader_notify, device,
descriptor_pool, guest_descriptor_queue, descriptor_buffer_ring, thread_worker, statistics,
render_pass_cache, key, std::move(modules), infos);
descriptor_pool, guest_descriptor_queue, thread_worker, statistics, render_pass_cache, key,
std::move(modules), infos);
} catch (const Shader::Exception& exception) {
auto hash = key.Hash();
@@ -916,7 +877,6 @@ std::unique_ptr<GraphicsPipeline> PipelineCache::CreateGraphicsPipeline() {
}
SerializePipeline(key, env_ptrs, pipeline_cache_filename, CACHE_VERSION);
});
QueueVulkanPipelineCacheFlush();
return pipeline;
}
@@ -936,7 +896,6 @@ std::unique_ptr<ComputePipeline> PipelineCache::CreateComputePipeline(
SerializePipeline(key, std::array<const GenericEnvironment*, 1>{&env_},
pipeline_cache_filename, CACHE_VERSION);
});
QueueVulkanPipelineCacheFlush();
return pipeline;
}
@@ -945,11 +904,11 @@ std::unique_ptr<ComputePipeline> PipelineCache::CreateComputePipeline(
PipelineStatistics* statistics, bool build_in_parallel) try {
auto hash = key.Hash();
if (device.HasBrokenCompute()) {
LOG_ERROR(Render_Vulkan, "Skipping {:#016x}", hash);
LOG_ERROR(Render_Vulkan, "Skipping 0x{:016x}", hash);
return nullptr;
}
LOG_DEBUG(Render_Vulkan, "{:#016x}", hash);
LOG_INFO(Render_Vulkan, "0x{:016x}", hash);
Shader::Maxwell::Flow::CFG cfg{env, pools.flow_block, env.StartAddress()};
@@ -966,7 +925,7 @@ std::unique_ptr<ComputePipeline> PipelineCache::CreateComputePipeline(
const u32 max_shared_memory = device.GetMaxComputeSharedMemorySize();
if (needs_shared_mem_clamp && program.shared_memory_size > max_shared_memory) {
LOG_WARNING(Render_Vulkan,
"Compute shader {:#016x} requests {}KB shared memory but device max is {}KB - clamping",
"Compute shader 0x{:016x} requests {}KB shared memory but device max is {}KB - clamping",
key.unique_hash,
program.shared_memory_size / 1024,
max_shared_memory / 1024);
@@ -998,8 +957,7 @@ std::unique_ptr<ComputePipeline> PipelineCache::CreateComputePipeline(
}
Common::ThreadWorker* const thread_worker{build_in_parallel ? &workers : nullptr};
return std::make_unique<ComputePipeline>(device, scheduler, vulkan_pipeline_cache, descriptor_pool,
guest_descriptor_queue, descriptor_buffer_ring,
thread_worker, statistics,
guest_descriptor_queue, thread_worker, statistics,
&shader_notify, program.info, std::move(spv_module),
key.unique_hash);
@@ -7,8 +7,6 @@
#pragma once
#include <array>
#include <atomic>
#include <chrono>
#include <cstddef>
#include <filesystem>
#include <memory>
@@ -107,7 +105,6 @@ public:
explicit PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, const Device& device,
Scheduler& scheduler, DescriptorPool& descriptor_pool,
GuestDescriptorQueue& guest_descriptor_queue,
DescriptorBufferRing& descriptor_buffer_ring,
RenderPassCache& render_pass_cache, BufferCache& buffer_cache,
TextureCache& texture_cache, VideoCore::ShaderNotify& shader_notify_);
~PipelineCache();
@@ -146,13 +143,10 @@ private:
vk::PipelineCache LoadVulkanPipelineCache(const std::filesystem::path& filename,
u32 expected_cache_version);
void QueueVulkanPipelineCacheFlush();
const Device& device;
Scheduler& scheduler;
DescriptorPool& descriptor_pool;
GuestDescriptorQueue& guest_descriptor_queue;
DescriptorBufferRing& descriptor_buffer_ring;
RenderPassCache& render_pass_cache;
BufferCache& buffer_cache;
TextureCache& texture_cache;
@@ -175,10 +169,6 @@ private:
std::filesystem::path vulkan_pipeline_cache_filename;
vk::PipelineCache vulkan_pipeline_cache;
size_t pipelines_since_flush{};
std::chrono::steady_clock::time_point last_flush{};
std::atomic<size_t> last_cache_size{};
std::atomic_bool flush_in_flight{};
Common::ThreadWorker workers;
Common::ThreadWorker serialization_thread;
@@ -296,6 +296,9 @@ void PresentManager::RecreateSwapchain(Frame* frame) {
}
void PresentManager::SetImageCount() {
// We cannot have more than 7 images in flight at any given time.
// FRAMES_IN_FLIGHT is 8, and the cache TICKS_TO_DESTROY is 8.
// Mali drivers will give us 6.
image_count = std::min<size_t>(swapchain.GetImageCount(), 7);
}
@@ -235,23 +235,11 @@ public:
return;
}
PauseCounter();
if (!CanResolveHostQueries()) {
sync_values_stash.clear();
scheduler.RequestOutsideRenderPassOperationContext();
scheduler.Record([buffer = *accumulation_buffer](vk::CommandBuffer cmdbuf) {
cmdbuf.FillBuffer(buffer, 0, 8, 0);
});
std::function<void()> func([this] {
amend_value = 0;
accumulation_value = 0;
});
rasterizer->SyncOperation(std::move(func));
AbandonCurrentQuery();
num_slots_used = 0;
first_accumulation_checkpoint = (std::numeric_limits<size_t>::max)();
last_accumulation_checkpoint = 0;
accumulation_since_last_sync = false;
const auto driver_id = device.GetDriverID();
if (driver_id == VK_DRIVER_ID_QUALCOMM_PROPRIETARY ||
driver_id == VK_DRIVER_ID_ARM_PROPRIETARY || driver_id == VK_DRIVER_ID_MESA_TURNIP) {
pending_sync.clear();
sync_values_stash.clear();
return;
}
sync_values_stash.clear();
@@ -418,13 +406,6 @@ public:
}
private:
bool CanResolveHostQueries() const {
const auto driver_id = device.GetDriverID();
return driver_id != VK_DRIVER_ID_QUALCOMM_PROPRIETARY &&
driver_id != VK_DRIVER_ID_ARM_PROPRIETARY &&
driver_id != VK_DRIVER_ID_MESA_TURNIP;
}
template <typename Func>
void ApplyBankOp(VideoCommon::HostQueryBase* query, Func&& func) {
size_t size_slots = query->size_slots;
@@ -938,7 +919,7 @@ private:
return;
}
has_flushed_end_pending = true;
scheduler.MarkTransformFeedbackUsed();
// Refresh buffers state before beginning transform feedback so counters are up-to-date
UpdateBuffers();
if (!has_started || buffers_count == 0) {
// No counter buffers available: begin without counters
@@ -1442,12 +1423,18 @@ void QueryCacheRuntime::HostConditionalRenderingCompareValueImpl(VideoCommon::Lo
return;
}
}
PauseHostConditionalRendering();
bool was_running = impl->is_hcr_running;
if (was_running) {
PauseHostConditionalRendering();
}
impl->hcr_setup.buffer = impl->hcr_buffer;
impl->hcr_setup.offset = impl->hcr_offset;
impl->hcr_setup.flags = is_equal ? VK_CONDITIONAL_RENDERING_INVERTED_BIT_EXT : 0;
impl->hcr_is_set = true;
impl->is_hcr_running = false;
if (was_running) {
ResumeHostConditionalRendering();
}
}
void QueryCacheRuntime::HostConditionalRenderingCompareBCImpl(DAddr address, bool is_equal,
@@ -1464,7 +1451,10 @@ void QueryCacheRuntime::HostConditionalRenderingCompareBCImpl(DAddr address, boo
to_resolve = buffer->Handle();
to_resolve_offset = static_cast<u32>(offset);
}
PauseHostConditionalRendering();
bool was_running = impl->is_hcr_running;
if (was_running) {
PauseHostConditionalRendering();
}
impl->conditional_resolve_pass->Resolve(*impl->hcr_resolve_buffer, to_resolve,
to_resolve_offset, compare_to_zero);
impl->hcr_setup.buffer = *impl->hcr_resolve_buffer;
@@ -1472,6 +1462,9 @@ void QueryCacheRuntime::HostConditionalRenderingCompareBCImpl(DAddr address, boo
impl->hcr_setup.flags = is_equal ? 0 : VK_CONDITIONAL_RENDERING_INVERTED_BIT_EXT;
impl->hcr_is_set = true;
impl->is_hcr_running = false;
if (was_running) {
ResumeHostConditionalRendering();
}
}
bool QueryCacheRuntime::HostConditionalRenderingCompareValue(VideoCommon::LookupData object_1,
@@ -1479,12 +1472,6 @@ bool QueryCacheRuntime::HostConditionalRenderingCompareValue(VideoCommon::Lookup
if (!impl->device.IsExtConditionalRendering()) {
return false;
}
const auto driver_id = impl->device.GetDriverID();
if (driver_id == VK_DRIVER_ID_QUALCOMM_PROPRIETARY ||
driver_id == VK_DRIVER_ID_ARM_PROPRIETARY || driver_id == VK_DRIVER_ID_MESA_TURNIP) {
EndHostConditionalRendering();
return true;
}
HostConditionalRenderingCompareBCImpl(object_1.address, true, true);
return true;
}
@@ -1531,12 +1518,10 @@ bool QueryCacheRuntime::HostConditionalRenderingCompareValues(VideoCommon::Looku
return false;
}
const auto driver_id = impl->device.GetDriverID();
auto driver_id = impl->device.GetDriverID();
const bool is_gpu_high = Settings::IsGPULevelHigh();
if ((!is_gpu_high && driver_id == VK_DRIVER_ID_INTEL_PROPRIETARY_WINDOWS) ||
driver_id == VK_DRIVER_ID_QUALCOMM_PROPRIETARY ||
driver_id == VK_DRIVER_ID_ARM_PROPRIETARY || driver_id == VK_DRIVER_ID_MESA_TURNIP) {
if ((!is_gpu_high && driver_id == VK_DRIVER_ID_INTEL_PROPRIETARY_WINDOWS) || driver_id == VK_DRIVER_ID_QUALCOMM_PROPRIETARY || driver_id == VK_DRIVER_ID_ARM_PROPRIETARY || driver_id == VK_DRIVER_ID_MESA_TURNIP) {
EndHostConditionalRendering();
return true;
}
@@ -6,8 +6,6 @@
#include <algorithm>
#include <array>
#include <atomic>
#include <limits>
#include <memory>
#include <mutex>
@@ -104,20 +102,8 @@ VkViewport GetViewportState(const Device& device, const Maxwell& regs, size_t in
.maxDepth = src.translate_z + src.scale_z,
};
if (!device.IsExtDepthRangeUnrestrictedSupported()) {
const float unclamped_min = viewport.minDepth;
const float unclamped_max = viewport.maxDepth;
viewport.minDepth = std::clamp(viewport.minDepth, 0.0f, 1.0f);
viewport.maxDepth = std::clamp(viewport.maxDepth, 0.0f, 1.0f);
if (viewport.minDepth != unclamped_min || viewport.maxDepth != unclamped_max) {
static std::atomic<u32> reported{0};
if (reported.fetch_add(1, std::memory_order_relaxed) < 32) {
LOG_WARNING(Render_Vulkan,
"Depth range clamped: viewport={} mode={} translate_z={} scale_z={} "
"range=[{}, {}] -> [{}, {}]",
index, static_cast<u32>(regs.depth_mode), src.translate_z, src.scale_z,
unclamped_min, unclamped_max, viewport.minDepth, viewport.maxDepth);
}
}
}
return viewport;
}
@@ -217,10 +203,7 @@ RasterizerVulkan::RasterizerVulkan(Core::Frontend::EmuWindow& emu_window_, Tegra
: gpu{gpu_}, device_memory{device_memory_}, device{device_},
memory_allocator{memory_allocator_}, state_tracker{state_tracker_}, scheduler{scheduler_},
staging_pool(device, memory_allocator, scheduler), descriptor_pool(device, scheduler),
guest_descriptor_queue(device, UpdateDescriptorQueue::GUEST_FRAME_PAYLOAD_SIZE,
device.IsExtDescriptorBufferSupported()),
compute_pass_descriptor_queue(device, UpdateDescriptorQueue::COMPUTE_FRAME_PAYLOAD_SIZE),
descriptor_buffer_ring(device, memory_allocator),
guest_descriptor_queue(device), compute_pass_descriptor_queue(device),
blit_image(device, scheduler, state_tracker, descriptor_pool), render_pass_cache(device),
texture_cache_runtime{
device, scheduler, memory_allocator, staging_pool,
@@ -233,35 +216,14 @@ RasterizerVulkan::RasterizerVulkan(Core::Frontend::EmuWindow& emu_window_, Tegra
staging_pool, compute_pass_descriptor_queue, descriptor_pool, texture_cache),
query_cache(gpu, *this, device_memory, query_cache_runtime),
pipeline_cache(device_memory, device, scheduler, descriptor_pool, guest_descriptor_queue,
descriptor_buffer_ring, render_pass_cache, buffer_cache, texture_cache,
gpu.ShaderNotify()),
render_pass_cache, buffer_cache, texture_cache, gpu.ShaderNotify()),
accelerate_dma(buffer_cache, texture_cache, scheduler),
fence_manager(*this, gpu, texture_cache, buffer_cache, query_cache, device, scheduler),
wfi_event(device.GetLogical().CreateEvent()) {
scheduler.SetQueryCache(query_cache);
if (Settings::values.use_unified_memory.GetValue()) {
buffer_cache_runtime.TryEnableUnifiedMemory(
device_memory.GetPhysicalBase(), device_memory.GetPhysicalSize(),
device_memory.GetBackingHardwareBuffers(),
device_memory.GetBackingHardwareBufferWindowSize());
}
memory_allocator.SetReclaimCallback([this](u64 bytes) -> u64 {
u64 freed = staging_pool.ReclaimMemory(bytes);
if (freed < bytes) {
freed += texture_cache.ReclaimMemory(bytes - freed, false);
}
if (freed < bytes) {
freed += buffer_cache.ReclaimMemory(bytes - freed, false);
}
auto& master_semaphore = scheduler.GetMasterSemaphore();
master_semaphore.Refresh();
vk::TickDeletionQueue(master_semaphore.KnownGpuTick());
return freed;
});
}
RasterizerVulkan::~RasterizerVulkan() {
memory_allocator.SetReclaimCallback(nullptr);
scheduler.WaitWorker();
scheduler.Finish();
}
@@ -279,13 +241,11 @@ void RasterizerVulkan::PrepareDraw(bool is_indexed, Func&& draw_func) {
if (!pipeline) {
return;
}
{
std::scoped_lock lock{buffer_cache.mutex, texture_cache.mutex};
pipeline->SetEngine(maxwell3d, gpu_memory);
if (!pipeline->Configure(is_indexed)) {
return;
}
}
std::scoped_lock lock{buffer_cache.mutex, texture_cache.mutex};
// update engine as channel may be different.
pipeline->SetEngine(maxwell3d, gpu_memory);
if (!pipeline->Configure(is_indexed))
return;
UpdateDynamicStates();
@@ -399,6 +359,7 @@ void RasterizerVulkan::DrawTexture() {
UpdateDynamicStates();
query_cache.NotifySegment(true);
query_cache.CounterEnable(VideoCommon::QueryType::ZPassPixelCount64, maxwell3d->regs.zpass_pixel_count_enable);
const auto& draw_texture_state = maxwell3d->draw_manager.draw_texture_state;
const auto& sampler = texture_cache.GetSampler(draw_texture_state.src_sampler, false);
@@ -459,21 +420,15 @@ void RasterizerVulkan::Clear(u32 layer_count) {
const bool ds_deferrable =
!ds_used || ((!framebuffer->HasAspectDepthBit() || use_depth) &&
(!framebuffer->HasAspectStencilBit() || use_stencil) && !stencil_partial);
const bool clear_shape_deferrable = ENABLE_DEFERRED_CLEAR &&
!regs.clear_control.use_scissor &&
regs.clear_surface.layer == 0 &&
(!use_color || color_full_channels) && ds_deferrable;
// An open pass normally blocks deferral, which drops the clear to ClearAttachments inside the
// pass and also loses the MSAA store discard, since that only applies when a clear is folded
// into the begin. A pass whose BeginRendering has not been recorded yet can be retracted for
// free, so the clear becomes a load op after all. Only retract when it will actually be used.
const bool can_defer_clear =
clear_shape_deferrable &&
(!scheduler.IsRenderPassActive() || scheduler.RetractUnrecordedRenderPass());
const bool can_defer_clear = ENABLE_DEFERRED_CLEAR && !regs.clear_control.use_scissor &&
regs.clear_surface.layer == 0 &&
!scheduler.IsRenderPassActive() &&
(!use_color || color_full_channels) && ds_deferrable;
if (!can_defer_clear) {
scheduler.RequestRenderpass(framebuffer);
}
query_cache.NotifySegment(true);
query_cache.CounterEnable(VideoCommon::QueryType::ZPassPixelCount64, maxwell3d->regs.zpass_pixel_count_enable);
u32 up_scale = 1;
u32 down_shift = 0;
@@ -718,6 +673,7 @@ void RasterizerVulkan::FlushRegion(DAddr addr, u64 size, VideoCommon::CacheType
texture_cache.DownloadMemory(addr, size);
}
if ((True(which & VideoCommon::CacheType::BufferCache))) {
std::scoped_lock lock{buffer_cache.mutex};
buffer_cache.DownloadMemory(addr, size);
}
if ((True(which & VideoCommon::CacheType::QueryCache))) {
@@ -815,15 +771,12 @@ bool RasterizerVulkan::OnCPUWrite(DAddr addr, u64 size) {
return false;
}
static constexpr bool ENABLE_TEXTURE_CACHE_INVALIDATION_SKIP = true;
void RasterizerVulkan::OnCacheInvalidation(DAddr addr, u64 size) {
if (addr == 0 || size == 0) {
return;
}
if (!ENABLE_TEXTURE_CACHE_INVALIDATION_SKIP ||
device_memory.IsRegionTextureCached(addr, size)) {
{
std::scoped_lock lock{texture_cache.mutex};
texture_cache.WriteMemory(addr, size);
}
@@ -927,12 +880,8 @@ void RasterizerVulkan::FlushCommands() {
void RasterizerVulkan::TickFrame() {
draw_counter = 0;
auto& master_semaphore = scheduler.GetMasterSemaphore();
master_semaphore.Refresh();
vk::TickDeletionQueue(master_semaphore.KnownGpuTick());
guest_descriptor_queue.TickFrame();
compute_pass_descriptor_queue.TickFrame();
descriptor_buffer_ring.TickFrame();
fence_manager.TickFrame();
staging_pool.TickFrame();
{
@@ -1020,11 +969,11 @@ void RasterizerVulkan::LoadDiskResources(u64 title_id, std::stop_token stop_load
void RasterizerVulkan::FlushWork() {
#ifdef __ANDROID__
static constexpr u32 DRAWS_TO_DISPATCH = 1024;
static constexpr u32 CHECK_MASK = 63;
static constexpr u32 DRAWS_TO_DISPATCH = 512;
static constexpr u32 CHECK_MASK = 3;
#else
static constexpr u32 DRAWS_TO_DISPATCH = 4096;
static constexpr u32 CHECK_MASK = 31;
static constexpr u32 CHECK_MASK = 7;
#endif // __ANDROID__
static_assert(DRAWS_TO_DISPATCH % (CHECK_MASK + 1) == 0);
@@ -1330,13 +1279,6 @@ void RasterizerVulkan::UpdateDepthBias(Tegra::Engines::Maxwell3D::Regs& regs) {
regs.zeta.format == Tegra::DepthFormat::S8Z24_UNORM ||
regs.zeta.format == Tegra::DepthFormat::V8Z24_UNORM;
const bool is_float_depth = regs.zeta.format == Tegra::DepthFormat::Z32_FLOAT ||
regs.zeta.format == Tegra::DepthFormat::Z32_FLOAT_X24S8_UINT;
if (is_float_depth && !device.IsExtDepthBiasControlSupported()) {
units /= static_cast<float>(1ULL << (32 - 24));
}
if (is_d24 && !device.SupportsD24DepthBuffer()) {
static constexpr const size_t length = sizeof(NEEDS_D24) / sizeof(NEEDS_D24[0]);
@@ -1509,10 +1451,7 @@ void RasterizerVulkan::UpdateLineWidth(Tegra::Engines::Maxwell3D::Regs& regs) {
}
const float width =
regs.line_anti_alias_enable ? regs.line_width_smooth : regs.line_width_aliased;
const float clamped_width = device.ClampLineWidth(width);
scheduler.Record([clamped_width](vk::CommandBuffer cmdbuf) {
cmdbuf.SetLineWidth(clamped_width);
});
scheduler.Record([width](vk::CommandBuffer cmdbuf) { cmdbuf.SetLineWidth(width); });
}
void RasterizerVulkan::UpdateCullMode(Tegra::Engines::Maxwell3D::Regs& regs) {
@@ -1609,10 +1548,7 @@ void RasterizerVulkan::UpdateLineStippleEnable(Tegra::Engines::Maxwell3D::Regs&
return;
}
const VkLineRasterizationModeEXT mode =
device.GetLineRasterizationMode(regs.line_anti_alias_enable != 0);
const bool enable = regs.line_stipple_enable != 0 && device.SupportsStippleForMode(mode);
scheduler.Record([enable](vk::CommandBuffer cmdbuf) {
scheduler.Record([enable = regs.line_stipple_enable](vk::CommandBuffer cmdbuf) {
cmdbuf.SetLineStippleEnableEXT(enable);
});
}
@@ -1626,24 +1562,28 @@ void RasterizerVulkan::UpdateLineRasterizationMode(Tegra::Engines::Maxwell3D::Re
}
if (!device.SupportsDynamicState3LineRasterizationMode()) {
static std::once_flag warn_missing_dynamic_state;
std::call_once(warn_missing_dynamic_state, [] {
static std::once_flag warn_missing_rect;
std::call_once(warn_missing_rect, [] {
LOG_WARNING(Render_Vulkan,
"Driver lacks dynamic line rasterization mode; the pipeline static value "
"is used instead");
"Driver lacks rectangular line rasterization support; skipping dynamic "
"line state updates");
});
return;
}
const bool wants_smooth = regs.line_anti_alias_enable != 0;
const VkLineRasterizationModeEXT mode = device.GetLineRasterizationMode(wants_smooth);
if (wants_smooth && mode != VK_LINE_RASTERIZATION_MODE_RECTANGULAR_SMOOTH_EXT) {
static std::once_flag warn_missing_smooth;
std::call_once(warn_missing_smooth, [] {
LOG_WARNING(Render_Vulkan,
"Line anti-aliasing requested but smoothLines feature unavailable; "
"falling back to the closest supported mode");
});
VkLineRasterizationModeEXT mode = VK_LINE_RASTERIZATION_MODE_RECTANGULAR_EXT;
if (wants_smooth) {
if (device.SupportsSmoothLines()) {
mode = VK_LINE_RASTERIZATION_MODE_RECTANGULAR_SMOOTH_EXT;
} else {
static std::once_flag warn_missing_smooth;
std::call_once(warn_missing_smooth, [] {
LOG_WARNING(Render_Vulkan,
"Line anti-aliasing requested but smoothLines feature unavailable; "
"using rectangular rasterization");
});
}
}
scheduler.Record([mode](vk::CommandBuffer cmdbuf) {
cmdbuf.SetLineRasterizationModeEXT(mode);
@@ -1703,7 +1643,12 @@ void RasterizerVulkan::UpdateDepthClampEnable(Tegra::Engines::Maxwell3D::Regs& r
if (!device.SupportsDynamicState3DepthClampEnable()) {
return;
}
const bool is_enabled = IsDepthClampEnabled(regs, device.IsExtDepthClipEnableSupported());
bool is_enabled = !(regs.viewport_clip_control.geometry_clip ==
Maxwell::ViewportClipControl::GeometryClip::Passthrough ||
regs.viewport_clip_control.geometry_clip ==
Maxwell::ViewportClipControl::GeometryClip::FrustumXYZ ||
regs.viewport_clip_control.geometry_clip ==
Maxwell::ViewportClipControl::GeometryClip::FrustumZ);
scheduler.Record(
[is_enabled](vk::CommandBuffer cmdbuf) { cmdbuf.SetDepthClampEnableEXT(is_enabled); });
}
@@ -17,7 +17,6 @@
#include "video_core/rasterizer_interface.h"
#include "video_core/renderer_vulkan/blit_image.h"
#include "video_core/renderer_vulkan/vk_buffer_cache.h"
#include "video_core/renderer_vulkan/vk_descriptor_buffer.h"
#include "video_core/renderer_vulkan/vk_descriptor_pool.h"
#include "video_core/renderer_vulkan/vk_fence_manager.h"
#include "video_core/renderer_vulkan/vk_pipeline_cache.h"
@@ -180,6 +179,7 @@ private:
void UpdateRasterizerDiscardEnable(Tegra::Engines::Maxwell3D::Regs& regs);
void UpdateConservativeRasterizationMode(Tegra::Engines::Maxwell3D::Regs& regs);
void UpdateLineStippleEnable(Tegra::Engines::Maxwell3D::Regs& regs);
void UpdateLineStipple(Tegra::Engines::Maxwell3D::Regs& regs);
void UpdateLineRasterizationMode(Tegra::Engines::Maxwell3D::Regs& regs);
void UpdateDepthBiasEnable(Tegra::Engines::Maxwell3D::Regs& regs);
void UpdateLogicOpEnable(Tegra::Engines::Maxwell3D::Regs& regs);
@@ -207,7 +207,6 @@ private:
DescriptorPool descriptor_pool;
GuestDescriptorQueue guest_descriptor_queue;
ComputePassDescriptorQueue compute_pass_descriptor_queue;
DescriptorBufferRing descriptor_buffer_ring;
BlitImageHelper blit_image;
RenderPassCache render_pass_cache;
@@ -12,7 +12,10 @@ ResourcePool::ResourcePool(MasterSemaphore& master_semaphore_, size_t grow_step_
: master_semaphore{&master_semaphore_}, grow_step{grow_step_} {}
size_t ResourcePool::CommitResource() {
const auto search = [this](size_t begin, size_t end, u64 gpu_tick) -> std::optional<size_t> {
// Refresh semaphore to query updated results
master_semaphore->Refresh();
const u64 gpu_tick = master_semaphore->KnownGpuTick();
const auto search = [this, gpu_tick](size_t begin, size_t end) -> std::optional<size_t> {
for (size_t iterator = begin; iterator < end; ++iterator) {
if (gpu_tick >= ticks[iterator]) {
ticks[iterator] = master_semaphore->CurrentTick();
@@ -21,17 +24,11 @@ size_t ResourcePool::CommitResource() {
}
return std::nullopt;
};
const auto find_free = [&](u64 gpu_tick) -> std::optional<size_t> {
std::optional<size_t> result = search(hint_iterator, ticks.size(), gpu_tick);
if (!result) {
result = search(0, hint_iterator, gpu_tick);
}
return result;
};
std::optional<size_t> found = find_free(master_semaphore->KnownGpuTick());
// Try to find a free resource from the hinted position to the end.
std::optional<size_t> found = search(hint_iterator, ticks.size());
if (!found) {
master_semaphore->Refresh();
found = find_free(master_semaphore->KnownGpuTick());
// Search from beginning to the hinted position.
found = search(0, hint_iterator);
if (!found) {
// Both searches failed, the pool is full; handle it.
const size_t free_resource = ManageOverflow();
+17 -332
View File
@@ -47,7 +47,6 @@ Scheduler::Scheduler(const Device& device_, StateTracker& state_tracker_)
master_semaphore{std::make_unique<MasterSemaphore>(device)},
command_pool{std::make_unique<CommandPool>(*master_semaphore, device)} {
vk::SetDeletionTimeline(master_semaphore->CurrentTick());
AcquireNewChunk();
AllocateWorkerCommandBuffer();
worker_thread = std::jthread([this](std::stop_token token) { WorkerThread(token); });
@@ -94,87 +93,13 @@ void Scheduler::DispatchWork() {
}
}
void Scheduler::BeginDynamicRendering(const Framebuffer* framebuffer, const DeferredClear* clear) {
const VkExtent2D render_area = framebuffer->RenderArea();
std::array<VkImageView, 9> attachment_views{};
const auto& color_views = framebuffer->ColorAttachments();
for (size_t index = 0; index < color_views.size(); ++index) {
attachment_views[index] = color_views[index];
}
attachment_views[8] = framebuffer->DepthAttachment();
state.renderpass = VkRenderPass{};
state.framebuffer = VkFramebuffer{};
state.attachment_views = attachment_views;
state.color_resolve_views = framebuffer->ColorResolveAttachments();
state.color_resolve_modes = framebuffer->ColorResolveModes();
state.depth_resolve_view = framebuffer->DepthResolveAttachment();
state.depth_resolve_mode = framebuffer->DepthResolveMode();
state.stencil_resolve_mode = framebuffer->StencilResolveMode();
state.discards_msaa_color = framebuffer->DiscardsMsaaColor();
state.discards_msaa_depth = framebuffer->DiscardsMsaaDepth();
state.render_area = render_area;
state.num_color = framebuffer->NumColorAttachments();
state.has_depth = framebuffer->HasAspectDepthBit();
state.has_stencil = framebuffer->HasAspectStencilBit();
state.layer_count = framebuffer->NumLayers();
state.rendering = true;
if (GPU::Logging::IsActive() && Settings::values.gpu_log_vulkan_calls.GetValue()) {
const std::string render_pass_info =
fmt::format("renderArea={}x{}, numImages={}", render_area.width, render_area.height,
framebuffer->NumImages());
GPU::Logging::GPULogger::GetInstance().LogRenderPassBegin(render_pass_info);
}
pending_begin = true;
has_pending_begin_clear = clear != nullptr;
pending_begin_clear = clear != nullptr ? *clear : DeferredClear{};
num_renderpass_images = framebuffer->NumImages();
renderpass_images = framebuffer->Images();
renderpass_image_ranges = framebuffer->ImageRanges();
}
void Scheduler::FlushPendingRenderPass() {
if (!pending_begin) {
return;
}
pending_begin = false;
const bool had_clear = has_pending_begin_clear;
has_pending_begin_clear = false;
RecordDynamicBegin(had_clear ? &pending_begin_clear : nullptr);
}
bool Scheduler::RetractUnrecordedRenderPass() {
if (!pending_begin) {
return false;
}
pending_begin = false;
if (has_pending_begin_clear) {
deferred_clear = pending_begin_clear;
has_pending_begin_clear = false;
}
state.renderpass = VkRenderPass{};
state.framebuffer = VkFramebuffer{};
state.attachment_views = {};
state.rendering = false;
state.uses_transform_feedback = false;
num_renderpass_images = 0;
return true;
}
void Scheduler::BeginRenderPassImpl(const Framebuffer* framebuffer, VkRenderPass renderpass,
const VkClearValue* clear_values, u32 clear_value_count) {
if (device.IsKhrDynamicRenderingSupported()) {
BeginDynamicRendering(framebuffer, nullptr);
return;
}
const VkExtent2D render_area = framebuffer->RenderArea();
const VkFramebuffer framebuffer_handle = framebuffer->Handle();
const VkExtent2D render_area = framebuffer->RenderArea();
state.renderpass = renderpass;
state.framebuffer = framebuffer_handle;
state.render_area = render_area;
state.rendering = true;
if (GPU::Logging::IsActive() && Settings::values.gpu_log_vulkan_calls.GetValue()) {
const std::string render_pass_info =
@@ -216,12 +141,6 @@ void Scheduler::RealizeDeferredClear() {
const DeferredClear dc = deferred_clear;
deferred_clear = {};
if (device.IsKhrDynamicRenderingSupported()) {
EndRenderPass();
BeginDynamicRendering(dc.framebuffer, &dc);
return;
}
std::array<VkClearValue, 9> clear_values{};
u32 count = 0;
const RenderPassKey& base = dc.framebuffer->RenderPassKeyBase();
@@ -276,25 +195,9 @@ void Scheduler::RequestRenderpass(const Framebuffer* framebuffer) {
RealizeDeferredClear();
return;
}
const VkExtent2D render_area = framebuffer->RenderArea();
if (device.IsKhrDynamicRenderingSupported()) {
std::array<VkImageView, 9> attachment_views{};
const auto& color_views = framebuffer->ColorAttachments();
for (size_t index = 0; index < color_views.size(); ++index) {
attachment_views[index] = color_views[index];
}
attachment_views[8] = framebuffer->DepthAttachment();
if (state.rendering && attachment_views == state.attachment_views &&
render_area.width == state.render_area.width &&
render_area.height == state.render_area.height) {
return;
}
EndRenderPass();
BeginDynamicRendering(framebuffer, nullptr);
return;
}
const VkRenderPass renderpass = framebuffer->RenderPass();
const VkFramebuffer framebuffer_handle = framebuffer->Handle();
const VkExtent2D render_area = framebuffer->RenderArea();
if (renderpass == state.renderpass && framebuffer_handle == state.framebuffer &&
render_area.width == state.render_area.width &&
render_area.height == state.render_area.height) {
@@ -419,7 +322,6 @@ u64 Scheduler::SubmitExecution(VkSemaphore signal_semaphore, VkSemaphore wait_se
InvalidateState();
const u64 signal_value = master_semaphore->NextTick();
vk::SetDeletionTimeline(master_semaphore->CurrentTick());
RecordWithUploadBuffer([signal_semaphore, wait_semaphore, signal_value,
this](vk::CommandBuffer cmdbuf, vk::CommandBuffer upload_cmdbuf) {
static constexpr VkMemoryBarrier WRITE_BARRIER{
@@ -470,155 +372,6 @@ void Scheduler::InvalidateState() {
state_tracker.InvalidateCommandBufferState();
}
namespace {
/// The part of a dynamic rendering begin that every pass needs.
struct DynamicRenderingBase {
std::array<VkImageView, 9> views;
VkExtent2D render_area;
u32 num_color;
u32 layers;
VkImageView ds_resolve_view;
VkResolveModeFlagBits depth_resolve_mode;
VkResolveModeFlagBits stencil_resolve_mode;
bool has_depth;
bool has_stencil;
bool ds_discard;
};
/// The part only a pass that actually clears something needs.
struct DynamicRenderingClears {
std::array<VkClearValue, 8> color_values;
VkClearValue ds_value;
u32 color_clear_mask;
u32 color_discard_mask;
bool ds_clear;
};
/// Builds the rendering info and begins the instance. `resolve_views`/`resolve_modes` and `clears`
/// are null for passes with no colour resolve targets and no clears, so those tables never have to
/// be copied into the recorded command.
void IssueBeginRendering(vk::CommandBuffer cmdbuf, const DynamicRenderingBase& base,
const std::array<VkImageView, 8>* resolve_views,
const std::array<VkResolveModeFlagBits, 8>* resolve_modes,
const DynamicRenderingClears* clears) {
std::array<VkRenderingAttachmentInfo, VideoCommon::NUM_RT> color_infos{};
for (u32 index = 0; index < base.num_color; ++index) {
const bool clear_slot =
clears != nullptr && ((clears->color_clear_mask >> index) & 1u) != 0;
const VkImageView resolve_view =
resolve_views != nullptr ? (*resolve_views)[index] : VK_NULL_HANDLE;
const bool has_resolve = resolve_view != VK_NULL_HANDLE;
const bool discard_slot =
has_resolve && clears != nullptr && ((clears->color_discard_mask >> index) & 1u) != 0;
color_infos[index] = VkRenderingAttachmentInfo{
.sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO,
.pNext = nullptr,
.imageView = base.views[index],
.imageLayout = VK_IMAGE_LAYOUT_GENERAL,
.resolveMode = has_resolve ? (*resolve_modes)[index] : VK_RESOLVE_MODE_NONE,
.resolveImageView = resolve_view,
.resolveImageLayout =
has_resolve ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_UNDEFINED,
.loadOp = clear_slot ? VK_ATTACHMENT_LOAD_OP_CLEAR : VK_ATTACHMENT_LOAD_OP_LOAD,
.storeOp = discard_slot ? VK_ATTACHMENT_STORE_OP_DONT_CARE
: VK_ATTACHMENT_STORE_OP_STORE,
.clearValue = clear_slot ? clears->color_values[index] : VkClearValue{},
};
}
const bool ds_clear = clears != nullptr && clears->ds_clear;
const VkClearValue ds_clear_value = clears != nullptr ? clears->ds_value : VkClearValue{};
const bool has_ds_resolve = base.ds_resolve_view != VK_NULL_HANDLE;
const VkAttachmentLoadOp ds_load_op = ds_clear ? VK_ATTACHMENT_LOAD_OP_CLEAR
: base.ds_discard ? VK_ATTACHMENT_LOAD_OP_DONT_CARE
: VK_ATTACHMENT_LOAD_OP_LOAD;
const VkAttachmentStoreOp ds_store_op =
base.ds_discard ? VK_ATTACHMENT_STORE_OP_DONT_CARE : VK_ATTACHMENT_STORE_OP_STORE;
const VkRenderingAttachmentInfo depth_info{
.sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO,
.pNext = nullptr,
.imageView = base.views[8],
.imageLayout = VK_IMAGE_LAYOUT_GENERAL,
.resolveMode = has_ds_resolve ? base.depth_resolve_mode : VK_RESOLVE_MODE_NONE,
.resolveImageView = has_ds_resolve ? base.ds_resolve_view : VK_NULL_HANDLE,
.resolveImageLayout = has_ds_resolve ? VK_IMAGE_LAYOUT_GENERAL
: VK_IMAGE_LAYOUT_UNDEFINED,
.loadOp = ds_load_op,
.storeOp = ds_store_op,
.clearValue = ds_clear ? ds_clear_value : VkClearValue{},
};
// Stencil gets its own struct because its resolve mode may differ from depth's.
const VkRenderingAttachmentInfo stencil_info{
.sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO,
.pNext = nullptr,
.imageView = base.views[8],
.imageLayout = VK_IMAGE_LAYOUT_GENERAL,
.resolveMode = has_ds_resolve ? base.stencil_resolve_mode : VK_RESOLVE_MODE_NONE,
.resolveImageView = has_ds_resolve ? base.ds_resolve_view : VK_NULL_HANDLE,
.resolveImageLayout = has_ds_resolve ? VK_IMAGE_LAYOUT_GENERAL
: VK_IMAGE_LAYOUT_UNDEFINED,
.loadOp = ds_load_op,
.storeOp = ds_store_op,
.clearValue = ds_clear ? ds_clear_value : VkClearValue{},
};
const VkRenderingInfo rendering_info{
.sType = VK_STRUCTURE_TYPE_RENDERING_INFO,
.pNext = nullptr,
.flags = 0,
.renderArea =
{
.offset = {.x = 0, .y = 0},
.extent = base.render_area,
},
.layerCount = base.layers,
.viewMask = 0,
.colorAttachmentCount = base.num_color,
.pColorAttachments = color_infos.data(),
.pDepthAttachment = base.has_depth ? &depth_info : nullptr,
.pStencilAttachment = base.has_stencil ? &stencil_info : nullptr,
};
cmdbuf.BeginRendering(rendering_info);
}
} // Anonymous namespace
void Scheduler::RecordDynamicBegin(const DeferredClear* clear) {
const DynamicRenderingBase base{
.views = state.attachment_views,
.render_area = state.render_area,
.num_color = state.num_color,
.layers = state.layer_count,
.ds_resolve_view = state.depth_resolve_view,
.depth_resolve_mode = state.depth_resolve_mode,
.stencil_resolve_mode = state.stencil_resolve_mode,
.has_depth = state.has_depth,
.has_stencil = state.has_stencil,
.ds_discard = state.discards_msaa_depth,
};
bool has_color_resolve = false;
for (const VkImageView resolve_view : state.color_resolve_views) {
has_color_resolve = has_color_resolve || resolve_view != VK_NULL_HANDLE;
}
if (clear == nullptr && !has_color_resolve) {
// Common case. Every byte captured here is copied into the command chunk on every single
// pass begin, and the clear values plus the two resolve tables are two thirds of them.
Record([base](vk::CommandBuffer cmdbuf) {
IssueBeginRendering(cmdbuf, base, nullptr, nullptr, nullptr);
});
return;
}
const DynamicRenderingClears clears{
.color_values = clear != nullptr ? clear->color_values : std::array<VkClearValue, 8>{},
.ds_value = clear != nullptr ? clear->depth_stencil_value : VkClearValue{},
.color_clear_mask = clear != nullptr ? clear->color_clear_mask : 0u,
.color_discard_mask =
clear != nullptr && state.discards_msaa_color ? clear->color_clear_mask : 0u,
.ds_clear = clear != nullptr && clear->depth_stencil,
};
Record([base, resolve_views = state.color_resolve_views,
resolve_modes = state.color_resolve_modes, clears](vk::CommandBuffer cmdbuf) {
IssueBeginRendering(cmdbuf, base, &resolve_views, &resolve_modes, &clears);
});
}
void Scheduler::EndPendingOperations() {
query_cache->CounterReset(VideoCommon::QueryType::ZPassPixelCount64);
EndRenderPass();
@@ -627,10 +380,7 @@ void Scheduler::EndPendingOperations() {
void Scheduler::EndRenderPass()
{
RealizeDeferredClear();
if (query_cache) {
query_cache->NotifySegment(false);
}
if (!state.rendering) {
if (!state.renderpass) {
return;
}
@@ -643,24 +393,14 @@ void Scheduler::EndRenderPass()
}
query_cache->CounterEnable(VideoCommon::QueryType::ZPassPixelCount64, false);
if (pending_begin) {
if (!has_pending_begin_clear) {
RetractUnrecordedRenderPass();
return;
}
FlushPendingRenderPass();
}
query_cache->NotifySegment(false);
Record([num_images = num_renderpass_images,
images = renderpass_images,
ranges = renderpass_image_ranges,
has_transform_feedback = state.uses_transform_feedback,
dynamic_rendering = device.IsKhrDynamicRenderingSupported()](
has_transform_feedback = device.IsExtTransformFeedbackSupported()](
vk::CommandBuffer cmdbuf) {
const bool use_sync2 = cmdbuf.HasPipelineBarrier2();
std::array<VkImageMemoryBarrier, 9> barriers;
std::array<VkImageMemoryBarrier2, 9> barriers2;
for (size_t i = 0; i < num_images; ++i) {
const VkImageSubresourceRange& range = ranges[i];
const bool is_color = (range.aspectMask & VK_IMAGE_ASPECT_COLOR_BIT) != 0;
@@ -678,57 +418,15 @@ void Scheduler::EndRenderPass()
src_access |= VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT
| VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT;
if (!use_sync2) {
barriers[i] = VkImageMemoryBarrier{
.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
.pNext = nullptr,
.srcAccessMask = src_access,
.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT
| VK_ACCESS_COLOR_ATTACHMENT_READ_BIT
| VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT
| VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT
| VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT,
.oldLayout = VK_IMAGE_LAYOUT_GENERAL,
.newLayout = VK_IMAGE_LAYOUT_GENERAL,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.image = images[i],
.subresourceRange = range,
};
continue;
}
VkPipelineStageFlags2 src_stage = 0;
VkAccessFlags2 dst_access =
VK_ACCESS_2_SHADER_READ_BIT | VK_ACCESS_2_SHADER_WRITE_BIT;
if (is_color) {
src_stage |= VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT;
dst_access |= VK_ACCESS_2_COLOR_ATTACHMENT_READ_BIT
| VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT;
}
if (is_depth_stencil) {
src_stage |= VK_PIPELINE_STAGE_2_EARLY_FRAGMENT_TESTS_BIT
| VK_PIPELINE_STAGE_2_LATE_FRAGMENT_TESTS_BIT;
dst_access |= VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_READ_BIT
| VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT;
}
if (!is_color && !is_depth_stencil) {
src_stage = VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT
| VK_PIPELINE_STAGE_2_EARLY_FRAGMENT_TESTS_BIT
| VK_PIPELINE_STAGE_2_LATE_FRAGMENT_TESTS_BIT;
dst_access |= VK_ACCESS_2_COLOR_ATTACHMENT_READ_BIT
| VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT
| VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_READ_BIT
| VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT;
}
barriers2[i] = VkImageMemoryBarrier2{
.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2,
barriers[i] = VkImageMemoryBarrier{
.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
.pNext = nullptr,
.srcStageMask = src_stage,
.srcAccessMask = static_cast<VkAccessFlags2>(src_access),
.dstStageMask = VK_PIPELINE_STAGE_2_ALL_GRAPHICS_BIT
| VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT,
.dstAccessMask = dst_access,
.srcAccessMask = src_access,
.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT
| VK_ACCESS_COLOR_ATTACHMENT_READ_BIT
| VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT
| VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT
| VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT,
.oldLayout = VK_IMAGE_LAYOUT_GENERAL,
.newLayout = VK_IMAGE_LAYOUT_GENERAL,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
@@ -737,19 +435,10 @@ void Scheduler::EndRenderPass()
.subresourceRange = range,
};
}
if (dynamic_rendering) {
cmdbuf.EndRendering();
} else {
cmdbuf.EndRenderPass();
}
if (use_sync2) {
cmdbuf.PipelineBarrier2(0, nullptr, nullptr,
vk::Span(barriers2.data(), num_images));
} else {
cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT |
VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, vk::PIPELINE_STAGE_GRAPHICS_COMPUTE,
0, nullptr, nullptr, vk::Span(barriers.data(), num_images));
}
cmdbuf.EndRenderPass();
cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT |
VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, vk::PIPELINE_STAGE_GRAPHICS_COMPUTE,
0, nullptr, nullptr, vk::Span(barriers.data(), num_images));
if (has_transform_feedback) {
static constexpr VkMemoryBarrier XFB_OUTPUT_BARRIER{
.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
@@ -764,10 +453,6 @@ void Scheduler::EndRenderPass()
});
state.renderpass = VkRenderPass{};
state.framebuffer = VkFramebuffer{};
state.attachment_views = {};
state.rendering = false;
state.uses_transform_feedback = false;
num_renderpass_images = 0;
}
+26 -73
View File
@@ -71,12 +71,7 @@ public:
/// Returns true when a render pass is currently active in the scheduler state.
bool IsRenderPassActive() const {
return state.rendering;
}
/// Flags that transform feedback writes have been recorded since the last render pass end.
void MarkTransformFeedbackUsed() noexcept {
state.uses_transform_feedback = true;
return state.renderpass != VK_NULL_HANDLE;
}
/// Update the pipeline to the current execution context.
@@ -112,17 +107,12 @@ public:
template <typename T>
requires std::is_invocable_v<T, vk::CommandBuffer>
void Record(T&& c) {
if (pending_begin) {
FlushPendingRenderPass();
}
this->RecordWithUploadBuffer(
[command = std::move(c)](vk::CommandBuffer cmdbuf, vk::CommandBuffer) {
command(cmdbuf);
});
}
bool RetractUnrecordedRenderPass();
/// Returns the current command buffer tick.
[[nodiscard]] u64 CurrentTick() const noexcept {
return master_semaphore->CurrentTick();
@@ -141,16 +131,33 @@ public:
}
master_semaphore->Wait(tick);
}
ApplyFramePacing(target_fps);
}
void WaitSubmitted(u64 tick, double target_fps = 0.0) {
if (tick > 0 && tick < master_semaphore->CurrentTick()) {
master_semaphore->Wait(tick);
if (Settings::values.use_speed_limit.GetValue() && target_fps > 0.0) {
auto now = std::chrono::steady_clock::now();
if (last_target_fps != target_fps) {
frame_interval = std::chrono::duration_cast<std::chrono::steady_clock::duration>(std::chrono::duration<double>(1.0 / target_fps));
max_frame_count = static_cast<int>(0.1 * target_fps);
last_target_fps = target_fps;
frame_counter = 0;
start_time = now;
}
frame_counter++;
auto target_time = start_time + frame_interval * frame_counter;
if (target_time >= now) {
auto sleep_time = target_time - now;
if (sleep_time > std::chrono::milliseconds(15)) {
std::this_thread::sleep_for(sleep_time - std::chrono::milliseconds(1));
}
while (std::chrono::steady_clock::now() < target_time) {
std::this_thread::yield();
}
} else if (frame_counter > max_frame_count) {
frame_counter = 0;
start_time = now;
}
}
ApplyFramePacing(target_fps);
}
/// Returns the master timeline semaphore.
[[nodiscard]] MasterSemaphore& GetMasterSemaphore() const noexcept {
return *master_semaphore;
}
@@ -158,35 +165,6 @@ public:
std::mutex submit_mutex;
private:
void ApplyFramePacing(double target_fps) {
if (!Settings::values.use_speed_limit.GetValue() || target_fps <= 0.0) {
return;
}
auto now = std::chrono::steady_clock::now();
if (last_target_fps != target_fps) {
frame_interval = std::chrono::duration_cast<std::chrono::steady_clock::duration>(
std::chrono::duration<double>(1.0 / target_fps));
max_frame_count = static_cast<int>(0.1 * target_fps);
last_target_fps = target_fps;
frame_counter = 0;
start_time = now;
}
frame_counter++;
auto target_time = start_time + frame_interval * frame_counter;
if (target_time >= now) {
auto sleep_time = target_time - now;
if (sleep_time > std::chrono::milliseconds(15)) {
std::this_thread::sleep_for(sleep_time - std::chrono::milliseconds(1));
}
while (std::chrono::steady_clock::now() < target_time) {
std::this_thread::yield();
}
} else if (frame_counter > max_frame_count) {
frame_counter = 0;
start_time = now;
}
}
class Command {
public:
virtual ~Command() = default;
@@ -266,31 +244,17 @@ private:
size_t command_offset = 0;
bool submit = false;
alignas(std::max_align_t) std::array<u8, 0x10000> data{};
alignas(std::max_align_t) std::array<u8, 0x8000> data{};
};
struct State {
VkRenderPass renderpass{};
VkFramebuffer framebuffer{};
std::array<VkImageView, 9> attachment_views{};
std::array<VkImageView, 8> color_resolve_views{};
std::array<VkResolveModeFlagBits, 8> color_resolve_modes{};
VkImageView depth_resolve_view{};
VkResolveModeFlagBits depth_resolve_mode = VK_RESOLVE_MODE_NONE;
VkResolveModeFlagBits stencil_resolve_mode = VK_RESOLVE_MODE_NONE;
VkExtent2D render_area = {0, 0};
GraphicsPipeline* graphics_pipeline = nullptr;
bool rendering = false;
bool discards_msaa_color = false;
bool discards_msaa_depth = false;
u32 num_color = 0;
bool has_depth = false;
bool has_stencil = false;
u32 layer_count = 1;
bool is_rescaling = false;
bool rescaling_defined = false;
bool needs_state_enable_refresh = false;
bool uses_transform_feedback = false;
};
struct DeferredClear {
@@ -305,9 +269,6 @@ private:
void BeginRenderPassImpl(const Framebuffer* framebuffer, VkRenderPass renderpass,
const VkClearValue* clear_values, u32 clear_value_count);
/// Begins a dynamic rendering pass, optionally realizing a deferred clear via load ops.
void BeginDynamicRendering(const Framebuffer* framebuffer, const DeferredClear* clear);
/// If a deferred clear is pending.
void RealizeDeferredClear();
@@ -321,10 +282,6 @@ private:
void EndPendingOperations();
void RecordDynamicBegin(const DeferredClear* clear);
void FlushPendingRenderPass();
void EndRenderPass();
void AcquireNewChunk();
@@ -342,10 +299,6 @@ private:
DeferredClear deferred_clear;
bool pending_begin = false;
bool has_pending_begin_clear = false;
DeferredClear pending_begin_clear;
std::unique_ptr<CommandChunk> chunk;
std::function<void()> on_submit;
@@ -84,16 +84,10 @@ StagingBufferPool::StagingBufferPool(const Device& device_, MemoryAllocator& mem
if (device.IsExtTransformFeedbackSupported()) {
stream_ci.usage |= VK_BUFFER_USAGE_TRANSFORM_FEEDBACK_BUFFER_BIT_EXT;
}
if (device.IsBufferDeviceAddressSupported()) {
stream_ci.usage |= VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT;
}
stream_buffer = memory_allocator.CreateBuffer(stream_ci, MemoryUsage::Stream);
if (device.HasDebuggingToolAttached()) {
stream_buffer.SetObjectNameEXT("Stream Buffer");
}
if (device.IsBufferDeviceAddressSupported()) {
stream_buffer_address = device.GetLogical().GetBufferDeviceAddress(*stream_buffer);
}
stream_pointer = stream_buffer.Mapped();
ASSERT_MSG(!stream_pointer.empty(), "Stream buffer must be host visible!");
}
@@ -155,7 +149,6 @@ StagingBufferRef StagingBufferPool::GetStreamBuffer(size_t size) {
iterator = Common::AlignUp(iterator + size, MAX_ALIGNMENT);
return StagingBufferRef{
.buffer = *stream_buffer,
.device_address = stream_buffer_address,
.offset = static_cast<VkDeviceSize>(offset),
.mapped_span = stream_pointer.subspan(offset, size),
.usage{},
@@ -219,22 +212,14 @@ StagingBufferRef StagingBufferPool::CreateStagingBuffer(size_t size, MemoryUsage
if (device.IsExtTransformFeedbackSupported()) {
buffer_ci.usage |= VK_BUFFER_USAGE_TRANSFORM_FEEDBACK_BUFFER_BIT_EXT;
}
if (device.IsBufferDeviceAddressSupported()) {
buffer_ci.usage |= VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT;
}
vk::Buffer buffer = memory_allocator.CreateBuffer(buffer_ci, usage);
if (device.HasDebuggingToolAttached()) {
++buffer_index;
buffer.SetObjectNameEXT(fmt::format("Staging Buffer {}", buffer_index).c_str());
}
const std::span<u8> mapped_span = buffer.Mapped();
const VkDeviceAddress buffer_address =
device.IsBufferDeviceAddressSupported()
? device.GetLogical().GetBufferDeviceAddress(*buffer)
: VkDeviceAddress{};
StagingBuffer& entry = GetCache(usage)[log2_size].entries.emplace_back(StagingBuffer{
.buffer = std::move(buffer),
.device_address = buffer_address,
.mapped_span = mapped_span,
.usage = usage,
.log2_level = log2_size,
@@ -267,62 +252,25 @@ void StagingBufferPool::ReleaseLevel(StagingBuffersCache& cache, size_t log2) {
constexpr size_t deletions_per_tick = 16;
auto& staging = cache[log2];
auto& entries = staging.entries;
if (entries.empty()) {
staging.delete_index = 0;
staging.iterate_index = 0;
return;
}
const size_t old_size = entries.size();
const auto is_deletable = [this](const StagingBuffer& entry) {
return scheduler.IsFree(entry.tick);
};
const size_t begin_offset = (std::min)(staging.delete_index, entries.size());
const size_t end_offset = (std::min)(begin_offset + deletions_per_tick, entries.size());
const size_t begin_offset = staging.delete_index;
const size_t end_offset = (std::min)(begin_offset + deletions_per_tick, old_size);
const auto begin = entries.begin() + begin_offset;
const auto end = entries.begin() + end_offset;
const auto surviving_end = std::remove_if(begin, end, is_deletable);
const size_t removed = static_cast<size_t>(std::distance(surviving_end, end));
entries.erase(surviving_end, end);
entries.erase(std::remove_if(begin, end, is_deletable), end);
staging.delete_index = end_offset - removed;
if (staging.delete_index >= entries.size()) {
const size_t new_size = entries.size();
staging.delete_index += deletions_per_tick;
if (staging.delete_index >= new_size) {
staging.delete_index = 0;
}
if (staging.iterate_index > entries.size()) {
if (staging.iterate_index > new_size) {
staging.iterate_index = 0;
}
}
u64 StagingBufferPool::ReclaimMemory(u64 target_bytes) {
u64 freed = 0;
const auto is_deletable = [this](const StagingBuffer& entry) {
return scheduler.IsFree(entry.tick);
};
const auto reclaim_cache = [&](StagingBuffersCache& cache) {
for (size_t level = NUM_LEVELS; level-- > 0 && freed < target_bytes;) {
auto& staging = cache[level];
auto& entries = staging.entries;
if (entries.empty()) {
continue;
}
const u64 entry_bytes = 1ULL << level;
auto it = entries.begin();
while (it != entries.end() && freed < target_bytes) {
if (is_deletable(*it)) {
it = entries.erase(it);
freed += entry_bytes;
} else {
++it;
}
}
staging.delete_index = 0;
staging.iterate_index = 0;
}
};
reclaim_cache(device_local_cache);
reclaim_cache(upload_cache);
reclaim_cache(download_cache);
return freed;
}
} // namespace Vulkan
@@ -1,6 +1,3 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
@@ -21,7 +18,6 @@ class Scheduler;
struct StagingBufferRef {
VkBuffer buffer;
VkDeviceAddress device_address;
VkDeviceSize offset;
std::span<u8> mapped_span;
MemoryUsage usage;
@@ -46,8 +42,6 @@ public:
void TickFrame();
u64 ReclaimMemory(u64 target_bytes);
private:
struct StreamBufferCommit {
size_t upper_bound;
@@ -56,7 +50,6 @@ private:
struct StagingBuffer {
vk::Buffer buffer;
VkDeviceAddress device_address;
std::span<u8> mapped_span;
MemoryUsage usage;
u32 log2_level;
@@ -67,7 +60,6 @@ private:
StagingBufferRef Ref() const noexcept {
return {
.buffer = *buffer,
.device_address = device_address,
.offset = 0,
.mapped_span = mapped_span,
.usage = usage,
@@ -111,7 +103,6 @@ private:
Scheduler& scheduler;
vk::Buffer stream_buffer;
VkDeviceAddress stream_buffer_address{};
std::span<u8> stream_pointer;
VkDeviceSize stream_buffer_size;
VkDeviceSize region_size;
@@ -9,6 +9,10 @@
#include <limits>
#include <vector>
#ifdef __ANDROID__
#include <android/api-level.h>
#endif
#include "common/logging.h"
#include "common/settings.h"
#include "common/settings_enums.h"
@@ -172,26 +176,34 @@ bool Swapchain::AcquireNextImage() {
break;
}
#ifdef __ANDROID__
scheduler.WaitSubmitted(resource_ticks[image_index]);
#else
const auto wait_with_frame_pacing = [this] {
switch (Settings::values.frame_pacing_mode.GetValue()) {
case Settings::FramePacingMode::Target_Auto:
scheduler.WaitSubmitted(resource_ticks[image_index]);
scheduler.Wait(resource_ticks[image_index]);
break;
case Settings::FramePacingMode::Target_30:
scheduler.WaitSubmitted(resource_ticks[image_index], 30.0);
scheduler.Wait(resource_ticks[image_index], 30.0);
break;
case Settings::FramePacingMode::Target_60:
scheduler.WaitSubmitted(resource_ticks[image_index], 60.0);
scheduler.Wait(resource_ticks[image_index], 60.0);
break;
case Settings::FramePacingMode::Target_90:
scheduler.WaitSubmitted(resource_ticks[image_index], 90.0);
scheduler.Wait(resource_ticks[image_index], 90.0);
break;
case Settings::FramePacingMode::Target_120:
scheduler.WaitSubmitted(resource_ticks[image_index], 120.0);
scheduler.Wait(resource_ticks[image_index], 120.0);
break;
}
};
#ifdef __ANDROID__
if (android_get_device_api_level() >= 30) {
scheduler.Wait(resource_ticks[image_index]);
} else {
wait_with_frame_pacing();
}
#else
wait_with_frame_pacing();
#endif
resource_ticks[image_index] = scheduler.CurrentTick();
@@ -54,8 +54,6 @@ using VideoCore::Surface::SurfaceType;
namespace {
constexpr bool ENABLE_MSAA_RESOLVE_CONSUME = true;
constexpr bool ENABLE_MSAA_COLOR_DISCARD = true;
constexpr bool ENABLE_MSAA_DEPTH_DISCARD = true;
constexpr bool ENABLE_MSAA_DEPTH_RESOLVE = false;
constexpr VkBorderColor ConvertBorderColor(const std::array<float, 4>& color) {
if (color == std::array<float, 4>{0, 0, 0, 0}) {
@@ -146,6 +144,11 @@ constexpr VkBorderColor ConvertBorderColor(const std::array<float, 4>& color) {
info.size.depth == 1;
}
[[nodiscard]] bool WillUseWidenedAstcFormat(const Device& device, const ImageInfo& info) {
return WillUseAcceleratedAstcDecode(device, info) &&
!VideoCore::Surface::IsPixelFormatSRGB(info.format);
}
[[nodiscard]] VkImageCreateInfo MakeImageCreateInfo(const Device& device, const ImageInfo& info,
std::optional<VkFormat> format_override = {}) {
auto format_info =
@@ -209,11 +212,7 @@ constexpr VkBorderColor ConvertBorderColor(const std::array<float, 4>& color) {
return device.IsFormatSupported(view_format, VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT,
FormatType::Optimal);
});
const bool storage_allowed_for_samples =
image_ci.samples == VK_SAMPLE_COUNT_1_BIT ||
(device.GetStorageImageSampleCounts() &
static_cast<VkSampleCountFlags>(image_ci.samples)) != 0;
if (has_storage_compatible_view && storage_allowed_for_samples) {
if (has_storage_compatible_view) {
image_ci.usage |= VK_IMAGE_USAGE_STORAGE_BIT;
}
@@ -270,10 +269,6 @@ constexpr VkBorderColor ConvertBorderColor(const std::array<float, 4>& color) {
}
}
[[nodiscard]] bool IsLdrAstcFormat(VkFormat format) {
return format >= VK_FORMAT_ASTC_4x4_UNORM_BLOCK && format <= VK_FORMAT_ASTC_12x12_SRGB_BLOCK;
}
[[nodiscard]] VkImageAspectFlags ImageViewAspectMask(const VideoCommon::ImageViewInfo& info) {
if (info.IsRenderTarget()) {
return ImageAspectMask(info.format);
@@ -901,15 +896,6 @@ void BlitScale(Scheduler& scheduler, VkImage src_image, VkImage dst_image, const
0, nullptr, nullptr, write_barriers);
});
}
[[nodiscard]] bool CanBlitNatively(const Device& device, PixelFormat format) {
static constexpr auto OPTIMAL_FORMAT = FormatType::Optimal;
static constexpr VkFormatFeatureFlags BLIT_USAGE =
VK_FORMAT_FEATURE_BLIT_SRC_BIT | VK_FORMAT_FEATURE_BLIT_DST_BIT;
const VkFormat vk_format =
MaxwellToVK::SurfaceFormat(device, OPTIMAL_FORMAT, false, format).format;
return device.IsFormatSupported(vk_format, BLIT_USAGE, OPTIMAL_FORMAT);
}
} // Anonymous namespace
TextureCacheRuntime::TextureCacheRuntime(const Device& device_, Scheduler& scheduler_,
@@ -948,10 +934,6 @@ TextureCacheRuntime::TextureCacheRuntime(const Device& device_, Scheduler& sched
bl3d_unswizzle_pass.emplace(device, scheduler, descriptor_pool,
staging_buffer_pool, compute_pass_descriptor_queue);
}
bl2d_unswizzle_pass.emplace(device, scheduler, descriptor_pool, staging_buffer_pool,
compute_pass_descriptor_queue);
bl3db_unswizzle_pass.emplace(device, scheduler, descriptor_pool, staging_buffer_pool,
compute_pass_descriptor_queue);
}
void TextureCacheRuntime::Finish() {
@@ -1014,21 +996,13 @@ VkBuffer TextureCacheRuntime::GetTemporaryBuffer(size_t needed_size) {
}
VkImageView TextureCacheRuntime::GetOrCreateResolveShadow(VkImage msaa_image, VkFormat format,
VkExtent2D extent, u32 layers,
VkImageAspectFlags aspect) {
const bool is_depth_stencil =
(aspect & (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)) != 0;
VkExtent2D extent, u32 layers) {
ResolveShadow& shadow = resolve_shadows[msaa_image];
if (shadow.image && shadow.format == format && shadow.extent.width == extent.width &&
shadow.extent.height == extent.height && shadow.layers == layers) {
shadow.up_to_date = true;
return *shadow.view;
}
const VkImageUsageFlags shadow_usage =
static_cast<VkImageUsageFlags>(is_depth_stencil
? VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT
: VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT) |
VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
shadow.image = memory_allocator.CreateImage(VkImageCreateInfo{
.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO,
.pNext = nullptr,
@@ -1040,7 +1014,8 @@ VkImageView TextureCacheRuntime::GetOrCreateResolveShadow(VkImage msaa_image, Vk
.arrayLayers = layers,
.samples = VK_SAMPLE_COUNT_1_BIT,
.tiling = VK_IMAGE_TILING_OPTIMAL,
.usage = shadow_usage,
.usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT |
VK_IMAGE_USAGE_TRANSFER_SRC_BIT,
.sharingMode = VK_SHARING_MODE_EXCLUSIVE,
.queueFamilyIndexCount = 0,
.pQueueFamilyIndices = nullptr,
@@ -1055,7 +1030,7 @@ VkImageView TextureCacheRuntime::GetOrCreateResolveShadow(VkImage msaa_image, Vk
.format = format,
.components{},
.subresourceRange{
.aspectMask = aspect,
.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
.baseMipLevel = 0,
.levelCount = 1,
.baseArrayLayer = 0,
@@ -1066,38 +1041,6 @@ VkImageView TextureCacheRuntime::GetOrCreateResolveShadow(VkImage msaa_image, Vk
shadow.extent = extent;
shadow.layers = layers;
shadow.up_to_date = true;
if (device.IsKhrDynamicRenderingSupported()) {
const VkAccessFlags dst_access =
is_depth_stencil ? VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT
: VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
const VkPipelineStageFlags dst_stage =
is_depth_stencil ? VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT
: VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
scheduler.RecordWithUploadBuffer(
[image = *shadow.image, layers, aspect, dst_access, dst_stage](
vk::CommandBuffer, vk::CommandBuffer upload_cmdbuf) {
const VkImageMemoryBarrier barrier{
.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
.pNext = nullptr,
.srcAccessMask = 0,
.dstAccessMask = dst_access,
.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED,
.newLayout = VK_IMAGE_LAYOUT_GENERAL,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.image = image,
.subresourceRange{
.aspectMask = aspect,
.baseMipLevel = 0,
.levelCount = 1,
.baseArrayLayer = 0,
.layerCount = layers,
},
};
upload_cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, dst_stage, 0,
barrier);
});
}
return *shadow.view;
}
@@ -1285,19 +1228,27 @@ void TextureCacheRuntime::BlitImage(Framebuffer* dst_framebuffer, ImageView& dst
blit_image_helper.ResolveDepthStencil(dst_framebuffer, src, dst_region, src_region);
return;
}
static constexpr VkImageAspectFlags DEPTH_STENCIL_ASPECTS =
VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT;
if ((aspect_mask & DEPTH_STENCIL_ASPECTS) != 0 && !CanBlitNatively(device, src.format)) {
if (aspect_mask != DEPTH_STENCIL_ASPECTS) {
UNIMPLEMENTED_MSG("Host cannot blit format {} and no helper path exists for aspect "
"mask 0x{:x}",
src.format, aspect_mask);
if (aspect_mask == (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)) {
const auto format = src.format;
const auto can_blit_depth_stencil = [this, format] {
switch (format) {
case VideoCore::Surface::PixelFormat::D24_UNORM_S8_UINT:
case VideoCore::Surface::PixelFormat::S8_UINT_D24_UNORM:
return device.IsBlitDepth24Stencil8Supported();
case VideoCore::Surface::PixelFormat::D32_FLOAT_S8_UINT:
return device.IsBlitDepth32Stencil8Supported();
default:
UNREACHABLE();
}
}();
// Use shader-based depth/stencil blits if hardware doesn't support the format
// Note: MSAA resolves (MSAA->single) use vkCmdResolveImage which works fine
if (!can_blit_depth_stencil) {
UNIMPLEMENTED_IF(is_src_msaa || is_dst_msaa);
blit_image_helper.BlitDepthStencil(dst_framebuffer, src, dst_region, src_region,
filter, operation);
return;
}
UNIMPLEMENTED_IF(is_src_msaa || is_dst_msaa);
blit_image_helper.BlitDepthStencil(dst_framebuffer, src, dst_region, src_region, filter,
operation);
return;
}
ASSERT(!(is_dst_msaa && !is_src_msaa));
ASSERT(operation == Fermi2D::Operation::SrcCopy);
@@ -1402,7 +1353,7 @@ void TextureCacheRuntime::BlitImage(Framebuffer* dst_framebuffer, ImageView& dst
}
void TextureCacheRuntime::ConvertImage(Framebuffer* dst, ImageView& dst_view, ImageView& src_view) {
if (!dst->RenderPass() && !device.IsKhrDynamicRenderingSupported()) {
if (!dst->RenderPass()) {
return;
}
@@ -1690,6 +1641,11 @@ void TextureCacheRuntime::CopyImageMSAA(Image& dst, Image& src,
std::span<const VideoCommon::ImageCopy> copies) {
const bool msaa_to_non_msaa = src.info.num_samples > 1 && dst.info.num_samples == 1;
const u32 num_samples = msaa_to_non_msaa ? src.info.num_samples : dst.info.num_samples;
if (dst.AspectMask() != VK_IMAGE_ASPECT_COLOR_BIT ||
VideoCore::Surface::IsPixelFormatInteger(dst.info.format)) {
UNIMPLEMENTED_MSG("Copying images with different samples is not supported.");
return;
}
if (ENABLE_MSAA_RESOLVE_CONSUME && msaa_to_non_msaa && copies.size() == 1 &&
src.info.format == dst.info.format) {
const VideoCommon::ImageCopy& copy = copies.front();
@@ -1700,19 +1656,16 @@ void TextureCacheRuntime::CopyImageMSAA(Image& dst, Image& src,
static_cast<u32>(copy.extent.height) <= shadow->extent.height) {
const VkImage shadow_image = *shadow->image;
const VkImage dst_image = dst.Handle();
const VkImageAspectFlags aspect = dst.AspectMask();
const bool is_depth_stencil =
(aspect & (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)) != 0;
const VkImageCopy region{
.srcSubresource{
.aspectMask = aspect,
.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
.mipLevel = 0,
.baseArrayLayer = static_cast<u32>(copy.src_subresource.base_layer),
.layerCount = static_cast<u32>(copy.src_subresource.num_layers),
},
.srcOffset = {0, 0, 0},
.dstSubresource{
.aspectMask = aspect,
.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
.mipLevel = static_cast<u32>(copy.dst_subresource.base_level),
.baseArrayLayer = static_cast<u32>(copy.dst_subresource.base_layer),
.layerCount = static_cast<u32>(copy.dst_subresource.num_layers),
@@ -1721,38 +1674,34 @@ void TextureCacheRuntime::CopyImageMSAA(Image& dst, Image& src,
.extent = {copy.extent.width, copy.extent.height, 1},
};
scheduler.RequestOutsideRenderPassOperationContext();
scheduler.Record([shadow_image, dst_image, region, aspect,
is_depth_stencil](vk::CommandBuffer cmdbuf) {
const VkAccessFlags attachment_write =
is_depth_stencil ? VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT
: VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
scheduler.Record([shadow_image, dst_image, region](vk::CommandBuffer cmdbuf) {
const std::array pre_barriers{
VkImageMemoryBarrier{
.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
.pNext = nullptr,
.srcAccessMask = attachment_write,
.srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT,
.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT,
.oldLayout = VK_IMAGE_LAYOUT_GENERAL,
.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.image = shadow_image,
.subresourceRange{aspect, 0, VK_REMAINING_MIP_LEVELS, 0,
.subresourceRange{VK_IMAGE_ASPECT_COLOR_BIT, 0, VK_REMAINING_MIP_LEVELS, 0,
VK_REMAINING_ARRAY_LAYERS},
},
VkImageMemoryBarrier{
.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
.pNext = nullptr,
.srcAccessMask =
VK_ACCESS_SHADER_WRITE_BIT | attachment_write |
VK_ACCESS_TRANSFER_WRITE_BIT,
.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT |
VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT |
VK_ACCESS_TRANSFER_WRITE_BIT,
.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,
.oldLayout = VK_IMAGE_LAYOUT_GENERAL,
.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.image = dst_image,
.subresourceRange{aspect, 0, VK_REMAINING_MIP_LEVELS, 0,
.subresourceRange{VK_IMAGE_ASPECT_COLOR_BIT, 0, VK_REMAINING_MIP_LEVELS, 0,
VK_REMAINING_ARRAY_LAYERS},
},
};
@@ -1767,7 +1716,7 @@ void TextureCacheRuntime::CopyImageMSAA(Image& dst, Image& src,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.image = shadow_image,
.subresourceRange{aspect, 0, VK_REMAINING_MIP_LEVELS, 0,
.subresourceRange{VK_IMAGE_ASPECT_COLOR_BIT, 0, VK_REMAINING_MIP_LEVELS, 0,
VK_REMAINING_ARRAY_LAYERS},
},
VkImageMemoryBarrier{
@@ -1777,20 +1726,17 @@ void TextureCacheRuntime::CopyImageMSAA(Image& dst, Image& src,
.dstAccessMask = VK_ACCESS_SHADER_READ_BIT |
VK_ACCESS_COLOR_ATTACHMENT_READ_BIT |
VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT |
VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT |
VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT |
VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT,
.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
.newLayout = VK_IMAGE_LAYOUT_GENERAL,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.image = dst_image,
.subresourceRange{aspect, 0, VK_REMAINING_MIP_LEVELS, 0,
.subresourceRange{VK_IMAGE_ASPECT_COLOR_BIT, 0, VK_REMAINING_MIP_LEVELS, 0,
VK_REMAINING_ARRAY_LAYERS},
},
};
cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT |
VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT |
VK_PIPELINE_STAGE_TRANSFER_BIT,
VK_PIPELINE_STAGE_TRANSFER_BIT, 0, nullptr, nullptr,
pre_barriers);
@@ -1803,36 +1749,10 @@ void TextureCacheRuntime::CopyImageMSAA(Image& dst, Image& src,
return;
}
}
if (dst.AspectMask() != VK_IMAGE_ASPECT_COLOR_BIT ||
VideoCore::Surface::IsPixelFormatInteger(dst.info.format)) {
const u64 key{(static_cast<u64>(dst.AspectMask()) << 32) |
static_cast<u64>(dst.info.format)};
if (unsupported_msaa_resolves.insert(key).second) {
LOG_WARNING(Render_Vulkan,
"MSAA resolve unsupported: format={}, aspect={:#x}, samples {}->{}",
dst.info.format, dst.AspectMask(), src.info.num_samples,
dst.info.num_samples);
}
return;
}
blit_image_helper.CopyMSAA(render_pass_cache, dst.Handle(), dst.info.format, src.Handle(),
src.info.format, num_samples, copies, msaa_to_non_msaa);
}
u64 TextureCacheRuntime::CurrentSyncPoint() const noexcept {
return scheduler.CurrentTick();
}
u64 TextureCacheRuntime::CompletedSyncPoint() const {
auto& master_semaphore = scheduler.GetMasterSemaphore();
master_semaphore.Refresh();
return master_semaphore.KnownGpuTick();
}
void TextureCacheRuntime::WaitSyncPoint(u64 sync_point) {
scheduler.Wait(sync_point);
}
u64 TextureCacheRuntime::GetDeviceLocalMemory() const {
return device.GetDeviceLocalMemory();
}
@@ -1841,10 +1761,6 @@ u64 TextureCacheRuntime::GetDeviceMemoryUsage() const {
return device.GetDeviceMemoryUsage();
}
u64 TextureCacheRuntime::GetDeviceAllocationUsage() const {
return device.GetMemoryBudgetInfo().allocation_bytes;
}
bool TextureCacheRuntime::CanReportMemoryUsage() const {
return device.CanReportMemoryUsage();
}
@@ -1854,7 +1770,6 @@ std::optional<size_t> TextureCacheRuntime::GetSamplerHeapBudget() const {
}
void TextureCacheRuntime::TickFrame() {
device.TickAllocatorFrame();
std::erase_if(pending_msaa_images, [this](const auto& pending) {
return scheduler.IsFree(pending.first);
});
@@ -1865,7 +1780,12 @@ Image::Image(TextureCacheRuntime& runtime_, const ImageInfo& info_, GPUVAddr gpu
: VideoCommon::ImageBase(info_, gpu_addr_, cpu_addr_), scheduler{&runtime_.scheduler},
runtime{&runtime_},
original_image(MakeImage(runtime_.device, runtime_.memory_allocator, info,
runtime->ViewFormats(info.format))),
WillUseWidenedAstcFormat(runtime_.device, info)
? std::span<const VkFormat>{}
: runtime->ViewFormats(info.format),
WillUseWidenedAstcFormat(runtime_.device, info)
? std::make_optional(VK_FORMAT_R32G32B32A32_SFLOAT)
: std::nullopt)),
aspect_mask(ImageAspectMask(info.format)) {
if (IsPixelFormatASTC(info.format) && !runtime->device.IsOptimalAstcSupported()) {
switch (Settings::values.accelerate_astc.GetValue()) {
@@ -1880,11 +1800,7 @@ Image::Image(TextureCacheRuntime& runtime_, const ImageInfo& info_, GPUVAddr gpu
default:
break;
}
flags |= VideoCommon::ImageFlagBits::AcceleratedUpload;
flags |= VideoCommon::ImageFlagBits::CostlyLoad;
} else if (runtime->bl3db_unswizzle_pass &&
BlockLinearUnswizzle3DBufferPass::IsSupported(runtime->device, info)) {
flags |= VideoCommon::ImageFlagBits::AcceleratedUpload;
flags |= VideoCommon::ImageFlagBits::Converted;
flags |= VideoCommon::ImageFlagBits::CostlyLoad;
}
if (IsPixelFormatBCn(info.format) && !runtime->device.IsOptimalBcnSupported()) {
@@ -1900,7 +1816,9 @@ Image::Image(TextureCacheRuntime& runtime_, const ImageInfo& info_, GPUVAddr gpu
Settings::values.astc_recompression.GetValue() ==
Settings::AstcRecompression::Uncompressed) {
const auto& device = runtime->device.GetLogical();
const VkFormat storage_format = VK_FORMAT_A8B8G8R8_UNORM_PACK32;
const VkFormat storage_format = WillUseWidenedAstcFormat(runtime->device, info)
? VK_FORMAT_R32G32B32A32_SFLOAT
: VK_FORMAT_A8B8G8R8_UNORM_PACK32;
for (s32 level = 0; level < info.resources.levels; ++level) {
storage_image_views[level] =
MakeStorageView(device, level, *original_image, storage_format);
@@ -1974,11 +1892,9 @@ void Image::UploadMemory(VkBuffer buffer, VkDeviceSize offset,
ScaleDown(true);
}
const bool is_color_upload = (aspect_mask & VK_IMAGE_ASPECT_COLOR_BIT) != 0
const bool wants_msaa_upload = info.num_samples > 1
&& (aspect_mask & VK_IMAGE_ASPECT_COLOR_BIT) != 0
&& !VideoCore::Surface::IsPixelFormatInteger(info.format);
const bool is_depth_upload = (aspect_mask & VK_IMAGE_ASPECT_DEPTH_BIT) != 0;
const bool wants_msaa_upload =
info.num_samples > 1 && (is_color_upload || is_depth_upload);
if (wants_msaa_upload) {
ImageInfo temp_info = info;
@@ -2017,10 +1933,10 @@ void Image::UploadMemory(VkBuffer buffer, VkDeviceSize offset,
image_copies.push_back(image_copy);
}
runtime->TransitionImageLayout(*this);
runtime->blit_image_helper.CopyMSAA(runtime->render_pass_cache, Handle(), info.format,
temp_vk_image, info.format, info.num_samples,
image_copies, false);
initialized = true;
runtime->pending_msaa_images.emplace_back(scheduler->CurrentTick(), std::move(temp_image));
if (is_rescaled) {
@@ -2031,9 +1947,6 @@ void Image::UploadMemory(VkBuffer buffer, VkDeviceSize offset,
if (info.num_samples > 1) {
LOG_WARNING(Render_Vulkan, "MSAA upload not implemented for format {}", info.format);
if (runtime != nullptr) {
runtime->TransitionImageLayout(*this);
}
if (is_rescaled) {
ScaleUp();
}
@@ -2291,7 +2204,9 @@ VkImageView Image::StorageImageView(s32 level) noexcept {
auto format_info =
MaxwellToVK::SurfaceFormat(runtime->device, FormatType::Optimal, true, info.format);
if (WillUseAcceleratedAstcDecode(runtime->device, info)) {
format_info.format = VK_FORMAT_A8B8G8R8_UNORM_PACK32;
format_info.format = WillUseWidenedAstcFormat(runtime->device, info)
? VK_FORMAT_R32G32B32A32_SFLOAT
: VK_FORMAT_A8B8G8R8_UNORM_PACK32;
}
view = MakeStorageView(runtime->device.GetLogical(), level, *(this->*current_image),
format_info.format);
@@ -2467,7 +2382,11 @@ ImageView::ImageView(TextureCacheRuntime& runtime, const VideoCommon::ImageViewI
SanitizeDepthStencilSwizzle(swizzle, device->SupportsDepthStencilSwizzleOne());
}
}
uses_widened_astc_format = WillUseWidenedAstcFormat(*device, image.info);
auto format_info = MaxwellToVK::SurfaceFormat(*device, FormatType::Optimal, true, format);
if (uses_widened_astc_format) {
format_info.format = VK_FORMAT_R32G32B32A32_SFLOAT;
}
if (device->ApiVersion() >= VK_API_VERSION_1_3) {
const VkFormatProperties3 properties3 =
device->GetPhysical().GetFormatProperties3(format_info.format);
@@ -2485,18 +2404,9 @@ ImageView::ImageView(TextureCacheRuntime& runtime, const VideoCommon::ImageViewI
.pNext = nullptr,
.usage = clamped_view_usage,
};
const VkImageViewASTCDecodeModeEXT astc_decode_mode{
.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_ASTC_DECODE_MODE_EXT,
.pNext = &image_view_usage,
.decodeMode = VK_FORMAT_R8G8B8A8_UNORM,
};
const void* view_next = &image_view_usage;
if (device->IsExtAstcDecodeModeSupported() && IsLdrAstcFormat(format_info.format)) {
view_next = &astc_decode_mode;
}
const VkImageViewCreateInfo create_info{
.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
.pNext = view_next,
.pNext = &image_view_usage,
.flags = 0,
.image = image.Handle(),
.viewType = VkImageViewType{},
@@ -2619,12 +2529,14 @@ VkImageView ImageView::StorageView(Shader::TextureType texture_type,
Shader::ImageFormat image_format) {
if (image_handle) {
if (image_format == Shader::ImageFormat::Typeless) {
auto& view{typeless_storage_views[static_cast<size_t>(texture_type)]};
if (!view) {
if (!typeless_storage_view) {
auto info = MaxwellToVK::SurfaceFormat(*device, FormatType::Optimal, true, format);
view = MakeView(info.format, VK_IMAGE_ASPECT_COLOR_BIT, texture_type);
if (uses_widened_astc_format) {
info.format = VK_FORMAT_R32G32B32A32_SFLOAT;
}
typeless_storage_view = MakeView(info.format, VK_IMAGE_ASPECT_COLOR_BIT, texture_type);
}
return *view;
return *typeless_storage_view;
}
const bool is_signed = image_format == Shader::ImageFormat::R8_SINT
|| image_format == Shader::ImageFormat::R16_SINT;
@@ -2771,7 +2683,7 @@ Framebuffer::Framebuffer(TextureCacheRuntime& runtime, std::span<ImageView*, NUM
.height = key.size.height,
}} {
CreateFramebuffer(runtime, color_buffers, depth_buffer, key.is_rescaled);
if (runtime.device.HasDebuggingToolAttached() && framebuffer) {
if (runtime.device.HasDebuggingToolAttached()) {
framebuffer.SetObjectNameEXT(VideoCommon::Name(key).c_str());
}
}
@@ -2814,12 +2726,6 @@ void Framebuffer::CreateFramebuffer(TextureCacheRuntime& runtime,
image_ranges[num_images] = MakeSubresourceRange(color_buffer);
rt_map[index] = num_images;
samples = color_buffer->Samples();
color_attachments[index] = color_buffer->RenderTarget();
color_attachment_formats[index] =
MaxwellToVK::SurfaceFormat(runtime.device, FormatType::Optimal, true,
color_buffer->format)
.format;
num_color_attachments = static_cast<u32>(index + 1);
++num_images;
}
const size_t num_colors = attachments.size();
@@ -2835,11 +2741,6 @@ void Framebuffer::CreateFramebuffer(TextureCacheRuntime& runtime,
const VkImageSubresourceRange subresource_range = MakeSubresourceRange(depth_buffer);
image_ranges[num_images] = subresource_range;
samples = depth_buffer->Samples();
depth_attachment = depth_buffer->RenderTarget();
depth_attachment_format =
MaxwellToVK::SurfaceFormat(runtime.device, FormatType::Optimal, true,
depth_buffer->format)
.format;
++num_images;
has_depth = (subresource_range.aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT) != 0;
has_stencil = (subresource_range.aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT) != 0;
@@ -2851,23 +2752,10 @@ void Framebuffer::CreateFramebuffer(TextureCacheRuntime& runtime,
samples != VK_SAMPLE_COUNT_1_BIT && num_colors > 0 && runtime.device.IsTiler();
renderpass_key.resolve_color = do_resolve_color;
const VkResolveModeFlagBits depth_mode = runtime.device.GetDepthResolveMode();
const VkResolveModeFlagBits stencil_mode = runtime.device.GetStencilResolveMode();
const bool resolve_modes_compatible =
depth_mode != VK_RESOLVE_MODE_NONE &&
(!has_stencil || (stencil_mode != VK_RESOLVE_MODE_NONE &&
(runtime.device.IsIndependentResolveSupported() ||
depth_mode == stencil_mode)));
const bool msaa_depth =
samples != VK_SAMPLE_COUNT_1_BIT && has_depth && runtime.device.IsTiler();
const bool do_resolve_depth = ENABLE_MSAA_DEPTH_RESOLVE && msaa_depth &&
runtime.device.IsKhrDynamicRenderingSupported() &&
resolve_modes_compatible;
discard_msaa_color =
ENABLE_MSAA_RESOLVE_CONSUME && ENABLE_MSAA_COLOR_DISCARD && do_resolve_color;
discard_msaa_depth = ENABLE_MSAA_RESOLVE_CONSUME && ENABLE_MSAA_DEPTH_DISCARD && msaa_depth;
renderpass = runtime.render_pass_cache.Get(renderpass_key);
render_pass_key = renderpass_key;
render_pass_cache = &runtime.render_pass_cache;
render_area.width = (std::min)(render_area.width, width);
@@ -2882,15 +2770,10 @@ void Framebuffer::CreateFramebuffer(TextureCacheRuntime& runtime,
}
const VkFormat vk_format =
MaxwellToVK::SurfaceFormat(runtime.device, FormatType::Optimal, true, format).format;
color_resolve_modes[index] = VideoCore::Surface::IsPixelFormatInteger(format)
? VK_RESOLVE_MODE_SAMPLE_ZERO_BIT
: VK_RESOLVE_MODE_AVERAGE_BIT;
if (ENABLE_MSAA_RESOLVE_CONSUME) {
const VkImage msaa_image = images[rt_map[index]];
const VkImageView shadow_view = runtime.GetOrCreateResolveShadow(
msaa_image, vk_format, render_area, layers, VK_IMAGE_ASPECT_COLOR_BIT);
color_resolve_attachments[index] = shadow_view;
attachments.push_back(shadow_view);
attachments.push_back(runtime.GetOrCreateResolveShadow(msaa_image, vk_format,
render_area, layers));
continue;
}
VkImageCreateInfo resolve_ci{
@@ -2929,61 +2812,13 @@ void Framebuffer::CreateFramebuffer(TextureCacheRuntime& runtime,
.layerCount = layers,
},
});
if (runtime.device.IsKhrDynamicRenderingSupported()) {
runtime.scheduler.RecordWithUploadBuffer(
[image = *resolve_image, layers](vk::CommandBuffer, vk::CommandBuffer upload_cmdbuf) {
const VkImageMemoryBarrier barrier{
.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
.pNext = nullptr,
.srcAccessMask = 0,
.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT,
.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED,
.newLayout = VK_IMAGE_LAYOUT_GENERAL,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.image = image,
.subresourceRange{
.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
.baseMipLevel = 0,
.levelCount = 1,
.baseArrayLayer = 0,
.layerCount = layers,
},
};
upload_cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT,
0, barrier);
});
}
color_resolve_attachments[index] = *resolve_view;
attachments.push_back(*resolve_view);
resolve_images.push_back(std::move(resolve_image));
resolve_image_views.push_back(std::move(resolve_view));
}
}
if (do_resolve_depth && depth_buffer != nullptr) {
const u32 layers = static_cast<u32>((std::max)(num_layers, 1));
VkImageAspectFlags aspect = 0;
if (has_depth) {
aspect |= VK_IMAGE_ASPECT_DEPTH_BIT;
}
if (has_stencil) {
aspect |= VK_IMAGE_ASPECT_STENCIL_BIT;
}
depth_resolve_attachment =
runtime.GetOrCreateResolveShadow(depth_buffer->ImageHandle(), depth_attachment_format,
render_area, layers, aspect);
depth_resolve_mode = depth_mode;
stencil_resolve_mode = has_stencil ? stencil_mode : VK_RESOLVE_MODE_NONE;
}
num_color_buffers = static_cast<u32>(num_colors);
layer_count = static_cast<u32>((std::max)(num_layers, 1));
if (runtime.device.IsKhrDynamicRenderingSupported()) {
return;
}
renderpass = runtime.render_pass_cache.Get(renderpass_key);
framebuffer = runtime.device.GetLogical().CreateFramebuffer({
.sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO,
.pNext = nullptr,
@@ -2993,7 +2828,7 @@ void Framebuffer::CreateFramebuffer(TextureCacheRuntime& runtime,
.pAttachments = attachments.data(),
.width = render_area.width,
.height = render_area.height,
.layers = layer_count,
.layers = static_cast<u32>((std::max)(num_layers, 1)),
});
}
@@ -3009,53 +2844,6 @@ VkRenderPass Framebuffer::RenderPassVariant(u32 color_clear_mask, bool depth_ste
return render_pass_cache->Get(key);
}
void Framebuffer::BeginRendering(vk::CommandBuffer cmdbuf) const {
std::array<VkRenderingAttachmentInfo, NUM_RT> color_attachment_infos{};
for (size_t index = 0; index < num_color_attachments; ++index) {
color_attachment_infos[index] = VkRenderingAttachmentInfo{
.sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO,
.pNext = nullptr,
.imageView = color_attachments[index],
.imageLayout = VK_IMAGE_LAYOUT_GENERAL,
.resolveMode = VK_RESOLVE_MODE_NONE,
.resolveImageView = VK_NULL_HANDLE,
.resolveImageLayout = VK_IMAGE_LAYOUT_UNDEFINED,
.loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
.storeOp = VK_ATTACHMENT_STORE_OP_STORE,
.clearValue = {},
};
}
const VkRenderingAttachmentInfo depth_attachment_info{
.sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO,
.pNext = nullptr,
.imageView = depth_attachment,
.imageLayout = VK_IMAGE_LAYOUT_GENERAL,
.resolveMode = VK_RESOLVE_MODE_NONE,
.resolveImageView = VK_NULL_HANDLE,
.resolveImageLayout = VK_IMAGE_LAYOUT_UNDEFINED,
.loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
.storeOp = VK_ATTACHMENT_STORE_OP_STORE,
.clearValue = {},
};
const VkRenderingInfo rendering_info{
.sType = VK_STRUCTURE_TYPE_RENDERING_INFO,
.pNext = nullptr,
.flags = 0,
.renderArea =
{
.offset = {.x = 0, .y = 0},
.extent = render_area,
},
.layerCount = layer_count,
.viewMask = 0,
.colorAttachmentCount = num_color_attachments,
.pColorAttachments = color_attachment_infos.data(),
.pDepthAttachment = has_depth ? &depth_attachment_info : nullptr,
.pStencilAttachment = has_stencil ? &depth_attachment_info : nullptr,
};
cmdbuf.BeginRendering(rendering_info);
}
void TextureCacheRuntime::AccelerateImageUpload(
Image& image, const StagingBufferRef& map,
std::span<const VideoCommon::SwizzleParameters> swizzles,
@@ -3065,15 +2853,6 @@ void TextureCacheRuntime::AccelerateImageUpload(
return astc_decoder_pass->Assemble(image, map, swizzles);
}
if (bl2d_unswizzle_pass && BlockLinearUnswizzle2DPass::IsSupported(image.info)) {
return bl2d_unswizzle_pass->Unswizzle(image, map, swizzles);
}
if (bl3db_unswizzle_pass &&
BlockLinearUnswizzle3DBufferPass::IsSupported(device, image.info)) {
return bl3db_unswizzle_pass->Unswizzle(image, map, swizzles);
}
if (!Settings::values.gpu_unswizzle_enabled.GetValue() || !bl3d_unswizzle_pass) {
if (IsPixelFormatBCn(image.info.format) && image.info.type == ImageType::e3D) {
ASSERT(false && "GPU unswizzle is disabled for BCn 3D texture");
@@ -60,18 +60,10 @@ public:
void TickFrame();
u64 CurrentSyncPoint() const noexcept;
u64 CompletedSyncPoint() const;
void WaitSyncPoint(u64 sync_point);
u64 GetDeviceLocalMemory() const;
u64 GetDeviceMemoryUsage() const;
u64 GetDeviceAllocationUsage() const;
bool CanReportMemoryUsage() const;
std::optional<size_t> GetSamplerHeapBudget() const;
@@ -129,8 +121,7 @@ public:
};
[[nodiscard]] VkImageView GetOrCreateResolveShadow(VkImage msaa_image, VkFormat format,
VkExtent2D extent, u32 layers,
VkImageAspectFlags aspect);
VkExtent2D extent, u32 layers);
[[nodiscard]] const ResolveShadow* GetValidResolveShadow(VkImage msaa_image) const;
@@ -158,8 +149,6 @@ public:
std::optional<ASTCDecoderPass> astc_decoder_pass;
std::optional<BlockLinearUnswizzle3DPass> bl3d_unswizzle_pass;
std::optional<BlockLinearUnswizzle2DPass> bl2d_unswizzle_pass;
std::optional<BlockLinearUnswizzle3DBufferPass> bl3db_unswizzle_pass;
const Settings::ResolutionScalingInfo& resolution;
std::array<std::vector<VkFormat>, VideoCore::Surface::MaxPixelFormat> view_formats;
@@ -167,7 +156,6 @@ public:
std::array<vk::Buffer, indexing_slots> buffers{};
std::vector<std::pair<u64, vk::Image>> pending_msaa_images;
ankerl::unordered_dense::map<VkImage, ResolveShadow> resolve_shadows;
ankerl::unordered_dense::set<u64> unsupported_msaa_resolves;
};
class Framebuffer {
@@ -190,8 +178,6 @@ public:
std::span<ImageView*, NUM_RT> color_buffers, ImageView* depth_buffer,
bool is_rescaled = false);
void BeginRendering(vk::CommandBuffer cmdbuf) const;
[[nodiscard]] VkFramebuffer Handle() const noexcept {
return *framebuffer;
}
@@ -207,51 +193,6 @@ public:
[[nodiscard]] VkRenderPass RenderPassVariant(u32 color_clear_mask, bool depth_stencil_clear,
u32 color_discard_mask) const;
[[nodiscard]] u32 NumColorAttachments() const noexcept {
return num_color_attachments;
}
[[nodiscard]] const std::array<VkImageView, NUM_RT>& ColorAttachments() const noexcept {
return color_attachments;
}
[[nodiscard]] const std::array<VkFormat, NUM_RT>& ColorAttachmentFormats() const noexcept {
return color_attachment_formats;
}
[[nodiscard]] const std::array<VkImageView, NUM_RT>& ColorResolveAttachments() const noexcept {
return color_resolve_attachments;
}
[[nodiscard]] const std::array<VkResolveModeFlagBits, NUM_RT>& ColorResolveModes()
const noexcept {
return color_resolve_modes;
}
[[nodiscard]] VkImageView DepthAttachment() const noexcept {
return depth_attachment;
}
[[nodiscard]] VkImageView DepthResolveAttachment() const noexcept {
return depth_resolve_attachment;
}
[[nodiscard]] VkResolveModeFlagBits DepthResolveMode() const noexcept {
return depth_resolve_mode;
}
[[nodiscard]] VkResolveModeFlagBits StencilResolveMode() const noexcept {
return stencil_resolve_mode;
}
[[nodiscard]] VkFormat DepthAttachmentFormat() const noexcept {
return depth_attachment_format;
}
[[nodiscard]] u32 NumLayers() const noexcept {
return layer_count;
}
[[nodiscard]] VkExtent2D RenderArea() const noexcept {
return render_area;
}
@@ -304,10 +245,6 @@ public:
return discard_msaa_color;
}
[[nodiscard]] bool DiscardsMsaaDepth() const noexcept {
return discard_msaa_depth;
}
private:
vk::Framebuffer framebuffer;
VkRenderPass renderpass{};
@@ -318,17 +255,6 @@ private:
std::array<VkImage, 9> images{};
std::array<VkImageSubresourceRange, 9> image_ranges{};
std::array<size_t, NUM_RT> rt_map{};
std::array<VkImageView, NUM_RT> color_attachments{};
std::array<VkFormat, NUM_RT> color_attachment_formats{};
std::array<VkImageView, NUM_RT> color_resolve_attachments{};
std::array<VkResolveModeFlagBits, NUM_RT> color_resolve_modes{};
VkImageView depth_attachment{};
VkImageView depth_resolve_attachment{};
VkResolveModeFlagBits depth_resolve_mode = VK_RESOLVE_MODE_NONE;
VkResolveModeFlagBits stencil_resolve_mode = VK_RESOLVE_MODE_NONE;
VkFormat depth_attachment_format = VK_FORMAT_UNDEFINED;
u32 num_color_attachments = 0;
u32 layer_count = 1;
bool has_depth{};
bool has_stencil{};
bool is_rescaled{};
@@ -337,7 +263,6 @@ private:
RenderPassKey render_pass_key{};
RenderPassCache* render_pass_cache{nullptr};
bool discard_msaa_color{};
bool discard_msaa_depth{};
};
class Image : public VideoCommon::ImageBase {
@@ -501,7 +426,7 @@ private:
std::array<vk::ImageView, Shader::NUM_TEXTURE_TYPES> image_views;
std::optional<StorageViews> storage_views;
std::array<vk::ImageView, Shader::NUM_TEXTURE_TYPES> typeless_storage_views;
vk::ImageView typeless_storage_view;
vk::ImageView depth_view;
vk::ImageView stencil_view;
vk::ImageView color_view;
@@ -511,6 +436,7 @@ private:
VkSampleCountFlagBits samples = VK_SAMPLE_COUNT_1_BIT;
u32 buffer_size = 0;
bool uses_widened_astc_format = false;
bool supports_depth_comparison = false;
};
@@ -561,7 +487,6 @@ struct TextureCacheParams {
static constexpr bool HAS_EMULATED_COPIES = false;
static constexpr bool HAS_DEVICE_MEMORY_INFO = true;
static constexpr bool IMPLEMENTS_ASYNC_DOWNLOADS = true;
static constexpr bool HAS_TIMELINE_SYNC_POINTS = true;
using Runtime = Vulkan::TextureCacheRuntime;
using Image = Vulkan::Image;
@@ -16,11 +16,8 @@
namespace Vulkan {
UpdateDescriptorQueue::UpdateDescriptorQueue(const Device& device_, size_t frame_payload_size_,
bool supports_descriptor_buffer_)
: device{device_}, frame_payload_size{frame_payload_size_},
supports_descriptor_buffer{supports_descriptor_buffer_},
payload(frame_payload_size_ * FRAMES_IN_FLIGHT)
UpdateDescriptorQueue::UpdateDescriptorQueue(const Device& device_)
: device{device_}
{
payload_start = payload.data();
payload_cursor = payload.data();
@@ -32,21 +29,19 @@ void UpdateDescriptorQueue::TickFrame() {
if (++frame_index >= FRAMES_IN_FLIGHT) {
frame_index = 0;
}
payload_start = payload.data() + frame_index * frame_payload_size;
payload_start = payload.data() + frame_index * FRAME_PAYLOAD_SIZE;
payload_cursor = payload_start;
}
void UpdateDescriptorQueue::Acquire(Scheduler& scheduler, size_t required_entries,
bool use_descriptor_buffer_) {
use_descriptor_buffer = supports_descriptor_buffer && use_descriptor_buffer_;
void UpdateDescriptorQueue::Acquire(Scheduler& scheduler, size_t required_entries) {
static constexpr size_t DEFAULT_REQUIRED_ENTRIES = 0x400;
const size_t reserve = required_entries > 0 ? required_entries : DEFAULT_REQUIRED_ENTRIES;
ASSERT_MSG(reserve < frame_payload_size, "Descriptor reservation {} >= frame capacity {}",
reserve, frame_payload_size);
ASSERT_MSG(reserve < FRAME_PAYLOAD_SIZE, "Descriptor reservation {} >= frame capacity {}",
reserve, FRAME_PAYLOAD_SIZE);
const size_t used = static_cast<size_t>(std::distance(payload_start, payload_cursor));
if (used + reserve >= frame_payload_size) {
if (used + reserve >= FRAME_PAYLOAD_SIZE) {
LOG_WARNING(Render_Vulkan, "Payload overflow (used={}, reserve={}, capacity={})",
used, reserve, frame_payload_size);
used, reserve, FRAME_PAYLOAD_SIZE);
scheduler.WaitWorker();
payload_cursor = payload_start;
}
@@ -6,8 +6,8 @@
#pragma once
#include <array>
#include <variant>
#include <vector>
#include "video_core/vulkan_common/vulkan_wrapper.h"
namespace Vulkan {
@@ -15,45 +15,30 @@ namespace Vulkan {
class Device;
class Scheduler;
struct DescriptorAddress {
VkDeviceAddress address;
VkDeviceSize range;
VkFormat format;
};
union DescriptorUpdateEntry {
DescriptorUpdateEntry() = default;
DescriptorUpdateEntry(VkDescriptorImageInfo image_) : image{image_} {}
DescriptorUpdateEntry(VkDescriptorBufferInfo buffer_) : buffer{buffer_} {}
DescriptorUpdateEntry(VkBufferView texel_buffer_) : texel_buffer{texel_buffer_} {}
DescriptorUpdateEntry(DescriptorAddress address_) : address{address_} {}
std::monostate empty{};
VkDescriptorImageInfo image;
VkDescriptorBufferInfo buffer;
VkBufferView texel_buffer;
DescriptorAddress address;
};
class UpdateDescriptorQueue final {
// This should be plenty for the vast majority of cases. Most desktop platforms only
// provide up to 3 swapchain images.
static constexpr size_t FRAMES_IN_FLIGHT = 8;
static constexpr size_t FRAME_PAYLOAD_SIZE = 0x20000;
static constexpr size_t PAYLOAD_SIZE = FRAME_PAYLOAD_SIZE * FRAMES_IN_FLIGHT;
public:
static constexpr size_t GUEST_FRAME_PAYLOAD_SIZE = 0x80000;
static constexpr size_t COMPUTE_FRAME_PAYLOAD_SIZE = 0x20000;
explicit UpdateDescriptorQueue(const Device& device_, size_t frame_payload_size_,
bool supports_descriptor_buffer_ = false);
explicit UpdateDescriptorQueue(const Device& device_);
~UpdateDescriptorQueue();
[[nodiscard]] bool UsesDescriptorBuffer() const noexcept {
return use_descriptor_buffer;
}
void TickFrame();
void Acquire(Scheduler& scheduler, size_t required_entries = 0,
bool use_descriptor_buffer_ = false);
void Acquire(Scheduler& scheduler, size_t required_entries = 0);
const DescriptorUpdateEntry* UpdateData() const noexcept {
return upload_start;
@@ -83,46 +68,17 @@ public:
};
}
void AddBuffer(VkBuffer buffer, VkDeviceAddress base_address, VkDeviceSize offset,
VkDeviceSize size) {
if (!use_descriptor_buffer) {
AddBuffer(buffer, offset, size);
return;
}
*(payload_cursor++) = DescriptorAddress{
.address = base_address == 0 ? 0 : base_address + offset,
.range = base_address == 0 ? VK_WHOLE_SIZE : size,
.format = VK_FORMAT_UNDEFINED,
};
}
void AddTexelBuffer(VkBufferView texel_buffer) {
*(payload_cursor++) = texel_buffer;
}
void AddTexelBuffer(VkBufferView texel_buffer, VkDeviceAddress base_address,
VkDeviceSize offset, VkDeviceSize size, VkFormat format) {
if (!use_descriptor_buffer) {
AddTexelBuffer(texel_buffer);
return;
}
*(payload_cursor++) = DescriptorAddress{
.address = base_address == 0 ? 0 : base_address + offset,
.range = base_address == 0 ? VK_WHOLE_SIZE : size,
.format = format,
};
}
private:
const Device& device;
const size_t frame_payload_size;
const bool supports_descriptor_buffer;
bool use_descriptor_buffer{false};
size_t frame_index{0};
DescriptorUpdateEntry* payload_cursor = nullptr;
DescriptorUpdateEntry* payload_start = nullptr;
const DescriptorUpdateEntry* upload_start = nullptr;
std::vector<DescriptorUpdateEntry> payload;
std::array<DescriptorUpdateEntry, PAYLOAD_SIZE> payload;
};
// TODO: should these be separate classes instead?
+1 -18
View File
@@ -284,24 +284,7 @@ std::optional<u64> GenericEnvironment::TryFindSize() {
Tegra::Texture::TICEntry GenericEnvironment::ReadTextureInfo(GPUVAddr tic_addr, u32 tic_limit,
bool via_header_index, u32 raw) {
const auto handle{Tegra::Texture::TexturePair(raw, via_header_index)};
if (handle.first > tic_limit) {
LOG_CRITICAL(Shader,
"TIC index out of range: raw=0x{:08x} tic_index={} tsc_index={} tic_limit={} "
"tic_addr=0x{:x} via_header_index={} stage={} program_base=0x{:x} "
"start_address=0x{:x}",
raw, handle.first, handle.second, tic_limit, tic_addr, via_header_index,
static_cast<u32>(stage), program_base, start_address);
ASSERT(handle.first <= tic_limit);
Tegra::Texture::TICEntry fallback{};
fallback.format.Assign(Tegra::Texture::TextureFormat::A8B8G8R8);
fallback.r_type.Assign(Tegra::Texture::ComponentType::UNORM);
fallback.g_type.Assign(Tegra::Texture::ComponentType::UNORM);
fallback.b_type.Assign(Tegra::Texture::ComponentType::UNORM);
fallback.a_type.Assign(Tegra::Texture::ComponentType::UNORM);
fallback.texture_type.Assign(Tegra::Texture::TextureType::Texture2D);
fallback.normalized_coords.Assign(1);
return fallback;
}
ASSERT(handle.first <= tic_limit);
const GPUVAddr descriptor_addr{tic_addr + handle.first * sizeof(Tegra::Texture::TICEntry)};
Tegra::Texture::TICEntry entry;
gpu_memory->ReadBlock(descriptor_addr, &entry, sizeof(entry));
+86 -222
View File
@@ -58,9 +58,23 @@ TextureCache<P>::TextureCache(Runtime& runtime_, Tegra::MaxwellDeviceMemoryManag
void(slot_samplers.insert(runtime, sampler_descriptor));
if constexpr (HAS_DEVICE_MEMORY_INFO) {
memory_budget = runtime.GetDeviceLocalMemory();
const s64 device_local_memory = static_cast<s64>(runtime.GetDeviceLocalMemory());
const s64 min_spacing_expected = device_local_memory - 1_GiB;
const s64 min_spacing_critical = device_local_memory - 512_MiB;
const s64 mem_threshold = (std::min)(device_local_memory, TARGET_THRESHOLD);
const s64 min_vacancy_expected = (6 * mem_threshold) / 10;
const s64 min_vacancy_critical = (2 * mem_threshold) / 10;
expected_memory = static_cast<u64>(
(std::max)((std::min)(device_local_memory - min_vacancy_expected, min_spacing_expected),
DEFAULT_EXPECTED_MEMORY));
critical_memory = static_cast<u64>(
(std::max)((std::min)(device_local_memory - min_vacancy_critical, min_spacing_critical),
DEFAULT_CRITICAL_MEMORY));
minimum_memory = static_cast<u64>((device_local_memory - mem_threshold) / 2);
} else {
memory_budget = FALLBACK_MEMORY_BUDGET;
expected_memory = DEFAULT_EXPECTED_MEMORY + 512_MiB;
critical_memory = DEFAULT_CRITICAL_MEMORY + 1_GiB;
minimum_memory = 0;
}
const bool gpu_unswizzle_enabled = Settings::values.gpu_unswizzle_enabled.GetValue();
@@ -100,162 +114,71 @@ TextureCache<P>::TextureCache(Runtime& runtime_, Tegra::MaxwellDeviceMemoryManag
}
template <class P>
void TextureCache<P>::QueueEvictionDownload(Image& image) {
auto copies = FullDownloadCopies(image.info);
auto staging = runtime.DownloadStagingBuffer(image.unswizzled_size_bytes, true);
image.DownloadMemory(staging, FixSmallVectorADL(copies));
pending_eviction_downloads.push_back(PendingEvictionDownload{
.staging = staging,
.gpu_memory = gpu_memory,
.copies = std::move(copies),
.info = image.info,
.gpu_addr = image.gpu_addr,
.sync_point = runtime.CurrentSyncPoint(),
});
}
template <class P>
void TextureCache<P>::TickEvictionDownloads(u64 completed_sync_point) {
while (!pending_eviction_downloads.empty() &&
pending_eviction_downloads.front().sync_point <= completed_sync_point) {
auto& entry = pending_eviction_downloads.front();
SwizzleImage(*entry.gpu_memory, entry.gpu_addr, entry.info, FixSmallVectorADL(entry.copies),
entry.staging.mapped_span.subspan(entry.staging.offset), swizzle_data_buffer);
runtime.FreeDeferredStagingBuffer(entry.staging);
pending_eviction_downloads.pop_front();
}
}
template <class P>
void TextureCache<P>::FlushEvictionDownloads() {
if (pending_eviction_downloads.empty()) {
return;
}
const u64 last_sync_point = pending_eviction_downloads.back().sync_point;
runtime.WaitSyncPoint(last_sync_point);
TickEvictionDownloads(last_sync_point);
}
template <class P>
u64 TextureCache<P>::ImageSizeBytes(const ImageBase& image) {
u64 tentative_size = (std::max)(image.guest_size_bytes, image.unswizzled_size_bytes);
if ((IsPixelFormatASTC(image.info.format) &&
True(image.flags & ImageFlagBits::AcceleratedUpload)) ||
True(image.flags & ImageFlagBits::Converted)) {
tentative_size = TranscodedAstcSize(tentative_size, image.info.format);
}
u64 size = Common::AlignUp(tentative_size, 1024);
if (image.HasScaled()) {
size += GetScaledImageSizeBytes(image);
}
return size;
}
template <class P>
u64 TextureCache<P>::DeviceUsage(bool force_refresh) {
if (!runtime.CanReportMemoryUsage()) {
return total_used_memory;
}
if (force_refresh || usage_refresh_countdown == 0) {
cached_device_usage = runtime.GetDeviceAllocationUsage();
usage_refresh_countdown = USAGE_REFRESH_INTERVAL;
} else {
--usage_refresh_countdown;
}
return cached_device_usage;
}
template <class P>
u64 TextureCache<P>::ReclaimMemory(u64 target_bytes, bool allow_download) {
if (target_bytes == 0 || in_reclaim) {
return 0;
}
in_reclaim = true;
u64 freed = 0;
const auto evict = [&](ImageId image_id) {
if (freed >= target_bytes) {
void TextureCache<P>::RunGarbageCollector() {
bool high_priority_mode = false;
bool aggressive_mode = false;
u64 ticks_to_destroy = 0;
size_t num_iterations = 0;
const auto Configure = [&](bool allow_aggressive) {
high_priority_mode = total_used_memory >= expected_memory;
aggressive_mode = allow_aggressive && total_used_memory >= critical_memory;
ticks_to_destroy = aggressive_mode ? 10ULL : high_priority_mode ? 25ULL : 50ULL;
num_iterations = aggressive_mode ? 40 : (high_priority_mode ? 20 : 10);
};
const auto Cleanup = [this, &num_iterations, &high_priority_mode, &aggressive_mode](ImageId image_id) {
if (num_iterations == 0) {
return true;
}
--num_iterations;
auto& image = slot_images[image_id];
if (True(image.flags & ImageFlagBits::IsDecoding)) {
return false;
}
const bool must_download = image.IsSafeDownload();
if (must_download && True(image.flags & ImageFlagBits::BadOverlap)) {
LOG_WARNING(HW_GPU,
"Recovering bad overlap on eviction: gpu_addr=0x{:x} fmt={} {}x{}x{}",
image.gpu_addr, static_cast<u32>(image.info.format), image.info.size.width,
image.info.size.height, image.info.size.depth);
const bool must_download = image.IsSafeDownload() && False(image.flags & ImageFlagBits::BadOverlap);
if ((!aggressive_mode && True(image.flags & ImageFlagBits::CostlyLoad)) || (!high_priority_mode && must_download)) {
return false;
}
bool queued_download = false;
if (must_download) {
if constexpr (HAS_TIMELINE_SYNC_POINTS) {
if (!allow_download) {
return false;
}
QueueEvictionDownload(image);
queued_download = true;
} else {
return false;
}
auto map = runtime.DownloadStagingBuffer(image.unswizzled_size_bytes);
const auto copies = FixSmallVectorADL(FullDownloadCopies(image.info));
image.DownloadMemory(map, copies);
runtime.Finish();
SwizzleImage(*gpu_memory, image.gpu_addr, image.info, copies, map.mapped_span, swizzle_data_buffer);
}
const u64 image_bytes = ImageSizeBytes(image);
if (True(image.flags & ImageFlagBits::Tracked)) {
UntrackImage(image, image_id);
}
UnregisterImage(image_id);
DeleteImage(image_id, !queued_download && image.scale_tick > frame_tick + 5);
freed += image_bytes;
DeleteImage(image_id, image.scale_tick > frame_tick + 5);
if (aggressive_mode && total_used_memory < critical_memory) {
num_iterations >>= 2;
aggressive_mode = false;
} else if (high_priority_mode && total_used_memory < expected_memory) {
num_iterations >>= 1;
high_priority_mode = false;
}
return false;
};
const u64 cold_tick =
frame_tick > RECLAIM_GUARD_FRAMES ? frame_tick - RECLAIM_GUARD_FRAMES : 0;
lru_cache.ForEachItemBelow(cold_tick, evict);
if (freed == 0) {
const u64 urgent_tick = frame_tick > RECLAIM_URGENT_GUARD_FRAMES
? frame_tick - RECLAIM_URGENT_GUARD_FRAMES
: 0;
lru_cache.ForEachItemBelow(urgent_tick, evict);
Configure(false);
lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, Cleanup);
if (total_used_memory >= critical_memory) {
Configure(true);
lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, Cleanup);
}
in_reclaim = false;
usage_refresh_countdown = 0;
reclaim_stalled = freed == 0;
if (freed > 0) {
reclaim_wait_sync_point = runtime.CurrentSyncPoint();
}
return freed;
}
template <class P>
void TextureCache<P>::EnsureHeadroom(bool allow_download) {
if (reclaim_stalled) {
return;
}
if (runtime.CompletedSyncPoint() < reclaim_wait_sync_point) {
return;
}
const u64 limit = memory_budget > RECLAIM_HEADROOM ? memory_budget - RECLAIM_HEADROOM : 0;
const u64 usage = DeviceUsage(false);
if (usage <= limit) {
return;
}
const u64 target = (limit / 100) * RECLAIM_TARGET_PERCENT;
const u64 excess = usage - target;
const u64 usage_mib = (std::max)(usage >> 20, u64{1});
const u64 share = (((excess >> 20) * (total_used_memory >> 20)) / usage_mib) << 20;
ReclaimMemory((std::min)(share, total_used_memory), allow_download);
}
template <class P>
void TextureCache<P>::TickFrame() {
usage_refresh_countdown = 0;
reclaim_stalled = false;
EnsureHeadroom(true);
const u64 completed_sync_point = runtime.CompletedSyncPoint();
TickEvictionDownloads(completed_sync_point);
sentenced_images.Reclaim(completed_sync_point);
sentenced_framebuffers.Reclaim(completed_sync_point);
sentenced_image_view.Reclaim(completed_sync_point);
// If we can obtain the memory info, use it instead of the estimate.
if (runtime.CanReportMemoryUsage()) {
total_used_memory = runtime.GetDeviceMemoryUsage();
}
if (total_used_memory > minimum_memory) {
RunGarbageCollector();
}
sentenced_images.Tick();
sentenced_framebuffers.Tick();
sentenced_image_view.Tick();
TickAsyncDecode();
TickAsyncUnswizzle();
@@ -589,24 +512,10 @@ void TextureCache<P>::UpdateRenderTargets(bool is_clear) {
up_scale = Settings::values.resolution_info.up_scale;
down_shift = Settings::values.resolution_info.down_shift;
}
u32 clip_width = (maxwell3d->regs.surface_clip.width * up_scale) >> down_shift;
u32 clip_height = (maxwell3d->regs.surface_clip.height * up_scale) >> down_shift;
const auto& resolution = Settings::values.resolution_info;
const auto clamp_to_attachment = [&](ImageViewId view_id) {
if (!view_id) {
return;
}
const ImageViewBase& view = slot_image_views[view_id];
clip_width = (std::min)(clip_width, is_rescaling ? resolution.ScaleUp(view.size.width)
: view.size.width);
clip_height = (std::min)(clip_height, is_rescaling ? resolution.ScaleUp(view.size.height)
: view.size.height);
render_targets.size = Extent2D{
(maxwell3d->regs.surface_clip.width * up_scale) >> down_shift,
(maxwell3d->regs.surface_clip.height * up_scale) >> down_shift,
};
for (size_t index = 0; index < NUM_RT; ++index) {
clamp_to_attachment(render_targets.color_buffer_ids[index]);
}
clamp_to_attachment(depth_buffer_id);
render_targets.size = Extent2D{clip_width, clip_height};
render_targets.is_rescaled = is_rescaling;
if (render_targets != previous_render_targets) {
@@ -687,7 +596,6 @@ void TextureCache<P>::WriteMemory(DAddr cpu_addr, size_t size) {
template <class P>
void TextureCache<P>::DownloadMemory(DAddr cpu_addr, size_t size) {
FlushEvictionDownloads();
boost::container::small_vector<ImageId, 16> images;
ForEachImageInRegion(cpu_addr, size, [&images](ImageId image_id, ImageBase& image) {
if (!image.IsSafeDownload()) {
@@ -986,7 +894,6 @@ void TextureCache<P>::CommitAsyncFlushes() {
template <class P>
void TextureCache<P>::PopAsyncFlushes() {
TickEvictionDownloads(runtime.CompletedSyncPoint());
if (committed_downloads.empty()) {
return;
}
@@ -1241,8 +1148,6 @@ void TextureCache<P>::UploadImageContents(Image& image, StagingBuffer& staging)
return;
}
gpu_memory->FlushRegion(gpu_addr, image.guest_size_bytes,
VideoCommon::CacheType::NoTextureCache);
Tegra::Memory::GpuGuestMemory<u8, Tegra::Memory::GuestMemoryFlags::UnsafeRead> swizzle_data(
*gpu_memory, gpu_addr, image.guest_size_bytes, &swizzle_data_buffer);
if (True(image.flags & ImageFlagBits::Converted)) {
@@ -1389,9 +1294,8 @@ void TextureCache<P>::InvalidateScale(Image& image) {
}
RemoveImageViewReferences(image_view_ids);
RemoveFramebuffers(image_view_ids);
const u64 sync_point = runtime.CurrentSyncPoint();
for (const ImageViewId image_view_id : image_view_ids) {
sentenced_image_view.Push(std::move(slot_image_views[image_view_id]), sync_point);
sentenced_image_view.Push(std::move(slot_image_views[image_view_id]));
slot_image_views.erase(image_view_id);
}
image.image_view_ids.clear();
@@ -1427,7 +1331,6 @@ void TextureCache<P>::QueueAsyncDecode(Image& image, ImageId image_id) {
LOG_INFO(HW_GPU, "Queuing async texture decode");
image.flags |= ImageFlagBits::IsDecoding;
runtime.TransitionImageLayout(image);
auto decode = std::make_unique<AsyncDecodeContext>();
auto* decode_ptr = decode.get();
decode->image_id = image_id;
@@ -1460,7 +1363,6 @@ void TextureCache<P>::QueueAsyncUnswizzle(Image& image, ImageId image_id) {
}
image.flags |= ImageFlagBits::IsDecoding;
runtime.TransitionImageLayout(image);
unswizzle_queue.push_back({
.image_id = image_id,
@@ -1621,7 +1523,6 @@ ImageId TextureCache<P>::InsertImage(const ImageInfo& info, GPUVAddr gpu_addr,
template <class P>
ImageId TextureCache<P>::JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, DAddr cpu_addr) {
EnsureHeadroom(false);
ImageInfo new_info = info;
const size_t size_bytes = CalculateGuestSizeInBytes(new_info);
const bool broken_views = runtime.HasBrokenTextureViewFormats();
@@ -1730,35 +1631,7 @@ ImageId TextureCache<P>::JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, DA
for (const ImageId overlap_id : join_ignore_textures) {
Image& overlap = slot_images[overlap_id];
if (True(overlap.flags & ImageFlagBits::GpuModified)) {
if (new_image.TryFindBase(overlap.gpu_addr) &&
(!can_rescale || ImageCanRescale(overlap))) {
if (can_rescale) {
ScaleUp(overlap);
} else {
ScaleDown(overlap);
}
join_copies_to_do.emplace_back(JoinCopy{false, overlap_id});
continue;
}
if (overlap.IsSafeDownload() &&
gpu_memory->GpuToCpuAddress(overlap.gpu_addr).has_value()) {
if (True(overlap.flags & ImageFlagBits::BadOverlap)) {
LOG_WARNING(HW_GPU,
"Recovering bad overlap on join: gpu_addr=0x{:x} fmt={} {}x{}x{}",
overlap.gpu_addr, static_cast<u32>(overlap.info.format),
overlap.info.size.width, overlap.info.size.height,
overlap.info.size.depth);
}
QueueEvictionDownload(overlap);
} else {
LOG_WARNING(HW_GPU,
"Dropping GPU modified overlap, contents are not recoverable: "
"gpu_addr=0x{:x} format={} size={}x{}x{} levels={} layers={}",
overlap.gpu_addr, static_cast<int>(overlap.info.format),
overlap.info.size.width, overlap.info.size.height,
overlap.info.size.depth, overlap.info.resources.levels,
overlap.info.resources.layers);
}
UNIMPLEMENTED();
}
if (True(overlap.flags & ImageFlagBits::Tracked)) {
UntrackImage(overlap, overlap_id);
@@ -1805,8 +1678,6 @@ ImageId TextureCache<P>::JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, DA
ImageBase& aliased = slot_images[aliased_id];
aliased.overlapping_images.push_back(new_image_id);
new_image.overlapping_images.push_back(aliased_id);
const bool aliased_was_bad = True(aliased.flags & ImageFlagBits::BadOverlap);
const bool new_was_bad = True(new_image.flags & ImageFlagBits::BadOverlap);
if (aliased.info.resources.levels == 1 && aliased.info.block.depth == 0 &&
aliased.overlapping_images.size() > 1) {
aliased.flags |= ImageFlagBits::BadOverlap;
@@ -1815,21 +1686,6 @@ ImageId TextureCache<P>::JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, DA
new_image.overlapping_images.size() > 1) {
new_image.flags |= ImageFlagBits::BadOverlap;
}
const bool aliased_is_bad = True(aliased.flags & ImageFlagBits::BadOverlap);
const bool new_is_bad = True(new_image.flags & ImageFlagBits::BadOverlap);
if ((!aliased_was_bad && aliased_is_bad) || (!new_was_bad && new_is_bad)) {
LOG_WARNING(HW_GPU,
"Bad overlap: existing gpu_addr={:#x} {}x{}x{} fmt={} type={} rt={} | "
"incoming gpu_addr={:#x} {}x{}x{} fmt={} type={} rt={}",
aliased.gpu_addr, aliased.info.size.width, aliased.info.size.height,
aliased.info.size.depth, static_cast<u32>(aliased.info.format),
static_cast<u32>(aliased.info.type),
True(aliased.flags & ImageFlagBits::GpuModified),
new_image.gpu_addr, new_image.info.size.width, new_image.info.size.height,
new_image.info.size.depth, static_cast<u32>(new_image.info.format),
static_cast<u32>(new_image.info.type),
True(new_image.flags & ImageFlagBits::GpuModified));
}
}
for (const auto& copy_object : join_copies_to_do) {
@@ -2329,7 +2185,13 @@ void TextureCache<P>::RegisterImage(ImageId image_id) {
ASSERT_MSG(False(image.flags & ImageFlagBits::Registered),
"Trying to register an already registered image");
image.flags |= ImageFlagBits::Registered;
total_used_memory += ImageSizeBytes(image);
u64 tentative_size = (std::max)(image.guest_size_bytes, image.unswizzled_size_bytes);
if ((IsPixelFormatASTC(image.info.format) &&
True(image.flags & ImageFlagBits::AcceleratedUpload)) ||
True(image.flags & ImageFlagBits::Converted)) {
tentative_size = TranscodedAstcSize(tentative_size, image.info.format);
}
total_used_memory += Common::AlignUp(tentative_size, 1024);
image.lru_index = lru_cache.Insert(image_id, frame_tick);
ForEachGPUPage(image.gpu_addr, image.guest_size_bytes, [this, image_id](u64 page) {
@@ -2446,7 +2308,6 @@ void TextureCache<P>::TrackImage(ImageBase& image, ImageId image_id) {
if (False(image.flags & ImageFlagBits::Sparse)) {
if (image.cpu_addr < ~(1ULL << 40)) {
device_memory.UpdatePagesCachedCount(image.cpu_addr, image.guest_size_bytes, 1);
device_memory.UpdateTexturePagesCount(image.cpu_addr, image.guest_size_bytes, 1);
}
return;
}
@@ -2459,14 +2320,12 @@ void TextureCache<P>::TrackImage(ImageBase& image, ImageId image_id) {
const DAddr cpu_addr = map.cpu_addr;
const std::size_t size = map.size;
device_memory.UpdatePagesCachedCount(cpu_addr, size, 1);
device_memory.UpdateTexturePagesCount(cpu_addr, size, 1);
}
return;
}
ForEachSparseSegment(image,
[this]([[maybe_unused]] GPUVAddr gpu_addr, DAddr cpu_addr, size_t size) {
device_memory.UpdatePagesCachedCount(cpu_addr, size, 1);
device_memory.UpdateTexturePagesCount(cpu_addr, size, 1);
});
}
@@ -2477,7 +2336,6 @@ void TextureCache<P>::UntrackImage(ImageBase& image, ImageId image_id) {
if (False(image.flags & ImageFlagBits::Sparse)) {
if (image.cpu_addr < ~(1ULL << 40)) {
device_memory.UpdatePagesCachedCount(image.cpu_addr, image.guest_size_bytes, -1);
device_memory.UpdateTexturePagesCount(image.cpu_addr, image.guest_size_bytes, -1);
}
return;
}
@@ -2490,14 +2348,22 @@ void TextureCache<P>::UntrackImage(ImageBase& image, ImageId image_id) {
const DAddr cpu_addr = map.cpu_addr;
const std::size_t size = map.size;
device_memory.UpdatePagesCachedCount(cpu_addr, size, -1);
device_memory.UpdateTexturePagesCount(cpu_addr, size, -1);
}
}
template <class P>
void TextureCache<P>::DeleteImage(ImageId image_id, bool immediate_delete) {
ImageBase& image = slot_images[image_id];
total_used_memory -= std::min<u64>(total_used_memory, ImageSizeBytes(image));
if (image.HasScaled()) {
total_used_memory -= GetScaledImageSizeBytes(image);
}
u64 tentative_size = (std::max)(image.guest_size_bytes, image.unswizzled_size_bytes);
if ((IsPixelFormatASTC(image.info.format) &&
True(image.flags & ImageFlagBits::AcceleratedUpload)) ||
True(image.flags & ImageFlagBits::Converted)) {
tentative_size = TranscodedAstcSize(tentative_size, image.info.format);
}
total_used_memory -= Common::AlignUp(tentative_size, 1024);
const GPUVAddr gpu_addr = image.gpu_addr;
const auto alloc_it = image_allocs_table.find(gpu_addr);
if (alloc_it == image_allocs_table.end()) {
@@ -2551,15 +2417,14 @@ void TextureCache<P>::DeleteImage(ImageId image_id, bool immediate_delete) {
ASSERT_MSG(num_removed_overlaps == 1, "Invalid number of removed overlapps: {}",
num_removed_overlaps);
}
const u64 sync_point = runtime.CurrentSyncPoint();
for (const ImageViewId image_view_id : image_view_ids) {
if (!immediate_delete) {
sentenced_image_view.Push(std::move(slot_image_views[image_view_id]), sync_point);
sentenced_image_view.Push(std::move(slot_image_views[image_view_id]));
}
slot_image_views.erase(image_view_id);
}
if (!immediate_delete) {
sentenced_images.Push(std::move(slot_images[image_id]), sync_point);
sentenced_images.Push(std::move(slot_images[image_id]));
}
slot_images.erase(image_id);
@@ -2605,8 +2470,7 @@ void TextureCache<P>::RemoveFramebuffers(std::span<const ImageViewId> removed_vi
last_framebuffer_id = {};
last_framebuffer_serial = 0;
}
sentenced_framebuffers.Push(std::move(slot_framebuffers[framebuffer_id]),
runtime.CurrentSyncPoint());
sentenced_framebuffers.Push(std::move(slot_framebuffers[framebuffer_id]));
it = framebuffers.erase(it);
} else {
++it;
@@ -30,7 +30,7 @@
#include "common/thread_worker.h"
#include "video_core/compatible_formats.h"
#include "video_core/control/channel_state_cache.h"
#include "video_core/deferred_destruction_queue.h"
#include "video_core/delayed_destruction_ring.h"
#include "video_core/engines/fermi_2d.h"
#include "video_core/surface.h"
#include "video_core/texture_cache/descriptor_table.h"
@@ -108,21 +108,18 @@ class TextureCache : public VideoCommon::ChannelSetupCaches<TextureCacheChannelI
static constexpr bool HAS_DEVICE_MEMORY_INFO = P::HAS_DEVICE_MEMORY_INFO;
/// True when the API can do asynchronous texture downloads.
static constexpr bool IMPLEMENTS_ASYNC_DOWNLOADS = P::IMPLEMENTS_ASYNC_DOWNLOADS;
static constexpr bool HAS_TIMELINE_SYNC_POINTS = P::HAS_TIMELINE_SYNC_POINTS;
static constexpr size_t UNSET_CHANNEL{(std::numeric_limits<size_t>::max)()};
#ifdef YUZU_LEGACY
static constexpr u64 RECLAIM_HEADROOM = 192_MiB;
static constexpr s64 TARGET_THRESHOLD = 3_GiB;
#else
static constexpr u64 RECLAIM_HEADROOM = 256_MiB;
static constexpr s64 TARGET_THRESHOLD = 4_GiB;
#endif
static constexpr u64 FALLBACK_MEMORY_BUDGET = 2_GiB;
static constexpr u32 USAGE_REFRESH_INTERVAL = 16;
static constexpr u64 RECLAIM_GUARD_FRAMES = 16;
static constexpr u64 RECLAIM_URGENT_GUARD_FRAMES = 4;
static constexpr u64 RECLAIM_TARGET_PERCENT = 90;
static constexpr s64 DEFAULT_EXPECTED_MEMORY = 1_GiB + 125_MiB;
static constexpr s64 DEFAULT_CRITICAL_MEMORY = 1_GiB + 625_MiB;
static constexpr size_t GC_EMERGENCY_COUNTS = 2;
using Runtime = typename P::Runtime;
using Image = typename P::Image;
@@ -157,8 +154,6 @@ public:
/// Notify the cache that a new frame has been queued
void TickFrame();
u64 ReclaimMemory(u64 target_bytes, bool allow_download);
/// Return a constant reference to the given image view id
[[nodiscard]] const ImageView& GetImageView(ImageViewId id) const noexcept;
@@ -262,7 +257,7 @@ public:
/// Prepare an image to be used
void PrepareImage(ImageId image_id, bool is_modification, bool invalidate);
std::mutex mutex;
std::recursive_mutex mutex;
private:
/// Iterate over all page indices in a range
@@ -298,17 +293,8 @@ private:
void OnGPUASRegister(size_t map_id) final override;
u64 ImageSizeBytes(const ImageBase& image);
u64 DeviceUsage(bool force_refresh);
void EnsureHeadroom(bool allow_download);
void QueueEvictionDownload(Image& image);
void TickEvictionDownloads(u64 completed_sync_point);
void FlushEvictionDownloads();
/// Runs the Garbage Collector.
void RunGarbageCollector();
/// Find or create an image view in the guest descriptor table
ImageViewId VisitImageView(u32 index, bool compute);
@@ -465,12 +451,9 @@ private:
bool has_deleted_images = false;
bool is_rescaling = false;
u64 total_used_memory = 0;
u64 memory_budget = 0;
u64 cached_device_usage = 0;
u64 reclaim_wait_sync_point = 0;
u32 usage_refresh_countdown = 0;
bool in_reclaim = false;
bool reclaim_stalled = false;
u64 minimum_memory;
u64 expected_memory;
u64 critical_memory;
size_t gpu_unswizzle_maxsize = 0;
size_t swizzle_chunk_size = 0;
u32 swizzle_slices_per_batch = 0;
@@ -508,19 +491,14 @@ private:
};
Common::LeastRecentlyUsedCache<LRUItemParams> lru_cache;
DeferredDestructionQueue<Image> sentenced_images;
DeferredDestructionQueue<ImageView> sentenced_image_view;
DeferredDestructionQueue<Framebuffer> sentenced_framebuffers;
struct PendingEvictionDownload {
AsyncBuffer staging;
Tegra::MemoryManager* gpu_memory;
boost::container::small_vector<VideoCommon::BufferImageCopy, 16> copies;
VideoCommon::ImageInfo info;
GPUVAddr gpu_addr;
u64 sync_point;
};
std::deque<PendingEvictionDownload> pending_eviction_downloads;
#ifdef YUZU_LEGACY
static constexpr size_t TICKS_TO_DESTROY = 6;
#else
static constexpr size_t TICKS_TO_DESTROY = 8;
#endif
DelayedDestructionRing<Image, TICKS_TO_DESTROY> sentenced_images;
DelayedDestructionRing<ImageView, TICKS_TO_DESTROY> sentenced_image_view;
DelayedDestructionRing<Framebuffer, TICKS_TO_DESTROY> sentenced_framebuffers;
ankerl::unordered_dense::map<GPUVAddr, ImageAllocId> image_allocs_table;
@@ -531,8 +509,7 @@ private:
u64 frame_tick = 0;
u64 last_sampler_gc_frame = (std::numeric_limits<u64>::max)();
Common::ThreadWorker texture_decode_worker{1, "TextureDecoder", {},
Common::ThreadPlacement::Background};
Common::ThreadWorker texture_decode_worker{1, "TextureDecoder"};
std::vector<std::unique_ptr<AsyncDecodeContext>> async_decodes;
std::deque<PendingUnswizzle> unswizzle_queue;
+2 -2
View File
@@ -1,4 +1,4 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
@@ -747,7 +747,7 @@ boost::container::small_vector<ImageCopy, 16> MakeShrinkImageCopies(const ImageI
const bool is_dst_3d = dst.type == ImageType::e3D;
if (is_dst_3d) {
ASSERT(src.type == ImageType::e3D || src.resources.layers == 1);
ASSERT(src.type == ImageType::e3D);
ASSERT(src.resources.levels == 1);
}
const bool both_2d{src.type == ImageType::e2D && dst.type == ImageType::e2D};
+2 -3
View File
@@ -1,4 +1,4 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project
@@ -10,8 +10,7 @@ namespace Tegra::Texture {
Common::ThreadWorker& GetThreadWorkers() {
static Common::ThreadWorker workers{(std::max)(std::thread::hardware_concurrency(), 2U) / 2,
"ImageTranscode", {},
Common::ThreadPlacement::Background};
"ImageTranscode"};
return workers;
}
+12 -268
View File
@@ -5,11 +5,8 @@
// SPDX-License-Identifier: GPL-2.0-or-later
#include <algorithm>
#include <array>
#include <bitset>
#include <chrono>
#include <filesystem>
#include <fstream>
#include <optional>
#include <thread>
#include <ankerl/unordered_dense.h>
@@ -19,9 +16,6 @@
#include <fmt/format.h>
#include "common/assert.h"
#include "common/fs/fs.h"
#include "common/fs/path_util.h"
#include "common/host_memory.h"
#include "common/literals.h"
#include <ranges>
#include "common/settings.h"
@@ -399,17 +393,6 @@ std::vector<const char*> ExtensionListForVulkan(
return output;
}
constexpr std::array<char, 8> STATIC_CACHE_MAGIC_NUMBER{'e', 'd', 'e', 'n', 's', 't', 'p', 'c'};
constexpr u32 STATIC_CACHE_VERSION = 1;
std::filesystem::path StaticPipelineCacheFilename() {
const auto shader_dir = Common::FS::GetEdenPath(Common::FS::EdenPath::ShaderDir);
if (!Common::FS::CreateDir(shader_dir)) {
return {};
}
return shader_dir / "vulkan_static_pipelines.bin";
}
} // Anonymous namespace
void Device::RemoveExtension(bool& extension, const std::string& extension_name) {
@@ -540,6 +523,10 @@ Device::Device(VkInstance instance_, vk::PhysicalDevice physical_, VkSurfaceKHR
features.shader_atomic_int64.shaderBufferInt64Atomics = false;
features.shader_atomic_int64.shaderSharedInt64Atomics = false;
features.features.shaderInt64 = false;
LOG_WARNING(Render_Vulkan, "Qualcomm drivers have broken workgroup memory explicit layout.");
RemoveExtensionFeature(extensions.workgroup_memory_explicit_layout,
features.workgroup_memory_explicit_layout,
VK_KHR_WORKGROUP_MEMORY_EXPLICIT_LAYOUT_EXTENSION_NAME);
#if defined(__ANDROID__) && defined(ARCHITECTURE_arm64)
// BCn patching only safe on Android 9+ (API 28+). Older versions crash on driver load.
@@ -726,17 +713,6 @@ Device::Device(VkInstance instance_, vk::PhysicalDevice physical_, VkSurfaceKHR
RemoveExtensionFeature(extensions.vertex_input_dynamic_state, features.vertex_input_dynamic_state, VK_EXT_VERTEX_INPUT_DYNAMIC_STATE_EXTENSION_NAME);
}
// VK_EXT_descriptor_buffer requires VK_KHR_buffer_device_address
if (extensions.descriptor_buffer && !features.buffer_device_address.bufferDeviceAddress) {
LOG_WARNING(Render_Vulkan, "Descriptor buffer needs buffer device address, disabling.");
RemoveExtensionFeature(extensions.descriptor_buffer, features.descriptor_buffer,
VK_EXT_DESCRIPTOR_BUFFER_EXTENSION_NAME);
}
if (!extensions.descriptor_buffer) {
RemoveExtensionFeature(extensions.buffer_device_address, features.buffer_device_address,
VK_KHR_BUFFER_DEVICE_ADDRESS_EXTENSION_NAME);
}
logical = vk::Device::Create(physical, queue_cis, ExtensionListForVulkan(loaded_extensions), first_next, dld);
graphics_queue = logical.GetQueue(graphics_family);
@@ -750,16 +726,13 @@ Device::Device(VkInstance instance_, vk::PhysicalDevice physical_, VkSurfaceKHR
if (extensions.memory_budget) {
flags |= VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT;
}
if (extensions.buffer_device_address) {
flags |= VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT;
}
const VmaAllocatorCreateInfo allocator_info{
.flags = flags,
.physicalDevice = physical,
.device = *logical,
.preferredLargeHeapBlockSize = is_integrated
? (64u * 1024u * 1024u)
: (128u * 1024u * 1024u),
: (256u * 1024u * 1024u),
.pAllocationCallbacks = nullptr,
.pDeviceMemoryCallbacks = nullptr,
.pHeapSizeLimit = nullptr,
@@ -771,124 +744,15 @@ Device::Device(VkInstance instance_, vk::PhysicalDevice physical_, VkSurfaceKHR
vk::Check(vmaCreateAllocator(&allocator_info, &allocator));
{
const auto& limits = properties.properties.limits;
LOG_INFO(Render_Vulkan, "MSAA sample count support:");
LOG_INFO(Render_Vulkan, " framebufferColorSampleCounts: {:#x}",
limits.framebufferColorSampleCounts);
LOG_INFO(Render_Vulkan, " framebufferDepthSampleCounts: {:#x}",
limits.framebufferDepthSampleCounts);
LOG_INFO(Render_Vulkan, " framebufferStencilSampleCounts: {:#x}",
limits.framebufferStencilSampleCounts);
LOG_INFO(Render_Vulkan, " sampledImageColorSampleCounts: {:#x}",
limits.sampledImageColorSampleCounts);
LOG_INFO(Render_Vulkan, " sampledImageDepthSampleCounts: {:#x}",
limits.sampledImageDepthSampleCounts);
LOG_INFO(Render_Vulkan, " sampledImageIntegerSampleCounts:{:#x}",
limits.sampledImageIntegerSampleCounts);
LOG_INFO(Render_Vulkan, " storageImageSampleCounts: {:#x}",
limits.storageImageSampleCounts);
}
owns_static_pipeline_cache = surface != VkSurfaceKHR{};
LoadStaticPipelineCache();
// Initialize GPU logging if enabled
InitializeGPULogging();
}
Device::~Device() {
SaveStaticPipelineCache();
ShutdownGPULogging();
vk::FlushDeletionQueue();
vmaDestroyAllocator(allocator);
}
void Device::LoadStaticPipelineCache() {
const auto create = [this](size_t size, const void* data) {
static_pipeline_cache = logical.CreatePipelineCache({
.sType = VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO,
.pNext = nullptr,
.flags = 0,
.initialDataSize = size,
.pInitialData = data,
});
};
if (!owns_static_pipeline_cache) {
create(0, nullptr);
return;
}
const auto filename = StaticPipelineCacheFilename();
if (filename.empty()) {
create(0, nullptr);
return;
}
std::vector<char> data;
try {
std::ifstream file(filename, std::ios::binary | std::ios::ate);
if (!file.is_open()) {
create(0, nullptr);
return;
}
file.exceptions(std::ifstream::failbit | std::ifstream::badbit);
const size_t total = static_cast<size_t>(file.tellg());
file.seekg(0, std::ios::beg);
std::array<char, 8> magic{};
u32 version{};
if (total < magic.size() + sizeof(version)) {
create(0, nullptr);
return;
}
file.read(magic.data(), magic.size())
.read(reinterpret_cast<char*>(&version), sizeof(version));
if (magic != STATIC_CACHE_MAGIC_NUMBER || version != STATIC_CACHE_VERSION) {
LOG_INFO(Render_Vulkan, "Discarding stale static pipeline cache");
create(0, nullptr);
return;
}
data.resize(total - magic.size() - sizeof(version));
file.read(data.data(), static_cast<std::streamsize>(data.size()));
} catch (const std::ios_base::failure& e) {
LOG_WARNING(Render_Vulkan, "Failed to read static pipeline cache: {}", e.what());
create(0, nullptr);
return;
}
create(data.size(), data.empty() ? nullptr : data.data());
LOG_INFO(Render_Vulkan, "Loaded {} bytes of static pipeline cache", data.size());
}
void Device::SaveStaticPipelineCache() const {
if (!owns_static_pipeline_cache || !static_pipeline_cache) {
return;
}
const auto filename = StaticPipelineCacheFilename();
if (filename.empty()) {
return;
}
size_t size = 0;
std::vector<char> data;
static_pipeline_cache.Read(&size, nullptr);
if (size == 0) {
return;
}
data.resize(size);
static_pipeline_cache.Read(&size, data.data());
try {
std::ofstream file(filename, std::ios::binary | std::ios::trunc);
file.exceptions(std::ofstream::failbit);
if (!file.is_open()) {
return;
}
file.write(STATIC_CACHE_MAGIC_NUMBER.data(), STATIC_CACHE_MAGIC_NUMBER.size())
.write(reinterpret_cast<const char*>(&STATIC_CACHE_VERSION),
sizeof(STATIC_CACHE_VERSION))
.write(data.data(), static_cast<std::streamsize>(size));
} catch (const std::ios_base::failure& e) {
LOG_WARNING(Render_Vulkan, "Failed to write static pipeline cache: {}", e.what());
Common::FS::RemoveFile(filename);
}
}
VkFormat Device::GetSupportedFormat(VkFormat wanted_format, VkFormatFeatureFlags wanted_usage,
FormatType format_type) const {
if (IsFormatSupported(wanted_format, wanted_usage, format_type)) {
@@ -1075,7 +939,6 @@ bool Device::GetSuitability(bool requires_swapchain) {
FOR_EACH_VK_FEATURE_EXT(FEATURE_EXTENSION);
FOR_EACH_VK_EXTENSION(EXTENSION);
FOR_EACH_VK_PLATFORM_EXTENSION(EXTENSION);
if (supported_extensions.contains(VK_KHR_ROBUSTNESS_2_EXTENSION_NAME)) {
loaded_extensions.erase(VK_EXT_ROBUSTNESS_2_EXTENSION_NAME);
@@ -1088,17 +951,6 @@ bool Device::GetSuitability(bool requires_swapchain) {
extensions.robustness_2 = false;
}
#ifdef __ANDROID__
if (extensions.external_memory_ahb && !extensions.queue_family_foreign) {
LOG_INFO(Render_Vulkan,
"Not loading {} because its dependency {} is unavailable",
VK_ANDROID_EXTERNAL_MEMORY_ANDROID_HARDWARE_BUFFER_EXTENSION_NAME,
VK_EXT_QUEUE_FAMILY_FOREIGN_EXTENSION_NAME);
loaded_extensions.erase(VK_ANDROID_EXTERNAL_MEMORY_ANDROID_HARDWARE_BUFFER_EXTENSION_NAME);
extensions.external_memory_ahb = false;
}
#endif
#undef FEATURE_EXTENSION
#undef EXTENSION
@@ -1231,11 +1083,6 @@ bool Device::GetSuitability(bool requires_swapchain) {
VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PUSH_DESCRIPTOR_PROPERTIES_KHR;
SetNext(next, properties.push_descriptor);
}
if (extensions.descriptor_buffer) {
properties.descriptor_buffer.sType =
VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DESCRIPTOR_BUFFER_PROPERTIES_EXT;
SetNext(next, properties.descriptor_buffer);
}
if (extensions.subgroup_size_control || features.subgroup_size_control.subgroupSizeControl) {
properties.subgroup_size_control.sType =
VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_SIZE_CONTROL_PROPERTIES;
@@ -1251,21 +1098,6 @@ bool Device::GetSuitability(bool requires_swapchain) {
VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MAINTENANCE_5_PROPERTIES_KHR;
SetNext(next, properties.maintenance5);
}
if (extensions.external_memory_host) {
properties.external_memory_host.sType =
VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_MEMORY_HOST_PROPERTIES_EXT;
SetNext(next, properties.external_memory_host);
}
if (extensions.maintenance4 || features.maintenance4.maintenance4) {
properties.maintenance4.sType =
VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MAINTENANCE_4_PROPERTIES;
SetNext(next, properties.maintenance4);
}
if (instance_version >= VK_API_VERSION_1_2) {
properties.depth_stencil_resolve.sType =
VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DEPTH_STENCIL_RESOLVE_PROPERTIES;
SetNext(next, properties.depth_stencil_resolve);
}
// Perform the property fetch.
physical.GetProperties2(properties2);
@@ -1377,20 +1209,10 @@ void Device::RemoveUnsuitableExtensions() {
RemoveExtensionFeatureIfUnsuitable(extensions.depth_bias_control, features.depth_bias_control,
VK_EXT_DEPTH_BIAS_CONTROL_EXTENSION_NAME);
// VK_EXT_depth_clamp_zero_one
extensions.depth_clamp_zero_one = features.depth_clamp_zero_one.depthClampZeroOne;
RemoveExtensionFeatureIfUnsuitable(extensions.depth_clamp_zero_one,
features.depth_clamp_zero_one,
VK_EXT_DEPTH_CLAMP_ZERO_ONE_EXTENSION_NAME);
// VK_EXT_depth_clip_control
extensions.depth_clip_control = features.depth_clip_control.depthClipControl;
RemoveExtensionFeatureIfUnsuitable(extensions.depth_clip_control, features.depth_clip_control,
VK_EXT_DEPTH_CLIP_CONTROL_EXTENSION_NAME);
// VK_EXT_depth_clip_enable
extensions.depth_clip_enable = features.depth_clip_enable.depthClipEnable;
RemoveExtensionFeatureIfUnsuitable(extensions.depth_clip_enable, features.depth_clip_enable,
VK_EXT_DEPTH_CLIP_ENABLE_EXTENSION_NAME);
// VK_EXT_extended_dynamic_state
extensions.extended_dynamic_state = features.extended_dynamic_state.extendedDynamicState;
@@ -1504,12 +1326,6 @@ void Device::RemoveUnsuitableExtensions() {
features.subgroup_size_control,
VK_EXT_SUBGROUP_SIZE_CONTROL_EXTENSION_NAME);
// VK_EXT_texture_compression_astc_hdr
extensions.texture_compression_astc_hdr = false;
RemoveExtensionFeatureIfUnsuitable(extensions.texture_compression_astc_hdr,
features.texture_compression_astc_hdr,
VK_EXT_TEXTURE_COMPRESSION_ASTC_HDR_EXTENSION_NAME);
// VK_EXT_transform_feedback
extensions.transform_feedback =
features.transform_feedback.transformFeedback &&
@@ -1537,18 +1353,7 @@ void Device::RemoveUnsuitableExtensions() {
VK_KHR_PIPELINE_EXECUTABLE_PROPERTIES_EXTENSION_NAME);
}
// VK_KHR_shader_quad_control
extensions.shader_quad_control = features.shader_quad_control.shaderQuadControl;
RemoveExtensionFeatureIfUnsuitable(extensions.shader_quad_control, features.shader_quad_control,
VK_KHR_SHADER_QUAD_CONTROL_EXTENSION_NAME);
// VK_KHR_workgroup_memory_explicit_layout
#ifdef __ANDROID__
if (!Settings::values.workgroup_memory_explicit_layout.GetValue()) {
LOG_INFO(Render_Vulkan, "Workgroup memory explicit layout disabled by user setting");
features.workgroup_memory_explicit_layout.workgroupMemoryExplicitLayout = false;
}
#endif
extensions.workgroup_memory_explicit_layout =
features.workgroup_memory_explicit_layout.workgroupMemoryExplicitLayout &&
features.workgroup_memory_explicit_layout.workgroupMemoryExplicitLayoutScalarBlockLayout;
@@ -1568,15 +1373,6 @@ void Device::RemoveUnsuitableExtensions() {
extensions.maintenance3 = loaded_extensions.contains(VK_KHR_MAINTENANCE_3_EXTENSION_NAME);
RemoveExtensionIfUnsuitable(extensions.maintenance3, VK_KHR_MAINTENANCE_3_EXTENSION_NAME);
// VK_KHR_dynamic_rendering
if (!Settings::values.dynamic_rendering.GetValue()) {
LOG_INFO(Render_Vulkan, "Dynamic rendering disabled by user setting");
features.dynamic_rendering.dynamicRendering = false;
}
extensions.dynamic_rendering = features.dynamic_rendering.dynamicRendering;
RemoveExtensionFeatureIfUnsuitable(extensions.dynamic_rendering, features.dynamic_rendering,
VK_KHR_DYNAMIC_RENDERING_EXTENSION_NAME);
// VK_KHR_maintenance4
extensions.maintenance4 = features.maintenance4.maintenance4;
RemoveExtensionFeatureIfUnsuitable(extensions.maintenance4, features.maintenance4,
@@ -1648,28 +1444,8 @@ std::optional<size_t> Device::GetSamplerHeapBudget() const {
return sampler_heap_budget;
}
Device::MemoryBudgetInfo Device::GetMemoryBudgetInfo() const {
std::array<VmaBudget, VK_MAX_MEMORY_HEAPS> budgets{};
vmaGetHeapBudgets(allocator, budgets.data());
MemoryBudgetInfo info{};
for (const size_t heap : valid_heap_memory) {
info.usage += budgets[heap].usage;
info.budget += budgets[heap].budget;
info.block_bytes += budgets[heap].statistics.blockBytes;
info.allocation_bytes += budgets[heap].statistics.allocationBytes;
}
return info;
}
void Device::TickAllocatorFrame() const {
vmaSetCurrentFrameIndex(allocator, ++allocator_frame_index);
}
u64 Device::GetDeviceMemoryUsage() const {
if (!extensions.memory_budget) {
return GetMemoryBudgetInfo().allocation_bytes;
}
VkPhysicalDeviceMemoryBudgetPropertiesEXT budget{};
VkPhysicalDeviceMemoryBudgetPropertiesEXT budget;
budget.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT;
budget.pNext = nullptr;
physical.GetMemoryProperties(&budget);
@@ -1691,27 +1467,12 @@ void Device::CollectPhysicalMemoryInfo() {
device_access_memory = 0;
u64 device_initial_usage = 0;
u64 local_memory = 0;
const auto heap_has_usable_type = [&mem_properties](size_t heap) {
for (u32 index = 0; index < mem_properties.memoryTypeCount; ++index) {
if (mem_properties.memoryTypes[index].heapIndex != heap) {
continue;
}
if ((mem_properties.memoryTypes[index].propertyFlags &
VK_MEMORY_PROPERTY_PROTECTED_BIT) == 0) {
return true;
}
}
return false;
};
for (size_t element = 0; element < num_properties; ++element) {
const bool is_heap_local =
(mem_properties.memoryHeaps[element].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0;
if (!is_integrated && !is_heap_local) {
continue;
}
if (!heap_has_usable_type(element)) {
continue;
}
valid_heap_memory.push_back(element);
if (is_heap_local) {
local_memory += mem_properties.memoryHeaps[element].size;
@@ -1723,35 +1484,18 @@ void Device::CollectPhysicalMemoryInfo() {
}
device_access_memory += mem_properties.memoryHeaps[element].size;
}
const u64 committed_backing = Common::GetCommittedBackingSize();
if (committed_backing != 0) {
LOG_INFO(Render_Vulkan, "Discounting {} MiB of guest memory committed by the host",
committed_backing >> 20);
local_memory -= std::min(local_memory, committed_backing);
device_access_memory -= std::min(device_access_memory, committed_backing);
}
if (is_integrated) {
const bool aggressive =
Settings::values.vram_usage_mode.GetValue() == Settings::VramUsageMode::Aggressive;
const s64 available_memory =
static_cast<s64>(device_access_memory) - static_cast<s64>(device_initial_usage);
const u64 reserve = std::min<u64>(local_memory / 8, 1_GiB);
const u64 usable = available_memory > static_cast<s64>(reserve)
? static_cast<u64>(available_memory) - reserve
: 0;
const u64 cap = (local_memory / 8) * (aggressive ? 6 : 5);
device_access_memory = std::max<u64>(std::min<u64>(usable, cap), 1_GiB);
const s64 available_memory = static_cast<s64>(device_access_memory - device_initial_usage);
const u64 memory_size = Settings::values.vram_usage_mode.GetValue() == Settings::VramUsageMode::Aggressive ? 6_GiB : 4_GiB;
device_access_memory = static_cast<u64>(std::max<s64>(std::min<s64>(available_memory - 8_GiB, memory_size), std::min<s64>(local_memory, memory_size)));
} else {
const u64 reserve_memory = std::min<u64>(device_access_memory / 8, 1_GiB);
device_access_memory -= reserve_memory;
if (Settings::values.vram_usage_mode.GetValue() != Settings::VramUsageMode::Aggressive) {
// Account for resolution scaling in memory limits
const u64 normal_memory = 6_GiB;
const u64 scaler_memory = 1_GiB * Settings::values.resolution_info.ScaleUp(1);
const u64 baseline = normal_memory + scaler_memory;
const u64 proportional = (device_access_memory / 4) * 3;
device_access_memory =
std::min<u64>(device_access_memory, std::max<u64>(baseline, proportional));
const size_t normal_memory = 6_GiB;
const size_t scaler_memory = 1_GiB * Settings::values.resolution_info.ScaleUp(1);
device_access_memory = std::min<u64>(device_access_memory, normal_memory + scaler_memory);
}
}
}
+3 -192
View File
@@ -36,7 +36,6 @@ VK_DEFINE_HANDLE(VmaAllocator)
FEATURE(EXT, DescriptorIndexing, DESCRIPTOR_INDEXING, descriptor_indexing) \
FEATURE(EXT, HostQueryReset, HOST_QUERY_RESET, host_query_reset) \
FEATURE(KHR, 8BitStorage, 8BIT_STORAGE, bit8_storage) \
FEATURE(KHR, BufferDeviceAddress, BUFFER_DEVICE_ADDRESS, buffer_device_address) \
FEATURE(KHR, TimelineSemaphore, TIMELINE_SEMAPHORE, timeline_semaphore)
#define FOR_EACH_VK_FEATURE_1_3(FEATURE) \
@@ -44,7 +43,6 @@ VK_DEFINE_HANDLE(VmaAllocator)
FEATURE(EXT, ShaderDemoteToHelperInvocation, SHADER_DEMOTE_TO_HELPER_INVOCATION, \
shader_demote_to_helper_invocation) \
FEATURE(EXT, SubgroupSizeControl, SUBGROUP_SIZE_CONTROL, subgroup_size_control) \
FEATURE(KHR, DynamicRendering, DYNAMIC_RENDERING, dynamic_rendering) \
FEATURE(KHR, Maintenance4, MAINTENANCE_4, maintenance4) \
FEATURE(KHR, Synchronization2, SYNCHRONIZATION_2, synchronization2)
@@ -56,10 +54,7 @@ VK_DEFINE_HANDLE(VmaAllocator)
FEATURE(EXT, ColorWriteEnable, COLOR_WRITE_ENABLE, color_write_enable) \
FEATURE(EXT, CustomBorderColor, CUSTOM_BORDER_COLOR, custom_border_color) \
FEATURE(EXT, DepthBiasControl, DEPTH_BIAS_CONTROL, depth_bias_control) \
FEATURE(EXT, DepthClampZeroOne, DEPTH_CLAMP_ZERO_ONE, depth_clamp_zero_one) \
FEATURE(EXT, DepthClipControl, DEPTH_CLIP_CONTROL, depth_clip_control) \
FEATURE(EXT, DepthClipEnable, DEPTH_CLIP_ENABLE, depth_clip_enable) \
FEATURE(EXT, DescriptorBuffer, DESCRIPTOR_BUFFER, descriptor_buffer) \
FEATURE(EXT, ExtendedDynamicState, EXTENDED_DYNAMIC_STATE, extended_dynamic_state) \
FEATURE(EXT, ExtendedDynamicState2, EXTENDED_DYNAMIC_STATE_2, extended_dynamic_state2) \
FEATURE(EXT, ExtendedDynamicState3, EXTENDED_DYNAMIC_STATE_3, extended_dynamic_state3) \
@@ -76,7 +71,6 @@ VK_DEFINE_HANDLE(VmaAllocator)
FEATURE(KHR, Maintenance6, MAINTENANCE_6, maintenance6) \
FEATURE(KHR, PipelineExecutableProperties, PIPELINE_EXECUTABLE_PROPERTIES, \
pipeline_executable_properties) \
FEATURE(KHR, ShaderQuadControl, SHADER_QUAD_CONTROL, shader_quad_control) \
FEATURE(KHR, WorkgroupMemoryExplicitLayout, WORKGROUP_MEMORY_EXPLICIT_LAYOUT, \
workgroup_memory_explicit_layout) \
FEATURE(EXT, TextureCompressionASTCHDR, TEXTURE_COMPRESSION_ASTC_HDR, \
@@ -85,13 +79,10 @@ VK_DEFINE_HANDLE(VmaAllocator)
// Define miscellaneous extensions which may be used by the implementation here.
#define FOR_EACH_VK_EXTENSION(EXTENSION) \
EXTENSION(EXT, ASTC_DECODE_MODE, astc_decode_mode) \
EXTENSION(EXT, CONDITIONAL_RENDERING, conditional_rendering) \
EXTENSION(EXT, CONSERVATIVE_RASTERIZATION, conservative_rasterization) \
EXTENSION(EXT, DEPTH_RANGE_UNRESTRICTED, depth_range_unrestricted) \
EXTENSION(EXT, EXTERNAL_MEMORY_HOST, external_memory_host) \
EXTENSION(EXT, MEMORY_BUDGET, memory_budget) \
EXTENSION(EXT, PIPELINE_CREATION_FEEDBACK, pipeline_creation_feedback) \
EXTENSION(EXT, ROBUSTNESS_2, robustness_2) \
EXTENSION(EXT, SAMPLER_FILTER_MINMAX, sampler_filter_minmax) \
EXTENSION(EXT, SHADER_STENCIL_EXPORT, shader_stencil_export) \
@@ -120,14 +111,6 @@ VK_DEFINE_HANDLE(VmaAllocator)
EXTENSION(IMG, FILTER_CUBIC, filter_cubic_img) \
EXTENSION(QCOM, FILTER_CUBIC_WEIGHTS, filter_cubic_weights)
#ifdef __ANDROID__
#define FOR_EACH_VK_PLATFORM_EXTENSION(EXTENSION) \
EXTENSION(EXT, QUEUE_FAMILY_FOREIGN, queue_family_foreign) \
EXTENSION(ANDROID, EXTERNAL_MEMORY_ANDROID_HARDWARE_BUFFER, external_memory_ahb)
#else
#define FOR_EACH_VK_PLATFORM_EXTENSION(EXTENSION)
#endif
// Define extensions which must be supported.
#define FOR_EACH_VK_MANDATORY_EXTENSION(EXTENSION_NAME) \
EXTENSION_NAME(VK_EXT_VERTEX_ATTRIBUTE_DIVISOR_EXTENSION_NAME) \
@@ -272,17 +255,6 @@ public:
return allocator;
}
struct MemoryBudgetInfo {
u64 usage;
u64 budget;
u64 block_bytes;
u64 allocation_bytes;
};
MemoryBudgetInfo GetMemoryBudgetInfo() const;
void TickAllocatorFrame() const;
/// Returns the logical device.
const vk::Device& GetLogical() const {
return logical;
@@ -293,10 +265,6 @@ public:
return physical;
}
VkPipelineCache StaticPipelineCache() const noexcept {
return *static_pipeline_cache;
}
/// Returns the main graphics queue.
vk::Queue GetGraphicsQueue() const {
return graphics_queue;
@@ -401,7 +369,8 @@ FN_MAX_LIMIT_LIST
}
bool IsOptimalAstcSupported() const {
return is_optimal_astc_supported;
return features.features.textureCompressionASTC_LDR &&
features.texture_compression_astc_hdr.textureCompressionASTC_HDR;
}
/// Returns true if BCn is natively supported.
@@ -499,29 +468,6 @@ FN_MAX_LIMIT_LIST
return properties.push_descriptor.maxPushDescriptors;
}
/// Returns true if the device supports descriptor buffers.
bool IsExtPipelineCreationFeedbackSupported() const {
return extensions.pipeline_creation_feedback;
}
bool IsRobustBufferAccessEnabled() const {
return features.features.robustBufferAccess == VK_TRUE;
}
bool IsExtDescriptorBufferSupported() const {
return extensions.descriptor_buffer;
}
/// Returns the descriptor buffer properties of the device.
const VkPhysicalDeviceDescriptorBufferPropertiesEXT& DescriptorBufferProperties() const {
return properties.descriptor_buffer;
}
/// Returns true if the device supports buffer device address.
bool IsBufferDeviceAddressSupported() const {
return extensions.buffer_device_address;
}
/// Returns true if formatless image load is supported.
bool IsFormatlessImageLoadSupported() const {
return features.features.shaderStorageImageReadWithoutFormat;
@@ -608,11 +554,6 @@ FN_MAX_LIMIT_LIST
features.workgroup_memory_explicit_layout.workgroupMemoryExplicitLayout16BitAccess &&
features.features.shaderInt16;
}
/// Returns true if the device supports VK_KHR_shader_quad_control.
bool IsKhrShaderQuadControlSupported() const {
return extensions.shader_quad_control && features.shader_quad_control.shaderQuadControl;
}
/// Returns true if the device supports VK_KHR_image_format_list.
bool IsKhrImageFormatListSupported() const {
@@ -645,6 +586,7 @@ FN_MAX_LIMIT_LIST
}
/// Returns true if depth/stencil operations can be performed efficiently.
/// Either through shader export or hardware blits.
bool CanPerformDepthStencilOperations() const {
return extensions.shader_stencil_export || is_blit_depth24_stencil8_supported ||
is_blit_depth32_stencil8_supported;
@@ -660,16 +602,6 @@ FN_MAX_LIMIT_LIST
return extensions.depth_clip_control;
}
/// Returns true if the device supports VK_EXT_depth_clamp_zero_one.
bool IsExtDepthClampZeroOneSupported() const {
return extensions.depth_clamp_zero_one;
}
/// Returns true if the device supports VK_EXT_depth_clip_enable.
bool IsExtDepthClipEnableSupported() const {
return extensions.depth_clip_enable;
}
/// Returns true if the device supports VK_EXT_depth_bias_control.
bool IsExtDepthBiasControlSupported() const {
return extensions.depth_bias_control;
@@ -802,38 +734,6 @@ FN_MAX_LIMIT_LIST
return features.line_rasterization.stippledRectangularLines != VK_FALSE;
}
VkLineRasterizationModeEXT GetLineRasterizationMode(bool wants_smooth) const {
if (wants_smooth && SupportsSmoothLines()) {
return VK_LINE_RASTERIZATION_MODE_RECTANGULAR_SMOOTH_EXT;
}
if (SupportsRectangularLines()) {
return VK_LINE_RASTERIZATION_MODE_RECTANGULAR_EXT;
}
return VK_LINE_RASTERIZATION_MODE_DEFAULT_EXT;
}
bool SupportsStippleForMode(VkLineRasterizationModeEXT mode) const {
switch (mode) {
case VK_LINE_RASTERIZATION_MODE_RECTANGULAR_SMOOTH_EXT:
return features.line_rasterization.stippledSmoothLines != VK_FALSE;
case VK_LINE_RASTERIZATION_MODE_BRESENHAM_EXT:
return features.line_rasterization.stippledBresenhamLines != VK_FALSE;
default:
return features.line_rasterization.stippledRectangularLines != VK_FALSE;
}
}
float ClampLineWidth(float width) const {
if (!features.features.wideLines) {
return 1.0f;
}
const auto& range = properties.properties.limits.lineWidthRange;
if (!(width >= range[0])) {
return range[0];
}
return width > range[1] ? range[1] : width;
}
bool SupportsAlphaToOne() const {
return features.features.alphaToOne != VK_FALSE;
}
@@ -912,41 +812,10 @@ FN_MAX_LIMIT_LIST
features.shader_atomic_int64.shaderSharedInt64Atomics;
}
/// Returns true if the device supports VK_EXT_conditional_rendering.
bool IsExtConditionalRendering() const {
return extensions.conditional_rendering;
}
/// Returns true if the device supports VK_EXT_astc_decode_mode.
bool IsExtAstcDecodeModeSupported() const {
return extensions.astc_decode_mode;
}
/// Returns true if the device supports VK_EXT_external_memory_host.
bool IsExtExternalMemoryHostSupported() const {
return extensions.external_memory_host;
}
bool IsExtExternalMemoryAhbSupported() const {
#ifdef __ANDROID__
return extensions.external_memory_ahb && extensions.queue_family_foreign;
#else
return false;
#endif
}
bool IsDescriptorBindingPartiallyBoundSupported() const {
return features.descriptor_indexing.descriptorBindingPartiallyBound;
}
u64 GetMinImportedHostPointerAlignment() const {
return properties.external_memory_host.minImportedHostPointerAlignment;
}
u64 GetMaxBufferSize() const {
return properties.maintenance4.maxBufferSize;
}
bool HasTimelineSemaphore() const;
/// Returns true if the device supports VK_KHR_synchronization2.
@@ -1007,10 +876,6 @@ FN_MAX_LIMIT_LIST
u64 GetDeviceMemoryUsage() const;
VkSampleCountFlags GetStorageImageSampleCounts() const {
return properties.properties.limits.storageImageSampleCounts;
}
u32 GetSetsPerPool() const {
return sets_per_pool;
}
@@ -1076,33 +941,6 @@ FN_MAX_LIMIT_LIST
return extensions.maintenance3;
}
static constexpr bool ENABLE_DYNAMIC_RENDERING = true;
/// Returns true if the device supports VK_KHR_dynamic_rendering.
bool IsKhrDynamicRenderingSupported() const {
return ENABLE_DYNAMIC_RENDERING && extensions.dynamic_rendering;
}
/// Returns the resolve mode to use for multisampled depth.
VkResolveModeFlagBits GetDepthResolveMode() const {
return PickResolveMode(properties.depth_stencil_resolve.supportedDepthResolveModes);
}
/// Same as GetDepthResolveMode, for the stencil aspect.
VkResolveModeFlagBits GetStencilResolveMode() const {
return PickResolveMode(properties.depth_stencil_resolve.supportedStencilResolveModes);
}
/// Returns true if depth and stencil may resolve with independent modes.
bool IsIndependentResolveSupported() const {
return properties.depth_stencil_resolve.independentResolve == VK_TRUE;
}
/// Returns true if one aspect may resolve while the other does not.
bool IsIndependentResolveNoneSupported() const {
return properties.depth_stencil_resolve.independentResolveNone == VK_TRUE;
}
/// Returns true if the device supports VK_KHR_maintenance4.
bool IsKhrMaintenance4Supported() const {
return extensions.maintenance4;
@@ -1184,22 +1022,6 @@ FN_MAX_LIMIT_LIST
void ShutdownGPULogging();
private:
/// Picks a usable resolve mode out of a supported-modes mask. SAMPLE_ZERO is preferred because
/// it is the only mode the spec guarantees whenever any depth/stencil resolve is supported,
/// and averaging depth samples is not meaningful for depth testing anyway.
static VkResolveModeFlagBits PickResolveMode(VkResolveModeFlags supported) {
if ((supported & VK_RESOLVE_MODE_SAMPLE_ZERO_BIT) != 0) {
return VK_RESOLVE_MODE_SAMPLE_ZERO_BIT;
}
if ((supported & VK_RESOLVE_MODE_MIN_BIT) != 0) {
return VK_RESOLVE_MODE_MIN_BIT;
}
if ((supported & VK_RESOLVE_MODE_MAX_BIT) != 0) {
return VK_RESOLVE_MODE_MAX_BIT;
}
return VK_RESOLVE_MODE_NONE;
}
/// Checks if the physical device is suitable and configures the object state
/// with all necessary info about its properties.
bool GetSuitability(bool requires_swapchain);
@@ -1235,20 +1057,14 @@ private:
/// Returns true if the device natively supports blitting depth stencil images.
bool TestDepthStencilBlits(VkFormat format) const;
void LoadStaticPipelineCache();
void SaveStaticPipelineCache() const;
private:
VkInstance instance; ///< Vulkan instance.
VmaAllocator allocator; ///< VMA allocator.
mutable u32 allocator_frame_index{};
vk::DeviceDispatch dld; ///< Device function pointers.
vk::PhysicalDevice physical; ///< Physical device.
vk::Device logical; ///< Logical device.
vk::Queue graphics_queue; ///< Main graphics queue.
vk::Queue present_queue; ///< Main present queue.
vk::PipelineCache static_pipeline_cache;
bool owns_static_pipeline_cache{};
u32 instance_version{}; ///< Vulkan instance version.
u32 graphics_family{}; ///< Main graphics queue family index.
u32 present_family{}; ///< Main present queue family index.
@@ -1263,7 +1079,6 @@ private:
FOR_EACH_VK_FEATURE_1_4(FEATURE);
FOR_EACH_VK_FEATURE_EXT(FEATURE);
FOR_EACH_VK_EXTENSION(EXTENSION);
FOR_EACH_VK_PLATFORM_EXTENSION(EXTENSION);
#undef EXTENSION
#undef FEATURE
@@ -1292,13 +1107,9 @@ private:
VkPhysicalDeviceSubgroupProperties subgroup_properties{};
VkPhysicalDeviceFloatControlsProperties float_controls{};
VkPhysicalDevicePushDescriptorPropertiesKHR push_descriptor{};
VkPhysicalDeviceDescriptorBufferPropertiesEXT descriptor_buffer{};
VkPhysicalDeviceSubgroupSizeControlProperties subgroup_size_control{};
VkPhysicalDeviceTransformFeedbackPropertiesEXT transform_feedback{};
VkPhysicalDeviceMaintenance4Properties maintenance4{};
VkPhysicalDeviceMaintenance5PropertiesKHR maintenance5{};
VkPhysicalDeviceDepthStencilResolveProperties depth_stencil_resolve{};
VkPhysicalDeviceExternalMemoryHostPropertiesEXT external_memory_host{};
VkPhysicalDeviceProperties properties{};
};
@@ -25,36 +25,31 @@
#include "video_core/gpu_logging/gpu_logging.h"
#include "common/settings.h"
#ifdef __ANDROID__
#include <android/hardware_buffer.h>
#endif
namespace Vulkan {
namespace {
[[nodiscard]] std::optional<u32> FindImportMemoryType(
const VkPhysicalDeviceMemoryProperties &props, u32 type_mask) {
const auto find = [&](VkMemoryPropertyFlags wanted) -> std::optional<u32> {
for (u32 i = 0; i < props.memoryTypeCount; ++i) {
if (((type_mask >> i) & 1u) != 0 &&
(props.memoryTypes[i].propertyFlags & wanted) == wanted) {
return i;
}
}
return std::nullopt;
};
auto type_index = find(VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
VK_MEMORY_PROPERTY_HOST_COHERENT_BIT |
VK_MEMORY_PROPERTY_HOST_CACHED_BIT);
if (!type_index) {
type_index = find(VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
}
return type_index;
}
// Helpers translating MemoryUsage to flags/usage
[[maybe_unused]] VkMemoryPropertyFlags MemoryUsagePropertyFlags(MemoryUsage usage) {
switch (usage) {
case MemoryUsage::DeviceLocal:
return VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
case MemoryUsage::Upload:
return VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
case MemoryUsage::Download:
return VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
VK_MEMORY_PROPERTY_HOST_COHERENT_BIT |
VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
case MemoryUsage::Stream:
return VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT |
VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
}
ASSERT_MSG(false, "Invalid memory usage={}", usage);
return VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
}
[[nodiscard]] VkMemoryPropertyFlags MemoryUsagePreferredVmaFlags(MemoryUsage usage) {
if (usage == MemoryUsage::Download) {
return VK_MEMORY_PROPERTY_HOST_CACHED_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
@@ -91,254 +86,125 @@ namespace Vulkan {
}
// This avoids calling vkGetBufferMemoryRequirements* directly.
template<typename T>
static VkBuffer GetVkHandleFromBuffer(const T &buf) {
if constexpr (requires { static_cast<VkBuffer>(buf); }) {
return static_cast<VkBuffer>(buf);
} else if constexpr (requires {{ buf.GetHandle() } -> std::convertible_to<VkBuffer>; }) {
return buf.GetHandle();
} else if constexpr (requires {{ buf.Handle() } -> std::convertible_to<VkBuffer>; }) {
return buf.Handle();
} else if constexpr (requires {{ buf.vk_handle() } -> std::convertible_to<VkBuffer>; }) {
return buf.vk_handle();
} else {
static_assert(sizeof(T) == 0, "Cannot extract VkBuffer handle from vk::Buffer");
return VK_NULL_HANDLE;
}
}
} // namespace
HostMemoryImport::HostMemoryImport(const Device &device_, void *base, size_t size,
std::span<AHardwareBuffer *const> hardware_buffers,
size_t hardware_buffer_window)
: device{device_} {
if (ImportHostPointer(base, size)) {
return;
}
ImportHardwareBuffers(hardware_buffers, hardware_buffer_window, size);
if (windows.empty()) {
LOG_INFO(Render_Vulkan, "Unified memory disabled, no host memory import path");
//MemoryCommit is now VMA-backed
MemoryCommit::MemoryCommit(VmaAllocator alloc, VmaAllocation a,
const VmaAllocationInfo &info) noexcept
: allocator{alloc}, allocation{a}, memory{info.deviceMemory},
offset{info.offset}, size{info.size}, mapped_ptr{info.pMappedData} {
// Log GPU memory allocation
if (GPU::Logging::IsActive() &&
Settings::values.gpu_log_memory_tracking.GetValue()) {
GPU::Logging::GPULogger::GetInstance().LogMemoryAllocation(
reinterpret_cast<uintptr_t>(memory),
static_cast<u64>(size),
0 // Memory property flags (not easily available from VMA)
);
}
}
bool HostMemoryImport::ImportHostPointer(void *base, size_t size) {
if (!device.IsExtExternalMemoryHostSupported()) {
LOG_INFO(Render_Vulkan,
"Unified memory disabled, VK_EXT_external_memory_host is not supported");
return false;
}
const u64 alignment = device.GetMinImportedHostPointerAlignment();
if (alignment == 0 || !Common::IsAligned(reinterpret_cast<uintptr_t>(base), alignment) ||
!Common::IsAligned(size, alignment)) {
LOG_INFO(Render_Vulkan,
"Unified memory disabled, host allocation does not satisfy alignment {}",
alignment);
return false;
}
using namespace Common::Literals;
VkDeviceSize candidate_window = 1_GiB;
const u64 max_buffer_size = device.GetMaxBufferSize();
if (max_buffer_size != 0 && max_buffer_size < candidate_window) {
candidate_window = Common::AlignDown(max_buffer_size, alignment);
}
if (candidate_window == 0) {
return false;
}
window_size = candidate_window;
MemoryCommit::~MemoryCommit() { Release(); }
const auto &logical = device.GetLogical();
const auto memory_props = device.GetPhysical().GetMemoryProperties().memoryProperties;
MemoryCommit::MemoryCommit(MemoryCommit &&rhs) noexcept
: allocator{std::exchange(rhs.allocator, nullptr)},
allocation{std::exchange(rhs.allocation, nullptr)},
memory{std::exchange(rhs.memory, VK_NULL_HANDLE)},
offset{std::exchange(rhs.offset, 0)},
size{std::exchange(rhs.size, 0)},
mapped_ptr{std::exchange(rhs.mapped_ptr, nullptr)} {}
for (size_t offset = 0; offset < size; offset += window_size) {
u8 *const window_base = static_cast<u8 *>(base) + offset;
const VkDeviceSize window_len =
(std::min)(static_cast<VkDeviceSize>(size - offset), window_size);
VkMemoryHostPointerPropertiesEXT host_props{
.sType = VK_STRUCTURE_TYPE_MEMORY_HOST_POINTER_PROPERTIES_EXT,
.pNext = nullptr,
.memoryTypeBits = 0,
};
if (logical.GetMemoryHostPointerPropertiesEXT(
VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, window_base,
&host_props) != VK_SUCCESS ||
host_props.memoryTypeBits == 0) {
break;
}
const VkExternalMemoryBufferCreateInfo external_info{
.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO,
.pNext = nullptr,
.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT,
};
const VkBufferCreateInfo buffer_ci{
.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,
.pNext = &external_info,
.flags = 0,
.size = window_len,
.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT,
.sharingMode = VK_SHARING_MODE_EXCLUSIVE,
.queueFamilyIndexCount = 0,
.pQueueFamilyIndices = nullptr,
};
VkBuffer new_buffer{};
if (logical.CreateBufferRaw(buffer_ci, &new_buffer) != VK_SUCCESS) {
break;
}
const VkMemoryRequirements requirements =
logical.GetBufferMemoryRequirements(new_buffer);
const u32 type_mask = requirements.memoryTypeBits & host_props.memoryTypeBits;
if (type_mask == 0 || requirements.size > window_len) {
logical.DestroyBufferRaw(new_buffer);
break;
}
const auto type_index = FindImportMemoryType(memory_props, type_mask);
if (!type_index) {
logical.DestroyBufferRaw(new_buffer);
break;
}
const u32 heap_index = memory_props.memoryTypes[*type_index].heapIndex;
const VkDeviceSize heap_size = memory_props.memoryHeaps[heap_index].size;
if (imported_size + window_len > heap_size / 2) {
LOG_INFO(Render_Vulkan,
"Stopping guest memory import at {} MiB to leave room on heap {} of {} MiB",
imported_size >> 20, heap_index, heap_size >> 20);
logical.DestroyBufferRaw(new_buffer);
break;
}
const VkImportMemoryHostPointerInfoEXT import_info{
.sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_HOST_POINTER_INFO_EXT,
.pNext = nullptr,
.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT,
.pHostPointer = window_base,
};
const VkMemoryAllocateInfo alloc_info{
.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,
.pNext = &import_info,
.allocationSize = window_len,
.memoryTypeIndex = *type_index,
};
vk::DeviceMemory memory = logical.TryAllocateMemory(alloc_info);
if (!memory) {
logical.DestroyBufferRaw(new_buffer);
break;
}
if (logical.BindBufferMemory(new_buffer, *memory, 0) != VK_SUCCESS) {
logical.DestroyBufferRaw(new_buffer);
break;
}
windows.push_back(Window{
.memory = std::move(memory),
.buffer = new_buffer,
});
imported_size += static_cast<size_t>(window_len);
MemoryCommit &MemoryCommit::operator=(MemoryCommit &&rhs) noexcept {
if (this != &rhs) {
Release();
allocator = std::exchange(rhs.allocator, nullptr);
allocation = std::exchange(rhs.allocation, nullptr);
memory = std::exchange(rhs.memory, VK_NULL_HANDLE);
offset = std::exchange(rhs.offset, 0);
size = std::exchange(rhs.size, 0);
mapped_ptr = std::exchange(rhs.mapped_ptr, nullptr);
}
if (windows.empty()) {
LOG_INFO(Render_Vulkan, "Host pointer import failed");
return false;
}
LOG_INFO(Render_Vulkan,
"Imported {} MiB of guest memory for unified memory access in {} windows",
imported_size >> 20, windows.size());
return true;
return *this;
}
void HostMemoryImport::ImportHardwareBuffers(
[[maybe_unused]] std::span<AHardwareBuffer *const> hardware_buffers,
[[maybe_unused]] size_t hardware_buffer_window, [[maybe_unused]] size_t size) {
#ifdef __ANDROID__
if (hardware_buffers.empty() || hardware_buffer_window == 0 ||
!device.IsExtExternalMemoryAhbSupported()) {
return;
std::span<u8> MemoryCommit::Map()
{
if (!allocation) return {};
if (!mapped_ptr) {
if (vmaMapMemory(allocator, allocation, &mapped_ptr) != VK_SUCCESS) return {};
}
const auto &logical = device.GetLogical();
const auto memory_props = device.GetPhysical().GetMemoryProperties().memoryProperties;
window_size = hardware_buffer_window;
for (size_t i = 0; i < hardware_buffers.size(); ++i) {
const size_t offset = i * hardware_buffer_window;
if (offset >= size) {
break;
}
const VkDeviceSize window_len = (std::min)(
static_cast<VkDeviceSize>(size - offset),
static_cast<VkDeviceSize>(hardware_buffer_window));
VkAndroidHardwareBufferPropertiesANDROID ahb_props{
.sType = VK_STRUCTURE_TYPE_ANDROID_HARDWARE_BUFFER_PROPERTIES_ANDROID,
.pNext = nullptr,
.allocationSize = 0,
.memoryTypeBits = 0,
};
if (logical.GetAndroidHardwareBufferPropertiesANDROID(hardware_buffers[i],
&ahb_props) != VK_SUCCESS ||
ahb_props.memoryTypeBits == 0 || ahb_props.allocationSize < window_len) {
break;
}
const VkExternalMemoryBufferCreateInfo external_info{
.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO,
.pNext = nullptr,
.handleTypes =
VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID,
};
const VkBufferCreateInfo buffer_ci{
.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,
.pNext = &external_info,
.flags = 0,
.size = window_len,
.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT,
.sharingMode = VK_SHARING_MODE_EXCLUSIVE,
.queueFamilyIndexCount = 0,
.pQueueFamilyIndices = nullptr,
};
VkBuffer new_buffer{};
if (logical.CreateBufferRaw(buffer_ci, &new_buffer) != VK_SUCCESS) {
break;
}
const VkMemoryRequirements requirements =
logical.GetBufferMemoryRequirements(new_buffer);
const u32 type_mask = requirements.memoryTypeBits & ahb_props.memoryTypeBits;
if (type_mask == 0 || requirements.size > ahb_props.allocationSize) {
logical.DestroyBufferRaw(new_buffer);
break;
}
const auto type_index = FindImportMemoryType(memory_props, type_mask);
if (!type_index) {
logical.DestroyBufferRaw(new_buffer);
break;
}
const VkImportAndroidHardwareBufferInfoANDROID import_info{
.sType = VK_STRUCTURE_TYPE_IMPORT_ANDROID_HARDWARE_BUFFER_INFO_ANDROID,
.pNext = nullptr,
.buffer = hardware_buffers[i],
};
const VkMemoryDedicatedAllocateInfo dedicated_info{
.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO,
.pNext = &import_info,
.image = VK_NULL_HANDLE,
.buffer = new_buffer,
};
const VkMemoryAllocateInfo alloc_info{
.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,
.pNext = &dedicated_info,
.allocationSize = ahb_props.allocationSize,
.memoryTypeIndex = *type_index,
};
vk::DeviceMemory memory = logical.TryAllocateMemory(alloc_info);
if (!memory) {
logical.DestroyBufferRaw(new_buffer);
break;
}
if (logical.BindBufferMemory(new_buffer, *memory, 0) != VK_SUCCESS) {
logical.DestroyBufferRaw(new_buffer);
break;
}
windows.push_back(Window{
.memory = std::move(memory),
.buffer = new_buffer,
});
imported_size += static_cast<size_t>(window_len);
}
if (!windows.empty()) {
LOG_INFO(Render_Vulkan,
"Imported {} MiB of guest memory via hardware buffers in {} windows",
imported_size >> 20, windows.size());
}
#endif
const size_t n = static_cast<size_t>(std::min<VkDeviceSize>(size,
(std::numeric_limits<size_t>::max)()));
return std::span<u8>{static_cast<u8 *>(mapped_ptr), n};
}
HostMemoryImport::~HostMemoryImport() {
for (Window &window : windows) {
if (window.buffer != VK_NULL_HANDLE) {
device.GetLogical().DestroyBufferRaw(window.buffer);
}
std::span<const u8> MemoryCommit::Map() const
{
if (!allocation) return {};
if (!mapped_ptr) {
void *p = nullptr;
if (vmaMapMemory(allocator, allocation, &p) != VK_SUCCESS) return {};
const_cast<MemoryCommit *>(this)->mapped_ptr = p;
}
const size_t n = static_cast<size_t>(std::min<VkDeviceSize>(size,
(std::numeric_limits<size_t>::max)()));
return std::span<const u8>{static_cast<const u8 *>(mapped_ptr), n};
}
void MemoryCommit::Unmap()
{
if (allocation && mapped_ptr) {
vmaUnmapMemory(allocator, allocation);
mapped_ptr = nullptr;
}
}
void MemoryCommit::Release() {
if (allocation && allocator) {
// Log GPU memory deallocation
if (GPU::Logging::IsActive() &&
Settings::values.gpu_log_memory_tracking.GetValue() &&
memory != VK_NULL_HANDLE) {
GPU::Logging::GPULogger::GetInstance().LogMemoryDeallocation(
reinterpret_cast<uintptr_t>(memory)
);
}
if (mapped_ptr) {
vmaUnmapMemory(allocator, allocation);
mapped_ptr = nullptr;
}
vmaFreeMemory(allocator, allocation);
}
allocation = nullptr;
allocator = nullptr;
memory = VK_NULL_HANDLE;
offset = 0;
size = 0;
}
MemoryAllocator::MemoryAllocator(const Device &device_)
: device{device_}, allocator{device.GetAllocator()},
properties{device_.GetPhysical().GetMemoryProperties().memoryProperties} {
properties{device_.GetPhysical().GetMemoryProperties().memoryProperties},
buffer_image_granularity{
device_.GetPhysical().GetProperties().limits.bufferImageGranularity} {
// Preserve the previous "RenderDoc small heap" trimming behavior that we had in original vma minus the heap bug
if (device.HasDebuggingToolAttached())
@@ -358,25 +224,10 @@ namespace Vulkan {
MemoryAllocator::~MemoryAllocator() = default;
void MemoryAllocator::SetReclaimCallback(ReclaimCallback callback) {
reclaim_callback = std::move(callback);
vk::SetAllocatorOwnerThread();
}
bool MemoryAllocator::ReclaimAtLeast(u64 hint_bytes) const {
if (!reclaim_callback || in_reclaim) {
return false;
}
in_reclaim = true;
const u64 freed = reclaim_callback(hint_bytes);
in_reclaim = false;
return freed > 0;
}
vk::Image MemoryAllocator::CreateImage(const VkImageCreateInfo &ci) const
{
const VmaAllocationCreateInfo alloc_ci = {
.flags = device.CanReportMemoryUsage() ? VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT : 0u,
.flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT,
.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE,
.requiredFlags = 0,
.preferredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT,
@@ -389,26 +240,7 @@ namespace Vulkan {
VkImage handle{};
VmaAllocation allocation{};
VmaAllocationInfo alloc_info{};
DEBUG_ASSERT(vk::OnAllocatorOwnerThread());
VkResult res = vmaCreateImage(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info);
if (res != VK_SUCCESS && ReclaimAtLeast(IMAGE_RECLAIM_HINT)) {
res = vmaCreateImage(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info);
}
if (res != VK_SUCCESS) {
auto relaxed_ci = alloc_ci;
relaxed_ci.flags &= ~VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT;
res = vmaCreateImage(allocator, &ci, &relaxed_ci, &handle, &allocation, &alloc_info);
if (res != VK_SUCCESS) {
relaxed_ci.preferredFlags &= ~VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
res = vmaCreateImage(allocator, &ci, &relaxed_ci, &handle, &allocation, &alloc_info);
}
}
vk::Check(res);
vk::Check(vmaCreateImage(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info));
// Log GPU memory allocation for images
if (GPU::Logging::IsActive() &&
@@ -430,8 +262,7 @@ namespace Vulkan {
&& device.GetDriverID() == VK_DRIVER_ID_INTEL_OPEN_SOURCE_MESA)
? VK_MEMORY_PROPERTY_HOST_CACHED_BIT : 0;
const VmaAllocationCreateInfo alloc_ci = {
.flags = (device.CanReportMemoryUsage() ? VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT : 0u) |
MemoryUsageVmaFlags(usage),
.flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | MemoryUsageVmaFlags(usage),
.usage = MemoryUsageVma(usage),
.requiredFlags = 0,
.preferredFlags = MemoryUsagePreferredVmaFlags(usage) | anv_flags,
@@ -446,28 +277,7 @@ namespace Vulkan {
VmaAllocation allocation{};
VkMemoryPropertyFlags property_flags{};
DEBUG_ASSERT(vk::OnAllocatorOwnerThread());
VkResult res = vmaCreateBuffer(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info);
if (res != VK_SUCCESS && ReclaimAtLeast(ci.size)) {
res = vmaCreateBuffer(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info);
}
if (res != VK_SUCCESS) {
auto relaxed_ci = alloc_ci;
relaxed_ci.flags &= ~VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT;
res = vmaCreateBuffer(allocator, &ci, &relaxed_ci, &handle, &allocation, &alloc_info);
if (res != VK_SUCCESS &&
(relaxed_ci.preferredFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)) {
relaxed_ci.preferredFlags &= ~VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
res = vmaCreateBuffer(allocator, &ci, &relaxed_ci, &handle, &allocation,
&alloc_info);
}
}
vk::Check(res);
vk::Check(vmaCreateBuffer(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info));
vmaGetAllocationMemoryProperties(allocator, allocation, &property_flags);
// Log GPU memory allocation for buffers
@@ -489,4 +299,77 @@ namespace Vulkan {
device.GetDispatchLoader());
}
MemoryCommit MemoryAllocator::Commit(const VkMemoryRequirements &reqs, MemoryUsage usage)
{
const auto vma_usage = MemoryUsageVma(usage);
VmaAllocationCreateInfo ci{};
ci.flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | MemoryUsageVmaFlags(usage);
ci.usage = vma_usage;
ci.memoryTypeBits = reqs.memoryTypeBits & valid_memory_types;
ci.requiredFlags = 0;
ci.preferredFlags = MemoryUsagePreferredVmaFlags(usage);
VmaAllocation a{};
VmaAllocationInfo info{};
VkResult res = vmaAllocateMemory(allocator, &reqs, &ci, &a, &info);
if (res != VK_SUCCESS) {
// Relax 1: drop budget constraint
auto ci2 = ci;
ci2.flags &= ~VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT;
res = vmaAllocateMemory(allocator, &reqs, &ci2, &a, &info);
// Relax 2: if we preferred DEVICE_LOCAL, drop that preference
if (res != VK_SUCCESS && (ci.preferredFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)) {
auto ci3 = ci2;
ci3.preferredFlags &= ~VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
res = vmaAllocateMemory(allocator, &reqs, &ci3, &a, &info);
}
}
vk::Check(res);
return MemoryCommit(allocator, a, info);
}
MemoryCommit MemoryAllocator::Commit(const vk::Buffer &buffer, MemoryUsage usage) {
// Allocate memory appropriate for this buffer automatically
const auto vma_usage = MemoryUsageVma(usage);
VmaAllocationCreateInfo ci{};
ci.flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | MemoryUsageVmaFlags(usage);
ci.usage = vma_usage;
ci.requiredFlags = 0;
ci.preferredFlags = MemoryUsagePreferredVmaFlags(usage);
ci.pool = VK_NULL_HANDLE;
ci.pUserData = nullptr;
ci.priority = 0.0f;
const VkBuffer raw = *buffer;
VmaAllocation a{};
VmaAllocationInfo info{};
// Let VMA infer memory requirements from the buffer
VkResult res = vmaAllocateMemoryForBuffer(allocator, raw, &ci, &a, &info);
if (res != VK_SUCCESS) {
auto ci2 = ci;
ci2.flags &= ~VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT;
res = vmaAllocateMemoryForBuffer(allocator, raw, &ci2, &a, &info);
if (res != VK_SUCCESS && (ci.preferredFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)) {
auto ci3 = ci2;
ci3.preferredFlags &= ~VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
res = vmaAllocateMemoryForBuffer(allocator, raw, &ci3, &a, &info);
}
}
vk::Check(res);
vk::Check(vmaBindBufferMemory2(allocator, a, 0, raw, nullptr));
return MemoryCommit(allocator, a, info);
}
} // namespace Vulkan
@@ -1,4 +1,4 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2019 yuzu Emulator Project
@@ -6,7 +6,6 @@
#pragma once
#include <functional>
#include <memory>
#include <span>
#include <vector>
@@ -16,8 +15,6 @@
#include "video_core/vulkan_common/vulkan_wrapper.h"
#include "video_core/vulkan_common/vma.h"
struct AHardwareBuffer;
namespace Vulkan {
class Device;
@@ -42,53 +39,49 @@ namespace Vulkan {
}
}
class HostMemoryImport {
/// Ownership handle of a memory commitment (real VMA allocation).
class MemoryCommit {
public:
explicit HostMemoryImport(const Device &device_, void *base, size_t size,
std::span<AHardwareBuffer *const> hardware_buffers,
size_t hardware_buffer_window);
MemoryCommit() noexcept = default;
~HostMemoryImport();
MemoryCommit(VmaAllocator allocator, VmaAllocation allocation,
const VmaAllocationInfo &info) noexcept;
HostMemoryImport(const HostMemoryImport &) = delete;
~MemoryCommit();
HostMemoryImport &operator=(const HostMemoryImport &) = delete;
MemoryCommit(const MemoryCommit &) = delete;
[[nodiscard]] bool IsValid() const noexcept {
return !windows.empty();
}
MemoryCommit &operator=(const MemoryCommit &) = delete;
[[nodiscard]] size_t GetSize() const noexcept {
return imported_size;
}
MemoryCommit(MemoryCommit &&) noexcept;
[[nodiscard]] VkDeviceSize GetWindowSize() const noexcept {
return window_size;
}
MemoryCommit &operator=(MemoryCommit &&) noexcept;
[[nodiscard]] VkBuffer GetWindowBuffer(size_t index) const noexcept {
return windows[index].buffer;
}
[[nodiscard]] std::span<u8> Map();
[[nodiscard]] size_t GetWindowCount() const noexcept {
return windows.size();
}
[[nodiscard]] std::span<const u8> Map() const;
void Unmap();
explicit operator bool() const noexcept { return allocation != nullptr; }
VkDeviceMemory Memory() const noexcept { return memory; }
VkDeviceSize Offset() const noexcept { return offset; }
VkDeviceSize Size() const noexcept { return size; }
VmaAllocation Allocation() const noexcept { return allocation; }
private:
struct Window {
vk::DeviceMemory memory;
VkBuffer buffer{};
};
void Release();
bool ImportHostPointer(void *base, size_t size);
void ImportHardwareBuffers(std::span<AHardwareBuffer *const> hardware_buffers,
size_t hardware_buffer_window, size_t size);
const Device &device;
std::vector<Window> windows;
VkDeviceSize window_size{};
size_t imported_size{};
VmaAllocator allocator{}; ///< VMA allocator
VmaAllocation allocation{}; ///< VMA allocation handle
VkDeviceMemory memory{}; ///< Underlying VkDeviceMemory chosen by VMA
VkDeviceSize offset{}; ///< Offset of this allocation inside VkDeviceMemory
VkDeviceSize size{}; ///< Size of the allocation
void *mapped_ptr{}; ///< Optional persistent mapped pointer
};
/// Memory allocator container.
@@ -114,21 +107,36 @@ namespace Vulkan {
vk::Buffer CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsage usage) const;
using ReclaimCallback = std::function<u64(u64)>;
/**
* Commits a memory with the specified requirements.
*
* @param requirements Requirements returned from a Vulkan call.
* @param usage Indicates how the memory will be used.
*
* @returns A memory commit.
*/
MemoryCommit Commit(const VkMemoryRequirements &requirements, MemoryUsage usage);
void SetReclaimCallback(ReclaimCallback callback);
/// Commits memory required by the buffer and binds it (for buffers created outside VMA).
MemoryCommit Commit(const vk::Buffer &buffer, MemoryUsage usage);
private:
bool ReclaimAtLeast(u64 hint_bytes) const;
static constexpr u64 IMAGE_RECLAIM_HINT = 64ULL * 1024 * 1024;
static bool IsAutoUsage(VmaMemoryUsage u) noexcept {
switch (u) {
case VMA_MEMORY_USAGE_AUTO:
case VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE:
case VMA_MEMORY_USAGE_AUTO_PREFER_HOST:
return true;
default:
return false;
}
}
const Device &device; ///< Device handle.
VmaAllocator allocator; ///< VMA allocator.
const VkPhysicalDeviceMemoryProperties properties; ///< Physical device memory properties.
VkDeviceSize buffer_image_granularity; ///< Adjacent buffer/image granularity
u32 valid_memory_types{~0u};
ReclaimCallback reclaim_callback;
mutable bool in_reclaim{false};
};
} // namespace Vulkan
+2 -102
View File
@@ -5,16 +5,11 @@
// SPDX-License-Identifier: GPL-2.0-or-later
#include <algorithm>
#include <atomic>
#include <limits>
#include <memory>
#include <mutex>
#include <optional>
#include <thread>
#include <utility>
#include <vector>
#include "common/assert.h"
#include "common/common_types.h"
#include "common/logging.h"
#include "video_core/vulkan_common/vk_enum_string_helper.h"
@@ -25,60 +20,6 @@ namespace Vulkan::vk {
namespace {
std::thread::id allocator_owner_thread;
template <typename HandleType>
struct PendingRelease {
VmaAllocator allocator;
HandleType handle;
VmaAllocation allocation;
u64 timeline;
};
std::mutex deletion_mutex;
std::atomic<u64> deletion_timeline{1};
std::vector<PendingRelease<VkImage>> pending_images;
std::vector<PendingRelease<VkBuffer>> pending_buffers;
template <typename HandleType>
void PushPendingRelease(std::vector<PendingRelease<HandleType>>& pending, VmaAllocator allocator,
HandleType handle, VmaAllocation allocation) noexcept {
std::scoped_lock lock{deletion_mutex};
pending.push_back(PendingRelease<HandleType>{
.allocator = allocator,
.handle = handle,
.allocation = allocation,
.timeline = deletion_timeline.load(std::memory_order_acquire),
});
}
template <typename HandleType>
void ExtractReleased(std::vector<PendingRelease<HandleType>>& pending,
std::vector<PendingRelease<HandleType>>& released, u64 completed_value) {
const auto split = std::partition(pending.begin(), pending.end(),
[completed_value](const PendingRelease<HandleType>& entry) {
return entry.timeline > completed_value;
});
released.assign(split, pending.end());
pending.erase(split, pending.end());
}
void DrainDeletionQueue(u64 completed_value) noexcept {
std::vector<PendingRelease<VkImage>> images;
std::vector<PendingRelease<VkBuffer>> buffers;
{
std::scoped_lock lock{deletion_mutex};
ExtractReleased(pending_images, images, completed_value);
ExtractReleased(pending_buffers, buffers, completed_value);
}
for (const auto& entry : images) {
vmaDestroyImage(entry.allocator, entry.handle, entry.allocation);
}
for (const auto& entry : buffers) {
vmaDestroyBuffer(entry.allocator, entry.handle, entry.allocation);
}
}
template <typename Func>
void SortPhysicalDevices(std::vector<VkPhysicalDevice>& devices, const InstanceDispatch& dld,
Func&& func) {
@@ -151,7 +92,6 @@ void Load(VkDevice device, DeviceDispatch& dld) noexcept {
X(vkCmdBeginConditionalRenderingEXT);
X(vkCmdBeginQuery);
X(vkCmdBeginRenderPass);
X(vkCmdBeginRendering);
X(vkCmdBeginTransformFeedbackEXT);
X(vkCmdBeginDebugUtilsLabelEXT);
X(vkCmdBindDescriptorSets);
@@ -179,7 +119,6 @@ void Load(VkDevice device, DeviceDispatch& dld) noexcept {
X(vkCmdEndConditionalRenderingEXT);
X(vkCmdEndQuery);
X(vkCmdEndRenderPass);
X(vkCmdEndRendering);
X(vkCmdEndTransformFeedbackEXT);
X(vkCmdEndDebugUtilsLabelEXT);
X(vkCmdFillBuffer);
@@ -277,16 +216,12 @@ void Load(VkDevice device, DeviceDispatch& dld) noexcept {
X(vkGetBufferMemoryRequirements2);
X(vkGetDeviceQueue);
X(vkGetEventStatus);
X(vkGetMemoryHostPointerPropertiesEXT);
X(vkGetFenceStatus);
X(vkGetImageMemoryRequirements);
X(vkGetPipelineCacheData);
X(vkGetMemoryFdKHR);
#ifdef _WIN32
X(vkGetMemoryWin32HandleKHR);
#endif
#ifdef __ANDROID__
X(vkGetAndroidHardwareBufferPropertiesANDROID);
#endif
X(vkGetQueryPoolResults);
X(vkGetPipelineExecutablePropertiesKHR);
@@ -295,7 +230,6 @@ void Load(VkDevice device, DeviceDispatch& dld) noexcept {
X(vkMapMemory);
X(vkQueueSubmit);
X(vkQueueSubmit2);
X(vkResetCommandPool);
X(vkResetFences);
X(vkResetQueryPool);
X(vkSetDebugUtilsObjectNameEXT);
@@ -303,12 +237,6 @@ void Load(VkDevice device, DeviceDispatch& dld) noexcept {
X(vkUnmapMemory);
X(vkUpdateDescriptorSetWithTemplate);
X(vkUpdateDescriptorSets);
X(vkGetBufferDeviceAddress);
X(vkGetDescriptorSetLayoutSizeEXT);
X(vkGetDescriptorSetLayoutBindingOffsetEXT);
X(vkGetDescriptorEXT);
X(vkCmdBindDescriptorBuffersEXT);
X(vkCmdSetDescriptorBufferOffsetsEXT);
X(vkWaitForFences);
X(vkWaitSemaphores);
@@ -329,12 +257,6 @@ void Load(VkDevice device, DeviceDispatch& dld) noexcept {
Proc(dld.vkCmdDrawIndexedIndirectCount, dld, "vkCmdDrawIndexedIndirectCountKHR", device);
}
// Support for dynamic rendering is optional until Vulkan 1.3
if (!dld.vkCmdBeginRendering) {
Proc(dld.vkCmdBeginRendering, dld, "vkCmdBeginRenderingKHR", device);
Proc(dld.vkCmdEndRendering, dld, "vkCmdEndRenderingKHR", device);
}
// Synchronization2 is core in Vulkan 1.3, otherwise requires VK_KHR_synchronization2
if (!dld.vkCmdPipelineBarrier2) {
Proc(dld.vkCmdPipelineBarrier2, dld, "vkCmdPipelineBarrier2KHR", device);
@@ -580,35 +502,13 @@ DebugReportCallback Instance::CreateDebugReportCallback(
return DebugReportCallback(object, handle, *dld);
}
void SetAllocatorOwnerThread() {
allocator_owner_thread = std::this_thread::get_id();
}
bool OnAllocatorOwnerThread() noexcept {
return allocator_owner_thread == std::thread::id{} ||
allocator_owner_thread == std::this_thread::get_id();
}
void SetDeletionTimeline(u64 value) noexcept {
deletion_timeline.store(value, std::memory_order_release);
}
void TickDeletionQueue(u64 completed_value) noexcept {
DEBUG_ASSERT(OnAllocatorOwnerThread());
DrainDeletionQueue(completed_value);
}
void FlushDeletionQueue() noexcept {
DrainDeletionQueue((std::numeric_limits<u64>::max)());
}
void Image::SetObjectNameEXT(const char* name) const {
SetObjectName(dld, owner, handle, VK_OBJECT_TYPE_IMAGE, name);
}
void Image::Release() const noexcept {
if (handle) {
PushPendingRelease(pending_images, allocator, handle, allocation);
vmaDestroyImage(allocator, handle, allocation);
}
}
@@ -630,7 +530,7 @@ void Buffer::SetObjectNameEXT(const char* name) const {
void Buffer::Release() const noexcept {
if (handle) {
PushPendingRelease(pending_buffers, allocator, handle, allocation);
vmaDestroyBuffer(allocator, handle, allocation);
}
}
@@ -131,16 +131,6 @@ private:
VkResult result;
};
void SetAllocatorOwnerThread();
[[nodiscard]] bool OnAllocatorOwnerThread() noexcept;
void SetDeletionTimeline(u64 value) noexcept;
void TickDeletionQueue(u64 completed_value) noexcept;
void FlushDeletionQueue() noexcept;
/// Throws a Vulkan exception if result is not success.
inline void Check(VkResult result) {
if (result != VK_SUCCESS) {
@@ -218,7 +208,6 @@ struct DeviceDispatch : InstanceDispatch {
PFN_vkCmdBeginDebugUtilsLabelEXT vkCmdBeginDebugUtilsLabelEXT{};
PFN_vkCmdBeginQuery vkCmdBeginQuery{};
PFN_vkCmdBeginRenderPass vkCmdBeginRenderPass{};
PFN_vkCmdBeginRendering vkCmdBeginRendering{};
PFN_vkCmdBeginTransformFeedbackEXT vkCmdBeginTransformFeedbackEXT{};
PFN_vkCmdBindDescriptorSets vkCmdBindDescriptorSets{};
PFN_vkCmdBindIndexBuffer vkCmdBindIndexBuffer{};
@@ -247,7 +236,6 @@ struct DeviceDispatch : InstanceDispatch {
PFN_vkCmdEndDebugUtilsLabelEXT vkCmdEndDebugUtilsLabelEXT{};
PFN_vkCmdEndQuery vkCmdEndQuery{};
PFN_vkCmdEndRenderPass vkCmdEndRenderPass{};
PFN_vkCmdEndRendering vkCmdEndRendering{};
PFN_vkCmdEndTransformFeedbackEXT vkCmdEndTransformFeedbackEXT{};
PFN_vkCmdFillBuffer vkCmdFillBuffer{};
PFN_vkCmdPipelineBarrier vkCmdPipelineBarrier{};
@@ -344,16 +332,12 @@ struct DeviceDispatch : InstanceDispatch {
PFN_vkGetBufferMemoryRequirements2 vkGetBufferMemoryRequirements2{};
PFN_vkGetDeviceQueue vkGetDeviceQueue{};
PFN_vkGetEventStatus vkGetEventStatus{};
PFN_vkGetMemoryHostPointerPropertiesEXT vkGetMemoryHostPointerPropertiesEXT{};
PFN_vkGetFenceStatus vkGetFenceStatus{};
PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements{};
PFN_vkGetPipelineCacheData vkGetPipelineCacheData{};
PFN_vkGetMemoryFdKHR vkGetMemoryFdKHR{};
#ifdef _WIN32
PFN_vkGetMemoryWin32HandleKHR vkGetMemoryWin32HandleKHR{};
#endif
#ifdef __ANDROID__
PFN_vkGetAndroidHardwareBufferPropertiesANDROID vkGetAndroidHardwareBufferPropertiesANDROID{};
#endif
PFN_vkGetPipelineExecutablePropertiesKHR vkGetPipelineExecutablePropertiesKHR{};
PFN_vkGetPipelineExecutableStatisticsKHR vkGetPipelineExecutableStatisticsKHR{};
@@ -362,19 +346,12 @@ struct DeviceDispatch : InstanceDispatch {
PFN_vkMapMemory vkMapMemory{};
PFN_vkQueueSubmit vkQueueSubmit{};
PFN_vkQueueSubmit2 vkQueueSubmit2{};
PFN_vkResetCommandPool vkResetCommandPool{};
PFN_vkResetFences vkResetFences{};
PFN_vkResetQueryPool vkResetQueryPool{};
PFN_vkSetDebugUtilsObjectNameEXT vkSetDebugUtilsObjectNameEXT{};
PFN_vkSetDebugUtilsObjectTagEXT vkSetDebugUtilsObjectTagEXT{};
PFN_vkUnmapMemory vkUnmapMemory{};
PFN_vkUpdateDescriptorSetWithTemplate vkUpdateDescriptorSetWithTemplate{};
PFN_vkGetBufferDeviceAddress vkGetBufferDeviceAddress{};
PFN_vkGetDescriptorSetLayoutSizeEXT vkGetDescriptorSetLayoutSizeEXT{};
PFN_vkGetDescriptorSetLayoutBindingOffsetEXT vkGetDescriptorSetLayoutBindingOffsetEXT{};
PFN_vkGetDescriptorEXT vkGetDescriptorEXT{};
PFN_vkCmdBindDescriptorBuffersEXT vkCmdBindDescriptorBuffersEXT{};
PFN_vkCmdSetDescriptorBufferOffsetsEXT vkCmdSetDescriptorBufferOffsetsEXT{};
PFN_vkUpdateDescriptorSets vkUpdateDescriptorSets{};
PFN_vkWaitForFences vkWaitForFences{};
PFN_vkWaitSemaphores vkWaitSemaphores{};
@@ -948,10 +925,6 @@ public:
CommandBuffers Allocate(std::size_t num_buffers,
VkCommandBufferLevel level = VK_COMMAND_BUFFER_LEVEL_PRIMARY) const;
void Reset(VkCommandPoolResetFlags flags = 0) const {
Check(dld->vkResetCommandPool(owner, handle, flags));
}
/// Set object name.
void SetObjectNameEXT(const char* name) const;
};
@@ -1098,34 +1071,6 @@ public:
VkMemoryRequirements GetBufferMemoryRequirements(VkBuffer buffer,
void* pnext = nullptr) const noexcept;
VkResult GetMemoryHostPointerPropertiesEXT(
VkExternalMemoryHandleTypeFlagBits handle_type, const void* host_pointer,
VkMemoryHostPointerPropertiesEXT* out_properties) const noexcept {
return dld->vkGetMemoryHostPointerPropertiesEXT(handle, handle_type, host_pointer,
out_properties);
}
#ifdef __ANDROID__
VkResult GetAndroidHardwareBufferPropertiesANDROID(
const struct AHardwareBuffer* buffer,
VkAndroidHardwareBufferPropertiesANDROID* out_properties) const noexcept {
return dld->vkGetAndroidHardwareBufferPropertiesANDROID(handle, buffer, out_properties);
}
#endif
VkResult CreateBufferRaw(const VkBufferCreateInfo& ci, VkBuffer* out_buffer) const noexcept {
return dld->vkCreateBuffer(handle, &ci, nullptr, out_buffer);
}
void DestroyBufferRaw(VkBuffer buffer) const noexcept {
dld->vkDestroyBuffer(handle, buffer, nullptr);
}
VkResult BindBufferMemory(VkBuffer buffer, VkDeviceMemory memory,
VkDeviceSize offset) const noexcept {
return dld->vkBindBufferMemory(handle, buffer, memory, offset);
}
VkMemoryRequirements GetImageMemoryRequirements(VkImage image) const noexcept;
std::vector<VkPipelineExecutablePropertiesKHR> GetPipelineExecutablePropertiesKHR(
@@ -1142,34 +1087,6 @@ public:
dld->vkUpdateDescriptorSetWithTemplate(handle, set, update_template, data);
}
[[nodiscard]] VkDeviceAddress GetBufferDeviceAddress(VkBuffer buffer) const noexcept {
const VkBufferDeviceAddressInfo info{
.sType = VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO,
.pNext = nullptr,
.buffer = buffer,
};
return dld->vkGetBufferDeviceAddress(handle, &info);
}
[[nodiscard]] VkDeviceSize GetDescriptorSetLayoutSizeEXT(
VkDescriptorSetLayout layout) const noexcept {
VkDeviceSize size{};
dld->vkGetDescriptorSetLayoutSizeEXT(handle, layout, &size);
return size;
}
[[nodiscard]] VkDeviceSize GetDescriptorSetLayoutBindingOffsetEXT(
VkDescriptorSetLayout layout, u32 binding) const noexcept {
VkDeviceSize offset{};
dld->vkGetDescriptorSetLayoutBindingOffsetEXT(handle, layout, binding, &offset);
return offset;
}
void GetDescriptorEXT(const VkDescriptorGetInfoEXT& info, size_t size,
void* descriptor) const noexcept {
dld->vkGetDescriptorEXT(handle, &info, size, descriptor);
}
VkResult AcquireNextImageKHR(VkSwapchainKHR swapchain, u64 timeout, VkSemaphore semaphore,
VkFence fence, u32* image_index) const noexcept {
return dld->vkAcquireNextImageKHR(handle, swapchain, timeout, semaphore, fence,
@@ -1269,14 +1186,6 @@ public:
dld->vkCmdEndRenderPass(handle);
}
void BeginRendering(const VkRenderingInfo& rendering_info) const noexcept {
dld->vkCmdBeginRendering(handle, &rendering_info);
}
void EndRendering() const noexcept {
dld->vkCmdEndRendering(handle);
}
void BeginQuery(VkQueryPool query_pool, u32 query, VkQueryControlFlags flags) const noexcept {
dld->vkCmdBeginQuery(handle, query_pool, query, flags);
}
@@ -1491,40 +1400,6 @@ public:
PipelineBarrier(src_stage_mask, dst_stage_mask, dependency_flags, {}, {}, image_barrier);
}
void BindDescriptorBuffersEXT(Span<VkDescriptorBufferBindingInfoEXT> bindings) const noexcept {
dld->vkCmdBindDescriptorBuffersEXT(handle, bindings.size(), bindings.data());
}
void SetDescriptorBufferOffsetsEXT(VkPipelineBindPoint bind_point, VkPipelineLayout layout,
u32 first_set, Span<u32> buffer_indices,
Span<VkDeviceSize> offsets) const noexcept {
dld->vkCmdSetDescriptorBufferOffsetsEXT(handle, bind_point, layout, first_set,
buffer_indices.size(), buffer_indices.data(),
offsets.data());
}
[[nodiscard]] bool HasPipelineBarrier2() const noexcept {
return dld->vkCmdPipelineBarrier2 != nullptr;
}
void PipelineBarrier2(VkDependencyFlags dependency_flags,
Span<VkMemoryBarrier2> memory_barriers,
Span<VkBufferMemoryBarrier2> buffer_barriers,
Span<VkImageMemoryBarrier2> image_barriers) const noexcept {
const VkDependencyInfo dependency_info{
.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
.pNext = nullptr,
.dependencyFlags = dependency_flags,
.memoryBarrierCount = memory_barriers.size(),
.pMemoryBarriers = memory_barriers.data(),
.bufferMemoryBarrierCount = buffer_barriers.size(),
.pBufferMemoryBarriers = buffer_barriers.data(),
.imageMemoryBarrierCount = image_barriers.size(),
.pImageMemoryBarriers = image_barriers.data(),
};
dld->vkCmdPipelineBarrier2(handle, &dependency_info);
}
void CopyBufferToImage(VkBuffer src_buffer, VkImage dst_image, VkImageLayout dst_image_layout,
Span<VkBufferImageCopy> regions) const noexcept {
dld->vkCmdCopyBufferToImage(handle, src_buffer, dst_image, dst_image_layout, regions.size(),