Compare commits

...

26 Commits

Author SHA1 Message Date
CamilleLaVey f2f05293a6 Pure and simply meow 2026-08-27 02:30:35 -04:00
CamilleLaVey ad075400a4 Let's try this on memory reclamations 2026-08-27 01:21:10 -04:00
CamilleLaVey 06dba55923 Extended the impact on AHB fallback 2026-08-26 22:10:00 -04:00
CamilleLaVey 8d19ee6dc5 What if we just rely entirely on SGSR common than using EdgeDir? 2026-08-26 19:06:23 -04:00
CamilleLaVey 3e91569d36 Another intent 2026-08-26 18:40:10 -04:00
CamilleLaVey 6010f60803 Just cries internally 2026-08-26 18:05:03 -04:00
CamilleLaVey c14b1de71e Meow. 2026-08-26 17:47:12 -04:00
CamilleLaVey 14cc49e727 Adjustments on upload for textures 2026-08-26 17:00:18 -04:00
CamilleLaVey e88923331c Fix build 2026-08-26 15:04:18 -04:00
CamilleLaVey 78918e995d Fix msvc build 2026-08-26 14:50:42 -04:00
CamilleLaVey 522440bcb2 [TEST] Bring the unswizzle work from tiled-gpu-v2 x2 + revert of vertex, indexes from AHB path 2026-08-26 14:49:01 -04:00
CamilleLaVey c1d8b7dd42 Just a quick test 2026-08-26 05:09:12 -04:00
CamilleLaVey 28d7c97d71 Added vertex, uniform and indexes on the upload path 2026-08-26 04:52:52 -04:00
CamilleLaVey 8a8763d975 Meowly The Build Destroyer 2026-08-26 04:11:59 -04:00
CamilleLaVey 54d8d5b0a8 Another intent on write/read on the fly 2026-08-26 04:06:25 -04:00
CamilleLaVey b9b9a0d704 Changes on the maps counts again 2026-08-26 03:12:06 -04:00
CamilleLaVey b8ce711575 Step back + keeping fixes from unified download + shared mem refactor 2026-08-25 21:36:17 -04:00
CamilleLaVey 46cd1e5d65 another take 2026-08-25 21:36:17 -04:00
CamilleLaVey ae561d49aa Keep AHB limitations 2026-08-25 21:36:17 -04:00
CamilleLaVey 9f45d357e9 [TEST] Another increased on windows size 2026-08-25 21:36:17 -04:00
CamilleLaVey 2c8c3062c4 Increased windows size 2026-08-25 21:36:17 -04:00
CamilleLaVey 51bdb62d74 [TEST] Increased window limiter on AHB 2026-08-25 21:36:17 -04:00
CamilleLaVey ed3838ccbb [TEST] Adjustment AHB to tiled-gpu-v2 state 2026-08-25 21:36:17 -04:00
CamilleLaVey e8c253f511 [memory, vulkan] Initial implementation for Unified Memory 2026-08-25 21:36:17 -04:00
CamilleLaVey 0295dc5fff [vulkan] Removal of QCOM sampler limiters + CustomBorderColor and ColorBorderSwizzle adjustments (#4301)
This PR removes the artificial limit added for QCOM drivers based on a sampler budget limit (based on #3280 work), removes the ban on CustomBorderColor/BorderColorSwizzle for also QCOM driver, reduce the amount of combination required to created a sampler with custom border color/ color border swizzle; adds cases on R16 formats not able to swizzle on BGR565 formats; degrades samplers and color combinations when there's no color border swizzle available; fixes the srgb (LUT) values from legacy and now all the color variations are cached in the same key, instead of having 7 different handlers for them. In resume, rather than performance (due to the reduced amount of process on duplicated/ synced code on pipeline/texture cache), it's a graphica accuracy work that will improve image quality at the cost of almost 0 performance hit, rather than certain hitch if the color combination wasn't cached before. This applies to all platform.

Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/4301
Reviewed-by: Samuel <lizzie@eden-emu.dev>
Reviewed-by: MaranBr <maranbr@eden-emu.dev>
2026-08-25 20:03:18 +02:00
CamilleLaVey 60a474b8df [vulkan, qcom] Fix shader float controls on QCOM driver (#4297)
Finishes my torture to find the culprit behind the broken behavior with float controls, even if device does support flush denorm on fp32 it doesn't really support it and provokes bad rounding modes by not flushing correctly denorms; this also returns the other working rounding modes on QCOM drivers.

_Special Thanks_

1.- Smoly The Big (@Gidoly)

Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/4297
Reviewed-by: Samuel <lizzie@eden-emu.dev>
Reviewed-by: MaranBr <maranbr@eden-emu.dev>
2026-08-24 17:57:12 +02:00
52 changed files with 3146 additions and 407 deletions
@@ -27,6 +27,7 @@ enum class BooleanSetting(override val key: String) : AbstractBooleanSetting {
RENDERER_ASYNCHRONOUS_GPU_EMULATION("use_asynchronous_gpu_emulation"),
RENDERER_ASYNC_PRESENTATION("async_presentation"),
RENDERER_ASYNCHRONOUS_SHADERS("use_asynchronous_shaders"),
RENDERER_UNIFIED_MEMORY("use_unified_memory"),
RENDERER_REACTIVE_FLUSHING("use_reactive_flushing"),
ENABLE_BUFFER_HISTORY("enable_buffer_history"),
USE_OPTIMIZED_VERTEX_BUFFERS("use_optimized_vertex_buffers"),
@@ -789,6 +789,13 @@ abstract class SettingsItem(
descriptionId = R.string.renderer_asynchronous_shaders_description
)
)
put(
SwitchSetting(
BooleanSetting.RENDERER_UNIFIED_MEMORY,
titleId = R.string.renderer_unified_memory,
descriptionId = R.string.renderer_unified_memory_description
)
)
put(
SingleChoiceSetting(
IntSetting.FAST_GPU_TIME,
@@ -344,6 +344,7 @@ class SettingsFragmentPresenter(
add(BooleanSetting.FIX_BLOOM_EFFECTS.key)
add(BooleanSetting.EMULATE_BGR565.key)
add(BooleanSetting.RENDERER_ASYNCHRONOUS_SHADERS.key)
add(BooleanSetting.RENDERER_UNIFIED_MEMORY.key)
add(IntSetting.ANDROID_PIPELINE_WORKERS.key)
add(BooleanSetting.RENDERER_ASYNCHRONOUS_GPU_EMULATION.key)
add(BooleanSetting.RENDERER_ASYNC_PRESENTATION.key)
@@ -585,6 +585,8 @@
<string name="rescale_hack_description">Enables a legacy handling for the rescale configuration pass for games by using a quick rescale path</string>
<string name="renderer_asynchronous_shaders">Use asynchronous shaders</string>
<string name="renderer_asynchronous_shaders_description">Compiles shaders asynchronously. This may reduce stutters but may also introduce glitches.</string>
<string name="renderer_unified_memory">Unified memory access</string>
<string name="renderer_unified_memory_description">Allows GPU write buffer readbacks directly into guest memory, skipping the CPU staging copy.</string>
<string name="gpu_unswizzle_settings">GPU Unswizzle Settings</string>
<string name="gpu_unswizzle_settings_description">Configure GPU-based texture unswizzling parameters or disable it entirely. Adjust these settings to balance performance and texture loading quality.</string>
<string name="gpu_unswizzle_enable">Enable GPU Unswizzle</string>
+22 -28
View File
@@ -4,34 +4,17 @@
// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#include <fstream>
#include "common/heap_tracker.h"
#include "common/logging.h"
#include "common/memory_detect.h"
#include "common/assert.h"
namespace Common {
namespace {
s64 GetMaxPermissibleResidentMapCount() {
// Default value.
s64 value = 65530;
// Try to read how many mappings we can make.
std::ifstream s("/proc/sys/vm/max_map_count");
s >> value;
// Print, for debug.
LOG_INFO(HW_Memory, "Current maximum map count: {}", value);
// Allow 20000 maps for other code and to account for split inaccuracy.
return std::max<s64>(value - 20000, 0);
}
} // namespace
HeapTracker::HeapTracker(Common::HostMemory& buffer)
: m_buffer(buffer), m_max_resident_map_count(GetMaxPermissibleResidentMapCount()) {}
: m_buffer(buffer),
m_has_hardware_buffer_backing(!buffer.BackingHardwareBuffers().empty()),
m_max_resident_map_count(static_cast<s64>(GetPermissibleMapCount())) {}
HeapTracker::~HeapTracker() = default;
void HeapTracker::Map(size_t virtual_offset, size_t host_offset, size_t length,
@@ -85,7 +68,8 @@ void HeapTracker::Unmap(size_t virtual_offset, size_t size, bool is_separate_hea
// If resident, erase from resident map.
if (item->is_resident) {
ASSERT(--m_resident_map_count >= 0);
m_resident_map_count -= this->HostMapCount(item->paddr, item->size);
ASSERT(m_resident_map_count >= 0);
m_resident_mappings.erase(m_resident_mappings.iterator_to(*item));
}
@@ -191,7 +175,7 @@ bool HeapTracker::DeferredMapSeparateHeap(size_t virtual_offset) {
// This map is now resident.
it->is_resident = true;
m_resident_map_count++;
m_resident_map_count += this->HostMapCount(it->paddr, it->size);
m_resident_mappings.insert(*it);
}
@@ -213,17 +197,17 @@ void HeapTracker::RebuildSeparateHeapAddressSpace() {
// Despite being worse in theory, this has proven to be better in practice than more
// regularly dumping a smaller amount, because it significantly reduces average case
// lock contention.
std::size_t const desired_count = (std::min)(m_resident_map_count, m_max_resident_map_count) / 2;
std::size_t const evict_count = m_resident_map_count - desired_count;
s64 const desired_count = (std::min)(m_resident_map_count, m_max_resident_map_count) / 2;
auto it = m_resident_mappings.begin();
for (size_t i = 0; i < evict_count && it != m_resident_mappings.end(); i++) {
while (m_resident_map_count > desired_count && it != m_resident_mappings.end()) {
// Unmark and unmap.
it->is_resident = false;
m_buffer.Unmap(it->vaddr, it->size, false);
// Advance.
ASSERT(--m_resident_map_count >= 0);
m_resident_map_count -= this->HostMapCount(it->paddr, it->size);
ASSERT(m_resident_map_count >= 0);
it = m_resident_mappings.erase(it);
}
}
@@ -245,6 +229,7 @@ void HeapTracker::SplitHeapMapLocked(VAddr offset) {
// Cache the original values.
auto* const left = std::addressof(*it);
const size_t orig_size = left->size;
const s64 orig_host_map_count = this->HostMapCount(left->paddr, orig_size);
// Adjust the left map.
const size_t left_size = offset - left->vaddr;
@@ -266,11 +251,20 @@ void HeapTracker::SplitHeapMapLocked(VAddr offset) {
// If resident, also insert into resident map.
if (right->is_resident) {
m_resident_map_count++;
m_resident_map_count += this->HostMapCount(left->paddr, left->size) +
this->HostMapCount(right->paddr, right->size) -
orig_host_map_count;
m_resident_mappings.insert(*right);
}
}
s64 HeapTracker::HostMapCount(PAddr paddr, size_t size) const {
if (!m_has_hardware_buffer_backing) {
return size != 0 ? 1 : 0;
}
return static_cast<s64>(m_buffer.BackingMapCount(paddr, size));
}
HeapTracker::AddrTree::iterator HeapTracker::GetNearestHeapMapLocked(VAddr offset) {
const SeparateHeapMap key{
.vaddr = offset,
+6
View File
@@ -1,3 +1,6 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
@@ -82,10 +85,13 @@ private:
AddrTree::iterator GetNearestHeapMapLocked(VAddr offset);
s64 HostMapCount(PAddr paddr, size_t size) const;
void RebuildSeparateHeapAddressSpace();
private:
Common::HostMemory& m_buffer;
const bool m_has_hardware_buffer_backing;
const s64 m_max_resident_map_count;
std::shared_mutex m_rebuild_lock{};
+393 -6
View File
@@ -51,14 +51,65 @@
#endif // ^^^ POSIX ^^^
#include <atomic>
#include <mutex>
#include <random>
#include <vector>
#include "common/alignment.h"
#include "common/assert.h"
#include "common/free_region_manager.h"
#include "common/host_memory.h"
#include "common/logging.h"
#include "common/memory_detect.h"
#include "common/settings.h"
#ifdef __ANDROID__
#include <cerrno>
#include <dlfcn.h>
#include <sys/ioctl.h>
#include <android/hardware_buffer.h>
namespace {
struct NativeHandle {
int version;
int numFds;
int numInts;
int data[1];
};
using PFN_AHardwareBuffer_getNativeHandle = const NativeHandle* (*)(const AHardwareBuffer*);
PFN_AHardwareBuffer_getNativeHandle ResolveGetNativeHandle() {
void* const lib = dlopen("libnativewindow.so", RTLD_NOW);
if (lib == nullptr) {
return nullptr;
}
return reinterpret_cast<PFN_AHardwareBuffer_getNativeHandle>(
dlsym(lib, "AHardwareBuffer_getNativeHandle"));
}
struct DmaBufSync {
u64 flags;
};
constexpr u64 DmaBufSyncRead = 1ULL << 0;
constexpr u64 DmaBufSyncWrite = 1ULL << 1;
constexpr u64 DmaBufSyncStart = 0ULL << 2;
constexpr u64 DmaBufSyncEnd = 1ULL << 2;
void SyncDmaBufCpuAccess(int fd, u64 phase) {
DmaBufSync sync{.flags = phase | DmaBufSyncRead | DmaBufSyncWrite};
while (ioctl(fd, _IOW('b', 0, DmaBufSync), &sync) != 0) {
if (errno != EINTR) {
return;
}
}
}
} // namespace
#endif
#if defined(__ANDROID__) && __ANDROID_API__ < 30
#include <sys/syscall.h>
@@ -75,6 +126,12 @@ namespace Common {
[[maybe_unused]] constexpr size_t PageAlignment = 0x1000;
[[maybe_unused]] constexpr size_t HugePageSize = 0x200000;
static std::atomic<u64> committed_backing_size{};
u64 GetCommittedBackingSize() noexcept {
return committed_backing_size.load(std::memory_order_relaxed);
}
#ifdef _WIN32
// Manually imported for MinGW compatibility
@@ -123,7 +180,7 @@ static void GetFuncAddress(Common::DynamicLibrary& dll, const char* name, T& pfn
class HostMemory::Impl {
public:
explicit Impl(size_t backing_size_, size_t virtual_size_)
explicit Impl(size_t backing_size_, size_t virtual_size_, size_t)
: backing_size{backing_size_}
, virtual_size{virtual_size_}
, process{GetCurrentProcess()}
@@ -229,6 +286,10 @@ public:
UNREACHABLE();
}
bool IsBackingShared() const noexcept {
return true;
}
const size_t backing_size; ///< Size of the backing memory in bytes
const size_t virtual_size; ///< Size of the virtual address placeholder in bytes
@@ -501,9 +562,10 @@ static int shm_open_anon(int flags, mode_t mode) {
class HostMemory::Impl {
public:
explicit Impl(size_t backing_size_, size_t virtual_size_)
explicit Impl(size_t backing_size_, size_t virtual_size_, size_t preferred_offset_)
: backing_size{backing_size_}
, virtual_size{virtual_size_}
, preferred_offset{preferred_offset_}
{}
bool Init() {
@@ -543,10 +605,15 @@ public:
LOG_WARNING(Common_Memory, "Using private mappings instead of shared ones");
backing_base = static_cast<u8*>(mmap(nullptr, backing_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0));
if (fd > 0) {
fd = -1;
close(fd);
}
fd = -1;
} else {
#ifdef __ANDROID__
if (InitAhbBacking()) {
return InitVirtual();
}
#endif
backing_base = static_cast<u8*>(mmap(nullptr, backing_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0));
}
if (backing_base == MAP_FAILED) {
@@ -554,7 +621,10 @@ public:
return false;
}
// Virtual memory initialization
return InitVirtual();
}
bool InitVirtual() {
virtual_base = virtual_map_base = static_cast<u8*>(ChooseVirtualBase(virtual_size));
if (virtual_base == MAP_FAILED) {
LOG_CRITICAL(HW_Memory, "mmap failed: {}", strerror(errno));
@@ -567,6 +637,244 @@ public:
return true;
}
#ifdef __ANDROID__
static AHardwareBuffer_Desc MakeBlobDesc(size_t len) {
return AHardwareBuffer_Desc{
.width = static_cast<u32>(len),
.height = 1,
.layers = 1,
.format = AHARDWAREBUFFER_FORMAT_BLOB,
.usage = AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN |
AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN |
AHARDWAREBUFFER_USAGE_GPU_DATA_BUFFER,
.stride = 0,
.rfu0 = 0,
.rfu1 = 0,
};
}
static bool ProbeAhbBacking(PFN_AHardwareBuffer_getNativeHandle get_native_handle) {
const AHardwareBuffer_Desc desc = MakeBlobDesc(PageAlignment * 2);
AHardwareBuffer* buffer{};
if (AHardwareBuffer_allocate(&desc, &buffer) != 0 || buffer == nullptr) {
return false;
}
const NativeHandle* const handle = get_native_handle(buffer);
if (handle == nullptr || handle->numFds < 1) {
AHardwareBuffer_release(buffer);
return false;
}
const int probe_fd = handle->data[0];
bool ok = true;
const auto try_map = [&](int prot, off_t offset) {
if (!ok) {
return;
}
void* const ptr = mmap(nullptr, PageAlignment, prot, MAP_SHARED, probe_fd, offset);
if (ptr == MAP_FAILED) {
ok = false;
return;
}
munmap(ptr, PageAlignment);
};
try_map(PROT_READ | PROT_WRITE, 0);
try_map(PROT_READ | PROT_WRITE, static_cast<off_t>(PageAlignment));
#ifdef ARCHITECTURE_arm64
try_map(PROT_READ | PROT_EXEC, 0);
#endif
AHardwareBuffer_release(buffer);
return ok;
}
size_t ComputeAhbBudget(size_t window_size) const {
const u64 total_physical = Common::GetMemInfo().TotalPhysicalMemory;
constexpr u64 BaselineFootprint = 6ULL << 30;
if (total_physical <= BaselineFootprint) {
return 0;
}
const u64 permissible_maps = Common::GetPermissibleMapCount();
if (permissible_maps == 0) {
return 0;
}
u64 budget = (total_physical - BaselineFootprint) / 2;
constexpr u64 MapSlotsPerWindow = 64;
const u64 affordable_windows = permissible_maps / MapSlotsPerWindow;
budget = (std::min)(budget, affordable_windows * window_size);
const u64 available = Common::GetAvailablePhysicalMemory();
if (available != 0) {
budget = (std::min)(budget, available / 2);
}
budget = (std::min)(budget, static_cast<u64>(backing_size));
budget = Common::AlignDown(budget, window_size);
constexpr u64 MinimumBudget = 256ULL << 20;
if (budget < MinimumBudget) {
return 0;
}
return static_cast<size_t>(budget);
}
bool InitAhbBacking() {
if (!Settings::values.use_unified_memory.GetValue()) {
return false;
}
static const PFN_AHardwareBuffer_getNativeHandle get_native_handle =
ResolveGetNativeHandle();
if (get_native_handle == nullptr) {
return false;
}
constexpr size_t window_size = 256ULL << 20;
const size_t budget = ComputeAhbBudget(window_size);
if (budget == 0) {
return false;
}
if (!ProbeAhbBacking(get_native_handle)) {
return false;
}
const size_t aligned_backing = Common::AlignDown(backing_size, window_size);
const size_t max_windows = (std::min)(budget, aligned_backing) / window_size;
std::vector<AHardwareBuffer*> buffers;
std::vector<int> buffer_fds;
const auto cleanup = [&] {
for (AHardwareBuffer* buffer : buffers) {
AHardwareBuffer_release(buffer);
}
buffers.clear();
buffer_fds.clear();
};
for (size_t i = 0; i < max_windows; ++i) {
const AHardwareBuffer_Desc desc = MakeBlobDesc(window_size);
AHardwareBuffer* buffer{};
if (AHardwareBuffer_allocate(&desc, &buffer) != 0 || buffer == nullptr) {
break;
}
const NativeHandle* const handle = get_native_handle(buffer);
if (handle == nullptr || handle->numFds < 1) {
AHardwareBuffer_release(buffer);
break;
}
const int buffer_fd = handle->data[0];
const off_t buffer_len = lseek(buffer_fd, 0, SEEK_END);
if (buffer_len < static_cast<off_t>(window_size)) {
AHardwareBuffer_release(buffer);
break;
}
buffers.push_back(buffer);
buffer_fds.push_back(buffer_fd);
}
const size_t num_windows = buffers.size();
if (num_windows == 0) {
return false;
}
const size_t region_size = num_windows * window_size;
const size_t region_base = Common::AlignDown(
(std::min)(preferred_offset, aligned_backing - region_size), window_size);
u8* const base = static_cast<u8*>(mmap(nullptr, backing_size, PROT_NONE,
MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0));
if (base == MAP_FAILED) {
cleanup();
return false;
}
const auto map_over_reservation = [&](size_t offset, size_t len, int map_fd,
off_t map_offset) {
if (len == 0) {
return true;
}
if (mmap(base + offset, len, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, map_fd,
map_offset) == MAP_FAILED) {
munmap(base, backing_size);
cleanup();
return false;
}
return true;
};
if (!map_over_reservation(0, region_base, fd, 0)) {
return false;
}
for (size_t i = 0; i < num_windows; ++i) {
if (!map_over_reservation(region_base + i * window_size, window_size, buffer_fds[i],
0)) {
return false;
}
}
const size_t tail_offset = region_base + region_size;
if (!map_over_reservation(tail_offset, backing_size - tail_offset, fd,
static_cast<off_t>(tail_offset))) {
return false;
}
backing_base = base;
ahb_windows = std::move(buffers);
ahb_fds = std::move(buffer_fds);
ahb_window_size = window_size;
ahb_base = region_base;
ahb_bytes = region_size;
committed_backing_size.store(region_size, std::memory_order_relaxed);
for (const int window_fd : ahb_fds) {
SyncDmaBufCpuAccess(window_fd, DmaBufSyncStart);
}
return true;
}
void MapBackingRange(size_t virtual_offset, size_t host_offset, size_t length, int prot_flags) {
while (length > 0) {
int map_fd = fd;
off_t map_offset = static_cast<off_t>(host_offset);
size_t chunk = length;
if (host_offset < ahb_base) {
chunk = (std::min)(chunk, ahb_base - host_offset);
} else if (host_offset < ahb_base + ahb_bytes) {
const size_t relative = host_offset - ahb_base;
const size_t window = relative / ahb_window_size;
const size_t local = relative % ahb_window_size;
map_fd = ahb_fds[window];
map_offset = static_cast<off_t>(local);
chunk = (std::min)(chunk, ahb_window_size - local);
}
void* const ret = mmap(virtual_base + virtual_offset, chunk, prot_flags,
MAP_SHARED | MAP_FIXED, map_fd, map_offset);
ASSERT_MSG(ret != MAP_FAILED, "mmap: {}", strerror(errno));
virtual_offset += chunk;
host_offset += chunk;
length -= chunk;
}
}
size_t BackingMapCount(size_t host_offset, size_t length) const noexcept {
if (length == 0) {
return 0;
}
if (ahb_bytes == 0) {
return 1;
}
size_t count = 0;
while (length > 0) {
size_t chunk = length;
if (host_offset < ahb_base) {
chunk = (std::min)(chunk, ahb_base - host_offset);
} else if (host_offset < ahb_base + ahb_bytes) {
const size_t local = (host_offset - ahb_base) % ahb_window_size;
chunk = (std::min)(chunk, ahb_window_size - local);
}
host_offset += chunk;
length -= chunk;
++count;
}
return count;
}
std::span<AHardwareBuffer* const> AhbWindows() const noexcept {
return ahb_windows;
}
size_t AhbWindowSize() const noexcept {
return ahb_bytes != 0 ? ahb_window_size : 0;
}
size_t AhbBase() const noexcept {
return ahb_base;
}
#endif
~Impl() {
Release();
}
@@ -587,6 +895,12 @@ public:
#ifdef ARCHITECTURE_arm64
if (True(perms & MemoryPermission::Execute))
prot_flags |= PROT_EXEC;
#endif
#ifdef __ANDROID__
if (ahb_bytes != 0) {
MapBackingRange(virtual_offset, host_offset, length, prot_flags);
return;
}
#endif
int flags = (fd >= 0 ? MAP_SHARED : MAP_PRIVATE) | MAP_FIXED;
void* ret = mmap(virtual_base + virtual_offset, length, prot_flags, flags, fd, host_offset);
@@ -632,8 +946,18 @@ public:
virtual_base = nullptr;
}
bool IsBackingShared() const noexcept {
#ifdef __ANDROID__
if (ahb_bytes != 0) {
return true;
}
#endif
return fd >= 0;
}
const size_t backing_size; ///< Size of the backing memory in bytes
const size_t virtual_size; ///< Size of the virtual address placeholder in bytes
const size_t preferred_offset;
u8* backing_base{reinterpret_cast<u8*>(MAP_FAILED)};
u8* virtual_base{reinterpret_cast<u8*>(MAP_FAILED)};
@@ -656,6 +980,21 @@ private:
int ret = close(fd);
ASSERT_MSG(ret == 0, "close failed: {}", strerror(errno));
}
#ifdef __ANDROID__
for (const int window_fd : ahb_fds) {
SyncDmaBufCpuAccess(window_fd, DmaBufSyncEnd);
}
for (AHardwareBuffer* buffer : ahb_windows) {
AHardwareBuffer_release(buffer);
}
ahb_windows.clear();
ahb_fds.clear();
if (ahb_bytes != 0) {
committed_backing_size.store(0, std::memory_order_relaxed);
ahb_bytes = 0;
}
#endif
}
void AdjustMap(size_t* virtual_offset, size_t* length) {
@@ -681,11 +1020,19 @@ private:
int fd{-1}; // memfd file descriptor, -1 is the error value of memfd_create
FreeRegionManager free_manager{};
#ifdef __ANDROID__
std::vector<AHardwareBuffer*> ahb_windows;
std::vector<int> ahb_fds;
size_t ahb_window_size{};
size_t ahb_base{};
size_t ahb_bytes{};
#endif
};
#endif // ^^^ POSIX ^^^
HostMemory::HostMemory(size_t backing_size_, size_t virtual_size_)
HostMemory::HostMemory(size_t backing_size_, size_t virtual_size_, size_t preferred_offset_)
: backing_size(backing_size_)
, virtual_size(virtual_size_)
{
@@ -697,7 +1044,7 @@ HostMemory::HostMemory(size_t backing_size_, size_t virtual_size_)
#else
// Try to allocate a fastmem arena.
// The implementation will fail with std::bad_alloc on errors.
impl = std::make_unique<HostMemory::Impl>(AlignUp(backing_size, PageAlignment), AlignUp(virtual_size, PageAlignment) + HugePageSize);
impl = std::make_unique<HostMemory::Impl>(AlignUp(backing_size, PageAlignment), AlignUp(virtual_size, PageAlignment) + HugePageSize, preferred_offset_);
if (impl->Init()) {
backing_base = impl->backing_base;
virtual_base = impl->virtual_base;
@@ -767,6 +1114,46 @@ void HostMemory::ClearBackingRegion(size_t physical_offset, size_t length, u32 f
std::memset(backing_base + physical_offset, fill_value, length);
}
std::span<AHardwareBuffer* const> HostMemory::BackingHardwareBuffers() const noexcept {
#ifdef __ANDROID__
return impl ? impl->AhbWindows() : std::span<AHardwareBuffer* const>{};
#else
return {};
#endif
}
size_t HostMemory::BackingMapCount(size_t host_offset, size_t length) const noexcept {
#ifdef __ANDROID__
return impl ? impl->BackingMapCount(host_offset, length) : (length != 0 ? 1 : 0);
#else
return length != 0 ? 1 : 0;
#endif
}
size_t HostMemory::BackingHardwareBufferWindowSize() const noexcept {
#ifdef __ANDROID__
return impl ? impl->AhbWindowSize() : 0;
#else
return 0;
#endif
}
bool HostMemory::IsBackingShared() const noexcept {
#if defined(__OPENORBIS__) || defined(__managarm__)
return false;
#else
return impl && impl->IsBackingShared();
#endif
}
size_t HostMemory::BackingHardwareBufferBase() const noexcept {
#ifdef __ANDROID__
return impl ? impl->AhbBase() : 0;
#else
return 0;
#endif
}
void HostMemory::EnableDirectMappedAddress() {
#if !(defined(__OPENORBIS__) || defined(__managarm__))
if (impl) {
+20 -1
View File
@@ -8,12 +8,17 @@
#include <memory>
#include <optional>
#include <span>
#include "common/common_funcs.h"
#include "common/common_types.h"
#include "common/virtual_buffer.h"
struct AHardwareBuffer;
namespace Common {
[[nodiscard]] u64 GetCommittedBackingSize() noexcept;
enum class MemoryPermission : u32 {
Read = 1 << 0,
Write = 1 << 1,
@@ -28,7 +33,7 @@ DECLARE_ENUM_FLAG_OPERATORS(MemoryPermission)
*/
class HostMemory {
public:
explicit HostMemory(size_t backing_size_, size_t virtual_size_);
explicit HostMemory(size_t backing_size_, size_t virtual_size_, size_t preferred_offset_ = 0);
~HostMemory();
/**
@@ -62,6 +67,20 @@ public:
return backing_base;
}
[[nodiscard]] size_t BackingSize() const noexcept {
return backing_size;
}
[[nodiscard]] size_t BackingMapCount(size_t host_offset, size_t length) const noexcept;
[[nodiscard]] std::span<AHardwareBuffer* const> BackingHardwareBuffers() const noexcept;
[[nodiscard]] size_t BackingHardwareBufferWindowSize() const noexcept;
[[nodiscard]] size_t BackingHardwareBufferBase() const noexcept;
[[nodiscard]] bool IsBackingShared() const noexcept;
[[nodiscard]] u8* VirtualBasePointer() noexcept {
return virtual_base;
}
+64
View File
@@ -1,3 +1,6 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
@@ -17,6 +20,10 @@
#endif
#endif
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include "common/memory_detect.h"
namespace Common {
@@ -69,4 +76,61 @@ const MemoryInfo& GetMemInfo() {
return mem_info;
}
u64 GetPermissibleMapCount() {
constexpr u64 DefaultMapCount = 65530;
constexpr u64 ReservedMaps = 20000;
u64 count = DefaultMapCount;
#ifdef __linux__
if (std::FILE* const file = std::fopen("/proc/sys/vm/max_map_count", "re")) {
char line[32];
if (std::fgets(line, sizeof(line), file) != nullptr) {
const u64 parsed = std::strtoull(line, nullptr, 10);
if (parsed != 0) {
count = parsed;
}
}
std::fclose(file);
}
#endif
if (count <= ReservedMaps) {
return 0;
}
return count - ReservedMaps;
}
u64 GetAvailablePhysicalMemory() {
#ifdef _WIN32
MEMORYSTATUSEX memorystatus;
memorystatus.dwLength = sizeof(memorystatus);
if (GlobalMemoryStatusEx(&memorystatus) == 0) {
return 0;
}
return memorystatus.ullAvailPhys;
#elif defined(__linux__)
static constexpr char AvailableKey[] = "MemAvailable:";
if (std::FILE* const file = std::fopen("/proc/meminfo", "re")) {
char line[256];
u64 available = 0;
while (std::fgets(line, sizeof(line), file) != nullptr) {
if (std::strncmp(line, AvailableKey, sizeof(AvailableKey) - 1) != 0) {
continue;
}
available = std::strtoull(line + sizeof(AvailableKey) - 1, nullptr, 10) * 1024;
break;
}
std::fclose(file);
if (available != 0) {
return available;
}
}
struct sysinfo meminfo;
if (sysinfo(&meminfo) != 0) {
return 0;
}
return static_cast<u64>(meminfo.freeram) * static_cast<u64>(meminfo.mem_unit);
#else
return 0;
#endif
}
} // namespace Common
+7
View File
@@ -1,3 +1,6 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
@@ -18,4 +21,8 @@ struct MemoryInfo {
*/
[[nodiscard]] const MemoryInfo& GetMemInfo();
[[nodiscard]] u64 GetPermissibleMapCount();
[[nodiscard]] u64 GetAvailablePhysicalMemory();
} // namespace Common
+3
View File
@@ -645,6 +645,9 @@ struct Values {
SwitchableSetting<bool> use_asynchronous_shaders{linkage, false, "use_asynchronous_shaders",
Category::RendererHacks};
SwitchableSetting<bool> use_unified_memory{linkage, false, "use_unified_memory",
Category::RendererHacks};
SwitchableSetting<GpuUnswizzleSize> gpu_unswizzle_texture_size{linkage,
GpuUnswizzleSize::Large,
"gpu_unswizzle_texture_size",
+5 -1
View File
@@ -119,6 +119,7 @@ struct System::Impl {
is_multicore = Settings::values.use_multi_core.GetValue();
extended_memory_layout = Settings::values.memory_layout_mode.GetValue() != Settings::MemoryLayout::Memory_4Gb;
unified_memory = Settings::values.use_unified_memory.GetValue();
core_timing.SetMulticore(is_multicore);
core_timing.Initialize([&system]() { system.RegisterHostThread(); });
@@ -146,7 +147,8 @@ struct System::Impl {
!device_memory.has_value() ||
is_multicore != Settings::values.use_multi_core.GetValue() ||
extended_memory_layout != (Settings::values.memory_layout_mode.GetValue() !=
Settings::MemoryLayout::Memory_4Gb);
Settings::MemoryLayout::Memory_4Gb) ||
unified_memory != Settings::values.use_unified_memory.GetValue();
if (!must_reinitialize) {
return;
@@ -157,6 +159,7 @@ struct System::Impl {
is_multicore = Settings::values.use_multi_core.GetValue();
extended_memory_layout =
Settings::values.memory_layout_mode.GetValue() != Settings::MemoryLayout::Memory_4Gb;
unified_memory = Settings::values.use_unified_memory.GetValue();
Initialize(system);
}
@@ -503,6 +506,7 @@ struct System::Impl {
std::atomic_bool is_powered_on{};
bool is_multicore : 1 = false;
bool extended_memory_layout : 1 = false;
bool unified_memory : 1 = false;
bool exit_locked : 1 = false;
bool exit_requested : 1 = false;
bool nvdec_active : 1 = false;
+13 -1
View File
@@ -1,3 +1,6 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
@@ -12,9 +15,18 @@ constexpr size_t VirtualReserveSize = 1ULL << 38;
constexpr size_t VirtualReserveSize = 1ULL << 39;
#endif
namespace {
size_t ApplicationPoolOffset() {
using Init = Kernel::Board::Nintendo::Nx::KSystemControl::Init;
const size_t dram_size = Init::GetIntendedMemorySize();
const size_t application_pool_size = Init::GetApplicationPoolSize();
return dram_size > application_pool_size ? dram_size - application_pool_size : 0;
}
}
DeviceMemory::DeviceMemory()
: buffer{Kernel::Board::Nintendo::Nx::KSystemControl::Init::GetIntendedMemorySize(),
VirtualReserveSize} {}
VirtualReserveSize, ApplicationPoolOffset()} {}
DeviceMemory::~DeviceMemory() = default;
+35
View File
@@ -20,6 +20,8 @@
#include "common/scratch_buffer.h"
#include "common/virtual_buffer.h"
struct AHardwareBuffer;
namespace Core {
constexpr size_t DEVICE_PAGEBITS = 12ULL;
@@ -95,6 +97,34 @@ public:
ApplyOpOnPAddr(address, buffer, operation);
}
u8* GetPhysicalBase() noexcept {
return reinterpret_cast<u8*>(physical_base);
}
const u8* GetPhysicalBase() const noexcept {
return reinterpret_cast<const u8*>(physical_base);
}
size_t GetPhysicalSize() const noexcept {
return physical_size;
}
std::span<AHardwareBuffer* const> GetBackingHardwareBuffers() const noexcept {
return ahb_windows;
}
size_t GetBackingHardwareBufferWindowSize() const noexcept {
return ahb_window_size;
}
size_t GetBackingHardwareBufferBase() const noexcept {
return ahb_base;
}
bool IsBackingShared() const noexcept {
return backing_is_shared;
}
PAddr GetPhysicalRawAddressFromDAddr(DAddr address) const {
PAddr subbits = PAddr(address & page_mask);
auto paddr = tracked_entries[(address >> page_bits)].compressed_physical_ptr;
@@ -171,6 +201,11 @@ private:
std::unique_ptr<DeviceMemoryManagerAllocator<Traits>> impl;
const uintptr_t physical_base;
const size_t physical_size;
const std::span<AHardwareBuffer* const> ahb_windows;
const size_t ahb_window_size;
const size_t ahb_base;
const bool backing_is_shared;
DeviceInterface* device_inter;
struct TrackedEntry {
+5
View File
@@ -171,6 +171,11 @@ struct DeviceMemoryManagerAllocator {
template <typename Traits>
DeviceMemoryManager<Traits>::DeviceMemoryManager(const DeviceMemory& device_memory_)
: physical_base{uintptr_t(device_memory_.buffer.BackingBasePointer())}
, physical_size{device_memory_.buffer.BackingSize()}
, ahb_windows{device_memory_.buffer.BackingHardwareBuffers()}
, ahb_window_size{device_memory_.buffer.BackingHardwareBufferWindowSize()}
, ahb_base{device_memory_.buffer.BackingHardwareBufferBase()}
, backing_is_shared{device_memory_.buffer.IsBackingShared()}
, device_inter{nullptr}
, compressed_device_addr(1ULL << ((Settings::values.memory_layout_mode.GetValue() == Settings::MemoryLayout::Memory_4Gb ? physical_min_bits : physical_max_bits) - Memory::YUZU_PAGEBITS))
, tracked_entries(device_as_size >> Memory::YUZU_PAGEBITS)
@@ -227,6 +227,8 @@ std::unique_ptr<TranslationMap> InitializeTranslations(QObject* parent) {
tr("Preserves GPU-modified data by reading it back before uploading.\nSome games require this to render certain effects properly."));
INSERT(Settings, use_asynchronous_shaders, tr("Enable asynchronous shader compilation"),
tr("May reduce shader stutter."));
INSERT(Settings, use_unified_memory, tr("Enable unified memory access"),
tr("Lets the GPU write buffer readbacks directly into guest memory."));
INSERT(Settings, gpu_clock, tr("GPU Clocks"),
tr("Makes the game believe GPU work finishes faster than it does, so it stops lowering "
"resolution and render distance to fit the Switch's clocks."));
@@ -335,7 +335,7 @@ void SetupDenormControl(const Profile& profile, const IR::Program& program, Emit
if (info.uses_fp32_denorms_flush && info.uses_fp32_denorms_preserve) {
LOG_DEBUG(Shader_SPIRV, "Fp32 denorm flush and preserve on the same shader");
} else if (info.uses_fp32_denorms_flush) {
if (profile.support_fp32_denorm_flush) {
if (profile.support_fp32_denorm_flush && !profile.has_broken_fp32_denorm_flush) {
ctx.AddCapability(spv::Capability::DenormFlushToZero);
ctx.AddExecutionMode(main_func, spv::ExecutionMode::DenormFlushToZero, 32U);
} else {
@@ -13,6 +13,7 @@
namespace Shader::Backend::SPIRV {
namespace {
Id SharedPointer(EmitContext& ctx, Id offset, u32 index_offset = 0) {
offset = ctx.BoundSharedOffset(offset, 4 + index_offset * 4);
const Id shift_id{ctx.Const(2U)};
Id index{ctx.OpShiftRightArithmetic(ctx.U32[1], offset, shift_id)};
if (index_offset > 0) {
@@ -160,7 +161,8 @@ Id EmitSharedAtomicExchange32(EmitContext& ctx, Id offset, Id value) {
Id EmitSharedAtomicExchange64(EmitContext& ctx, Id offset, Id value) {
if (ctx.profile.support_shared_int64_atomics && ctx.uses_explicit_workgroup_layout) {
const Id shift_id{ctx.Const(3U)};
const Id index{ctx.OpShiftRightArithmetic(ctx.U32[1], offset, shift_id)};
const Id index{
ctx.OpShiftRightArithmetic(ctx.U32[1], ctx.BoundSharedOffset(offset, 8), shift_id)};
const Id pointer{
ctx.OpAccessChain(ctx.shared_u64, ctx.shared_memory_u64, ctx.u32_zero_value, index)};
const auto [scope, semantics]{AtomicArgs(ctx)};
@@ -31,6 +31,7 @@ std::pair<Id, Id> ExtractArgs(EmitContext& ctx, Id offset, u32 mask, u32 count)
} // Anonymous namespace
Id EmitLoadSharedU8(EmitContext& ctx, Id offset) {
offset = ctx.BoundSharedOffset(offset, 1);
if (ctx.uses_explicit_workgroup_layout) {
const Id pointer{
ctx.OpAccessChain(ctx.shared_u8, ctx.shared_memory_u8, ctx.u32_zero_value, offset)};
@@ -42,6 +43,7 @@ Id EmitLoadSharedU8(EmitContext& ctx, Id offset) {
}
Id EmitLoadSharedS8(EmitContext& ctx, Id offset) {
offset = ctx.BoundSharedOffset(offset, 1);
if (ctx.uses_explicit_workgroup_layout) {
const Id pointer{
ctx.OpAccessChain(ctx.shared_u8, ctx.shared_memory_u8, ctx.u32_zero_value, offset)};
@@ -53,6 +55,7 @@ Id EmitLoadSharedS8(EmitContext& ctx, Id offset) {
}
Id EmitLoadSharedU16(EmitContext& ctx, Id offset) {
offset = ctx.BoundSharedOffset(offset, 2);
if (ctx.uses_explicit_workgroup_layout) {
const Id pointer{Pointer(ctx, ctx.shared_u16, ctx.shared_memory_u16, offset, 1)};
return ctx.OpUConvert(ctx.U32[1], ctx.OpLoad(ctx.U16, pointer));
@@ -63,6 +66,7 @@ Id EmitLoadSharedU16(EmitContext& ctx, Id offset) {
}
Id EmitLoadSharedS16(EmitContext& ctx, Id offset) {
offset = ctx.BoundSharedOffset(offset, 2);
if (ctx.uses_explicit_workgroup_layout) {
const Id pointer{Pointer(ctx, ctx.shared_u16, ctx.shared_memory_u16, offset, 1)};
return ctx.OpSConvert(ctx.U32[1], ctx.OpLoad(ctx.U16, pointer));
@@ -73,6 +77,7 @@ Id EmitLoadSharedS16(EmitContext& ctx, Id offset) {
}
Id EmitLoadSharedU32(EmitContext& ctx, Id offset) {
offset = ctx.BoundSharedOffset(offset, 4);
if (ctx.uses_explicit_workgroup_layout) {
const Id pointer{Pointer(ctx, ctx.shared_u32, ctx.shared_memory_u32, offset, 2)};
return ctx.OpLoad(ctx.U32[1], pointer);
@@ -82,6 +87,7 @@ Id EmitLoadSharedU32(EmitContext& ctx, Id offset) {
}
Id EmitLoadSharedU64(EmitContext& ctx, Id offset) {
offset = ctx.BoundSharedOffset(offset, 8);
if (ctx.uses_explicit_workgroup_layout) {
const Id pointer{Pointer(ctx, ctx.shared_u32x2, ctx.shared_memory_u32x2, offset, 3)};
return ctx.OpLoad(ctx.U32[2], pointer);
@@ -97,6 +103,7 @@ Id EmitLoadSharedU64(EmitContext& ctx, Id offset) {
}
Id EmitLoadSharedU128(EmitContext& ctx, Id offset) {
offset = ctx.BoundSharedOffset(offset, 16);
if (ctx.uses_explicit_workgroup_layout) {
const Id pointer{Pointer(ctx, ctx.shared_u32x4, ctx.shared_memory_u32x4, offset, 4)};
return ctx.OpLoad(ctx.U32[4], pointer);
@@ -113,6 +120,7 @@ Id EmitLoadSharedU128(EmitContext& ctx, Id offset) {
}
void EmitWriteSharedU8(EmitContext& ctx, Id offset, Id value) {
offset = ctx.BoundSharedOffset(offset, 1);
if (ctx.uses_explicit_workgroup_layout) {
const Id pointer{
ctx.OpAccessChain(ctx.shared_u8, ctx.shared_memory_u8, ctx.u32_zero_value, offset)};
@@ -123,6 +131,7 @@ void EmitWriteSharedU8(EmitContext& ctx, Id offset, Id value) {
}
void EmitWriteSharedU16(EmitContext& ctx, Id offset, Id value) {
offset = ctx.BoundSharedOffset(offset, 2);
if (ctx.uses_explicit_workgroup_layout) {
const Id pointer{Pointer(ctx, ctx.shared_u16, ctx.shared_memory_u16, offset, 1)};
ctx.OpStore(pointer, ctx.OpUConvert(ctx.U16, value));
@@ -132,6 +141,7 @@ void EmitWriteSharedU16(EmitContext& ctx, Id offset, Id value) {
}
void EmitWriteSharedU32(EmitContext& ctx, Id offset, Id value) {
offset = ctx.BoundSharedOffset(offset, 4);
Id pointer{};
if (ctx.uses_explicit_workgroup_layout) {
pointer = Pointer(ctx, ctx.shared_u32, ctx.shared_memory_u32, offset, 2);
@@ -144,6 +154,7 @@ void EmitWriteSharedU32(EmitContext& ctx, Id offset, Id value) {
}
void EmitWriteSharedU64(EmitContext& ctx, Id offset, Id value) {
offset = ctx.BoundSharedOffset(offset, 8);
if (ctx.uses_explicit_workgroup_layout) {
const Id pointer{Pointer(ctx, ctx.shared_u32x2, ctx.shared_memory_u32x2, offset, 3)};
ctx.OpStore(pointer, value);
@@ -159,6 +170,7 @@ void EmitWriteSharedU64(EmitContext& ctx, Id offset, Id value) {
}
void EmitWriteSharedU128(EmitContext& ctx, Id offset, Id value) {
offset = ctx.BoundSharedOffset(offset, 16);
if (ctx.uses_explicit_workgroup_layout) {
const Id pointer{Pointer(ctx, ctx.shared_u32x4, ctx.shared_memory_u32x4, offset, 4)};
ctx.OpStore(pointer, value);
@@ -600,6 +600,16 @@ void EmitContext::DefineLocalMemory(const IR::Program& program) {
}
}
Id EmitContext::BoundSharedOffset(Id offset, u32 access_bytes) {
if (shared_memory_declared_bytes == 0) {
return offset;
}
const u32 last_valid{shared_memory_declared_bytes > access_bytes
? shared_memory_declared_bytes - access_bytes
: 0U};
return OpUMin(U32[1], offset, Const(last_valid));
}
void EmitContext::DefineSharedMemory(const IR::Program& program) {
uses_explicit_workgroup_layout =
profile.support_explicit_workgroup_layout &&
@@ -608,8 +618,15 @@ void EmitContext::DefineSharedMemory(const IR::Program& program) {
if (program.shared_memory_size == 0) {
return;
}
const u32 device_limit{profile.max_shared_memory_size};
const u32 shared_memory_size{device_limit != 0 && program.shared_memory_size > device_limit
? device_limit
: program.shared_memory_size};
if (shared_memory_size != program.shared_memory_size) {
shared_memory_declared_bytes = shared_memory_size;
}
const auto make{[&](Id element_type, u32 element_size) {
const u32 num_elements{Common::DivCeil(program.shared_memory_size, element_size)};
const u32 num_elements{Common::DivCeil(shared_memory_size, element_size)};
const Id array_type{TypeArray(element_type, Const(num_elements))};
Decorate(array_type, spv::Decoration::ArrayStride, element_size);
@@ -644,7 +661,7 @@ void EmitContext::DefineSharedMemory(const IR::Program& program) {
std::tie(shared_memory_u32x4, shared_u32x4, std::ignore) = make(U32[4], 16);
return;
}
const u32 num_elements{Common::DivCeil(program.shared_memory_size, 4U)};
const u32 num_elements{Common::DivCeil(shared_memory_size, 4U)};
const Id type{TypeArray(U32[1], Const(num_elements))};
shared_memory_u32_type = TypePointer(spv::StorageClass::Workgroup, type);
@@ -312,6 +312,8 @@ public:
Id local_memory{};
bool uses_explicit_workgroup_layout{};
u32 shared_memory_declared_bytes{};
[[nodiscard]] Id BoundSharedOffset(Id offset, u32 access_bytes);
Id shared_memory_u8{};
Id shared_memory_u16{};
Id shared_memory_u32{};
+5
View File
@@ -86,6 +86,8 @@ struct Profile {
bool has_broken_signed_operations{};
/// Float controls break when fp16 is enabled
bool has_broken_fp16_float_controls{};
/// Declaring fp32 denorm flush to zero miscompiles on some drivers
bool has_broken_fp32_denorm_flush{};
/// Dynamic vec4 indexing is broken on some OpenGL drivers
bool has_gl_component_indexing_bug{};
/// The precise type qualifier is broken in the fragment stage of some drivers
@@ -101,6 +103,9 @@ struct Profile {
u32 gl_max_compute_smem_size{};
/// Largest workgroup shared memory allocation the device accepts, 0 when unconstrained
u32 max_shared_memory_size{};
/// Maxwell and earlier nVidia architectures have broken robust support
bool has_broken_robust{};
+272 -37
View File
@@ -571,7 +571,11 @@ void BufferCache<P>::AccumulateFlushes() {
template <class P>
bool BufferCache<P>::ShouldWaitAsyncFlushes() const noexcept {
return (!async_buffers.empty() && async_buffers.front().has_value());
if (async_buffers.empty()) {
return false;
}
return async_buffers.front().has_value() ||
!pending_downloads.front().unified_copies.empty();
}
template <class P>
@@ -579,6 +583,7 @@ void BufferCache<P>::CommitAsyncFlushesHigh() {
AccumulateFlushes();
if (committed_gpu_modified_ranges.empty()) {
pending_downloads.emplace_back();
async_buffers.emplace_back(std::optional<Async_Buffer>{});
return;
}
@@ -638,27 +643,84 @@ void BufferCache<P>::CommitAsyncFlushesHigh() {
}
committed_gpu_modified_ranges.clear();
if (downloads.empty()) {
pending_downloads.emplace_back();
async_buffers.emplace_back(std::optional<Async_Buffer>{});
return;
}
auto download_staging = runtime.DownloadStagingBuffer(total_size_bytes, true);
boost::container::small_vector<BufferCopy, 4> normalized_copies;
runtime.PreCopyBarrier();
struct QueuedUnifiedCopy {
u64 window;
BufferId buffer_id;
boost::container::small_vector<BufferCopy, 16> copies;
};
AsyncDownloadBatch batch;
boost::container::small_vector<std::pair<BufferCopy, BufferId>, 16> staging_downloads;
boost::container::small_vector<QueuedUnifiedCopy, 4> unified_copy_queue;
boost::container::small_vector<u64, 4> window_ids;
UnifiedWindowGroups groups;
u64 staging_size_bytes = 0;
for (auto& [copy, buffer_id] : downloads) {
copy.dst_offset += download_staging.offset;
const std::array copies{copy};
BufferCopy second_copy{copy};
Buffer& buffer = slot_buffers[buffer_id];
second_copy.src_offset = static_cast<size_t>(buffer.CpuAddr()) + copy.src_offset;
const DAddr orig_device_addr = static_cast<DAddr>(second_copy.src_offset);
const DAddr orig_device_addr = buffer.CpuAddr() + copy.src_offset;
bool unified = false;
if constexpr (USE_UNIFIED_MEMORY) {
if (runtime.HasUnifiedMemory()) {
window_ids.clear();
groups.clear();
unified = ResolveUnifiedWindows(orig_device_addr, copy.src_offset, copy.size,
window_ids, groups);
}
}
BufferCopy record{copy};
record.src_offset = static_cast<size_t>(orig_device_addr);
if (unified) {
async_downloads.Add(orig_device_addr, copy.size);
buffer.MarkUsage(copy.src_offset, copy.size);
for (size_t i = 0; i < window_ids.size(); ++i) {
unified_copy_queue.push_back(
QueuedUnifiedCopy{window_ids[i], buffer_id, std::move(groups[i])});
}
batch.unified_copies.push_back(record);
continue;
}
copy.dst_offset = staging_size_bytes;
constexpr u64 align = 64ULL;
staging_size_bytes += (copy.size + align - 1) & ~(align - 1ULL);
staging_downloads.push_back({copy, buffer_id});
}
std::optional<Async_Buffer> download_staging;
if (!staging_downloads.empty()) {
download_staging = runtime.DownloadStagingBuffer(staging_size_bytes, true);
}
runtime.PreCopyBarrier();
for (auto& [copy, buffer_id] : staging_downloads) {
copy.dst_offset += download_staging->offset;
const std::array copies{copy};
Buffer& buffer = slot_buffers[buffer_id];
BufferCopy record{copy};
record.src_offset = static_cast<size_t>(buffer.CpuAddr()) + copy.src_offset;
const DAddr orig_device_addr = static_cast<DAddr>(record.src_offset);
async_downloads.Add(orig_device_addr, copy.size);
buffer.MarkUsage(copy.src_offset, copy.size);
runtime.CopyBuffer(download_staging.buffer, buffer, copies, false);
normalized_copies.push_back(second_copy);
runtime.CopyBuffer(download_staging->buffer, buffer, copies, false);
batch.staging_copies.push_back(record);
}
if constexpr (USE_UNIFIED_MEMORY) {
for (const auto& queued : unified_copy_queue) {
const std::span<const BufferCopy> group_span(queued.copies.data(),
queued.copies.size());
runtime.CopyToUnifiedMemory(queued.window, slot_buffers[queued.buffer_id], group_span);
}
if (!unified_copy_queue.empty()) {
runtime.FlushUnifiedMemoryCopies();
runtime.UnifiedMemoryHostBarrier();
}
}
runtime.PostCopyBarrier();
pending_downloads.emplace_back(std::move(normalized_copies));
async_buffers.emplace_back(download_staging);
pending_downloads.emplace_back(std::move(batch));
async_buffers.emplace_back(std::move(download_staging));
}
template <class P>
@@ -673,32 +735,49 @@ void BufferCache<P>::PopAsyncFlushes() {
template <class P>
void BufferCache<P>::PopAsyncBuffers() {
if (async_buffers.empty()) {
return;
}
if (!async_buffers.front().has_value()) {
struct Writeback {
DAddr addr;
const u8* src;
u64 size;
};
boost::container::small_vector<Writeback, 8> writebacks;
{
std::scoped_lock lock{mutex};
if (async_buffers.empty()) {
return;
}
auto& batch = pending_downloads.front();
auto& async_buffer = async_buffers.front();
if (async_buffer.has_value()) {
const u8* base = async_buffer->mapped_span.data();
const size_t base_offset = async_buffer->offset;
for (const auto& copy : batch.staging_copies) {
const DAddr device_addr = static_cast<DAddr>(copy.src_offset);
const u64 dst_offset = copy.dst_offset - base_offset;
const u8* read_mapped_memory = base + dst_offset;
async_downloads.ForEachInRange(
device_addr, copy.size, [&](DAddr start, DAddr end, s32) {
writebacks.push_back(
{start, &read_mapped_memory[start - device_addr], end - start});
});
async_downloads.Subtract(device_addr, copy.size, [&](DAddr start, DAddr end) {
gpu_modified_ranges.Subtract(start, end - start);
});
}
async_buffers_death_ring.emplace_back(*async_buffer);
}
for (const auto& copy : batch.unified_copies) {
const DAddr device_addr = static_cast<DAddr>(copy.src_offset);
async_downloads.Subtract(device_addr, copy.size, [&](DAddr start, DAddr end) {
gpu_modified_ranges.Subtract(start, end - start);
});
}
async_buffers.pop_front();
return;
pending_downloads.pop_front();
}
auto& downloads = pending_downloads.front();
auto& async_buffer = async_buffers.front();
u8* base = async_buffer->mapped_span.data();
const size_t base_offset = async_buffer->offset;
for (const auto& copy : downloads) {
const DAddr device_addr = static_cast<DAddr>(copy.src_offset);
const u64 dst_offset = copy.dst_offset - base_offset;
const u8* read_mapped_memory = base + dst_offset;
async_downloads.ForEachInRange(device_addr, copy.size, [&](DAddr start, DAddr end, s32) {
device_memory.WriteBlockUnsafe(start, &read_mapped_memory[start - device_addr],
end - start);
});
async_downloads.Subtract(device_addr, copy.size, [&](DAddr start, DAddr end) {
gpu_modified_ranges.Subtract(start, end - start);
});
for (const auto& wb : writebacks) {
device_memory.WriteBlockUnsafe(wb.addr, wb.src, wb.size);
}
async_buffers_death_ring.emplace_back(*async_buffer);
async_buffers.pop_front();
pending_downloads.pop_front();
}
template <class P>
@@ -1006,11 +1085,25 @@ void BufferCache<P>::BindHostGraphicsStorageBuffers(size_t stage) {
Buffer& buffer = slot_buffers[binding.buffer_id];
TouchBuffer(buffer, binding.buffer_id);
const u32 size = binding.size;
const bool is_written = ((channel_state->written_storage_buffers[stage] >> index) & 1) != 0;
if constexpr (USE_UNIFIED_MEMORY) {
const auto window = TryResolveUnifiedRange(binding.device_addr, size);
if (window && runtime.IsUnifiedStorageRange(size, window->offset)) {
if (is_written) {
memory_tracker.MarkRegionAsCpuModified(binding.device_addr, size);
}
runtime.BindStorageBuffer(runtime.UnifiedWindowBuffer(window->window),
runtime.UnifiedWindowAddress(window->window),
static_cast<u32>(window->offset), size, is_written);
return;
}
}
SynchronizeBuffer(buffer, binding.device_addr, size);
const u32 offset = buffer.Offset(binding.device_addr);
buffer.MarkUsage(offset, size);
const bool is_written = ((channel_state->written_storage_buffers[stage] >> index) & 1) != 0;
if (is_written) {
MarkWrittenBuffer(binding.buffer_id, binding.device_addr, size);
@@ -1699,6 +1792,142 @@ void BufferCache<P>::ImmediateUploadMemory([[maybe_unused]] Buffer& buffer,
}
}
template <class P>
bool BufferCache<P>::ResolveUnifiedWindows(
[[maybe_unused]] DAddr device_addr, [[maybe_unused]] u64 buffer_offset,
[[maybe_unused]] u64 size, [[maybe_unused]] boost::container::small_vector<u64, 4>& window_ids,
[[maybe_unused]] UnifiedWindowGroups& groups) {
if constexpr (USE_UNIFIED_MEMORY) {
const u8* const physical_base = device_memory.GetPhysicalBase();
const u64 unified_base = runtime.UnifiedMemoryBase();
const u64 unified_size = runtime.UnifiedMemorySize();
const u64 window_size = runtime.UnifiedMemoryWindowSize();
if (window_size == 0) {
return false;
}
const auto group_for = [&](u64 window) -> boost::container::small_vector<BufferCopy, 16>& {
for (size_t i = 0; i < window_ids.size(); ++i) {
if (window_ids[i] == window) {
return groups[i];
}
}
window_ids.push_back(window);
groups.emplace_back();
return groups.back();
};
u64 downloaded = 0;
while (downloaded < size) {
const DAddr page_addr = device_addr + downloaded;
const u8* const ptr = device_memory.GetPointer<u8>(page_addr);
if (ptr == nullptr) {
return false;
}
const u64 page_offset = page_addr & Core::DEVICE_PAGEMASK;
u64 chunk = (std::min)(size - downloaded,
static_cast<u64>(Core::DEVICE_PAGESIZE) - page_offset);
const u64 phys_offset = static_cast<u64>(ptr - physical_base);
if (phys_offset < unified_base || phys_offset - unified_base + chunk > unified_size) {
return false;
}
const u64 relative = phys_offset - unified_base;
const u64 window = relative / window_size;
const u64 local_offset = relative % window_size;
chunk = (std::min)(chunk, window_size - local_offset);
auto& group = group_for(window);
if (!group.empty()) {
BufferCopy& last = group.back();
if (last.src_offset + last.size == buffer_offset + downloaded &&
last.dst_offset + last.size == local_offset) {
last.size += chunk;
downloaded += chunk;
continue;
}
}
group.push_back(BufferCopy{
.src_offset = buffer_offset + downloaded,
.dst_offset = local_offset,
.size = chunk,
});
downloaded += chunk;
}
return true;
} else {
return false;
}
}
template <class P>
std::optional<typename BufferCache<P>::UnifiedWindowRange>
BufferCache<P>::TryResolveUnifiedRange([[maybe_unused]] DAddr device_addr,
[[maybe_unused]] u64 size) {
if constexpr (USE_UNIFIED_MEMORY) {
if (size == 0 || !runtime.IsUnifiedMemoryBindable()) {
return std::nullopt;
}
const u64 window_size = runtime.UnifiedMemoryWindowSize();
if (window_size == 0) {
return std::nullopt;
}
const u8* const first = device_memory.GetSpan(device_addr, size);
if (first == nullptr) {
return std::nullopt;
}
const u64 phys_offset = static_cast<u64>(first - device_memory.GetPhysicalBase());
const u64 unified_base = runtime.UnifiedMemoryBase();
if (phys_offset < unified_base) {
return std::nullopt;
}
const u64 relative = phys_offset - unified_base;
const u64 unified_size = runtime.UnifiedMemorySize();
if (relative >= unified_size || unified_size - relative < size) {
return std::nullopt;
}
const u64 local_offset = relative % window_size;
if (window_size - local_offset < size) {
return std::nullopt;
}
if (memory_tracker.IsRegionGpuModified(device_addr, size) ||
IsRegionGpuModified(device_addr, size)) {
return std::nullopt;
}
return UnifiedWindowRange{
.window = static_cast<size_t>(relative / window_size),
.offset = local_offset,
};
} else {
return std::nullopt;
}
}
template <class P>
bool BufferCache<P>::TryUnifiedDownloadMemory([[maybe_unused]] Buffer& buffer,
[[maybe_unused]] std::span<BufferCopy> copies) {
if constexpr (USE_UNIFIED_MEMORY) {
boost::container::small_vector<u64, 4> window_ids;
UnifiedWindowGroups groups;
for (const BufferCopy& copy : copies) {
if (!ResolveUnifiedWindows(buffer.CpuAddr() + copy.src_offset, copy.src_offset,
copy.size, window_ids, groups)) {
return false;
}
}
for (const BufferCopy& copy : copies) {
buffer.MarkUsage(copy.src_offset, copy.size);
}
runtime.PreCopyBarrier();
for (size_t i = 0; i < window_ids.size(); ++i) {
const std::span<const BufferCopy> group_span(groups[i].data(), groups[i].size());
runtime.CopyToUnifiedMemory(window_ids[i], buffer, group_span);
}
runtime.FlushUnifiedMemoryCopies();
runtime.UnifiedMemoryHostBarrier();
runtime.Finish();
return true;
} else {
return false;
}
}
template <class P>
void BufferCache<P>::MappedUploadMemory([[maybe_unused]] Buffer& buffer,
[[maybe_unused]] u64 total_size_bytes,
@@ -1802,6 +2031,12 @@ void BufferCache<P>::DownloadBufferMemory(Buffer& buffer, DAddr device_addr, u64
}
if constexpr (USE_MEMORY_MAPS) {
if constexpr (USE_UNIFIED_MEMORY) {
if (runtime.HasUnifiedMemory() &&
TryUnifiedDownloadMemory(buffer, std::span(copies.data(), copies.size()))) {
return;
}
}
auto download_staging = runtime.DownloadStagingBuffer(total_size_bytes);
const u8* const mapped_memory = download_staging.mapped_span.data();
const std::span<BufferCopy> copies_span(copies.data(), copies.data() + copies.size());
@@ -12,6 +12,7 @@
#include <functional>
#include <memory>
#include <mutex>
#include <optional>
#include <numeric>
#include <span>
#include <vector>
@@ -180,6 +181,7 @@ class BufferCache : public VideoCommon::ChannelSetupCaches<BufferCacheChannelInf
static constexpr bool USE_MEMORY_MAPS = P::USE_MEMORY_MAPS;
static constexpr bool SEPARATE_IMAGE_BUFFERS_BINDINGS = P::SEPARATE_IMAGE_BUFFER_BINDINGS;
static constexpr bool USE_MEMORY_MAPS_FOR_UPLOADS = P::USE_MEMORY_MAPS_FOR_UPLOADS;
static constexpr bool USE_UNIFIED_MEMORY = P::USE_UNIFIED_MEMORY;
#ifdef YUZU_LEGACY
static constexpr s64 TARGET_THRESHOLD = 3_GiB;
@@ -443,6 +445,22 @@ private:
void MappedUploadMemory(Buffer& buffer, u64 total_size_bytes, std::span<BufferCopy> copies);
bool TryUnifiedDownloadMemory(Buffer& buffer, std::span<BufferCopy> copies);
struct UnifiedWindowRange {
size_t window;
u64 offset;
};
std::optional<UnifiedWindowRange> TryResolveUnifiedRange(DAddr device_addr, u64 size);
using UnifiedWindowGroups =
boost::container::small_vector<boost::container::small_vector<BufferCopy, 16>, 4>;
bool ResolveUnifiedWindows(DAddr device_addr, u64 buffer_offset, u64 size,
boost::container::small_vector<u64, 4>& window_ids,
UnifiedWindowGroups& groups);
void DownloadBufferMemory(Buffer& buffer_id);
void DownloadBufferMemory(Buffer& buffer_id, DAddr device_addr, u64 size);
@@ -498,9 +516,14 @@ private:
std::deque<Common::RangeSet<DAddr>> committed_gpu_modified_ranges;
// Async Buffers
struct AsyncDownloadBatch {
boost::container::small_vector<BufferCopy, 4> staging_copies;
boost::container::small_vector<BufferCopy, 4> unified_copies;
};
Common::OverlapRangeSet<DAddr> async_downloads;
std::deque<std::optional<Async_Buffer>> async_buffers;
std::deque<boost::container::small_vector<BufferCopy, 4>> pending_downloads;
std::deque<AsyncDownloadBatch> pending_downloads;
std::optional<Async_Buffer> current_buffer;
std::deque<Async_Buffer> async_buffers_death_ring;
@@ -17,12 +17,14 @@ set(SHADER_FILES
${CMAKE_CURRENT_SOURCE_DIR}/astc_decoder.comp
${CMAKE_CURRENT_SOURCE_DIR}/blit_color_float.frag
${CMAKE_CURRENT_SOURCE_DIR}/block_linear_unswizzle_2d.comp
${CMAKE_CURRENT_SOURCE_DIR}/block_linear_unswizzle_2d_buffer.comp
${CMAKE_CURRENT_SOURCE_DIR}/blit_color_msaa.frag
${CMAKE_CURRENT_SOURCE_DIR}/blit_depth.frag
${CMAKE_CURRENT_SOURCE_DIR}/blit_depth_msaa.frag
${CMAKE_CURRENT_SOURCE_DIR}/blit_depth_stencil_msaa.frag
${CMAKE_CURRENT_SOURCE_DIR}/block_linear_unswizzle_3d.comp
${CMAKE_CURRENT_SOURCE_DIR}/block_linear_unswizzle_3d_bcn.comp
${CMAKE_CURRENT_SOURCE_DIR}/block_linear_unswizzle_3d_buffer.comp
${CMAKE_CURRENT_SOURCE_DIR}/convert_abgr8_to_d24s8.frag
${CMAKE_CURRENT_SOURCE_DIR}/convert_abgr8_to_d32f.frag
${CMAKE_CURRENT_SOURCE_DIR}/convert_d32f_to_abgr8.frag
@@ -0,0 +1,104 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
#version 430
#extension GL_EXT_shader_16bit_storage : require
#extension GL_EXT_shader_8bit_storage : require
#define BINDING_INPUT_BUFFER 0
#define BINDING_OUTPUT_BUFFER 1
layout(push_constant) uniform PushConstants {
uvec3 dim;
uint bytes_per_block_log2;
uvec3 origin;
uint layer_stride;
uint block_size;
uint x_shift;
uint block_height;
uint block_height_mask;
} pc;
layout(binding = BINDING_INPUT_BUFFER, std430) buffer InputBufferU32 { uint u32data[]; };
layout(binding = BINDING_INPUT_BUFFER, std430) buffer InputBufferU64 { uvec2 u64data[]; };
layout(binding = BINDING_INPUT_BUFFER, std430) buffer InputBufferU128 { uvec4 u128data[]; };
layout(binding = BINDING_OUTPUT_BUFFER, std430) writeonly buffer OutputBuffer {
uint out_u32[];
};
layout(local_size_x = 16, local_size_y = 8, local_size_z = 1) in;
const uint GOB_SIZE_X = 64;
const uint GOB_SIZE_Y = 8;
const uint GOB_SIZE_X_SHIFT = 6;
const uint GOB_SIZE_Y_SHIFT = 3;
const uint GOB_SIZE_SHIFT = GOB_SIZE_X_SHIFT + GOB_SIZE_Y_SHIFT;
const uvec2 SWIZZLE_MASK = uvec2(GOB_SIZE_X - 1u, GOB_SIZE_Y - 1u);
uint SwizzleTable(uint pos) {
const uint t[8] = uint[](
0x12100200, 0x13110301, 0x16140604, 0x17150705,
0x1a180a08, 0x1b190b09, 0x1e1c0e0c, 0x1f1d0f0d
);
const uint i = pos >> 4;
const uint h = (t[i / 4] >> ((i % 4) * 8)) & 0xff;
return (h << 4) | (pos & 0xf);
}
uint SwizzleOffset(uvec2 pos) {
pos = pos & SWIZZLE_MASK;
return SwizzleTable(pos.y * 64u + pos.x);
}
uvec4 ReadTexel(uint offset) {
switch (pc.bytes_per_block_log2) {
case 2u:
return uvec4(u32data[offset / 4u], 0u, 0u, 0u);
case 3u:
return uvec4(u64data[offset / 8u], 0u, 0u);
case 4u:
return u128data[offset / 16u];
}
return uvec4(0u);
}
void main() {
uvec3 coord = gl_GlobalInvocationID;
if (coord.x >= pc.dim.x || coord.y >= pc.dim.y || coord.z >= pc.dim.z) {
return;
}
uvec3 pos = coord + pc.origin;
pos.x <<= pc.bytes_per_block_log2;
uint swizzle = SwizzleOffset(pos.xy);
uint block_y = pos.y >> GOB_SIZE_Y_SHIFT;
uint offset = 0u;
offset += pos.z * pc.layer_stride;
offset += (block_y >> pc.block_height) * pc.block_size;
offset += (block_y & pc.block_height_mask) << GOB_SIZE_SHIFT;
offset += (pos.x >> GOB_SIZE_X_SHIFT) << pc.x_shift;
offset += swizzle;
uvec4 texel = ReadTexel(offset);
uint words = 1u << (pc.bytes_per_block_log2 - 2u);
uint linear_index = coord.x + coord.y * pc.dim.x + coord.z * pc.dim.x * pc.dim.y;
uint out_idx = linear_index * words;
out_u32[out_idx] = texel.x;
if (words > 1u) {
out_u32[out_idx + 1u] = texel.y;
}
if (words > 2u) {
out_u32[out_idx + 2u] = texel.z;
out_u32[out_idx + 3u] = texel.w;
}
}
@@ -0,0 +1,105 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
#version 430
#define BINDING_INPUT_BUFFER 0
#define BINDING_OUTPUT_BUFFER 1
layout(push_constant) uniform PushConstants {
uvec3 dim;
uint bytes_per_block_log2;
uvec3 origin;
uint slice_size;
uint block_size;
uint x_shift;
uint block_height;
uint block_height_mask;
uint block_depth;
uint block_depth_mask;
} pc;
layout(binding = BINDING_INPUT_BUFFER, std430) buffer InputBufferU32 { uint u32data[]; };
layout(binding = BINDING_INPUT_BUFFER, std430) buffer InputBufferU64 { uvec2 u64data[]; };
layout(binding = BINDING_INPUT_BUFFER, std430) buffer InputBufferU128 { uvec4 u128data[]; };
layout(binding = BINDING_OUTPUT_BUFFER, std430) writeonly buffer OutputBuffer {
uint out_u32[];
};
layout(local_size_x = 8, local_size_y = 8, local_size_z = 4) in;
const uint GOB_SIZE_X = 64;
const uint GOB_SIZE_Y = 8;
const uint GOB_SIZE_X_SHIFT = 6;
const uint GOB_SIZE_Y_SHIFT = 3;
const uint GOB_SIZE_SHIFT = GOB_SIZE_X_SHIFT + GOB_SIZE_Y_SHIFT;
const uvec2 SWIZZLE_MASK = uvec2(GOB_SIZE_X - 1u, GOB_SIZE_Y - 1u);
uint SwizzleTable(uint pos) {
const uint t[8] = uint[](
0x12100200, 0x13110301, 0x16140604, 0x17150705,
0x1a180a08, 0x1b190b09, 0x1e1c0e0c, 0x1f1d0f0d
);
const uint i = pos >> 4;
const uint h = (t[i / 4] >> ((i % 4) * 8)) & 0xff;
return (h << 4) | (pos & 0xf);
}
uint SwizzleOffset(uvec2 pos) {
pos = pos & SWIZZLE_MASK;
return SwizzleTable(pos.y * 64u + pos.x);
}
uvec4 ReadTexel(uint offset) {
switch (pc.bytes_per_block_log2) {
case 2u:
return uvec4(u32data[offset / 4u], 0u, 0u, 0u);
case 3u:
return uvec4(u64data[offset / 8u], 0u, 0u);
case 4u:
return u128data[offset / 16u];
}
return uvec4(0u);
}
void main() {
uvec3 coord = gl_GlobalInvocationID;
if (coord.x >= pc.dim.x || coord.y >= pc.dim.y || coord.z >= pc.dim.z) {
return;
}
uvec3 pos = coord + pc.origin;
pos.x <<= pc.bytes_per_block_log2;
uint swizzle = SwizzleOffset(pos.xy);
uint block_y = pos.y >> GOB_SIZE_Y_SHIFT;
uint offset = 0u;
offset += (pos.z >> pc.block_depth) * pc.slice_size;
offset += (pos.z & pc.block_depth_mask) << (GOB_SIZE_SHIFT + pc.block_height);
offset += (block_y >> pc.block_height) * pc.block_size;
offset += (block_y & pc.block_height_mask) << GOB_SIZE_SHIFT;
offset += (pos.x >> GOB_SIZE_X_SHIFT) << pc.x_shift;
offset += swizzle;
uvec4 texel = ReadTexel(offset);
uint words = 1u << (pc.bytes_per_block_log2 - 2u);
uint linear_index = coord.x + coord.y * pc.dim.x + coord.z * pc.dim.x * pc.dim.y;
uint out_idx = linear_index * words;
out_u32[out_idx] = texel.x;
if (words > 1u) {
out_u32[out_idx + 1u] = texel.y;
}
if (words > 2u) {
out_u32[out_idx + 2u] = texel.z;
out_u32[out_idx + 3u] = texel.w;
}
}
@@ -6,9 +6,9 @@
precision highp float;
precision highp int;
// Operation modes: RGBA -> 1, RGBY -> 3, LERP -> 4
#define OPERATION_MODE 1
#define EDGE_THRESHOLD (8.0 / 255.0)
#define DIRECTION_EPSILON 6.5e-05
#define DEVIATION_FLOOR 6.0e-02
layout(push_constant) uniform constants {
vec2 scale;
@@ -21,20 +21,37 @@ layout(set = 0, binding = 0) uniform sampler2D sampler0;
layout(location=0) in vec2 texcoord;
layout(location=0) out vec4 frag_color;
vec4 weightY(vec4 dx, vec4 dy, vec4 std) {
vec4 x = ((dx * dx) + (dy * dy)) * 0.55f + std;
return (x - 1.f) * (x - 4.f) * 3.8125f; // approx. of (x - 1) * (x - 4)^3
mediump vec4 fastLanczos2(mediump vec4 x) {
mediump vec4 wA = x - 4.0f;
mediump vec4 wB = x * wA - wA;
wA *= wA;
return wB * wA;
}
mediump vec2 edgeDirection(mediump vec4 left, mediump vec4 right) {
mediump float RxLz = right.x - left.z;
mediump float RwLy = right.w - left.y;
mediump vec2 delta = vec2(RxLz + RwLy, RxLz - RwLy);
mediump float length_inv =
inversesqrt((delta.x * delta.x + DIRECTION_EPSILON) + delta.y * delta.y);
return delta * length_inv;
}
mediump vec4 weightY(mediump vec4 dx, mediump vec4 dy, mediump vec4 c, mediump float std,
mediump vec2 dir) {
mediump vec4 edge_dis = dx * dir.y + dy * dir.x;
mediump vec4 x = (dx * dx + dy * dy) +
(edge_dis * edge_dis) * (clamp((c * c) * std, 0.0f, 1.0f) * 0.7f - 1.0f);
return fastLanczos2(x);
}
void main() {
vec4 color = textureLod(sampler0, texcoord.xy, 0.0f);
// image coord
vec2 icoord = (texcoord * size + vec2(-0.5f, 0.5f));
vec2 icoord_pixel = floor(icoord);
vec2 coord = icoord_pixel * scale;
vec2 pl = icoord - icoord_pixel;
// left: 0, right: 1, upDown: 2
mat3x4 dg = mat3x4(
mediump vec4 color = textureLod(sampler0, texcoord.xy, 0.0f);
highp vec2 icoord = (texcoord * size + vec2(-0.5f, 0.5f));
highp vec2 icoord_pixel = floor(icoord);
highp vec2 coord = icoord_pixel * scale;
mediump vec2 pl = icoord - icoord_pixel;
mediump mat3x4 dg = mat3x4(
textureGather(sampler0, coord, 1),
textureGather(sampler0, coord + vec2(2.f * scale.x, 0.0f), 1),
vec4(
@@ -42,42 +59,40 @@ void main() {
textureGather(sampler0, coord + vec2(scale.x, +scale.y), 1).yx
)
);
float edgeVote = abs(dg[0].z - dg[0].y) + abs(color.y - dg[0].y) + abs(color.y - dg[0].z);
mediump float edgeVote =
abs(dg[0].z - dg[0].y) + abs(color.y - dg[0].y) + abs(color.y - dg[0].z);
if (edgeVote > EDGE_THRESHOLD) {
float mean = (dg[0].y + dg[0].z + dg[1].x + dg[1].w) * 0.25f;
mediump float mean = (dg[0].y + dg[0].z + dg[1].x + dg[1].w) * 0.25f;
dg = dg - mean;
vec4 sum = abs(dg[0]) + abs(dg[1]) + abs(dg[2]);
float std = 2.181818f / (sum.x + sum.y + sum.z + sum.w);
mat2x4 w = mat2x4(
weightY(
pl.xxxx + vec4(+1.0f, +0.0f, +0.0f, +1.0f),
pl.yyyy + vec4(-1.0f, -1.0f, +0.0f, +0.0f),
clamp(abs(dg[0]) * std, 0.0f, 1.0f)
) + weightY(
pl.xxxx + vec4(-1.0f, -2.0f, -2.0f, -1.0f),
pl.yyyy + vec4(-1.0f, -1.0f, +0.0f, +0.0f),
clamp(abs(dg[1]) * std, 0.0f, 1.0f)
) + weightY(
pl.xxxx + vec4(+0.0f, -1.0f, -1.0f, +0.0f),
pl.yyyy + vec4(+1.0f, +1.0f, -2.0f, -2.0f),
clamp(abs(dg[2]) * std, 0.0f, 1.0f)
),
dg[0] + dg[1] + dg[2]
mediump float sum = dot(abs(dg[0]) + abs(dg[1]) + abs(dg[2]), vec4(1.0f));
mediump float sum_mean = 1.014185e+01f / max(sum, DEVIATION_FLOOR);
mediump float std = sum_mean * sum_mean;
mediump vec2 dir = edgeDirection(dg[0], dg[1]);
mediump vec4 w0 = weightY(
pl.xxxx + vec4(+1.0f, +0.0f, +0.0f, +1.0f),
pl.yyyy + vec4(-1.0f, -1.0f, +0.0f, +0.0f),
dg[0], std, dir
);
// compute final y with bounds
vec2 yb = vec2(
min(min(dg[0].y, dg[0].z), min(dg[1].x, dg[1].w)), // min
max(max(dg[0].y, dg[0].z), max(dg[1].x, dg[1].w)) // max
mediump vec4 w1 = weightY(
pl.xxxx + vec4(-1.0f, -2.0f, -2.0f, -1.0f),
pl.yyyy + vec4(-1.0f, -1.0f, +0.0f, +0.0f),
dg[1], std, dir
);
vec2 fvy = vec2(
w[0].x + w[0].y + w[0].z + w[0].w,
w[1].x + w[1].y + w[1].z + w[1].w
mediump vec4 w2 = weightY(
pl.xxxx + vec4(+0.0f, -1.0f, -1.0f, +0.0f),
pl.yyyy + vec4(+1.0f, +1.0f, -2.0f, -2.0f),
dg[2], std, dir
);
float fy = clamp((fvy.y / fvy.x) * edge_sharpness, yb[0], yb[1]);
// Smooth high contrast input
float dy = clamp(fy - color.y + mean, -23.0f / 255.0f, 23.0f / 255.0f);
mediump float sum_w = dot(w0 + w1 + w2, vec4(1.0f));
mediump float sum_wc = dot(w0 * dg[0] + w1 * dg[1] + w2 * dg[2], vec4(1.0f));
mediump vec2 yb = vec2(
min(min(dg[0].y, dg[0].z), min(dg[1].x, dg[1].w)),
max(max(dg[0].y, dg[0].z), max(dg[1].x, dg[1].w))
);
mediump float fy = clamp((sum_wc / sum_w) * edge_sharpness, yb[0], yb[1]);
mediump float dy = clamp(fy - color.y + mean, -23.0f / 255.0f, 23.0f / 255.0f);
color = clamp(color + dy, 0.0f, 1.0f);
}
color.w = 1.0f; //assume alpha channel is not used
color.w = 1.0f;
frag_color.xyzw = color;
}
}
@@ -261,6 +261,7 @@ struct BufferCacheParams {
// TODO: Investigate why OpenGL seems to perform worse with persistently mapped buffer uploads
static constexpr bool USE_MEMORY_MAPS_FOR_UPLOADS = false;
static constexpr bool USE_UNIFIED_MEMORY = false;
};
using BufferCache = VideoCommon::BufferCache<BufferCacheParams>;
@@ -231,12 +231,14 @@ ShaderCache::ShaderCache(Tegra::MaxwellDeviceMemoryManager& device_memory_,
.has_broken_unsigned_image_offsets = true,
.has_broken_signed_operations = true,
.has_broken_fp16_float_controls = false,
.has_broken_fp32_denorm_flush = false,
.has_gl_component_indexing_bug = device.HasComponentIndexingBug(),
.has_gl_precise_bug = device.HasPreciseBug(),
.has_gl_cbuf_ftou_bug = device.HasCbufFtouBug(),
.has_gl_bool_ref_bug = device.HasBoolRefBug(),
.ignore_nan_fp_comparisons = true,
.gl_max_compute_smem_size = device.GetMaxComputeSharedMemorySize(),
.max_shared_memory_size = device.GetMaxComputeSharedMemorySize(),
.min_ssbo_alignment = device.GetShaderStorageBufferAlignment(),
// Use the host limit, but never more than the guest can produce. Maxwell exposes 8 clip
// distances and the SPIR-V output array is sized for at most 8, so clamping here keeps a
@@ -373,6 +373,7 @@ struct TextureCacheParams {
static constexpr bool HAS_DEVICE_MEMORY_INFO = true;
static constexpr bool IMPLEMENTS_ASYNC_DOWNLOADS = true;
static constexpr bool HAS_MSAA_DOWNLOADS = false;
static constexpr bool USE_UNIFIED_MEMORY = false;
using Runtime = OpenGL::TextureCacheRuntime;
using Image = OpenGL::Image;
@@ -370,20 +370,9 @@ inline void PushImageDescriptors(TextureCache& texture_cache,
const VkImageView null_image_view{texture_cache.GetImageView(VideoCommon::NULL_IMAGE_VIEW_ID).Handle(desc.type)};
if (null_image_view != VK_NULL_HANDLE) vk_image_view = null_image_view;
}
const Sampler& sampler{texture_cache.GetSampler(sampler_id)};
const bool use_fallback_sampler{sampler.HasAddedAnisotropy() &&
!image_view.SupportsAnisotropy()};
VkSampler vk_sampler{use_fallback_sampler ? sampler.HandleWithDefaultAnisotropy()
: sampler.Handle()};
if (sampler.HasLinearFiltering() &&
VideoCore::Surface::IsPixelFormatInteger(image_view.format)) {
vk_sampler = sampler.HandleWithNearestFilter();
}
if (desc.is_depth && sampler.HasDepthComparison() &&
!image_view.SupportsDepthComparison()) {
vk_sampler = sampler.HandleWithoutDepthComparison();
}
guest_descriptor_queue.AddSampledImage(vk_image_view, vk_sampler);
Sampler& sampler{texture_cache.GetSampler(sampler_id)};
guest_descriptor_queue.AddSampledImage(vk_image_view,
sampler.HandleFor(image_view, desc.is_depth));
const bool element_rescaled{texture_cache.IsRescaling(image_view)};
is_rescaled |= element_rescaled;
}
@@ -7,7 +7,9 @@
#include <algorithm>
#include <array>
#include <cstring>
#include <limits>
#include <span>
#include <utility>
#include <vector>
#include "video_core/buffer_cache/buffer_cache_base.h"
@@ -32,6 +34,32 @@ VkBufferCopy MakeBufferCopy(const VideoCommon::BufferCopy& copy) {
};
}
constexpr size_t MAX_WINDOW_BARRIER_RANGES = 8;
using WindowRange = std::pair<VkDeviceSize, VkDeviceSize>;
using WindowRanges = boost::container::small_vector<WindowRange, MAX_WINDOW_BARRIER_RANGES>;
void CoalesceWindowRanges(WindowRanges& ranges) {
if (ranges.size() < 2) {
return;
}
std::sort(ranges.begin(), ranges.end());
size_t merged = 0;
for (size_t index = 1; index < ranges.size(); ++index) {
if (ranges[index].first <= ranges[merged].second) {
ranges[merged].second = (std::max)(ranges[merged].second, ranges[index].second);
} else {
ranges[++merged] = ranges[index];
}
}
ranges.resize(merged + 1);
if (ranges.size() > MAX_WINDOW_BARRIER_RANGES) {
const WindowRange bounding{ranges.front().first, ranges.back().second};
ranges.clear();
ranges.push_back(bounding);
}
}
VkIndexType IndexTypeFromNumElements(const Device& device, u32 num_elements) {
if (num_elements <= 0xff && device.IsExtIndexTypeUint8Supported()) {
return VK_INDEX_TYPE_UINT8_EXT;
@@ -42,6 +70,16 @@ VkIndexType IndexTypeFromNumElements(const Device& device, u32 num_elements) {
return VK_INDEX_TYPE_UINT32;
}
u32 GrowIndexCount(u32 current, u32 requested) {
constexpr u32 MinimumIndices = 4096;
constexpr u32 GrowthLimit = (std::numeric_limits<u32>::max)() / 2;
u32 grown = (std::max)(requested, MinimumIndices);
if (current <= GrowthLimit) {
grown = (std::max)(grown, current * 2);
}
return grown;
}
size_t BytesPerIndex(VkIndexType index_type) {
switch (index_type) {
case VK_INDEX_TYPE_UINT8_EXT:
@@ -158,13 +196,12 @@ public:
virtual ~QuadIndexBuffer() = default;
void UpdateBuffer(u32 num_indices_) {
ReleaseRetiredBuffers();
if (num_indices_ <= num_indices) {
return;
}
scheduler.Finish();
num_indices = num_indices_;
num_indices = GrowIndexCount(num_indices, num_indices_);
index_type = IndexTypeFromNumElements(device, num_indices);
const u32 num_quads = GetQuadsNum(num_indices);
@@ -182,6 +219,12 @@ public:
.queueFamilyIndexCount = 0,
.pQueueFamilyIndices = nullptr,
};
if (buffer) {
retired_buffers.push_back(RetiredBuffer{
.buffer = std::move(buffer),
.tick = scheduler.CurrentTick(),
});
}
buffer = memory_allocator.CreateBuffer(buffer_ci, MemoryUsage::DeviceLocal);
if (device.HasDebuggingToolAttached()) {
buffer.SetObjectNameEXT("Quad LUT");
@@ -249,6 +292,17 @@ protected:
virtual void MakeAndUpdateIndices(u8* staging_data, size_t quad_size, u32 quad, u32 first) = 0;
struct RetiredBuffer {
vk::Buffer buffer;
u64 tick;
};
void ReleaseRetiredBuffers() {
std::erase_if(retired_buffers, [this](const RetiredBuffer& entry) {
return scheduler.IsFree(entry.tick);
});
}
const Device& device;
MemoryAllocator& memory_allocator;
Scheduler& scheduler;
@@ -256,6 +310,7 @@ protected:
vk::Buffer buffer{};
MemoryCommit memory_commit{};
std::vector<RetiredBuffer> retired_buffers;
VkIndexType index_type{};
u32 num_indices = 0;
};
@@ -365,6 +420,138 @@ BufferCacheRuntime::BufferCacheRuntime(const Device& device_, MemoryAllocator& m
scheduler_, staging_pool_);
}
void BufferCacheRuntime::TryEnableUnifiedMemory(void* base, size_t size,
std::span<AHardwareBuffer* const> hardware_buffers,
size_t hardware_buffer_window,
size_t hardware_buffer_base) {
unified_memory = memory_allocator.CreateHostMemoryImport(
base, size, hardware_buffers, hardware_buffer_window, hardware_buffer_base);
}
void BufferCacheRuntime::CopyToUnifiedMemory(
size_t window_index, VkBuffer src_buffer,
std::span<const VideoCommon::BufferCopy> copies) {
if (!unified_memory || src_buffer == VK_NULL_HANDLE || copies.empty() ||
window_index >= unified_memory->GetWindowCount() ||
unified_memory->GetWindowBuffer(window_index) == VK_NULL_HANDLE) {
return;
}
PendingUnifiedCopy& pending = pending_unified_copies.emplace_back();
pending.window = window_index;
pending.buffer = src_buffer;
pending.copies.resize(copies.size());
std::ranges::transform(copies, pending.copies.begin(), MakeBufferCopy);
}
void BufferCacheRuntime::FlushUnifiedMemoryCopies() {
if (pending_unified_copies.empty()) {
return;
}
struct UnifiedCopyCommand {
VkBuffer buffer;
boost::container::small_vector<VkBufferCopy, 8> copies;
};
std::stable_sort(pending_unified_copies.begin(), pending_unified_copies.end(),
[](const PendingUnifiedCopy& lhs, const PendingUnifiedCopy& rhs) {
return lhs.window < rhs.window;
});
const bool foreign = unified_memory->NeedsForeignOwnershipTransfer();
const u32 queue_family = device.GetGraphicsFamily();
size_t group_begin = 0;
while (group_begin < pending_unified_copies.size()) {
const size_t window = pending_unified_copies[group_begin].window;
size_t group_end = group_begin;
while (group_end < pending_unified_copies.size() &&
pending_unified_copies[group_end].window == window) {
++group_end;
}
const VkBuffer window_buffer = unified_memory->GetWindowBuffer(window);
WindowRanges ranges;
for (size_t index = group_begin; index < group_end; ++index) {
for (const VkBufferCopy& copy : pending_unified_copies[index].copies) {
ranges.emplace_back(copy.dstOffset, copy.dstOffset + copy.size);
}
}
CoalesceWindowRanges(ranges);
boost::container::small_vector<VkBufferMemoryBarrier, MAX_WINDOW_BARRIER_RANGES> acquire;
boost::container::small_vector<VkBufferMemoryBarrier, MAX_WINDOW_BARRIER_RANGES> release;
if (foreign) {
for (const WindowRange& range : ranges) {
acquire.push_back(VkBufferMemoryBarrier{
.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER,
.pNext = nullptr,
.srcAccessMask = 0,
.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_FOREIGN_EXT,
.dstQueueFamilyIndex = queue_family,
.buffer = window_buffer,
.offset = range.first,
.size = range.second - range.first,
});
release.push_back(VkBufferMemoryBarrier{
.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER,
.pNext = nullptr,
.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,
.dstAccessMask = 0,
.srcQueueFamilyIndex = queue_family,
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_FOREIGN_EXT,
.buffer = window_buffer,
.offset = range.first,
.size = range.second - range.first,
});
}
}
boost::container::small_vector<UnifiedCopyCommand, 4> commands;
commands.reserve(group_end - group_begin);
for (size_t index = group_begin; index < group_end; ++index) {
PendingUnifiedCopy& pending = pending_unified_copies[index];
commands.push_back(UnifiedCopyCommand{pending.buffer, std::move(pending.copies)});
}
scheduler.RequestOutsideRenderPassOperationContext();
scheduler.Record([window_buffer, acquire = std::move(acquire), release = std::move(release),
commands = std::move(commands)](vk::CommandBuffer cmdbuf) {
if (!acquire.empty()) {
cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
VK_PIPELINE_STAGE_TRANSFER_BIT, 0, {},
VideoCommon::FixSmallVectorADL(acquire), {});
}
for (const UnifiedCopyCommand& command : commands) {
cmdbuf.CopyBuffer(command.buffer, window_buffer,
VideoCommon::FixSmallVectorADL(command.copies));
}
if (!release.empty()) {
cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT,
VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 0, {},
VideoCommon::FixSmallVectorADL(release), {});
}
});
group_begin = group_end;
}
pending_unified_copies.clear();
}
void BufferCacheRuntime::UnifiedMemoryHostBarrier() {
static constexpr VkMemoryBarrier HOST_BARRIER{
.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
.pNext = nullptr,
.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,
.dstAccessMask = VK_ACCESS_HOST_READ_BIT,
};
scheduler.RequestOutsideRenderPassOperationContext();
scheduler.Record([](vk::CommandBuffer cmdbuf) {
cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, 0,
HOST_BARRIER);
});
}
StagingBufferRef BufferCacheRuntime::UploadStagingBuffer(size_t size) {
return staging_pool.Request(size, MemoryUsage::Upload);
}
@@ -402,6 +589,7 @@ u32 BufferCacheRuntime::GetStorageBufferAlignment() const {
}
void BufferCacheRuntime::TickFrame(Common::SlotVector<Buffer>& slot_buffers) noexcept {
FlushUnifiedMemoryCopies();
for (auto it = slot_buffers.begin(); it != slot_buffers.end(); it++) {
if (scheduler.IsFree(it->LastUsageTick())) {
it->ResetUsageTracking();
@@ -7,6 +7,10 @@
#pragma once
#include <limits>
#include <memory>
#include <span>
#include <boost/container/small_vector.hpp>
#include "video_core/buffer_cache/buffer_cache_base.h"
#include "video_core/buffer_cache/memory_tracker_base.h"
@@ -97,6 +101,33 @@ public:
void TickFrame(Common::SlotVector<Buffer>& slot_buffers) noexcept;
void TryEnableUnifiedMemory(void* base, size_t size,
std::span<AHardwareBuffer* const> hardware_buffers,
size_t hardware_buffer_window, size_t hardware_buffer_base);
[[nodiscard]] bool HasUnifiedMemory() const noexcept {
return unified_memory != nullptr && unified_memory->IsValid();
}
[[nodiscard]] u64 UnifiedMemorySize() const noexcept {
return unified_memory ? unified_memory->GetSize() : 0;
}
[[nodiscard]] u64 UnifiedMemoryBase() const noexcept {
return unified_memory ? unified_memory->GetBaseOffset() : 0;
}
[[nodiscard]] u64 UnifiedMemoryWindowSize() const noexcept {
return unified_memory ? unified_memory->GetWindowSize() : 0;
}
void CopyToUnifiedMemory(size_t window_index, VkBuffer src_buffer,
std::span<const VideoCommon::BufferCopy> copies);
void FlushUnifiedMemoryCopies();
void UnifiedMemoryHostBarrier();
u64 CurrentTick();
u64 KnownGpuTick();
@@ -164,6 +195,29 @@ public:
BindBuffer(buffer, offset, size);
}
void BindStorageBuffer(VkBuffer buffer, VkDeviceAddress address, u32 offset, u32 size,
[[maybe_unused]] bool is_written) {
guest_descriptor_queue.AddBuffer(buffer, address, offset, size);
}
[[nodiscard]] bool IsUnifiedMemoryBindable() const noexcept {
return unified_memory != nullptr && unified_memory->IsValid() &&
unified_memory->IsBindable();
}
[[nodiscard]] VkBuffer UnifiedWindowBuffer(size_t index) const noexcept {
return unified_memory->GetWindowBuffer(index);
}
[[nodiscard]] VkDeviceAddress UnifiedWindowAddress(size_t index) const noexcept {
return unified_memory->GetWindowAddress(index);
}
[[nodiscard]] bool IsUnifiedStorageRange(u32 size, u64 offset) const {
return size <= device.GetMaxStorageBufferRange() &&
(offset % device.GetStorageBufferAlignment()) == 0;
}
void BindTextureBuffer(Buffer& buffer, u32 offset, u32 size,
VideoCore::Surface::PixelFormat format) {
guest_descriptor_queue.AddTexelBuffer(buffer.View(offset, size, format),
@@ -180,6 +234,12 @@ public:
}
private:
struct PendingUnifiedCopy {
size_t window;
VkBuffer buffer;
boost::container::small_vector<VkBufferCopy, 8> copies;
};
void BindBuffer(const Buffer& buffer, u32 offset, u32 size) {
const VkBuffer handle = buffer.Handle();
if (handle == VK_NULL_HANDLE) {
@@ -204,6 +264,8 @@ private:
std::shared_ptr<QuadStripIndexBuffer> quad_strip_index_buffer;
vk::Buffer null_buffer;
HostMemoryImport* unified_memory{};
boost::container::small_vector<PendingUnifiedCopy, 8> pending_unified_copies;
std::unique_ptr<Uint8Pass> uint8_pass;
QuadIndexedPass quad_index_pass;
@@ -226,6 +288,7 @@ struct BufferCacheParams {
static constexpr bool USE_MEMORY_MAPS = true;
static constexpr bool SEPARATE_IMAGE_BUFFER_BINDINGS = false;
static constexpr bool USE_MEMORY_MAPS_FOR_UPLOADS = true;
static constexpr bool USE_UNIFIED_MEMORY = true;
};
using BufferCache = VideoCommon::BufferCache<BufferCacheParams>;
@@ -4,6 +4,7 @@
// SPDX-FileCopyrightText: Copyright 2019 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#include <algorithm>
#include <array>
#include <memory>
#include <numeric>
@@ -12,6 +13,7 @@
#include "video_core/renderer_vulkan/vk_texture_cache.h"
#include "common/alignment.h"
#include "common/assert.h"
#include "common/common_types.h"
#include "common/div_ceil.h"
@@ -22,7 +24,9 @@
#include "video_core/host_shaders/resolve_conditional_render_comp_spv.h"
#include "video_core/host_shaders/vulkan_quad_indexed_comp_spv.h"
#include "video_core/host_shaders/vulkan_uint8_comp_spv.h"
#include "video_core/host_shaders/block_linear_unswizzle_2d_buffer_comp_spv.h"
#include "video_core/host_shaders/block_linear_unswizzle_3d_bcn_comp_spv.h"
#include "video_core/host_shaders/block_linear_unswizzle_3d_buffer_comp_spv.h"
#include "video_core/renderer_vulkan/vk_compute_pass.h"
#include "video_core/surface.h"
#include "video_core/renderer_vulkan/vk_descriptor_pool.h"
@@ -872,4 +876,537 @@ void BlockLinearUnswizzle3DPass::UnswizzleChunk(
});
}
namespace {
constexpr u32 BL2D_BINDING_INPUT_BUFFER = 0;
constexpr u32 BL2D_BINDING_OUTPUT_BUFFER = 1;
struct alignas(16) BlockLinearUnswizzle2DPushConstants {
std::array<u32, 3> dim;
u32 bytes_per_block_log2;
std::array<u32, 3> origin;
u32 layer_stride;
u32 block_size;
u32 x_shift;
u32 block_height;
u32 block_height_mask;
};
static_assert(sizeof(BlockLinearUnswizzle2DPushConstants) <= 128);
constexpr std::array<VkDescriptorSetLayoutBinding, 2> BL2D_BINDINGS{{
{
.binding = BL2D_BINDING_INPUT_BUFFER,
.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
.descriptorCount = 1,
.stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
.pImmutableSamplers = nullptr,
},
{
.binding = BL2D_BINDING_OUTPUT_BUFFER,
.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
.descriptorCount = 1,
.stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
.pImmutableSamplers = nullptr,
},
}};
constexpr std::array<VkDescriptorUpdateTemplateEntry, 2> BL2D_TEMPLATE{{
{
.dstBinding = BL2D_BINDING_INPUT_BUFFER,
.dstArrayElement = 0,
.descriptorCount = 1,
.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
.offset = BL2D_BINDING_INPUT_BUFFER * sizeof(DescriptorUpdateEntry),
.stride = sizeof(DescriptorUpdateEntry),
},
{
.dstBinding = BL2D_BINDING_OUTPUT_BUFFER,
.dstArrayElement = 0,
.descriptorCount = 1,
.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
.offset = BL2D_BINDING_OUTPUT_BUFFER * sizeof(DescriptorUpdateEntry),
.stride = sizeof(DescriptorUpdateEntry),
},
}};
constexpr DescriptorBankInfo BL2D_BANK_INFO{
.uniform_buffers = 0,
.storage_buffers = 2,
.texture_buffers = 0,
.image_buffers = 0,
.textures = 0,
.images = 0,
.score = 2,
};
} // Anonymous namespace
BlockLinearUnswizzle2DPass::BlockLinearUnswizzle2DPass(
const Device& device_, Scheduler& scheduler_, DescriptorPool& descriptor_pool_,
StagingBufferPool& staging_buffer_pool_,
ComputePassDescriptorQueue& compute_pass_descriptor_queue_)
: ComputePass(device_, scheduler_, descriptor_pool_, BL2D_BINDINGS, BL2D_TEMPLATE,
BL2D_BANK_INFO,
COMPUTE_PUSH_CONSTANT_RANGE<sizeof(BlockLinearUnswizzle2DPushConstants)>,
BLOCK_LINEAR_UNSWIZZLE_2D_BUFFER_COMP_SPV),
scheduler{scheduler_}, staging_buffer_pool{staging_buffer_pool_},
compute_pass_descriptor_queue{compute_pass_descriptor_queue_} {}
BlockLinearUnswizzle2DPass::~BlockLinearUnswizzle2DPass() = default;
bool BlockLinearUnswizzle2DPass::IsSupported(const Device& device,
const VideoCommon::ImageInfo& info) {
if (info.type != VideoCommon::ImageType::e2D) {
return false;
}
if (info.num_samples > 1) {
return false;
}
if (device.GetStorageBufferAlignment() > Tegra::Texture::GOB_SIZE) {
return false;
}
if (VideoCore::Surface::GetFormatType(info.format) !=
VideoCore::Surface::SurfaceType::ColorTexture) {
return false;
}
if (VideoCore::Surface::IsPixelFormatASTC(info.format) && !device.IsOptimalAstcSupported()) {
return false;
}
if (VideoCore::Surface::IsPixelFormatBCn(info.format) && !device.IsOptimalBcnSupported()) {
return false;
}
const u32 bytes_per_block = VideoCore::Surface::BytesPerBlock(info.format);
return bytes_per_block == 4 || bytes_per_block == 8 || bytes_per_block == 16;
}
void BlockLinearUnswizzle2DPass::Unswizzle(
Image& image, const StagingBufferRef& swizzled,
std::span<const VideoCommon::SwizzleParameters> swizzles) {
UnswizzleFrom(image, swizzled.buffer, swizzled.offset, swizzles);
}
void BlockLinearUnswizzle2DPass::UnswizzleFrom(
Image& image, VkBuffer source_buffer, VkDeviceSize source_offset,
std::span<const VideoCommon::SwizzleParameters> swizzles) {
const VkImage dst_image = image.Handle();
if (swizzles.empty() || source_buffer == VK_NULL_HANDLE || dst_image == VK_NULL_HANDLE) {
return;
}
const u32 layers = image.info.resources.layers;
const VkImageAspectFlags aspect = image.AspectMask();
const VkDeviceSize output_alignment =
(std::max)(device.GetStorageBufferAlignment(), VkDeviceSize{16});
VkDeviceSize total_output = 0;
for (const VideoCommon::SwizzleParameters& sw : swizzles) {
const auto params =
VideoCommon::Accelerated::MakeBlockLinearSwizzle2DParams(sw, image.info);
const VkDeviceSize level_size = static_cast<VkDeviceSize>(sw.num_tiles.width) *
sw.num_tiles.height * layers *
(1ULL << params.bytes_per_block_log2);
if (level_size == 0) {
continue;
}
total_output = Common::AlignUp(total_output, output_alignment) + level_size;
}
if (total_output == 0) {
return;
}
const StagingBufferRef output =
staging_buffer_pool.Request(static_cast<size_t>(total_output), MemoryUsage::DeviceLocal);
const VkBuffer out_buffer = output.buffer;
if (out_buffer == VK_NULL_HANDLE) {
return;
}
VkDeviceSize level_offset = 0;
scheduler.RequestOutsideRenderPassOperationContext();
VkAccessFlags pre_access = VK_ACCESS_NONE;
VkImageLayout pre_layout = VK_IMAGE_LAYOUT_UNDEFINED;
VkPipelineStageFlags pre_stage = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT;
if (image.ExchangeInitialization()) {
pre_access = VK_ACCESS_SHADER_READ_BIT;
pre_layout = VK_IMAGE_LAYOUT_GENERAL;
pre_stage = vk::PIPELINE_STAGE_GRAPHICS_COMPUTE;
}
scheduler.Record([dst_image, aspect, pre_access, pre_layout,
pre_stage](vk::CommandBuffer cmdbuf) {
const VkImageMemoryBarrier barrier{
.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
.pNext = nullptr,
.srcAccessMask = pre_access,
.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,
.oldLayout = pre_layout,
.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.image = dst_image,
.subresourceRange{
.aspectMask = aspect,
.baseMipLevel = 0,
.levelCount = VK_REMAINING_MIP_LEVELS,
.baseArrayLayer = 0,
.layerCount = VK_REMAINING_ARRAY_LAYERS,
},
};
cmdbuf.PipelineBarrier(pre_stage, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, {}, {}, barrier);
});
for (const VideoCommon::SwizzleParameters& sw : swizzles) {
const auto params =
VideoCommon::Accelerated::MakeBlockLinearSwizzle2DParams(sw, image.info);
const u32 width = sw.num_tiles.width;
const u32 height = sw.num_tiles.height;
const u32 bytes_per_block = 1u << params.bytes_per_block_log2;
const VkDeviceSize output_size =
static_cast<VkDeviceSize>(width) * height * layers * bytes_per_block;
if (output_size == 0) {
continue;
}
const u32 level = static_cast<u32>(sw.level);
const u32 texel_width = (std::max)(1u, image.info.size.width >> level);
const u32 texel_height = (std::max)(1u, image.info.size.height >> level);
level_offset = Common::AlignUp(level_offset, output_alignment);
const VkDeviceSize out_offset = output.offset + level_offset;
level_offset += output_size;
BlockLinearUnswizzle2DPushConstants pc{};
pc.dim = {width, height, layers};
pc.bytes_per_block_log2 = params.bytes_per_block_log2;
pc.origin = params.origin;
pc.layer_stride = params.layer_stride;
pc.block_size = params.block_size;
pc.x_shift = params.x_shift;
pc.block_height = params.block_height;
pc.block_height_mask = params.block_height_mask;
compute_pass_descriptor_queue.Acquire(scheduler, 2);
compute_pass_descriptor_queue.AddBuffer(source_buffer, sw.buffer_offset + source_offset,
image.guest_size_bytes - sw.buffer_offset);
compute_pass_descriptor_queue.AddBuffer(out_buffer, out_offset, output_size);
const void* descriptor_data = compute_pass_descriptor_queue.UpdateData();
const VkDescriptorSet set = descriptor_allocator.Commit();
const u32 gx = Common::DivCeil(width, 16u);
const u32 gy = Common::DivCeil(height, 8u);
scheduler.Record([this, set, descriptor_data, pc, gx, gy, layers, output_size, out_buffer,
out_offset, dst_image, aspect, texel_width, texel_height,
level](vk::CommandBuffer cmdbuf) {
device.GetLogical().UpdateDescriptorSet(set, *descriptor_template, descriptor_data);
cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_COMPUTE, *pipeline);
cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_COMPUTE, *layout, 0, set, {});
cmdbuf.PushConstants(*layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc);
cmdbuf.Dispatch(gx, gy, layers);
const VkBufferMemoryBarrier buffer_barrier{
.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER,
.pNext = nullptr,
.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT,
.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.buffer = out_buffer,
.offset = out_offset,
.size = output_size,
};
cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
VK_PIPELINE_STAGE_TRANSFER_BIT, 0, {}, buffer_barrier, {});
const VkBufferImageCopy copy{
.bufferOffset = out_offset,
.bufferRowLength = 0,
.bufferImageHeight = 0,
.imageSubresource{
.aspectMask = aspect,
.mipLevel = level,
.baseArrayLayer = 0,
.layerCount = layers,
},
.imageOffset = {0, 0, 0},
.imageExtent = {texel_width, texel_height, 1},
};
cmdbuf.CopyBufferToImage(out_buffer, dst_image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
copy);
});
}
scheduler.Record([dst_image, aspect](vk::CommandBuffer cmdbuf) {
const VkImageMemoryBarrier barrier{
.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
.pNext = nullptr,
.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,
.dstAccessMask = VK_ACCESS_SHADER_READ_BIT,
.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
.newLayout = VK_IMAGE_LAYOUT_GENERAL,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.image = dst_image,
.subresourceRange{
.aspectMask = aspect,
.baseMipLevel = 0,
.levelCount = VK_REMAINING_MIP_LEVELS,
.baseArrayLayer = 0,
.layerCount = VK_REMAINING_ARRAY_LAYERS,
},
};
cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT,
vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, 0, {}, {}, barrier);
});
}
namespace {
constexpr u32 BL3DB_BINDING_INPUT_BUFFER = 0;
constexpr u32 BL3DB_BINDING_OUTPUT_BUFFER = 1;
struct alignas(16) BlockLinearUnswizzle3DBufferPushConstants {
std::array<u32, 3> dim;
u32 bytes_per_block_log2;
std::array<u32, 3> origin;
u32 slice_size;
u32 block_size;
u32 x_shift;
u32 block_height;
u32 block_height_mask;
u32 block_depth;
u32 block_depth_mask;
};
static_assert(sizeof(BlockLinearUnswizzle3DBufferPushConstants) <= 128);
constexpr std::array<VkDescriptorSetLayoutBinding, 2> BL3DB_BINDINGS{{
{
.binding = BL3DB_BINDING_INPUT_BUFFER,
.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
.descriptorCount = 1,
.stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
.pImmutableSamplers = nullptr,
},
{
.binding = BL3DB_BINDING_OUTPUT_BUFFER,
.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
.descriptorCount = 1,
.stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
.pImmutableSamplers = nullptr,
},
}};
constexpr std::array<VkDescriptorUpdateTemplateEntry, 2> BL3DB_TEMPLATE{{
{
.dstBinding = BL3DB_BINDING_INPUT_BUFFER,
.dstArrayElement = 0,
.descriptorCount = 1,
.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
.offset = BL3DB_BINDING_INPUT_BUFFER * sizeof(DescriptorUpdateEntry),
.stride = sizeof(DescriptorUpdateEntry),
},
{
.dstBinding = BL3DB_BINDING_OUTPUT_BUFFER,
.dstArrayElement = 0,
.descriptorCount = 1,
.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
.offset = BL3DB_BINDING_OUTPUT_BUFFER * sizeof(DescriptorUpdateEntry),
.stride = sizeof(DescriptorUpdateEntry),
},
}};
constexpr DescriptorBankInfo BL3DB_BANK_INFO{
.uniform_buffers = 0,
.storage_buffers = 2,
.texture_buffers = 0,
.image_buffers = 0,
.textures = 0,
.images = 0,
.score = 2,
};
} // Anonymous namespace
BlockLinearUnswizzle3DBufferPass::BlockLinearUnswizzle3DBufferPass(
const Device& device_, Scheduler& scheduler_, DescriptorPool& descriptor_pool_,
StagingBufferPool& staging_buffer_pool_,
ComputePassDescriptorQueue& compute_pass_descriptor_queue_)
: ComputePass(device_, scheduler_, descriptor_pool_, BL3DB_BINDINGS, BL3DB_TEMPLATE,
BL3DB_BANK_INFO,
COMPUTE_PUSH_CONSTANT_RANGE<sizeof(BlockLinearUnswizzle3DBufferPushConstants)>,
BLOCK_LINEAR_UNSWIZZLE_3D_BUFFER_COMP_SPV),
scheduler{scheduler_}, staging_buffer_pool{staging_buffer_pool_},
compute_pass_descriptor_queue{compute_pass_descriptor_queue_} {}
BlockLinearUnswizzle3DBufferPass::~BlockLinearUnswizzle3DBufferPass() = default;
bool BlockLinearUnswizzle3DBufferPass::IsSupported(const Device& device,
const VideoCommon::ImageInfo& info) {
if (info.type != VideoCommon::ImageType::e3D) {
return false;
}
if (info.resources.levels != 1 || info.resources.layers != 1) {
return false;
}
if (info.num_samples > 1) {
return false;
}
if (info.size.depth <= 1) {
return false;
}
if (VideoCore::Surface::GetFormatType(info.format) !=
VideoCore::Surface::SurfaceType::ColorTexture) {
return false;
}
if (VideoCore::Surface::IsPixelFormatASTC(info.format)) {
return false;
}
if (VideoCore::Surface::IsPixelFormatBCn(info.format) && !device.IsOptimalBcnSupported()) {
return false;
}
const u32 bytes_per_block = VideoCore::Surface::BytesPerBlock(info.format);
return bytes_per_block == 4 || bytes_per_block == 8 || bytes_per_block == 16;
}
void BlockLinearUnswizzle3DBufferPass::Unswizzle(
Image& image, const StagingBufferRef& swizzled,
std::span<const VideoCommon::SwizzleParameters> swizzles) {
if (swizzles.empty()) {
return;
}
const VideoCommon::SwizzleParameters& sw = swizzles.front();
const auto params = VideoCommon::Accelerated::MakeBlockLinearSwizzle3DParams(sw, image.info);
const u32 blocks_x = sw.num_tiles.width;
const u32 blocks_y = sw.num_tiles.height;
const u32 blocks_z = sw.num_tiles.depth;
const u32 bytes_per_block = 1u << params.bytes_per_block_log2;
const VkDeviceSize output_size =
static_cast<VkDeviceSize>(blocks_x) * blocks_y * blocks_z * bytes_per_block;
const StagingBufferRef output =
staging_buffer_pool.Request(static_cast<size_t>(output_size), MemoryUsage::DeviceLocal);
BlockLinearUnswizzle3DBufferPushConstants pc{};
pc.dim = {blocks_x, blocks_y, blocks_z};
pc.bytes_per_block_log2 = params.bytes_per_block_log2;
pc.origin = params.origin;
pc.slice_size = params.slice_size;
pc.block_size = params.block_size;
pc.x_shift = params.x_shift;
pc.block_height = params.block_height;
pc.block_height_mask = params.block_height_mask;
pc.block_depth = params.block_depth;
pc.block_depth_mask = params.block_depth_mask;
scheduler.RequestOutsideRenderPassOperationContext();
compute_pass_descriptor_queue.Acquire(scheduler, 2);
compute_pass_descriptor_queue.AddBuffer(swizzled.buffer, sw.buffer_offset + swizzled.offset,
image.guest_size_bytes - sw.buffer_offset);
compute_pass_descriptor_queue.AddBuffer(output.buffer, output.offset, output_size);
const void* descriptor_data = compute_pass_descriptor_queue.UpdateData();
const VkDescriptorSet set = descriptor_allocator.Commit();
const u32 gx = Common::DivCeil(blocks_x, 8u);
const u32 gy = Common::DivCeil(blocks_y, 8u);
const u32 gz = Common::DivCeil(blocks_z, 4u);
const bool is_initialized = image.ExchangeInitialization();
const VkBuffer out_buffer = output.buffer;
const VkDeviceSize out_offset = output.offset;
const VkImage dst_image = image.Handle();
const VkImageAspectFlags aspect = image.AspectMask();
const VkExtent3D extent{
.width = image.info.size.width,
.height = image.info.size.height,
.depth = image.info.size.depth,
};
scheduler.Record([this, set, descriptor_data, pc, gx, gy, gz, output_size, out_buffer,
out_offset, dst_image, aspect, extent,
is_initialized](vk::CommandBuffer cmdbuf) {
if (dst_image == VK_NULL_HANDLE || out_buffer == VK_NULL_HANDLE) {
return;
}
device.GetLogical().UpdateDescriptorSet(set, *descriptor_template, descriptor_data);
cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_COMPUTE, *pipeline);
cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_COMPUTE, *layout, 0, set, {});
cmdbuf.PushConstants(*layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc);
cmdbuf.Dispatch(gx, gy, gz);
const VkBufferMemoryBarrier buffer_barrier{
.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER,
.pNext = nullptr,
.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT,
.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.buffer = out_buffer,
.offset = out_offset,
.size = output_size,
};
VkAccessFlags pre_copy_access = VK_ACCESS_NONE;
VkImageLayout pre_copy_layout = VK_IMAGE_LAYOUT_UNDEFINED;
VkPipelineStageFlags pre_copy_stage = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT;
if (is_initialized) {
pre_copy_access = VK_ACCESS_SHADER_READ_BIT;
pre_copy_layout = VK_IMAGE_LAYOUT_GENERAL;
pre_copy_stage = vk::PIPELINE_STAGE_GRAPHICS_COMPUTE;
}
const VkImageMemoryBarrier pre_copy{
.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
.pNext = nullptr,
.srcAccessMask = pre_copy_access,
.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,
.oldLayout = pre_copy_layout,
.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.image = dst_image,
.subresourceRange{
.aspectMask = aspect,
.baseMipLevel = 0,
.levelCount = VK_REMAINING_MIP_LEVELS,
.baseArrayLayer = 0,
.layerCount = VK_REMAINING_ARRAY_LAYERS,
},
};
cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | pre_copy_stage,
VK_PIPELINE_STAGE_TRANSFER_BIT, 0, {}, buffer_barrier, pre_copy);
const VkBufferImageCopy copy{
.bufferOffset = out_offset,
.bufferRowLength = 0,
.bufferImageHeight = 0,
.imageSubresource{
.aspectMask = aspect,
.mipLevel = 0,
.baseArrayLayer = 0,
.layerCount = 1,
},
.imageOffset = {0, 0, 0},
.imageExtent = extent,
};
cmdbuf.CopyBufferToImage(out_buffer, dst_image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, copy);
const VkImageMemoryBarrier post_copy{
.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
.pNext = nullptr,
.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,
.dstAccessMask = VK_ACCESS_SHADER_READ_BIT,
.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
.newLayout = VK_IMAGE_LAYOUT_GENERAL,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.image = dst_image,
.subresourceRange{
.aspectMask = aspect,
.baseMipLevel = 0,
.levelCount = VK_REMAINING_MIP_LEVELS,
.baseArrayLayer = 0,
.layerCount = VK_REMAINING_ARRAY_LAYERS,
},
};
cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, vk::PIPELINE_STAGE_GRAPHICS_COMPUTE,
0, {}, {}, post_copy);
});
}
} // namespace Vulkan
@@ -164,4 +164,46 @@ private:
ComputePassDescriptorQueue& compute_pass_descriptor_queue;
};
class BlockLinearUnswizzle2DPass final : public ComputePass {
public:
explicit BlockLinearUnswizzle2DPass(const Device& device_, Scheduler& scheduler_,
DescriptorPool& descriptor_pool_,
StagingBufferPool& staging_buffer_pool_,
ComputePassDescriptorQueue& compute_pass_descriptor_queue_);
~BlockLinearUnswizzle2DPass();
[[nodiscard]] static bool IsSupported(const Device& device,
const VideoCommon::ImageInfo& info);
void Unswizzle(Image& image, const StagingBufferRef& swizzled,
std::span<const VideoCommon::SwizzleParameters> swizzles);
void UnswizzleFrom(Image& image, VkBuffer source_buffer, VkDeviceSize source_offset,
std::span<const VideoCommon::SwizzleParameters> swizzles);
private:
Scheduler& scheduler;
StagingBufferPool& staging_buffer_pool;
ComputePassDescriptorQueue& compute_pass_descriptor_queue;
};
class BlockLinearUnswizzle3DBufferPass final : public ComputePass {
public:
explicit BlockLinearUnswizzle3DBufferPass(
const Device& device_, Scheduler& scheduler_, DescriptorPool& descriptor_pool_,
StagingBufferPool& staging_buffer_pool_,
ComputePassDescriptorQueue& compute_pass_descriptor_queue_);
~BlockLinearUnswizzle3DBufferPass();
[[nodiscard]] static bool IsSupported(const Device& device, const VideoCommon::ImageInfo& info);
void Unswizzle(Image& image, const StagingBufferRef& swizzled,
std::span<const VideoCommon::SwizzleParameters> swizzles);
private:
Scheduler& scheduler;
StagingBufferPool& staging_buffer_pool;
ComputePassDescriptorQueue& compute_pass_descriptor_queue;
};
} // namespace Vulkan
@@ -444,8 +444,10 @@ PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_,
.has_broken_unsigned_image_offsets = false,
.has_broken_signed_operations = false,
.has_broken_fp16_float_controls = driver_id == VK_DRIVER_ID_NVIDIA_PROPRIETARY,
.has_broken_fp32_denorm_flush = driver_id == VK_DRIVER_ID_QUALCOMM_PROPRIETARY,
.ignore_nan_fp_comparisons = false,
.has_broken_spirv_subgroup_mask_vector_extract_dynamic = false,
.max_shared_memory_size = device.GetMaxComputeSharedMemorySize(),
.has_broken_robust =
device.IsNvidia() && device.GetNvidiaArch() <= NvidiaArchitecture::Arch_Pascal,
.min_ssbo_alignment = device.GetStorageBufferAlignment(),
@@ -921,19 +923,6 @@ std::unique_ptr<ComputePipeline> PipelineCache::CreateComputePipeline(
}
auto program{TranslateProgram(pools.inst, pools.block, env, cfg, host_info)};
const VkDriverIdKHR driver_id = device.GetDriverID();
const bool needs_shared_mem_clamp =
driver_id == VK_DRIVER_ID_QUALCOMM_PROPRIETARY ||
driver_id == VK_DRIVER_ID_ARM_PROPRIETARY;
const u32 max_shared_memory = device.GetMaxComputeSharedMemorySize();
if (needs_shared_mem_clamp && program.shared_memory_size > max_shared_memory) {
LOG_WARNING(Render_Vulkan,
"Compute shader {:#016x} requests {}KB shared memory but device max is {}KB - clamping",
key.unique_hash,
program.shared_memory_size / 1024,
max_shared_memory / 1024);
program.shared_memory_size = max_shared_memory;
}
const std::vector<u32> code{EmitSPIRV(profile, program)};
device.SaveShader(code);
vk::ShaderModule spv_module{BuildShader(device, code)};
@@ -225,6 +225,13 @@ RasterizerVulkan::RasterizerVulkan(Core::Frontend::EmuWindow& emu_window_, Tegra
fence_manager(*this, gpu, texture_cache, buffer_cache, query_cache, device, scheduler),
wfi_event(device.GetLogical().CreateEvent()) {
scheduler.SetQueryCache(query_cache);
if (Settings::values.use_unified_memory.GetValue() && device_memory.IsBackingShared()) {
buffer_cache_runtime.TryEnableUnifiedMemory(
device_memory.GetPhysicalBase(), device_memory.GetPhysicalSize(),
device_memory.GetBackingHardwareBuffers(),
device_memory.GetBackingHardwareBufferWindowSize(),
device_memory.GetBackingHardwareBufferBase());
}
}
RasterizerVulkan::~RasterizerVulkan() {
@@ -129,6 +129,10 @@ public:
return master_semaphore->IsFree(tick);
}
void RefreshTick() {
master_semaphore->Refresh();
}
/// Waits for the given GPU tick, optionally pacing frames.
void Wait(u64 tick, double target_fps = 0.0) {
if (tick > 0) {
@@ -51,6 +51,7 @@ using VideoCore::Surface::BytesPerBlock;
using VideoCore::Surface::HasAlpha;
using VideoCore::Surface::IsPixelFormatASTC;
using VideoCore::Surface::IsPixelFormatInteger;
using VideoCore::Surface::IsPixelFormatSRGB;
using VideoCore::Surface::SurfaceType;
namespace {
@@ -59,6 +60,17 @@ constexpr bool ENABLE_MSAA_RESOLVE_CONSUME = true;
constexpr bool ENABLE_MSAA_COLOR_DISCARD = true;
constexpr bool ENABLE_MSAA_DEPTH_STENCIL_DISCARD = true;
[[nodiscard]] constexpr bool NeedsExplicitBorderColorFormat(VkFormat format) {
switch (format) {
case VK_FORMAT_B4G4R4A4_UNORM_PACK16:
case VK_FORMAT_B5G6R5_UNORM_PACK16:
case VK_FORMAT_B5G5R5A1_UNORM_PACK16:
return true;
default:
return false;
}
}
constexpr VkBorderColor ConvertBorderColor(const std::array<float, 4>& color) {
if (color == std::array<float, 4>{0, 0, 0, 0}) {
return VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK;
@@ -956,6 +968,10 @@ TextureCacheRuntime::TextureCacheRuntime(const Device& device_, Scheduler& sched
bl3d_unswizzle_pass.emplace(device, scheduler, descriptor_pool,
staging_buffer_pool, compute_pass_descriptor_queue);
}
bl2d_unswizzle_pass.emplace(device, scheduler, descriptor_pool, staging_buffer_pool,
compute_pass_descriptor_queue);
bl3db_unswizzle_pass.emplace(device, scheduler, descriptor_pool, staging_buffer_pool,
compute_pass_descriptor_queue);
}
void TextureCacheRuntime::Finish() {
@@ -1802,10 +1818,6 @@ bool TextureCacheRuntime::CanReportMemoryUsage() const {
return device.CanReportMemoryUsage();
}
std::optional<size_t> TextureCacheRuntime::GetSamplerHeapBudget() const {
return device.GetSamplerHeapBudget();
}
void TextureCacheRuntime::FlushDeferredClear() {
scheduler.FlushDeferredClear();
}
@@ -1886,6 +1898,12 @@ Image::Image(TextureCacheRuntime& runtime_, const ImageInfo& info_, GPUVAddr gpu
}
flags |= VideoCommon::ImageFlagBits::Converted;
flags |= VideoCommon::ImageFlagBits::CostlyLoad;
} else if (runtime->bl2d_unswizzle_pass &&
BlockLinearUnswizzle2DPass::IsSupported(runtime->device, info)) {
flags |= VideoCommon::ImageFlagBits::AcceleratedUpload;
} else if (runtime->bl3db_unswizzle_pass &&
BlockLinearUnswizzle3DBufferPass::IsSupported(runtime->device, info)) {
flags |= VideoCommon::ImageFlagBits::AcceleratedUpload;
}
if (IsPixelFormatBCn(info.format) && !runtime->device.IsOptimalBcnSupported()) {
flags |= VideoCommon::ImageFlagBits::Converted;
@@ -2517,9 +2535,23 @@ ImageView::ImageView(TextureCacheRuntime& runtime, const VideoCommon::ImageViewI
supports_depth_comparison =
(properties3.optimalTilingFeatures &
VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_DEPTH_COMPARISON_BIT) != 0;
supports_minmax_filter = (properties3.optimalTilingFeatures &
VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_FILTER_MINMAX_BIT) != 0;
} else {
supports_depth_comparison = true;
supports_minmax_filter =
(device->GetPhysical().GetFormatProperties(format_info.format).optimalTilingFeatures &
VK_FORMAT_FEATURE_SAMPLED_IMAGE_FILTER_MINMAX_BIT) != 0;
}
requires_border_color_format = NeedsExplicitBorderColorFormat(format_info.format);
swizzle_mapping = VkComponentMapping{
.r = ComponentSwizzle(swizzle[0]),
.g = ComponentSwizzle(swizzle[1]),
.b = ComponentSwizzle(swizzle[2]),
.a = ComponentSwizzle(swizzle[3]),
};
has_identity_swizzle = swizzle[0] == SwizzleSource::R && swizzle[1] == SwizzleSource::G &&
swizzle[2] == SwizzleSource::B && swizzle[3] == SwizzleSource::A;
const VkImageUsageFlags requested_view_usage = ImageUsageFlags(format_info, format);
const VkImageUsageFlags image_usage = image.UsageFlags();
const VkImageUsageFlags clamped_view_usage = requested_view_usage & image_usage;
@@ -2544,12 +2576,7 @@ ImageView::ImageView(TextureCacheRuntime& runtime, const VideoCommon::ImageViewI
.image = image.Handle(),
.viewType = VkImageViewType{},
.format = format_info.format,
.components{
.r = ComponentSwizzle(swizzle[0]),
.g = ComponentSwizzle(swizzle[1]),
.b = ComponentSwizzle(swizzle[2]),
.a = ComponentSwizzle(swizzle[3]),
},
.components = swizzle_mapping,
.subresourceRange = MakeSubresourceRange(aspect_mask, info.range),
};
const auto create = [&](TextureType tex_type, std::optional<u32> num_layers) {
@@ -2718,92 +2745,223 @@ vk::ImageView ImageView::MakeView(VkFormat vk_format, VkImageAspectFlags aspect_
});
}
CustomBorderColorBudget::~CustomBorderColorBudget() {
Release();
}
CustomBorderColorBudget::CustomBorderColorBudget(CustomBorderColorBudget&& rhs) noexcept
: device_ptr{std::exchange(rhs.device_ptr, nullptr)}, held{std::exchange(rhs.held, 0)} {}
CustomBorderColorBudget& CustomBorderColorBudget::operator=(
CustomBorderColorBudget&& rhs) noexcept {
if (this != &rhs) {
Release();
device_ptr = std::exchange(rhs.device_ptr, nullptr);
held = std::exchange(rhs.held, 0);
}
return *this;
}
bool CustomBorderColorBudget::TryAcquire(const Device& device, size_t count) {
if (!device.TryReserveCustomBorderColorSamplers(count)) {
return false;
}
device_ptr = &device;
held += count;
return true;
}
void CustomBorderColorBudget::Release() noexcept {
if (device_ptr != nullptr) {
device_ptr->ReleaseCustomBorderColorSamplers(held);
}
device_ptr = nullptr;
held = 0;
}
Sampler::Sampler(TextureCacheRuntime& runtime, const Tegra::Texture::TSCEntry& tsc) {
const auto& device = runtime.device;
const bool has_custom_border_extension = runtime.device.IsExtCustomBorderColorSupported();
const bool has_format_undefined =
has_custom_border_extension && runtime.device.IsCustomBorderColorWithoutFormatSupported();
const bool has_custom_border_colors =
has_format_undefined && runtime.device.IsCustomBorderColorsSupported();
const auto color = tsc.BorderColor();
device_ptr = &device;
border_color = tsc.BorderColor();
srgb_border_color = tsc.SrgbBorderColor();
const f32 max_anisotropy = std::clamp(tsc.MaxAnisotropy(), 1.0f, 16.0f);
default_anisotropy = static_cast<f32>(1U << tsc.max_anisotropy);
const VkFilter mag_filter{MaxwellToVK::Sampler::Filter(tsc.mag_filter)};
const VkFilter min_filter{MaxwellToVK::Sampler::Filter(tsc.min_filter)};
const VkSamplerMipmapMode mipmap_mode{MaxwellToVK::Sampler::MipmapMode(tsc.mipmap_filter)};
const VkSamplerAddressMode wrap_u{
MaxwellToVK::Sampler::WrapMode(device, tsc.wrap_u, tsc.mag_filter)};
const VkSamplerAddressMode wrap_v{
MaxwellToVK::Sampler::WrapMode(device, tsc.wrap_v, tsc.mag_filter)};
const VkSamplerAddressMode wrap_p{
MaxwellToVK::Sampler::WrapMode(device, tsc.wrap_p, tsc.mag_filter)};
const bool samples_border = wrap_u == VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER ||
wrap_v == VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER ||
wrap_p == VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER;
reduction_mode = MaxwellToVK::SamplerReduction(tsc.reduction_filter);
has_added_anisotropy = max_anisotropy > default_anisotropy;
has_linear_filtering = mag_filter == VK_FILTER_LINEAR || min_filter == VK_FILTER_LINEAR ||
mipmap_mode == VK_SAMPLER_MIPMAP_MODE_LINEAR;
has_depth_comparison = tsc.depth_compare_enabled != 0;
has_minmax_reduction = reduction_mode != VK_SAMPLER_REDUCTION_MODE_WEIGHTED_AVERAGE_EXT;
has_srgb_border_color = tsc.srgb_conversion != 0 && srgb_border_color != border_color;
if (has_minmax_reduction && !device.IsExtSamplerFilterMinmaxSupported()) {
LOG_WARNING(Render_Vulkan, "VK_EXT_sampler_filter_minmax is required");
has_minmax_reduction = false;
}
has_custom_border_colors = samples_border && device.IsCustomBorderColorUsable();
needs_swizzle_mapping = has_custom_border_colors && device.NeedsBorderColorSwizzleMapping();
if (has_custom_border_colors && GPU::Logging::IsActive()) {
GPU::Logging::GPULogger::GetInstance().LogExtensionUsage(
"VK_EXT_custom_border_color", "Sampler::Sampler");
}
if (device.IsExtBorderColorSwizzleSupported() && GPU::Logging::IsActive()) {
GPU::Logging::GPULogger::GetInstance().LogExtensionUsage(
"VK_EXT_border_color_swizzle", "Sampler::Sampler");
}
f32 min_lod = 0.0f;
f32 max_lod = 0.25f;
if (tsc.mipmap_filter != TextureMipmapFilter::None) {
min_lod = tsc.MinLod();
max_lod = tsc.MaxLod();
}
base_ci = VkSamplerCreateInfo{
.sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO,
.pNext = nullptr,
.flags = 0,
.magFilter = mag_filter,
.minFilter = min_filter,
.mipmapMode = mipmap_mode,
.addressModeU = wrap_u,
.addressModeV = wrap_v,
.addressModeW = wrap_p,
.mipLodBias = tsc.LodBias(),
.anisotropyEnable = static_cast<VkBool32>(max_anisotropy > 1.0f),
.maxAnisotropy = max_anisotropy,
.compareEnable = static_cast<VkBool32>(tsc.depth_compare_enabled),
.compareOp = MaxwellToVK::Sampler::DepthCompareFunction(tsc.depth_compare_func),
.minLod = min_lod,
.maxLod = max_lod,
.borderColor = VK_BORDER_COLOR_FLOAT_CUSTOM_EXT,
.unnormalizedCoordinates = VK_FALSE,
};
variants.reserve(1);
Emplace(VariantKey{});
}
Sampler::VariantKey Sampler::MakeKey(const ImageView& image_view, bool is_depth) const noexcept {
VariantKey key{};
key.reduce_anisotropy = has_added_anisotropy && !image_view.SupportsAnisotropy();
key.force_nearest = has_linear_filtering && IsPixelFormatInteger(image_view.format);
key.drop_depth_comparison =
is_depth && has_depth_comparison && !image_view.SupportsDepthComparison();
key.drop_reduction = has_minmax_reduction && !image_view.SupportsMinmaxFilter();
key.drop_custom_border = has_custom_border_colors && image_view.RequiresBorderColorFormat();
key.srgb_border = has_srgb_border_color && IsPixelFormatSRGB(image_view.format);
if (needs_swizzle_mapping && !key.drop_custom_border && !image_view.HasIdentitySwizzle()) {
const VkComponentMapping& mapping = image_view.Swizzle();
key.swizzle = {mapping.r, mapping.g, mapping.b, mapping.a};
}
return key;
}
VkSampler Sampler::Find(const VariantKey& key) const noexcept {
const auto it = std::ranges::find(variants, key, &Variant::key);
if (it == variants.end()) {
return VK_NULL_HANDLE;
}
return *it->sampler;
}
VkSampler Sampler::Emplace(VariantKey key) {
bool custom_border = has_custom_border_colors && !key.drop_custom_border;
if (custom_border && !custom_border_color_budget.TryAcquire(*device_ptr, 1)) {
static bool warned_budget = false;
if (!warned_budget) {
warned_budget = true;
}
custom_border = false;
key.drop_custom_border = true;
key.swizzle = {};
if (const VkSampler existing = Find(key); existing != VK_NULL_HANDLE) {
return existing;
}
}
std::array<float, 4> color = border_color;
if (key.srgb_border) {
color = srgb_border_color;
}
const VkSamplerCustomBorderColorCreateInfoEXT border_ci{
.sType = VK_STRUCTURE_TYPE_SAMPLER_CUSTOM_BORDER_COLOR_CREATE_INFO_EXT,
.pNext = nullptr,
.customBorderColor = std::bit_cast<VkClearColorValue>(color),
.format = VK_FORMAT_UNDEFINED,
};
const void* pnext = nullptr;
if (has_custom_border_colors) {
pnext = &border_ci;
if (GPU::Logging::IsActive()) {
GPU::Logging::GPULogger::GetInstance().LogExtensionUsage(
"VK_EXT_custom_border_color", "Sampler::Sampler");
const VkSamplerBorderColorComponentMappingCreateInfoEXT mapping_ci{
.sType = VK_STRUCTURE_TYPE_SAMPLER_BORDER_COLOR_COMPONENT_MAPPING_CREATE_INFO_EXT,
.pNext = &border_ci,
.components{key.swizzle[0], key.swizzle[1], key.swizzle[2], key.swizzle[3]},
.srgb = VK_FALSE,
};
const void* chain = nullptr;
if (custom_border) {
chain = &border_ci;
if (key.HasSwizzle()) {
chain = &mapping_ci;
}
}
if (device.IsExtBorderColorSwizzleSupported() && GPU::Logging::IsActive()) {
GPU::Logging::GPULogger::GetInstance().LogExtensionUsage(
"VK_EXT_border_color_swizzle", "Sampler::Sampler");
}
const VkSamplerReductionModeCreateInfoEXT reduction_ci{
.sType = VK_STRUCTURE_TYPE_SAMPLER_REDUCTION_MODE_CREATE_INFO_EXT,
.pNext = pnext,
.reductionMode = MaxwellToVK::SamplerReduction(tsc.reduction_filter),
.pNext = chain,
.reductionMode = reduction_mode,
};
if (runtime.device.IsExtSamplerFilterMinmaxSupported()) {
pnext = &reduction_ci;
} else if (reduction_ci.reductionMode != VK_SAMPLER_REDUCTION_MODE_WEIGHTED_AVERAGE_EXT) {
LOG_WARNING(Render_Vulkan, "VK_EXT_sampler_filter_minmax is required");
if (has_minmax_reduction && !key.drop_reduction) {
chain = &reduction_ci;
}
// Some games have samplers with garbage. Sanitize them here.
const f32 max_anisotropy = std::clamp(tsc.MaxAnisotropy(), 1.0f, 16.0f);
const VkFilter mag_filter{MaxwellToVK::Sampler::Filter(tsc.mag_filter)};
const VkFilter min_filter{MaxwellToVK::Sampler::Filter(tsc.min_filter)};
const VkSamplerMipmapMode mipmap_mode{MaxwellToVK::Sampler::MipmapMode(tsc.mipmap_filter)};
const bool has_linear_filtering{mag_filter == VK_FILTER_LINEAR ||
min_filter == VK_FILTER_LINEAR ||
mipmap_mode == VK_SAMPLER_MIPMAP_MODE_LINEAR};
const auto create_sampler = [&](const f32 anisotropy, bool force_nearest,
bool disable_compare = false) {
return device.GetLogical().CreateSampler(VkSamplerCreateInfo{
.sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO,
.pNext = pnext,
.flags = 0,
.magFilter = force_nearest ? VK_FILTER_NEAREST : mag_filter,
.minFilter = force_nearest ? VK_FILTER_NEAREST : min_filter,
.mipmapMode = force_nearest ? VK_SAMPLER_MIPMAP_MODE_NEAREST : mipmap_mode,
.addressModeU = MaxwellToVK::Sampler::WrapMode(device, tsc.wrap_u, tsc.mag_filter),
.addressModeV = MaxwellToVK::Sampler::WrapMode(device, tsc.wrap_v, tsc.mag_filter),
.addressModeW = MaxwellToVK::Sampler::WrapMode(device, tsc.wrap_p, tsc.mag_filter),
.mipLodBias = tsc.LodBias(),
.anisotropyEnable =
static_cast<VkBool32>(!force_nearest && anisotropy > 1.0f ? VK_TRUE : VK_FALSE),
.maxAnisotropy = force_nearest ? 1.0f : anisotropy,
.compareEnable = disable_compare ? VK_FALSE
: static_cast<VkBool32>(tsc.depth_compare_enabled),
.compareOp = MaxwellToVK::Sampler::DepthCompareFunction(tsc.depth_compare_func),
.minLod = tsc.mipmap_filter == TextureMipmapFilter::None ? 0.0f : tsc.MinLod(),
.maxLod = tsc.mipmap_filter == TextureMipmapFilter::None ? 0.25f : tsc.MaxLod(),
.borderColor = has_custom_border_colors ? VK_BORDER_COLOR_FLOAT_CUSTOM_EXT
: ConvertBorderColor(color),
.unnormalizedCoordinates = VK_FALSE,
});
};
sampler = create_sampler(max_anisotropy, false);
const f32 max_anisotropy_default = static_cast<f32>(1U << tsc.max_anisotropy);
if (max_anisotropy > max_anisotropy_default) {
sampler_default_anisotropy = create_sampler(max_anisotropy_default, false);
VkSamplerCreateInfo create_info = base_ci;
create_info.pNext = chain;
if (key.force_nearest) {
create_info.magFilter = VK_FILTER_NEAREST;
create_info.minFilter = VK_FILTER_NEAREST;
create_info.mipmapMode = VK_SAMPLER_MIPMAP_MODE_NEAREST;
create_info.anisotropyEnable = VK_FALSE;
create_info.maxAnisotropy = 1.0f;
} else if (key.reduce_anisotropy) {
create_info.anisotropyEnable = static_cast<VkBool32>(default_anisotropy > 1.0f);
create_info.maxAnisotropy = default_anisotropy;
}
if (has_linear_filtering) {
sampler_nearest = create_sampler(1.0f, true);
if (key.drop_depth_comparison) {
create_info.compareEnable = VK_FALSE;
}
if (tsc.depth_compare_enabled) {
sampler_noncompare = create_sampler(max_anisotropy, false, true);
if (!custom_border) {
create_info.borderColor = ConvertBorderColor(color);
}
variants.push_back(Variant{
.key = key,
.sampler = device_ptr->GetLogical().CreateSampler(create_info),
});
return *variants.back().sampler;
}
VkSampler Sampler::HandleFor(const ImageView& image_view, bool is_depth) {
VariantKey key = MakeKey(image_view, is_depth);
if (variants.size() >= MAX_VARIANTS) {
key.srgb_border = false;
key.swizzle = {};
}
if (const VkSampler existing = Find(key); existing != VK_NULL_HANDLE) {
return existing;
}
return Emplace(key);
}
Framebuffer::Framebuffer(TextureCacheRuntime& runtime, std::span<ImageView*, NUM_RT> color_buffers,
@@ -2989,10 +3147,19 @@ void TextureCacheRuntime::AccelerateImageUpload(
std::span<const VideoCommon::SwizzleParameters> swizzles,
u32 z_start, u32 z_count) {
if (IsPixelFormatASTC(image.info.format)) {
if (astc_decoder_pass && WillUseAcceleratedAstcDecode(device, image.info)) {
return astc_decoder_pass->Assemble(image, map, swizzles);
}
if (bl2d_unswizzle_pass && BlockLinearUnswizzle2DPass::IsSupported(device, image.info)) {
return bl2d_unswizzle_pass->Unswizzle(image, map, swizzles);
}
if (bl3db_unswizzle_pass && z_count == 0 &&
BlockLinearUnswizzle3DBufferPass::IsSupported(device, image.info)) {
return bl3db_unswizzle_pass->Unswizzle(image, map, swizzles);
}
if (!Settings::values.gpu_unswizzle_enabled.GetValue() || !bl3d_unswizzle_pass) {
if (IsPixelFormatBCn(image.info.format) && image.info.type == ImageType::e3D) {
ASSERT(false && "GPU unswizzle is disabled for BCn 3D texture");
@@ -3008,6 +3175,74 @@ void TextureCacheRuntime::AccelerateImageUpload(
ASSERT(false);
}
bool TextureCacheRuntime::IsUnifiedMemoryBindable() const noexcept {
const HostMemoryImport* const import = memory_allocator.GetHostMemoryImport();
return import != nullptr && import->IsValid() && import->IsBindable();
}
u64 TextureCacheRuntime::UnifiedMemoryBase() const noexcept {
const HostMemoryImport* const import = memory_allocator.GetHostMemoryImport();
if (import == nullptr) {
return 0;
}
return import->GetBaseOffset();
}
u64 TextureCacheRuntime::UnifiedMemorySize() const noexcept {
const HostMemoryImport* const import = memory_allocator.GetHostMemoryImport();
if (import == nullptr) {
return 0;
}
return import->GetSize();
}
u64 TextureCacheRuntime::UnifiedMemoryWindowSize() const noexcept {
const HostMemoryImport* const import = memory_allocator.GetHostMemoryImport();
if (import == nullptr) {
return 0;
}
return import->GetWindowSize();
}
bool TextureCacheRuntime::CanUploadImageDirectly(const VideoCommon::ImageInfo& info) const {
return bl2d_unswizzle_pass.has_value() &&
BlockLinearUnswizzle2DPass::IsSupported(device, info);
}
bool TextureCacheRuntime::UploadImageDirectly(
Image& image, size_t window_index, u64 window_offset,
std::span<const VideoCommon::SwizzleParameters> swizzles) {
if ((window_offset % device.GetStorageBufferAlignment()) != 0) {
return false;
}
if (image.guest_size_bytes > device.GetMaxStorageBufferRange()) {
return false;
}
const HostMemoryImport* const import = memory_allocator.GetHostMemoryImport();
if (import == nullptr || window_index >= import->GetWindowCount()) {
return false;
}
const VkBuffer window_buffer = import->GetWindowBuffer(window_index);
if (window_buffer == VK_NULL_HANDLE) {
return false;
}
bl2d_unswizzle_pass->UnswizzleFrom(image, window_buffer,
static_cast<VkDeviceSize>(window_offset), swizzles);
return true;
}
u64 TextureCacheRuntime::CurrentTick() const noexcept {
return scheduler.CurrentTick();
}
bool TextureCacheRuntime::IsDirectUploadRetired(u64 tick) {
if (scheduler.IsFree(tick)) {
return true;
}
scheduler.RefreshTick();
return scheduler.IsFree(tick);
}
void TextureCacheRuntime::TransitionImageLayout(Image& image) {
if (!image.ExchangeInitialization()) {
VkImageMemoryBarrier barrier{
+107 -30
View File
@@ -68,8 +68,6 @@ public:
bool CanReportMemoryUsage() const;
std::optional<size_t> GetSamplerHeapBudget() const;
bool CanDownloadMsaa(const VideoCommon::ImageInfo& info) const;
[[nodiscard]] VkImage AcquireMsaaScratchImage(const VkImageCreateInfo& image_ci);
@@ -103,6 +101,23 @@ public:
std::span<const VideoCommon::SwizzleParameters>,
u32 z_start, u32 z_count);
[[nodiscard]] bool IsUnifiedMemoryBindable() const noexcept;
[[nodiscard]] u64 UnifiedMemoryBase() const noexcept;
[[nodiscard]] u64 UnifiedMemorySize() const noexcept;
[[nodiscard]] u64 UnifiedMemoryWindowSize() const noexcept;
[[nodiscard]] bool CanUploadImageDirectly(const VideoCommon::ImageInfo& info) const;
bool UploadImageDirectly(Image& image, size_t window_index, u64 window_offset,
std::span<const VideoCommon::SwizzleParameters> swizzles);
[[nodiscard]] u64 CurrentTick() const noexcept;
[[nodiscard]] bool IsDirectUploadRetired(u64 tick);
void InsertUploadMemoryBarrier() {}
void TransitionImageLayout(Image& image);
@@ -161,6 +176,8 @@ public:
std::optional<ASTCDecoderPass> astc_decoder_pass;
std::optional<BlockLinearUnswizzle3DPass> bl3d_unswizzle_pass;
std::optional<BlockLinearUnswizzle2DPass> bl2d_unswizzle_pass;
std::optional<BlockLinearUnswizzle3DBufferPass> bl3db_unswizzle_pass;
const Settings::ResolutionScalingInfo& resolution;
std::array<std::vector<VkFormat>, VideoCore::Surface::MaxPixelFormat> view_formats;
@@ -446,6 +463,22 @@ public:
return supports_depth_comparison;
}
[[nodiscard]] bool RequiresBorderColorFormat() const noexcept {
return requires_border_color_format;
}
[[nodiscard]] bool SupportsMinmaxFilter() const noexcept {
return supports_minmax_filter;
}
[[nodiscard]] const VkComponentMapping& Swizzle() const noexcept {
return swizzle_mapping;
}
[[nodiscard]] bool HasIdentitySwizzle() const noexcept {
return has_identity_swizzle;
}
[[nodiscard]] GPUVAddr GpuAddr() const noexcept {
return gpu_addr;
}
@@ -478,48 +511,91 @@ private:
VkSampleCountFlagBits samples = VK_SAMPLE_COUNT_1_BIT;
u32 buffer_size = 0;
VkComponentMapping swizzle_mapping{};
bool supports_depth_comparison = false;
bool requires_border_color_format = false;
bool supports_minmax_filter = false;
bool has_identity_swizzle = true;
};
class ImageAlloc : public VideoCommon::ImageAllocBase {};
class CustomBorderColorBudget {
public:
CustomBorderColorBudget() = default;
~CustomBorderColorBudget();
CustomBorderColorBudget(const CustomBorderColorBudget&) = delete;
CustomBorderColorBudget& operator=(const CustomBorderColorBudget&) = delete;
CustomBorderColorBudget(CustomBorderColorBudget&& rhs) noexcept;
CustomBorderColorBudget& operator=(CustomBorderColorBudget&& rhs) noexcept;
bool TryAcquire(const Device& device, size_t count);
private:
void Release() noexcept;
const Device* device_ptr = nullptr;
size_t held = 0;
};
class Sampler {
public:
explicit Sampler(TextureCacheRuntime&, const Tegra::Texture::TSCEntry&);
[[nodiscard]] VkSampler Handle() const noexcept {
return *sampler;
return *variants.front().sampler;
}
[[nodiscard]] VkSampler HandleWithDefaultAnisotropy() const noexcept {
return *sampler_default_anisotropy;
}
[[nodiscard]] bool HasAddedAnisotropy() const noexcept {
return static_cast<bool>(sampler_default_anisotropy);
}
[[nodiscard]] VkSampler HandleWithNearestFilter() const noexcept {
return *sampler_nearest;
}
[[nodiscard]] bool HasLinearFiltering() const noexcept {
return static_cast<bool>(sampler_nearest);
}
[[nodiscard]] VkSampler HandleWithoutDepthComparison() const noexcept {
return *sampler_noncompare;
}
[[nodiscard]] bool HasDepthComparison() const noexcept {
return static_cast<bool>(sampler_noncompare);
}
[[nodiscard]] VkSampler HandleFor(const ImageView& image_view, bool is_depth);
private:
vk::Sampler sampler;
vk::Sampler sampler_default_anisotropy;
vk::Sampler sampler_nearest;
vk::Sampler sampler_noncompare;
struct VariantKey {
bool reduce_anisotropy;
bool force_nearest;
bool drop_depth_comparison;
bool drop_reduction;
bool drop_custom_border;
bool srgb_border;
std::array<VkComponentSwizzle, 4> swizzle;
bool operator==(const VariantKey&) const noexcept = default;
[[nodiscard]] bool HasSwizzle() const noexcept {
return swizzle != std::array<VkComponentSwizzle, 4>{};
}
};
struct Variant {
VariantKey key;
vk::Sampler sampler;
};
static constexpr size_t MAX_VARIANTS = 32;
[[nodiscard]] VariantKey MakeKey(const ImageView& image_view, bool is_depth) const noexcept;
[[nodiscard]] VkSampler Find(const VariantKey& key) const noexcept;
VkSampler Emplace(VariantKey key);
CustomBorderColorBudget custom_border_color_budget;
std::vector<Variant> variants;
const Device* device_ptr{nullptr};
VkSamplerCreateInfo base_ci{};
VkSamplerReductionModeEXT reduction_mode{VK_SAMPLER_REDUCTION_MODE_WEIGHTED_AVERAGE_EXT};
std::array<float, 4> border_color{};
std::array<float, 4> srgb_border_color{};
f32 default_anisotropy{1.0f};
bool has_added_anisotropy{};
bool has_linear_filtering{};
bool has_depth_comparison{};
bool has_minmax_reduction{};
bool has_custom_border_colors{};
bool has_srgb_border_color{};
bool needs_swizzle_mapping{};
};
struct TextureCacheParams {
@@ -529,6 +605,7 @@ struct TextureCacheParams {
static constexpr bool HAS_DEVICE_MEMORY_INFO = true;
static constexpr bool IMPLEMENTS_ASYNC_DOWNLOADS = true;
static constexpr bool HAS_MSAA_DOWNLOADS = true;
static constexpr bool USE_UNIFIED_MEMORY = true;
using Runtime = Vulkan::TextureCacheRuntime;
using Image = Vulkan::Image;
@@ -95,6 +95,8 @@ struct ImageBase {
u32 scale_rating = 0;
u64 scale_tick = 0;
bool has_scaled = false;
u64 direct_upload_tick = 0;
bool direct_upload_blocked = false;
size_t channel = 0;
+65 -58
View File
@@ -119,13 +119,16 @@ void TextureCache<P>::RunGarbageCollector() {
bool aggressive_mode = false;
u64 ticks_to_destroy = 0;
size_t num_iterations = 0;
size_t num_downloads = 0;
const auto Configure = [&](bool allow_aggressive) {
high_priority_mode = total_used_memory >= expected_memory;
aggressive_mode = allow_aggressive && total_used_memory >= critical_memory;
ticks_to_destroy = aggressive_mode ? 10ULL : high_priority_mode ? 25ULL : 50ULL;
num_iterations = aggressive_mode ? 40 : (high_priority_mode ? 20 : 10);
num_downloads = MAX_GC_DOWNLOADS_PER_PASS;
};
const auto Cleanup = [this, &num_iterations, &high_priority_mode, &aggressive_mode](ImageId image_id) {
const auto Cleanup = [this, &num_iterations, &num_downloads, &high_priority_mode,
&aggressive_mode](ImageId image_id) {
if (num_iterations == 0) {
return true;
}
@@ -139,6 +142,10 @@ void TextureCache<P>::RunGarbageCollector() {
return false;
}
if (must_download) {
if (num_downloads == 0) {
return false;
}
--num_downloads;
auto map = runtime.DownloadStagingBuffer(image.unswizzled_size_bytes);
const auto copies = FixSmallVectorADL(FullDownloadCopies(image.info));
image.DownloadMemory(map, copies);
@@ -585,6 +592,15 @@ FramebufferId TextureCache<P>::GetFramebufferId(const RenderTargets& key) {
template <class P>
void TextureCache<P>::WriteMemory(DAddr cpu_addr, size_t size) {
ForEachImageInRegion(cpu_addr, size, [this](ImageId image_id, Image& image) {
if constexpr (USE_UNIFIED_MEMORY) {
if (image.direct_upload_tick != 0) {
const u64 upload_tick = image.direct_upload_tick;
image.direct_upload_tick = 0;
if (!runtime.IsDirectUploadRetired(upload_tick)) {
image.direct_upload_blocked = true;
}
}
}
if (True(image.flags & ImageFlagBits::CpuModified)) {
return;
}
@@ -1145,11 +1161,59 @@ void TextureCache<P>::RefreshContents(Image& image, ImageId image_id) {
QueueAsyncUnswizzle(image, image_id);
return;
}
if (True(image.flags & ImageFlagBits::AcceleratedUpload) &&
TryUploadFromUnifiedMemory(image)) {
runtime.InsertUploadMemoryBarrier();
return;
}
auto staging = runtime.UploadStagingBuffer(MapSizeBytes(image));
UploadImageContents(image, staging);
runtime.InsertUploadMemoryBarrier();
}
template <class P>
bool TextureCache<P>::TryUploadFromUnifiedMemory([[maybe_unused]] Image& image) {
if constexpr (USE_UNIFIED_MEMORY) {
if (image.direct_upload_blocked || image.guest_size_bytes == 0) {
return false;
}
if (!runtime.IsUnifiedMemoryBindable() || !runtime.CanUploadImageDirectly(image.info)) {
return false;
}
const u64 window_size = runtime.UnifiedMemoryWindowSize();
if (window_size == 0) {
return false;
}
const u8* const first = gpu_memory->GetSpan(image.gpu_addr, image.guest_size_bytes);
if (first == nullptr) {
return false;
}
const u64 phys_offset = static_cast<u64>(first - device_memory.GetPhysicalBase());
const u64 unified_base = runtime.UnifiedMemoryBase();
if (phys_offset < unified_base) {
return false;
}
const u64 relative = phys_offset - unified_base;
const u64 unified_size = runtime.UnifiedMemorySize();
if (relative >= unified_size || unified_size - relative < image.guest_size_bytes) {
return false;
}
const u64 local_offset = relative % window_size;
if (window_size - local_offset < image.guest_size_bytes) {
return false;
}
const auto swizzles = FullUploadSwizzles(image.info);
if (!runtime.UploadImageDirectly(image, static_cast<size_t>(relative / window_size),
local_offset, FixSmallVectorADL(swizzles))) {
return false;
}
image.direct_upload_tick = runtime.CurrentTick();
return true;
} else {
return false;
}
}
template <class P>
template <typename StagingBuffer>
void TextureCache<P>::UploadImageContents(Image& image, StagingBuffer& staging) {
@@ -1896,67 +1960,10 @@ SamplerId TextureCache<P>::FindSampler(const TSCEntry& config, bool compute) {
const auto [pair, is_new] = channel_state->samplers.try_emplace(config);
if (is_new) {
pair->second = slot_samplers.insert(runtime, config);
EnforceSamplerBudget();
}
return pair->second;
}
template <class P>
std::optional<size_t> TextureCache<P>::QuerySamplerBudget() const {
if constexpr (requires { runtime.GetSamplerHeapBudget(); }) {
return runtime.GetSamplerHeapBudget();
} else {
return std::nullopt;
}
}
template <class P>
void TextureCache<P>::EnforceSamplerBudget() {
if (auto const budget = QuerySamplerBudget(); budget) {
if (slot_samplers.size() < *budget) {
return;
}
if (!channel_state) {
return;
}
if (last_sampler_gc_frame == frame_tick) {
return;
}
last_sampler_gc_frame = frame_tick;
TrimInactiveSamplers(*budget);
}
}
template <class P>
void TextureCache<P>::TrimInactiveSamplers(size_t budget) {
if (channel_state->samplers.size() > 0) {
constexpr size_t SAMPLER_GC_SLACK = 1024;
ankerl::unordered_dense::set<SamplerId> active_sampler_ids;
for (auto const& e : channel_state->sampler_ids)
active_sampler_ids.insert(e.second);
// Elements in the map must be necesarily valid
size_t removed = 0;
for (auto it = channel_state->samplers.begin(); it != channel_state->samplers.end();) {
const SamplerId sampler_id = it->second;
if (!sampler_id || sampler_id == CORRUPT_ID) {
it = channel_state->samplers.erase(it);
} else if (std::ranges::find(active_sampler_ids, sampler_id) != active_sampler_ids.end()) {
++it;
} else {
slot_samplers.erase(sampler_id);
it = channel_state->samplers.erase(it);
++removed;
if (slot_samplers.size() + SAMPLER_GC_SLACK <= budget) {
break;
}
}
}
if (removed != 0) {
LOG_WARNING(HW_GPU, "Sampler cache exceeded {} entries on this driver; reclaimed {} inactive samplers", budget, removed);
}
}
}
template <class P>
ImageViewId TextureCache<P>::FindColorBuffer(size_t index) {
const auto& regs = maxwell3d->regs;
@@ -108,6 +108,7 @@ class TextureCache : public VideoCommon::ChannelSetupCaches<TextureCacheChannelI
static constexpr bool HAS_DEVICE_MEMORY_INFO = P::HAS_DEVICE_MEMORY_INFO;
/// True when the API can do asynchronous texture downloads.
static constexpr bool IMPLEMENTS_ASYNC_DOWNLOADS = P::IMPLEMENTS_ASYNC_DOWNLOADS;
static constexpr bool USE_UNIFIED_MEMORY = P::USE_UNIFIED_MEMORY;
static constexpr size_t UNSET_CHANNEL{(std::numeric_limits<size_t>::max)()};
@@ -120,6 +121,7 @@ class TextureCache : public VideoCommon::ChannelSetupCaches<TextureCacheChannelI
static constexpr s64 DEFAULT_EXPECTED_MEMORY = 1_GiB + 125_MiB;
static constexpr s64 DEFAULT_CRITICAL_MEMORY = 1_GiB + 625_MiB;
static constexpr size_t GC_EMERGENCY_COUNTS = 2;
static constexpr size_t MAX_GC_DOWNLOADS_PER_PASS = 4;
using Runtime = typename P::Runtime;
using Image = typename P::Image;
@@ -307,6 +309,8 @@ private:
void RefreshContents(Image& image, ImageId image_id);
bool TryUploadFromUnifiedMemory(Image& image);
/// Upload data from guest to an image
template <typename StagingBuffer>
void UploadImageContents(Image& image, StagingBuffer& staging_buffer);
@@ -420,9 +424,6 @@ private:
void QueueAsyncDecode(Image& image, ImageId image_id);
void TickAsyncDecode();
void EnforceSamplerBudget();
void TrimInactiveSamplers(size_t budget);
std::optional<size_t> QuerySamplerBudget() const;
void QueueAsyncUnswizzle(Image& image, ImageId image_id);
void TickAsyncUnswizzle();
@@ -509,7 +510,6 @@ private:
u64 modification_tick = 0;
u64 frame_tick = 0;
u64 last_sampler_gc_frame = (std::numeric_limits<u64>::max)();
Common::ThreadWorker texture_decode_worker{1, "TextureDecoder", {},
Common::ThreadPlacement::Efficiency};
+13 -40
View File
@@ -5,6 +5,7 @@
// SPDX-License-Identifier: GPL-2.0-or-later
#include <array>
#include <cmath>
#include "common/cityhash.h"
#include "common/settings.h"
@@ -17,53 +18,25 @@ namespace Tegra::Texture {
namespace {
[[maybe_unused]] constexpr std::array<float, 256> SRGB_CONVERSION_LUT = {
0.000000f, 0.000000f, 0.000000f, 0.000012f, 0.000021f, 0.000033f, 0.000046f, 0.000062f,
0.000081f, 0.000102f, 0.000125f, 0.000151f, 0.000181f, 0.000214f, 0.000251f, 0.000293f,
0.000338f, 0.000388f, 0.000443f, 0.000503f, 0.000568f, 0.000639f, 0.000715f, 0.000798f,
0.000887f, 0.000983f, 0.001085f, 0.001195f, 0.001312f, 0.001437f, 0.001569f, 0.001710f,
0.001860f, 0.002019f, 0.002186f, 0.002364f, 0.002551f, 0.002748f, 0.002955f, 0.003174f,
0.003403f, 0.003643f, 0.003896f, 0.004160f, 0.004436f, 0.004725f, 0.005028f, 0.005343f,
0.005672f, 0.006015f, 0.006372f, 0.006744f, 0.007130f, 0.007533f, 0.007950f, 0.008384f,
0.008834f, 0.009301f, 0.009785f, 0.010286f, 0.010805f, 0.011342f, 0.011898f, 0.012472f,
0.013066f, 0.013680f, 0.014313f, 0.014967f, 0.015641f, 0.016337f, 0.017054f, 0.017793f,
0.018554f, 0.019337f, 0.020144f, 0.020974f, 0.021828f, 0.022706f, 0.023609f, 0.024536f,
0.025489f, 0.026468f, 0.027473f, 0.028504f, 0.029563f, 0.030649f, 0.031762f, 0.032904f,
0.034074f, 0.035274f, 0.036503f, 0.037762f, 0.039050f, 0.040370f, 0.041721f, 0.043103f,
0.044518f, 0.045964f, 0.047444f, 0.048956f, 0.050503f, 0.052083f, 0.053699f, 0.055349f,
0.057034f, 0.058755f, 0.060513f, 0.062307f, 0.064139f, 0.066008f, 0.067915f, 0.069861f,
0.071845f, 0.073869f, 0.075933f, 0.078037f, 0.080182f, 0.082369f, 0.084597f, 0.086867f,
0.089180f, 0.091535f, 0.093935f, 0.096378f, 0.098866f, 0.101398f, 0.103977f, 0.106601f,
0.109271f, 0.111988f, 0.114753f, 0.117565f, 0.120426f, 0.123335f, 0.126293f, 0.129301f,
0.132360f, 0.135469f, 0.138629f, 0.141841f, 0.145105f, 0.148421f, 0.151791f, 0.155214f,
0.158691f, 0.162224f, 0.165810f, 0.169453f, 0.173152f, 0.176907f, 0.180720f, 0.184589f,
0.188517f, 0.192504f, 0.196549f, 0.200655f, 0.204820f, 0.209046f, 0.213334f, 0.217682f,
0.222093f, 0.226567f, 0.231104f, 0.235704f, 0.240369f, 0.245099f, 0.249894f, 0.254754f,
0.259681f, 0.264674f, 0.269736f, 0.274864f, 0.280062f, 0.285328f, 0.290664f, 0.296070f,
0.301546f, 0.307094f, 0.312713f, 0.318404f, 0.324168f, 0.330006f, 0.335916f, 0.341902f,
0.347962f, 0.354097f, 0.360309f, 0.366597f, 0.372961f, 0.379403f, 0.385924f, 0.392524f,
0.399202f, 0.405960f, 0.412798f, 0.419718f, 0.426719f, 0.433802f, 0.440967f, 0.448216f,
0.455548f, 0.462965f, 0.470465f, 0.478052f, 0.485725f, 0.493484f, 0.501329f, 0.509263f,
0.517285f, 0.525396f, 0.533595f, 0.541885f, 0.550265f, 0.558736f, 0.567299f, 0.575954f,
0.584702f, 0.593542f, 0.602477f, 0.611507f, 0.620632f, 0.629852f, 0.639168f, 0.648581f,
0.658092f, 0.667700f, 0.677408f, 0.687214f, 0.697120f, 0.707127f, 0.717234f, 0.727443f,
0.737753f, 0.748167f, 0.758685f, 0.769305f, 0.780031f, 0.790861f, 0.801798f, 0.812839f,
0.823989f, 0.835246f, 0.846611f, 0.858085f, 0.869668f, 0.881360f, 0.893164f, 0.905078f,
0.917104f, 0.929242f, 0.941493f, 0.953859f, 0.966338f, 1.000000f, 1.000000f, 1.000000f,
};
float SrgbToLinear(u32 value) {
const float encoded = static_cast<float>(value) / 255.0f;
if (encoded <= 0.04045f) {
return encoded / 12.92f;
}
return std::pow((encoded + 0.055f) / 1.055f, 2.4f);
}
} // Anonymous namespace
std::array<float, 4> TSCEntry::BorderColor() const noexcept {
// TODO: Handle SRGB correctly. Using this breaks shadows in some games (Xenoblade).
// if (!srgb_conversion) {
// return border_color;
//}
// return {SRGB_CONVERSION_LUT[srgb_border_color_r], SRGB_CONVERSION_LUT[srgb_border_color_g],
// SRGB_CONVERSION_LUT[srgb_border_color_b], border_color[3]};
return border_color;
}
std::array<float, 4> TSCEntry::SrgbBorderColor() const noexcept {
return {SrgbToLinear(srgb_border_color_r), SrgbToLinear(srgb_border_color_g),
SrgbToLinear(srgb_border_color_b), border_color[3]};
}
float TSCEntry::MaxAnisotropy() const noexcept {
const bool is_suitable_mipmap_filter = mipmap_filter != TextureMipmapFilter::None;
const bool has_regular_lods = min_lod_clamp == 0 && max_lod_clamp >= 256;
+5
View File
@@ -1,3 +1,6 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
@@ -378,6 +381,8 @@ struct TSCEntry {
std::array<float, 4> BorderColor() const noexcept;
std::array<float, 4> SrgbBorderColor() const noexcept;
float MaxAnisotropy() const noexcept;
float MinLod() const {
+71 -30
View File
@@ -16,6 +16,7 @@
#include <fmt/format.h>
#include "common/assert.h"
#include "common/host_memory.h"
#include "common/literals.h"
#include <ranges>
#include "common/settings.h"
@@ -508,17 +509,9 @@ Device::Device(VkInstance instance_, vk::PhysicalDevice physical_, VkSurfaceKHR
LOG_WARNING(Render_Vulkan, "Qualcomm drivers require scaled vertex format emulation.");
has_broken_descriptor_aliasing = true;
LOG_WARNING(Render_Vulkan, "Qualcomm drivers have broken descriptor aliasing.");
LOG_WARNING(Render_Vulkan, "Qualcomm drivers have broken custom border color.");
RemoveExtensionFeature(extensions.custom_border_color, features.custom_border_color,
VK_EXT_CUSTOM_BORDER_COLOR_EXTENSION_NAME);
LOG_WARNING(Render_Vulkan, "Qualcomm drivers have broken border color swizzle.");
RemoveExtensionFeature(extensions.border_color_swizzle, features.border_color_swizzle,
VK_EXT_BORDER_COLOR_SWIZZLE_EXTENSION_NAME);
LOG_WARNING(Render_Vulkan, "Qualcomm drivers have broken color write enable.");
RemoveExtensionFeature(extensions.color_write_enable, features.color_write_enable,
VK_EXT_COLOR_WRITE_ENABLE_EXTENSION_NAME);
LOG_WARNING(Render_Vulkan, "Qualcomm drivers have broken shader float controls.");
RemoveExtension(extensions.shader_float_controls, VK_KHR_SHADER_FLOAT_CONTROLS_EXTENSION_NAME);
LOG_WARNING(Render_Vulkan, "Qualcomm drivers have broken shader atomic int64.");
RemoveExtensionFeature(extensions.shader_atomic_int64, features.shader_atomic_int64,
VK_KHR_SHADER_ATOMIC_INT64_EXTENSION_NAME);
@@ -619,21 +612,6 @@ Device::Device(VkInstance instance_, vk::PhysicalDevice physical_, VkSurfaceKHR
}
}
if (is_qualcomm) {
const size_t sampler_limit = properties.properties.limits.maxSamplerAllocationCount;
if (sampler_limit > 0) {
constexpr size_t MIN_SAMPLER_BUDGET = 1024U;
const size_t reserved = sampler_limit / 4U;
const size_t derived_budget =
(std::max)(MIN_SAMPLER_BUDGET, sampler_limit - reserved);
sampler_heap_budget = derived_budget;
LOG_WARNING(Render_Vulkan,
"Qualcomm driver reports max {} samplers; reserving {} (25%) and "
"allowing Eden to use {} (75%) to avoid heap exhaustion",
sampler_limit, reserved, sampler_heap_budget);
}
}
if (extensions.sampler_filter_minmax && is_amd) {
// Disable ext_sampler_filter_minmax on AMD GCN4 and lower as it is broken.
if (!features.shader_float16_int8.shaderFloat16) {
@@ -967,6 +945,7 @@ bool Device::GetSuitability(bool requires_swapchain) {
FOR_EACH_VK_FEATURE_EXT(FEATURE_EXTENSION);
FOR_EACH_VK_EXTENSION(EXTENSION);
FOR_EACH_VK_PLATFORM_EXTENSION(EXTENSION);
extensions.depth_stencil_resolve =
extensions.depth_stencil_resolve &&
@@ -985,6 +964,13 @@ bool Device::GetSuitability(bool requires_swapchain) {
extensions.robustness_2 = false;
}
#ifdef __ANDROID__
if (extensions.external_memory_ahb && !extensions.queue_family_foreign) {
loaded_extensions.erase(VK_ANDROID_EXTERNAL_MEMORY_ANDROID_HARDWARE_BUFFER_EXTENSION_NAME);
extensions.external_memory_ahb = false;
}
#endif
#undef FEATURE_EXTENSION
#undef EXTENSION
@@ -1146,6 +1132,26 @@ bool Device::GetSuitability(bool requires_swapchain) {
VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MAINTENANCE_5_PROPERTIES_KHR;
SetNext(next, properties.maintenance5);
}
if (extensions.custom_border_color) {
properties.custom_border_color.sType =
VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_CUSTOM_BORDER_COLOR_PROPERTIES_EXT;
SetNext(next, properties.custom_border_color);
}
if (extensions.external_memory_host) {
properties.external_memory_host.sType =
VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_MEMORY_HOST_PROPERTIES_EXT;
SetNext(next, properties.external_memory_host);
}
if (extensions.maintenance3 || instance_version >= VK_API_VERSION_1_1) {
properties.maintenance3.sType =
VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MAINTENANCE_3_PROPERTIES;
SetNext(next, properties.maintenance3);
}
if (extensions.maintenance4 || features.maintenance4.maintenance4) {
properties.maintenance4.sType =
VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MAINTENANCE_4_PROPERTIES;
SetNext(next, properties.maintenance4);
}
// Perform the property fetch.
physical.GetProperties2(properties2);
@@ -1242,9 +1248,7 @@ void Device::RemoveUnsuitableExtensions() {
// VK_EXT_border_color_swizzle
if (extensions.border_color_swizzle) {
extensions.border_color_swizzle =
extensions.custom_border_color &&
features.border_color_swizzle.borderColorSwizzle &&
features.border_color_swizzle.borderColorSwizzleFromImage;
extensions.custom_border_color && features.border_color_swizzle.borderColorSwizzle;
}
RemoveExtensionFeatureIfUnsuitable(extensions.border_color_swizzle,
features.border_color_swizzle,
@@ -1490,11 +1494,26 @@ void Device::SetupFamilies(VkSurfaceKHR surface) {
}
}
std::optional<size_t> Device::GetSamplerHeapBudget() const {
if (sampler_heap_budget == 0) {
return std::nullopt;
bool Device::TryReserveCustomBorderColorSamplers(size_t count) const {
const size_t limit = properties.custom_border_color.maxCustomBorderColorSamplers;
if (limit == 0) {
return true;
}
return sampler_heap_budget;
size_t used = custom_border_color_samplers_used.load(std::memory_order_relaxed);
while (used + count <= limit) {
if (custom_border_color_samplers_used.compare_exchange_weak(
used, used + count, std::memory_order_relaxed, std::memory_order_relaxed)) {
return true;
}
}
return false;
}
void Device::ReleaseCustomBorderColorSamplers(size_t count) const {
if (count == 0) {
return;
}
custom_border_color_samplers_used.fetch_sub(count, std::memory_order_relaxed);
}
u64 Device::GetDeviceMemoryUsage() const {
@@ -1520,12 +1539,27 @@ void Device::CollectPhysicalMemoryInfo() {
device_access_memory = 0;
u64 device_initial_usage = 0;
u64 local_memory = 0;
const auto heap_has_usable_type = [&mem_properties](size_t heap) {
for (u32 index = 0; index < mem_properties.memoryTypeCount; ++index) {
if (mem_properties.memoryTypes[index].heapIndex != heap) {
continue;
}
if ((mem_properties.memoryTypes[index].propertyFlags &
VK_MEMORY_PROPERTY_PROTECTED_BIT) == 0) {
return true;
}
}
return false;
};
for (size_t element = 0; element < num_properties; ++element) {
const bool is_heap_local =
(mem_properties.memoryHeaps[element].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0;
if (!is_integrated && !is_heap_local) {
continue;
}
if (!heap_has_usable_type(element)) {
continue;
}
valid_heap_memory.push_back(element);
if (is_heap_local) {
local_memory += mem_properties.memoryHeaps[element].size;
@@ -1537,6 +1571,12 @@ void Device::CollectPhysicalMemoryInfo() {
}
device_access_memory += mem_properties.memoryHeaps[element].size;
}
const u64 committed_backing = Common::GetCommittedBackingSize();
if (committed_backing != 0) {
LOG_INFO(Render_Vulkan, "Discounting {} MiB of guest memory committed by the host",
committed_backing >> 20);
local_memory -= (std::min)(local_memory, committed_backing);
}
if (is_integrated) {
const s64 available_memory = static_cast<s64>(device_access_memory - device_initial_usage);
const u64 memory_size = Settings::values.vram_usage_mode.GetValue() == Settings::VramUsageMode::Aggressive ? 6_GiB : 4_GiB;
@@ -1551,6 +1591,7 @@ void Device::CollectPhysicalMemoryInfo() {
device_access_memory = std::min<u64>(device_access_memory, normal_memory + scaler_memory);
}
}
device_access_memory -= (std::min)(device_access_memory, committed_backing);
}
void Device::CollectToolingInfo() {
+55 -14
View File
@@ -6,6 +6,7 @@
#pragma once
#include <atomic>
#include <optional>
#include <set>
#include <span>
@@ -84,6 +85,7 @@ VK_DEFINE_HANDLE(VmaAllocator)
EXTENSION(EXT, CONDITIONAL_RENDERING, conditional_rendering) \
EXTENSION(EXT, CONSERVATIVE_RASTERIZATION, conservative_rasterization) \
EXTENSION(EXT, DEPTH_RANGE_UNRESTRICTED, depth_range_unrestricted) \
EXTENSION(EXT, EXTERNAL_MEMORY_HOST, external_memory_host) \
EXTENSION(EXT, MEMORY_BUDGET, memory_budget) \
EXTENSION(EXT, ROBUSTNESS_2, robustness_2) \
EXTENSION(EXT, SAMPLER_FILTER_MINMAX, sampler_filter_minmax) \
@@ -115,6 +117,14 @@ VK_DEFINE_HANDLE(VmaAllocator)
EXTENSION(IMG, FILTER_CUBIC, filter_cubic_img) \
EXTENSION(QCOM, FILTER_CUBIC_WEIGHTS, filter_cubic_weights)
#ifdef __ANDROID__
#define FOR_EACH_VK_PLATFORM_EXTENSION(EXTENSION) \
EXTENSION(EXT, QUEUE_FAMILY_FOREIGN, queue_family_foreign) \
EXTENSION(ANDROID, EXTERNAL_MEMORY_ANDROID_HARDWARE_BUFFER, external_memory_ahb)
#else
#define FOR_EACH_VK_PLATFORM_EXTENSION(EXTENSION)
#endif
// Define extensions which must be supported.
#define FOR_EACH_VK_MANDATORY_EXTENSION(EXTENSION_NAME) \
EXTENSION_NAME(VK_EXT_VERTEX_ATTRIBUTE_DIVISOR_EXTENSION_NAME) \
@@ -699,20 +709,18 @@ FN_MAX_LIMIT_LIST
return features.transform_feedback.geometryStreams;
}
/// Returns true if the device supports VK_EXT_custom_border_color.
bool IsExtCustomBorderColorSupported() const {
return extensions.custom_border_color;
/// Returns true if custom border colors can be created without a format.
bool IsCustomBorderColorUsable() const {
return extensions.custom_border_color &&
features.custom_border_color.customBorderColors &&
features.custom_border_color.customBorderColorWithoutFormat;
}
/// Returns true if customBorderColors feature is available.
bool IsCustomBorderColorsSupported() const {
return features.custom_border_color.customBorderColors;
}
/// Takes budget for samplers carrying a custom border color, false when exhausted.
bool TryReserveCustomBorderColorSamplers(size_t count) const;
/// Returns true if customBorderColorWithoutFormat feature is available.
bool IsCustomBorderColorWithoutFormatSupported() const {
return features.custom_border_color.customBorderColorWithoutFormat;
}
/// Gives back budget taken by TryReserveCustomBorderColorSamplers.
void ReleaseCustomBorderColorSamplers(size_t count) const;
/// Returns true if the device supports VK_EXT_color_write_enable.
bool IsExtColorWriteEnableSupported() const {
@@ -724,6 +732,12 @@ FN_MAX_LIMIT_LIST
return extensions.border_color_swizzle;
}
/// Returns true if samplers must be carried with border color swizzle mapping.
bool NeedsBorderColorSwizzleMapping() const {
return extensions.border_color_swizzle &&
!features.border_color_swizzle.borderColorSwizzleFromImage;
}
/// Returns true if borderColorSwizzleFromImage is available.
bool IsBorderColorSwizzleFromImageSupported() const {
return features.border_color_swizzle.borderColorSwizzleFromImage;
@@ -872,6 +886,30 @@ FN_MAX_LIMIT_LIST
return extensions.conditional_rendering;
}
bool IsExtExternalMemoryHostSupported() const {
return extensions.external_memory_host;
}
bool IsExtExternalMemoryAhbSupported() const {
#ifdef __ANDROID__
return extensions.external_memory_ahb && extensions.queue_family_foreign;
#else
return false;
#endif
}
u64 GetMinImportedHostPointerAlignment() const {
return properties.external_memory_host.minImportedHostPointerAlignment;
}
u64 GetMaxBufferSize() const {
return properties.maintenance4.maxBufferSize;
}
u64 GetMaxMemoryAllocationSize() const {
return properties.maintenance3.maxMemoryAllocationSize;
}
bool IsExtAstcDecodeModeSupported() const {
return extensions.astc_decode_mode;
}
@@ -919,8 +957,6 @@ FN_MAX_LIMIT_LIST
return has_broken_parallel_compiling;
}
std::optional<size_t> GetSamplerHeapBudget() const;
/// Returns the vendor name reported from Vulkan.
std::string_view GetVendorName() const {
return properties.driver.driverName;
@@ -1140,6 +1176,7 @@ private:
FOR_EACH_VK_FEATURE_1_4(FEATURE);
FOR_EACH_VK_FEATURE_EXT(FEATURE);
FOR_EACH_VK_EXTENSION(EXTENSION);
FOR_EACH_VK_PLATFORM_EXTENSION(EXTENSION);
#undef EXTENSION
#undef FEATURE
@@ -1171,8 +1208,12 @@ private:
VkPhysicalDeviceDescriptorBufferPropertiesEXT descriptor_buffer{};
VkPhysicalDeviceSubgroupSizeControlProperties subgroup_size_control{};
VkPhysicalDeviceTransformFeedbackPropertiesEXT transform_feedback{};
VkPhysicalDeviceMaintenance3Properties maintenance3{};
VkPhysicalDeviceMaintenance4Properties maintenance4{};
VkPhysicalDeviceMaintenance5PropertiesKHR maintenance5{};
VkPhysicalDeviceDepthStencilResolveProperties depth_stencil_resolve{};
VkPhysicalDeviceCustomBorderColorPropertiesEXT custom_border_color{};
VkPhysicalDeviceExternalMemoryHostPropertiesEXT external_memory_host{};
VkPhysicalDeviceProperties properties{};
};
@@ -1211,7 +1252,7 @@ private:
bool dynamic_state3_alpha_to_coverage{};
bool dynamic_state3_alpha_to_one{};
bool supports_conditional_barriers{}; ///< Allows barriers in conditional control flow.
size_t sampler_heap_budget{}; ///< Sampler budget for buggy drivers (0 = unlimited).
mutable std::atomic<size_t> custom_border_color_samplers_used{};
u64 device_access_memory{}; ///< Total size of device local memory in bytes.
u32 sets_per_pool{}; ///< Sets per Description Pool
NvidiaArchitecture nvidia_arch{NvidiaArchitecture::Arch_AmpereOrNewer};
@@ -25,9 +25,34 @@
#include "video_core/gpu_logging/gpu_logging.h"
#include "common/settings.h"
#ifdef __ANDROID__
#include <android/hardware_buffer.h>
#endif
namespace Vulkan {
namespace {
[[nodiscard]] std::optional<u32> FindImportMemoryType(
const VkPhysicalDeviceMemoryProperties &props, u32 type_mask) {
const auto find = [&](VkMemoryPropertyFlags wanted) -> std::optional<u32> {
for (u32 i = 0; i < props.memoryTypeCount; ++i) {
if (((type_mask >> i) & 1u) != 0 &&
(props.memoryTypes[i].propertyFlags & wanted) == wanted) {
return i;
}
}
return std::nullopt;
};
auto type_index = find(VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
VK_MEMORY_PROPERTY_HOST_COHERENT_BIT |
VK_MEMORY_PROPERTY_HOST_CACHED_BIT);
if (!type_index) {
type_index = find(VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
}
return type_index;
}
// Helpers translating MemoryUsage to flags/usage
[[maybe_unused]] VkMemoryPropertyFlags MemoryUsagePropertyFlags(MemoryUsage usage) {
@@ -200,6 +225,316 @@ namespace Vulkan {
size = 0;
}
HostMemoryImport::HostMemoryImport(const Device &device_, void *base, size_t size,
std::span<AHardwareBuffer *const> hardware_buffers,
size_t hardware_buffer_window, size_t hardware_buffer_base)
: device{device_} {
if (ImportHardwareBuffers(hardware_buffers, hardware_buffer_window, hardware_buffer_base,
size)) {
return;
}
if (device.IsTiler()) {
return;
}
if (!hardware_buffers.empty()) {
return;
}
if (ImportHostPointer(base, size)) {
return;
}
LOG_INFO(Render_Vulkan, "Unified memory disabled, no host memory import path");
}
bool HostMemoryImport::ImportHostPointer(void *base, size_t size) {
if (!device.IsExtExternalMemoryHostSupported()) {
return false;
}
const u64 alignment = device.GetMinImportedHostPointerAlignment();
if (alignment == 0 || !Common::IsAligned(reinterpret_cast<uintptr_t>(base), alignment) ||
!Common::IsAligned(size, alignment)) {
return false;
}
using namespace Common::Literals;
constexpr VkDeviceSize DesktopWindowSize = 4_GiB;
VkDeviceSize candidate_window = DesktopWindowSize;
const u64 max_buffer_size = device.GetMaxBufferSize();
if (max_buffer_size != 0 && max_buffer_size < candidate_window) {
candidate_window = max_buffer_size;
}
const u64 max_allocation_size = device.GetMaxMemoryAllocationSize();
if (max_allocation_size != 0 && max_allocation_size < candidate_window) {
candidate_window = max_allocation_size;
}
candidate_window = Common::AlignDown(candidate_window, alignment);
if (candidate_window == 0) {
return false;
}
window_size = candidate_window;
const auto &logical = device.GetLogical();
const auto memory_props = device.GetPhysical().GetMemoryProperties().memoryProperties;
for (size_t offset = 0; offset < size; offset += window_size) {
u8 *const window_base = static_cast<u8 *>(base) + offset;
const VkDeviceSize window_len =
(std::min)(static_cast<VkDeviceSize>(size - offset), window_size);
VkMemoryHostPointerPropertiesEXT host_props{
.sType = VK_STRUCTURE_TYPE_MEMORY_HOST_POINTER_PROPERTIES_EXT,
.pNext = nullptr,
.memoryTypeBits = 0,
};
if (logical.GetMemoryHostPointerPropertiesEXT(
VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, window_base,
&host_props) != VK_SUCCESS ||
host_props.memoryTypeBits == 0) {
break;
}
const VkExternalMemoryBufferCreateInfo external_info{
.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO,
.pNext = nullptr,
.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT,
};
const VkBufferCreateInfo buffer_ci{
.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,
.pNext = &external_info,
.flags = 0,
.size = window_len,
.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT,
.sharingMode = VK_SHARING_MODE_EXCLUSIVE,
.queueFamilyIndexCount = 0,
.pQueueFamilyIndices = nullptr,
};
VkBuffer new_buffer{};
if (logical.CreateBufferRaw(buffer_ci, &new_buffer) != VK_SUCCESS) {
break;
}
const VkMemoryRequirements requirements =
logical.GetBufferMemoryRequirements(new_buffer);
const u32 type_mask = requirements.memoryTypeBits & host_props.memoryTypeBits;
if (type_mask == 0 || requirements.size > window_len) {
logical.DestroyBufferRaw(new_buffer);
break;
}
const auto type_index = FindImportMemoryType(memory_props, type_mask);
if (!type_index) {
logical.DestroyBufferRaw(new_buffer);
break;
}
const u32 heap_index = memory_props.memoryTypes[*type_index].heapIndex;
const VkDeviceSize heap_size = memory_props.memoryHeaps[heap_index].size;
if (imported_size + window_len > heap_size / 2) {
logical.DestroyBufferRaw(new_buffer);
break;
}
const VkImportMemoryHostPointerInfoEXT import_info{
.sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_HOST_POINTER_INFO_EXT,
.pNext = nullptr,
.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT,
.pHostPointer = window_base,
};
const VkMemoryAllocateInfo alloc_info{
.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,
.pNext = &import_info,
.allocationSize = window_len,
.memoryTypeIndex = *type_index,
};
vk::DeviceMemory memory = logical.TryAllocateMemory(alloc_info);
if (!memory) {
logical.DestroyBufferRaw(new_buffer);
break;
}
if (logical.BindBufferMemory(new_buffer, *memory, 0) != VK_SUCCESS) {
logical.DestroyBufferRaw(new_buffer);
break;
}
windows.push_back(Window{
.memory = std::move(memory),
.buffer = new_buffer,
});
imported_size += static_cast<size_t>(window_len);
}
if (windows.empty()) {
return false;
}
return true;
}
bool HostMemoryImport::ImportHardwareBuffers(
[[maybe_unused]] std::span<AHardwareBuffer *const> hardware_buffers,
[[maybe_unused]] size_t hardware_buffer_window,
[[maybe_unused]] size_t hardware_buffer_base, [[maybe_unused]] size_t size) {
#ifdef __ANDROID__
if (hardware_buffers.empty() || hardware_buffer_window == 0 ||
!device.IsExtExternalMemoryAhbSupported()) {
return false;
}
const u64 max_allocation_size = device.GetMaxMemoryAllocationSize();
if (max_allocation_size != 0 && hardware_buffer_window > max_allocation_size) {
return false;
}
if (hardware_buffer_base >= size) {
return false;
}
const auto &logical = device.GetLogical();
const auto memory_props = device.GetPhysical().GetMemoryProperties().memoryProperties;
window_size = hardware_buffer_window;
base_offset = hardware_buffer_base;
const auto import_all = [&](VkBufferUsageFlags usage, bool want_address) {
for (size_t i = 0; i < hardware_buffers.size(); ++i) {
const size_t offset = hardware_buffer_base + i * hardware_buffer_window;
if (offset >= size) {
break;
}
const VkDeviceSize window_len = (std::min)(
static_cast<VkDeviceSize>(size - offset),
static_cast<VkDeviceSize>(hardware_buffer_window));
VkAndroidHardwareBufferPropertiesANDROID ahb_props{
.sType = VK_STRUCTURE_TYPE_ANDROID_HARDWARE_BUFFER_PROPERTIES_ANDROID,
.pNext = nullptr,
.allocationSize = 0,
.memoryTypeBits = 0,
};
if (logical.GetAndroidHardwareBufferPropertiesANDROID(hardware_buffers[i],
&ahb_props) != VK_SUCCESS ||
ahb_props.memoryTypeBits == 0 || ahb_props.allocationSize < window_len) {
break;
}
const VkExternalMemoryBufferCreateInfo external_info{
.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO,
.pNext = nullptr,
.handleTypes =
VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID,
};
const VkBufferCreateInfo buffer_ci{
.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,
.pNext = &external_info,
.flags = 0,
.size = window_len,
.usage = usage,
.sharingMode = VK_SHARING_MODE_EXCLUSIVE,
.queueFamilyIndexCount = 0,
.pQueueFamilyIndices = nullptr,
};
VkBuffer new_buffer{};
if (logical.CreateBufferRaw(buffer_ci, &new_buffer) != VK_SUCCESS) {
break;
}
const VkMemoryRequirements requirements =
logical.GetBufferMemoryRequirements(new_buffer);
const u32 type_mask = requirements.memoryTypeBits & ahb_props.memoryTypeBits;
if (type_mask == 0 || requirements.size > ahb_props.allocationSize) {
logical.DestroyBufferRaw(new_buffer);
break;
}
const auto type_index = FindImportMemoryType(memory_props, type_mask);
if (!type_index) {
logical.DestroyBufferRaw(new_buffer);
break;
}
const VkImportAndroidHardwareBufferInfoANDROID import_info{
.sType = VK_STRUCTURE_TYPE_IMPORT_ANDROID_HARDWARE_BUFFER_INFO_ANDROID,
.pNext = nullptr,
.buffer = hardware_buffers[i],
};
const VkMemoryDedicatedAllocateInfo dedicated_info{
.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO,
.pNext = &import_info,
.image = VK_NULL_HANDLE,
.buffer = new_buffer,
};
const VkMemoryAllocateFlagsInfo flags_info{
.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO,
.pNext = &dedicated_info,
.flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT,
.deviceMask = 0,
};
const void *alloc_next = &dedicated_info;
if (want_address) {
alloc_next = &flags_info;
}
const VkMemoryAllocateInfo alloc_info{
.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,
.pNext = alloc_next,
.allocationSize = ahb_props.allocationSize,
.memoryTypeIndex = *type_index,
};
vk::DeviceMemory memory = logical.TryAllocateMemory(alloc_info);
if (!memory) {
logical.DestroyBufferRaw(new_buffer);
break;
}
if (logical.BindBufferMemory(new_buffer, *memory, 0) != VK_SUCCESS) {
logical.DestroyBufferRaw(new_buffer);
break;
}
VkDeviceAddress address = 0;
if (want_address) {
address = logical.GetBufferDeviceAddress(new_buffer);
}
windows.push_back(Window{
.memory = std::move(memory),
.buffer = new_buffer,
.address = address,
});
imported_size += static_cast<size_t>(window_len);
}
return !windows.empty();
};
constexpr VkBufferUsageFlags TransferUsage =
VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
VkBufferUsageFlags shader_usage = TransferUsage |
VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT |
VK_BUFFER_USAGE_STORAGE_TEXEL_BUFFER_BIT | VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT |
VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT |
VK_BUFFER_USAGE_VERTEX_BUFFER_BIT | VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT;
const bool want_address = device.IsBufferDeviceAddressSupported();
VkBufferUsageFlags minimal_usage = TransferUsage | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT;
if (want_address) {
shader_usage |= VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT;
minimal_usage |= VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT;
}
const auto reset_windows = [&] {
for (Window &window : windows) {
if (window.buffer != VK_NULL_HANDLE) {
logical.DestroyBufferRaw(window.buffer);
}
}
windows.clear();
imported_size = 0;
};
bindable = import_all(shader_usage, want_address);
if (!bindable) {
reset_windows();
bindable = import_all(minimal_usage, want_address);
}
if (!bindable) {
reset_windows();
import_all(TransferUsage, false);
}
if (windows.empty()) {
window_size = 0;
base_offset = 0;
return false;
}
foreign_ownership = true;
return true;
#else
return false;
#endif
}
HostMemoryImport::~HostMemoryImport() {
for (Window &window : windows) {
if (window.buffer != VK_NULL_HANDLE) {
device.GetLogical().DestroyBufferRaw(window.buffer);
}
}
}
MemoryAllocator::MemoryAllocator(const Device &device_)
: device{device_}, allocator{device.GetAllocator()},
properties{device_.GetPhysical().GetMemoryProperties().memoryProperties},
@@ -332,6 +667,17 @@ namespace Vulkan {
return MemoryCommit(allocator, a, info);
}
HostMemoryImport *MemoryAllocator::CreateHostMemoryImport(
void *base, size_t size, std::span<AHardwareBuffer *const> hardware_buffers,
size_t hardware_buffer_window, size_t hardware_buffer_base) {
unified_memory = std::make_unique<HostMemoryImport>(
device, base, size, hardware_buffers, hardware_buffer_window, hardware_buffer_base);
if (!unified_memory->IsValid()) {
unified_memory.reset();
}
return unified_memory.get();
}
MemoryCommit MemoryAllocator::Commit(const vk::Buffer &buffer, MemoryUsage usage) {
// Allocate memory appropriate for this buffer automatically
const auto vma_usage = MemoryUsageVma(usage);
@@ -15,6 +15,8 @@
#include "video_core/vulkan_common/vulkan_wrapper.h"
#include "video_core/vulkan_common/vma.h"
struct AHardwareBuffer;
namespace Vulkan {
class Device;
@@ -84,6 +86,76 @@ namespace Vulkan {
void *mapped_ptr{}; ///< Optional persistent mapped pointer
};
class HostMemoryImport {
public:
explicit HostMemoryImport(const Device &device_, void *base, size_t size,
std::span<AHardwareBuffer *const> hardware_buffers,
size_t hardware_buffer_window, size_t hardware_buffer_base);
~HostMemoryImport();
HostMemoryImport(const HostMemoryImport &) = delete;
HostMemoryImport &operator=(const HostMemoryImport &) = delete;
[[nodiscard]] bool IsValid() const noexcept {
return !windows.empty();
}
[[nodiscard]] size_t GetSize() const noexcept {
return imported_size;
}
[[nodiscard]] size_t GetBaseOffset() const noexcept {
return base_offset;
}
[[nodiscard]] bool NeedsForeignOwnershipTransfer() const noexcept {
return foreign_ownership;
}
[[nodiscard]] VkDeviceSize GetWindowSize() const noexcept {
return window_size;
}
[[nodiscard]] VkBuffer GetWindowBuffer(size_t index) const noexcept {
return windows[index].buffer;
}
[[nodiscard]] VkDeviceAddress GetWindowAddress(size_t index) const noexcept {
return windows[index].address;
}
[[nodiscard]] size_t GetWindowCount() const noexcept {
return windows.size();
}
[[nodiscard]] bool IsBindable() const noexcept {
return bindable;
}
private:
struct Window {
vk::DeviceMemory memory;
VkBuffer buffer{};
VkDeviceAddress address{};
};
bool ImportHostPointer(void *base, size_t size);
bool ImportHardwareBuffers(std::span<AHardwareBuffer *const> hardware_buffers,
size_t hardware_buffer_window, size_t hardware_buffer_base,
size_t size);
const Device &device;
std::vector<Window> windows;
VkDeviceSize window_size{};
size_t imported_size{};
size_t base_offset{};
bool foreign_ownership{};
bool bindable{};
};
/// Memory allocator container.
/// Allocates and releases memory allocations on demand.
class MemoryAllocator {
@@ -120,6 +192,15 @@ namespace Vulkan {
/// Commits memory required by the buffer and binds it (for buffers created outside VMA).
MemoryCommit Commit(const vk::Buffer &buffer, MemoryUsage usage);
HostMemoryImport *CreateHostMemoryImport(void *base, size_t size,
std::span<AHardwareBuffer *const> hardware_buffers,
size_t hardware_buffer_window,
size_t hardware_buffer_base);
[[nodiscard]] HostMemoryImport *GetHostMemoryImport() const noexcept {
return unified_memory.get();
}
private:
static bool IsAutoUsage(VmaMemoryUsage u) noexcept {
switch (u) {
@@ -137,6 +218,7 @@ namespace Vulkan {
const VkPhysicalDeviceMemoryProperties properties; ///< Physical device memory properties.
VkDeviceSize buffer_image_granularity; ///< Adjacent buffer/image granularity
u32 valid_memory_types{~0u};
std::unique_ptr<HostMemoryImport> unified_memory;
};
} // namespace Vulkan
@@ -217,12 +217,16 @@ void Load(VkDevice device, DeviceDispatch& dld) noexcept {
X(vkGetBufferMemoryRequirements2);
X(vkGetDeviceQueue);
X(vkGetEventStatus);
X(vkGetMemoryHostPointerPropertiesEXT);
X(vkGetFenceStatus);
X(vkGetImageMemoryRequirements);
X(vkGetPipelineCacheData);
X(vkGetMemoryFdKHR);
#ifdef _WIN32
X(vkGetMemoryWin32HandleKHR);
#endif
#ifdef __ANDROID__
X(vkGetAndroidHardwareBufferPropertiesANDROID);
#endif
X(vkGetQueryPoolResults);
X(vkGetPipelineExecutablePropertiesKHR);
@@ -333,12 +333,16 @@ struct DeviceDispatch : InstanceDispatch {
PFN_vkGetBufferMemoryRequirements2 vkGetBufferMemoryRequirements2{};
PFN_vkGetDeviceQueue vkGetDeviceQueue{};
PFN_vkGetEventStatus vkGetEventStatus{};
PFN_vkGetMemoryHostPointerPropertiesEXT vkGetMemoryHostPointerPropertiesEXT{};
PFN_vkGetFenceStatus vkGetFenceStatus{};
PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements{};
PFN_vkGetPipelineCacheData vkGetPipelineCacheData{};
PFN_vkGetMemoryFdKHR vkGetMemoryFdKHR{};
#ifdef _WIN32
PFN_vkGetMemoryWin32HandleKHR vkGetMemoryWin32HandleKHR{};
#endif
#ifdef __ANDROID__
PFN_vkGetAndroidHardwareBufferPropertiesANDROID vkGetAndroidHardwareBufferPropertiesANDROID{};
#endif
PFN_vkGetPipelineExecutablePropertiesKHR vkGetPipelineExecutablePropertiesKHR{};
PFN_vkGetPipelineExecutableStatisticsKHR vkGetPipelineExecutableStatisticsKHR{};
@@ -1085,6 +1089,34 @@ public:
VkMemoryRequirements GetBufferMemoryRequirements(VkBuffer buffer,
void* pnext = nullptr) const noexcept;
VkResult GetMemoryHostPointerPropertiesEXT(
VkExternalMemoryHandleTypeFlagBits handle_type, const void* host_pointer,
VkMemoryHostPointerPropertiesEXT* out_properties) const noexcept {
return dld->vkGetMemoryHostPointerPropertiesEXT(handle, handle_type, host_pointer,
out_properties);
}
#ifdef __ANDROID__
VkResult GetAndroidHardwareBufferPropertiesANDROID(
const struct AHardwareBuffer* buffer,
VkAndroidHardwareBufferPropertiesANDROID* out_properties) const noexcept {
return dld->vkGetAndroidHardwareBufferPropertiesANDROID(handle, buffer, out_properties);
}
#endif
VkResult CreateBufferRaw(const VkBufferCreateInfo& ci, VkBuffer* out_buffer) const noexcept {
return dld->vkCreateBuffer(handle, &ci, nullptr, out_buffer);
}
void DestroyBufferRaw(VkBuffer buffer) const noexcept {
dld->vkDestroyBuffer(handle, buffer, nullptr);
}
VkResult BindBufferMemory(VkBuffer buffer, VkDeviceMemory memory,
VkDeviceSize offset) const noexcept {
return dld->vkBindBufferMemory(handle, buffer, memory, offset);
}
VkMemoryRequirements GetImageMemoryRequirements(VkImage image) const noexcept;
std::vector<VkPipelineExecutablePropertiesKHR> GetPipelineExecutablePropertiesKHR(