[video_core, settings] readback for nce invalidations (#4490) optimization and nuke toggles (#4502) (#4531)

- [x] I have read and followed the [Contribution Guidelines](https://git.eden-emu.dev/eden-emu/eden/src/branch/master/CONTRIBUTING.md#code-contributions).
- [x] I have read and followed the [AI Policy](https://git.eden-emu.dev/eden-emu/eden/src/branch/master/docs/policies/AI.md).
- [x] I have read and followed the [Coding Guidelines](https://git.eden-emu.dev/eden-emu/eden/src/branch/master/docs/policies/Coding.md) to the best of my ability.

-------------------

Follow up for #4490: Originally i was racing locks between dirty and clean invalidations, but +94% were clean so it was causing useless locks. Solutioin was to share locks between both.

P.S.: THIS IS INTENDED TO FIX RANDOM CRASHES IN UE, OBJECTIVELY REPRODUCIBLE IN ENDER MAGNOLIA IN ANDROID.

Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/4531
Reviewed-by: CamilleLaVey <camillelavey99@gmail.com>
Reviewed-by: Shinmegumi <shinmegumi@eden-emu.dev>

src/video_core/gpu.cpp

@@ -227,17 +227,17 @@ struct GPU::Impl {
 
     /// Notify rasterizer that any caches of the specified region should be invalidated
     void InvalidateRegion(DAddr addr, u64 size, bool preserve_gpu_writes) {
-        if (Settings::values.nce_invalidation_gpu_readback.GetValue()) {
+        if (preserve_gpu_writes) {
             VideoCore::RasterizerInterface* rasterizer = renderer->ReadRasterizer();
-            if (preserve_gpu_writes && rasterizer->MustFlushRegion(addr, size, VideoCommon::CacheType::BufferCache)) {
+            if (!rasterizer->TryCacheInvalidation(addr, size)) {
                 const u64 fence = RequestSyncOperation([rasterizer, addr, size] {
                     rasterizer->FlushRegion(addr, size, VideoCommon::CacheType::BufferCache);
                     rasterizer->OnCacheInvalidation(addr, size);
                 });
                 gpu_thread.TickGPU(is_async);
                 WaitForSyncOperation(fence);
-                return;
             }
+            return;
         }
         gpu_thread.InvalidateRegion(addr, size);
     }

src/video_core/rasterizer_interface.h

@@ -1,3 +1,6 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
 // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
@@ -111,6 +114,12 @@ public:
     /// Notify rasterizer that any caches of the specified region are desync with guest
     virtual void OnCacheInvalidation(PAddr addr, u64 size) = 0;
 
+    virtual bool TryCacheInvalidation(DAddr addr, u64 size) {
+        if (MustFlushRegion(addr, size, VideoCommon::CacheType::BufferCache)) return false;
+        OnCacheInvalidation(addr, size);
+        return true;
+    }
+
     virtual bool OnCPUWrite(PAddr addr, u64 size) = 0;
 
     /// Sync memory between guest and host.

src/video_core/renderer_vulkan/vk_rasterizer.cpp

@@ -798,6 +798,18 @@ void RasterizerVulkan::OnCacheInvalidation(DAddr addr, u64 size) {
     pipeline_cache.InvalidateRegion(addr, size);
 }
 
+bool RasterizerVulkan::TryCacheInvalidation(DAddr addr, u64 size) {
+    if (addr == 0 || size == 0) return true;
+    {
+        std::scoped_lock lock{buffer_cache.mutex, texture_cache.mutex};
+        if (buffer_cache.IsRegionGpuModified(addr, size)) return false;
+        texture_cache.WriteMemory(addr, size);
+        buffer_cache.WriteMemory(addr, size);
+    }
+    pipeline_cache.InvalidateRegion(addr, size);
+    return true;
+}
+
 void RasterizerVulkan::InvalidateGPUCache() {
     gpu.InvalidateGPUCache();
 }

src/video_core/renderer_vulkan/vk_rasterizer.h

@@ -106,6 +106,7 @@ public:
                           VideoCommon::CacheType which = VideoCommon::CacheType::All) override;
     void InnerInvalidation(std::span<const std::pair<DAddr, std::size_t>> sequences) override;
     void OnCacheInvalidation(DAddr addr, u64 size) override;
+    bool TryCacheInvalidation(DAddr addr, u64 size) override;
     bool OnCPUWrite(DAddr addr, u64 size) override;
     void InvalidateGPUCache() override;
     void UnmapMemory(DAddr addr, u64 size) override;