[vulkan, vk_pipeline_cache] PSO Optimizations (#4294)

PR/Commit Owner: CamilleLaVey

This PR prevents the recompilation of shaders when a new session has been started, based on data found on the recompiler, each new session does still recompile already cached shader cache, reducing the reaction and speed of games by at certain level, this prevent the warm-up with post sessions, only needed the initial warm-up and works globally for all games.

Co-authored-by: CamilleLaVey <camillelavey99@gmail.com>
Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/4294
Reviewed-by: CamilleLaVey <camillelavey99@gmail.com>
Reviewed-by: Maufeat <sahyno1996@gmail.com>

src/video_core/renderer_vulkan/blit_image.cpp

@@ -1281,7 +1281,7 @@ VkPipeline BlitImageHelper::FindOrEmplaceColorPipeline(const BlitImagePipelineKe
         .subpass = 0,
         .basePipelineHandle = VK_NULL_HANDLE,
         .basePipelineIndex = 0,
-    }));
+    }, device.StaticPipelineCache()));
     return *blit_color_pipelines.back();
 }
 
@@ -1313,7 +1313,7 @@ VkPipeline BlitImageHelper::FindOrEmplaceDepthStencilPipeline(const BlitImagePip
         .subpass = 0,
         .basePipelineHandle = VK_NULL_HANDLE,
         .basePipelineIndex = 0,
-    }));
+    }, device.StaticPipelineCache()));
     return *blit_depth_stencil_pipelines.back();
 }
 
@@ -1366,7 +1366,7 @@ VkPipeline BlitImageHelper::FindOrEmplaceClearColorPipeline(const BlitImagePipel
         .subpass = 0,
         .basePipelineHandle = VK_NULL_HANDLE,
         .basePipelineIndex = 0,
-    }));
+    }, device.StaticPipelineCache()));
     return *clear_color_pipelines.back();
 }
 
@@ -1422,7 +1422,7 @@ VkPipeline BlitImageHelper::FindOrEmplaceClearStencilPipeline(
         .subpass = 0,
         .basePipelineHandle = VK_NULL_HANDLE,
         .basePipelineIndex = 0,
-    }));
+    }, device.StaticPipelineCache()));
     return *clear_stencil_pipelines.back();
 }
 
@@ -1465,7 +1465,7 @@ VkPipeline BlitImageHelper::FindOrEmplaceBlitColorMSAAPipeline(const BlitMSAAPip
         .subpass = 0,
         .basePipelineHandle = VK_NULL_HANDLE,
         .basePipelineIndex = 0,
-    }));
+    }, device.StaticPipelineCache()));
     return *blit_msaa_color_pipelines.back();
 }
 
@@ -1584,7 +1584,7 @@ VkPipeline BlitImageHelper::FindOrEmplaceResolveDepthStencilPipeline(VkRenderPas
         .subpass = 0,
         .basePipelineHandle = VK_NULL_HANDLE,
         .basePipelineIndex = 0,
-    }));
+    }, device.StaticPipelineCache()));
     return *pipelines.back();
 }
 
@@ -1697,7 +1697,7 @@ VkPipeline BlitImageHelper::FindOrEmplaceMSAACopyPipeline(const MSAACopyPipeline
         .subpass = 0,
         .basePipelineHandle = VK_NULL_HANDLE,
         .basePipelineIndex = 0,
-    }));
+    }, device.StaticPipelineCache()));
     return *msaa_copy_pipelines.back();
 }
 
@@ -1817,7 +1817,7 @@ void BlitImageHelper::ConvertPipelineEx(vk::Pipeline& pipeline, VkRenderPass ren
         .subpass = 0,
         .basePipelineHandle = VK_NULL_HANDLE,
         .basePipelineIndex = 0,
-    });
+    }, device.StaticPipelineCache());
 }
 
 void BlitImageHelper::ConvertPipelineColorTargetEx(vk::Pipeline& pipeline, VkRenderPass renderpass,
@@ -1860,7 +1860,7 @@ void BlitImageHelper::ConvertPipeline(vk::Pipeline& pipeline, VkRenderPass rende
         .subpass = 0,
         .basePipelineHandle = VK_NULL_HANDLE,
         .basePipelineIndex = 0,
-    });
+    }, device.StaticPipelineCache());
 }
 
 } // namespace Vulkan

src/video_core/renderer_vulkan/present/util.cpp

@@ -521,7 +521,7 @@ static vk::Pipeline CreateWrappedPipelineImpl(
         .subpass = 0,
         .basePipelineHandle = 0,
         .basePipelineIndex = 0,
-    });
+    }, device.StaticPipelineCache());
 }
 
 vk::Pipeline CreateWrappedPipeline(const Device& device, vk::RenderPass& renderpass,

src/video_core/renderer_vulkan/vk_compute_pass.cpp

@@ -268,7 +268,7 @@ ComputePass::ComputePass(const Device& device_, Scheduler& scheduler, Descriptor
         .layout = *layout,
         .basePipelineHandle = {},
         .basePipelineIndex = 0,
-    });
+    }, device.StaticPipelineCache());
 }
 
 ComputePass::~ComputePass() = default;

src/video_core/renderer_vulkan/vk_pipeline_cache.cpp

@@ -63,6 +63,8 @@ using VideoCommon::GenericEnvironment;
 using VideoCommon::GraphicsEnvironment;
 
 constexpr u32 CACHE_VERSION = 18;
+constexpr size_t VULKAN_CACHE_FLUSH_PIPELINES = 128;
+constexpr size_t VULKAN_CACHE_FLUSH_MIN_SECONDS = 30;
 constexpr std::array<char, 8> VULKAN_CACHE_MAGIC_NUMBER{'y', 'u', 'z', 'u', 'v', 'k', 'c', 'h'};
 
 template <typename Container>
@@ -710,6 +712,10 @@ void PipelineCache::LoadDiskResources(u64 title_id, std::stop_token stop_loading
     if (use_vulkan_pipeline_cache) {
         SerializeVulkanPipelineCache(vulkan_pipeline_cache_filename, vulkan_pipeline_cache,
                                      CACHE_VERSION);
+        size_t size = 0;
+        vulkan_pipeline_cache.Read(&size, nullptr);
+        last_cache_size.store(size, std::memory_order_relaxed);
+        last_flush = std::chrono::steady_clock::now();
     }
 
     if (state.statistics) {
@@ -717,6 +723,35 @@ void PipelineCache::LoadDiskResources(u64 title_id, std::stop_token stop_loading
     }
 }
 
+void PipelineCache::QueueVulkanPipelineCacheFlush() {
+    if (!use_vulkan_pipeline_cache || vulkan_pipeline_cache_filename.empty()) {
+        return;
+    }
+    if (++pipelines_since_flush < VULKAN_CACHE_FLUSH_PIPELINES) {
+        return;
+    }
+    const auto now = std::chrono::steady_clock::now();
+    const auto megabytes = last_cache_size.load(std::memory_order_relaxed) / (1024 * 1024);
+    const std::chrono::seconds interval{
+        std::max<size_t>(VULKAN_CACHE_FLUSH_MIN_SECONDS, megabytes)};
+    if (last_flush.time_since_epoch().count() != 0 && now - last_flush < interval) {
+        return;
+    }
+    if (flush_in_flight.exchange(true, std::memory_order_acq_rel)) {
+        return;
+    }
+    pipelines_since_flush = 0;
+    last_flush = now;
+    serialization_thread.QueueWork([this] {
+        SerializeVulkanPipelineCache(vulkan_pipeline_cache_filename, vulkan_pipeline_cache,
+                                     CACHE_VERSION);
+        size_t size = 0;
+        vulkan_pipeline_cache.Read(&size, nullptr);
+        last_cache_size.store(size, std::memory_order_relaxed);
+        flush_in_flight.store(false, std::memory_order_release);
+    });
+}
+
 GraphicsPipeline* PipelineCache::CurrentGraphicsPipelineSlowPath() {
     const auto [pair, is_new]{graphics_cache.try_emplace(graphics_key)};
     auto& pipeline{pair->second};
@@ -755,7 +790,7 @@ std::unique_ptr<GraphicsPipeline> PipelineCache::CreateGraphicsPipeline(
     std::span<Shader::Environment* const> envs, PipelineStatistics* statistics,
     bool build_in_parallel) try {
     auto hash = key.Hash();
-    LOG_INFO(Render_Vulkan, "{:#016x}", hash);
+    LOG_DEBUG(Render_Vulkan, "{:#016x}", hash);
     size_t env_index{0};
     std::array<Shader::IR::Program, Maxwell::MaxShaderProgram> programs;
     const bool uses_vertex_a{key.unique_hashes[0] != 0};
@@ -891,6 +926,7 @@ std::unique_ptr<GraphicsPipeline> PipelineCache::CreateGraphicsPipeline() {
         }
         SerializePipeline(key, env_ptrs, pipeline_cache_filename, CACHE_VERSION);
     });
+    QueueVulkanPipelineCacheFlush();
     return pipeline;
 }
 
@@ -910,6 +946,7 @@ std::unique_ptr<ComputePipeline> PipelineCache::CreateComputePipeline(
         SerializePipeline(key, std::array<const GenericEnvironment*, 1>{&env_},
                           pipeline_cache_filename, CACHE_VERSION);
     });
+    QueueVulkanPipelineCacheFlush();
     return pipeline;
 }
 
@@ -922,7 +959,7 @@ std::unique_ptr<ComputePipeline> PipelineCache::CreateComputePipeline(
         return nullptr;
     }
 
-    LOG_INFO(Render_Vulkan, "{:#016x}", hash);
+    LOG_DEBUG(Render_Vulkan, "{:#016x}", hash);
 
     Shader::Maxwell::Flow::CFG cfg{env, pools.flow_block, env.StartAddress()};
 

src/video_core/renderer_vulkan/vk_pipeline_cache.h

@@ -7,6 +7,8 @@
 #pragma once
 
 #include <array>
+#include <atomic>
+#include <chrono>
 #include <cstddef>
 #include <filesystem>
 #include <memory>
@@ -144,6 +146,8 @@ private:
     vk::PipelineCache LoadVulkanPipelineCache(const std::filesystem::path& filename,
                                               u32 expected_cache_version);
 
+    void QueueVulkanPipelineCacheFlush();
+
     const Device& device;
     Scheduler& scheduler;
     DescriptorPool& descriptor_pool;
@@ -171,6 +175,10 @@ private:
 
     std::filesystem::path vulkan_pipeline_cache_filename;
     vk::PipelineCache vulkan_pipeline_cache;
+    size_t pipelines_since_flush{};
+    std::chrono::steady_clock::time_point last_flush{};
+    std::atomic<size_t> last_cache_size{};
+    std::atomic_bool flush_in_flight{};
 
     Common::ThreadWorker workers;
     Common::ThreadWorker serialization_thread;