[core, hle, video_core, memory] Improve multi-process, display layers, dynamic shader cache and rework overlay (#4238)

- [x] I have read and followed the [Contribution Guidelines](https://git.eden-emu.dev/eden-emu/eden/src/branch/master/CONTRIBUTING.md#code-contributions).
- [x] I have read and followed the [AI Policy](https://git.eden-emu.dev/eden-emu/eden/src/branch/master/docs/policies/AI.md)
- [x] I have read and followed the [Coding Guidelines](https://git.eden-emu.dev/eden-emu/eden/src/branch/master/docs/policies/Coding.md) to the best of my ability.

-------------------

- Dynamic shader cache reloading capability for qlaunch
- Multi-process improvements (thanks to @frank1734), instead of just using the main application we now respect caller process (also did it for HID devices while I was at it)
- Layer stack masks & shared buffer screenshot for video core, added different masks (screenshot, recording, etc.) this was discovered as an issue due to how in qlaunch the transition was not right between applications. May not be perfect but also fixes screenshots while using qlaunch
- Reworked overlay display management (input and visibility) - instead of random numbers as I've previously did, I decided to add an AppletZIndex enum for better readability. Also split capability of input by touch and gamepad.
- etc.

Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/4238
Reviewed-by: Samuel <lizzie@eden-emu.dev>
Reviewed-by: CamilleLaVey <camillelavey99@gmail.com>
Reviewed-by: MaranBr <maranbr@eden-emu.dev>

src/video_core/framebuffer_config.cpp

@@ -1,11 +1,35 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
 // SPDX-FileCopyrightText: Copyright 2024 yuzu Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
+#include <algorithm>
+
 #include "common/assert.h"
 #include "video_core/framebuffer_config.h"
 
 namespace Tegra {
 
+std::span<const FramebufferConfig> FilterLayerStack(
+    std::span<const FramebufferConfig> layers, Service::Nvnflinger::LayerStackId stack,
+    std::vector<FramebufferConfig>& scratch) {
+    const u32 bit = Service::Nvnflinger::LayerStackBit(stack);
+
+    if (std::ranges::all_of(layers,
+                            [bit](const auto& layer) { return (layer.layer_stack_mask & bit) != 0; }))
+        return layers;
+
+    scratch.clear();
+    for (const auto& layer : layers) {
+        if ((layer.layer_stack_mask & bit) != 0) {
+            scratch.push_back(layer);
+        }
+    }
+
+    return scratch;
+}
+
 Common::Rectangle<f32> NormalizeCrop(const FramebufferConfig& framebuffer, u32 texture_width,
                                      u32 texture_height) {
     f32 left, top, right, bottom;

src/video_core/framebuffer_config.h

@@ -1,11 +1,18 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
 // SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
 #pragma once
 
+#include <span>
+#include <vector>
+
 #include "common/common_types.h"
 #include "common/math_util.h"
 #include "core/hle/service/nvnflinger/buffer_transform_flags.h"
+#include "core/hle/service/nvnflinger/hwc_layer.h"
 #include "core/hle/service/nvnflinger/pixel_format.h"
 #include "core/hle/service/nvnflinger/ui/fence.h"
 
@@ -30,9 +37,17 @@ struct FramebufferConfig {
     Service::android::BufferTransformFlags transform_flags{};
     Common::Rectangle<int> crop_rect{};
     BlendMode blending{};
+    u32 layer_stack_mask{Service::Nvnflinger::DefaultLayerStackMask};
 };
 
 Common::Rectangle<f32> NormalizeCrop(const FramebufferConfig& framebuffer, u32 texture_width,
                                      u32 texture_height);
 
+/**
+ * Returns the subset of layers belonging to a stack.
+ */
+std::span<const FramebufferConfig> FilterLayerStack(std::span<const FramebufferConfig> layers,
+                                                    Service::Nvnflinger::LayerStackId stack,
+                                                    std::vector<FramebufferConfig>& scratch);
+
 } // namespace Tegra

src/video_core/gpu.cpp

@@ -144,6 +144,12 @@ struct GPU::Impl {
         sync_request_cv.wait(lck, [this, fence] { return CurrentSyncRequestFence() >= fence; });
     }
 
+    void WaitForIdle() {
+        const u64 fence = RequestSyncOperation([] {});
+        gpu_thread.TickGPU(is_async);
+        WaitForSyncOperation(fence);
+    }
+
     /// Tick pending requests within the GPU.
     void TickWork() {
         std::unique_lock lck{sync_request_mutex};
@@ -484,6 +490,10 @@ void GPU::NotifyShutdown() {
     impl->NotifyShutdown();
 }
 
+void GPU::WaitForIdle() {
+    impl->WaitForIdle();
+}
+
 void GPU::ObtainContext() {
     impl->ObtainContext();
 }

src/video_core/gpu.h

@@ -168,6 +168,8 @@ public:
 
     void WaitForSyncOperation(u64 fence);
 
+    void WaitForIdle();
+
     /// Tick pending requests within the GPU.
     void TickWork();
 

src/video_core/renderer_base.cpp

@@ -36,7 +36,8 @@ bool RendererBase::IsScreenshotPending() const {
 }
 
 void RendererBase::RequestScreenshot(void* data, std::function<void(bool)> callback,
-                                     const Layout::FramebufferLayout& layout) {
+                                     const Layout::FramebufferLayout& layout,
+                                     Service::Nvnflinger::LayerStackId layer_stack) {
     if (renderer_settings.screenshot_requested) {
         LOG_ERROR(Render, "A screenshot is already requested or in progress, ignoring the request");
         return;
@@ -48,6 +49,7 @@ void RendererBase::RequestScreenshot(void* data, std::function<void(bool)> callb
     renderer_settings.screenshot_bits = data;
     renderer_settings.screenshot_complete_callback = async_callback;
     renderer_settings.screenshot_framebuffer_layout = layout;
+    renderer_settings.screenshot_layer_stack = layer_stack;
     renderer_settings.screenshot_requested = true;
 }
 

src/video_core/renderer_base.h

@@ -1,3 +1,6 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
 // SPDX-FileCopyrightText: 2014 Citra Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
@@ -26,6 +29,7 @@ struct RendererSettings {
     void* screenshot_bits{};
     std::function<void(bool)> screenshot_complete_callback;
     Layout::FramebufferLayout screenshot_framebuffer_layout;
+    Service::Nvnflinger::LayerStackId screenshot_layer_stack{Service::Nvnflinger::LayerStackId::Default};
 };
 
 class RendererBase {
@@ -88,9 +92,11 @@ public:
     /// Returns true if a screenshot is being processed
     bool IsScreenshotPending() const;
 
-    /// Request a screenshot of the next frame
+    /// Request a screenshot of the next frame.
     void RequestScreenshot(void* data, std::function<void(bool)> callback,
-                           const Layout::FramebufferLayout& layout);
+                           const Layout::FramebufferLayout& layout,
+                           Service::Nvnflinger::LayerStackId layer_stack =
+                               Service::Nvnflinger::LayerStackId::Default);
 
 protected:
     Core::Frontend::EmuWindow& render_window; ///< Reference to the render window handle.

src/video_core/renderer_opengl/renderer_opengl.cpp

@@ -199,7 +199,10 @@ void RendererOpenGL::RenderScreenshot(std::span<const Tegra::FramebufferConfig>
         return;
     }
 
-    RenderToBuffer(framebuffers, renderer_settings.screenshot_framebuffer_layout,
+    const auto screenshot_layers = Tegra::FilterLayerStack(
+        framebuffers, renderer_settings.screenshot_layer_stack, screenshot_layer_scratch);
+
+    RenderToBuffer(screenshot_layers, renderer_settings.screenshot_framebuffer_layout,
                    renderer_settings.screenshot_bits);
 
     renderer_settings.screenshot_complete_callback(true);
@@ -208,6 +211,12 @@ void RendererOpenGL::RenderScreenshot(std::span<const Tegra::FramebufferConfig>
 
 void RendererOpenGL::RenderAppletCaptureLayer(
     std::span<const Tegra::FramebufferConfig> framebuffers) {
+    const auto capture_layers = Tegra::FilterLayerStack(
+        framebuffers, Service::Nvnflinger::LayerStackId::LastFrame, applet_capture_layers);
+
+    if (capture_layers.empty())
+        return;
+
     GLint old_read_fb;
     GLint old_draw_fb;
     glGetIntegerv(GL_READ_FRAMEBUFFER_BINDING, &old_read_fb);
@@ -217,7 +226,7 @@ void RendererOpenGL::RenderAppletCaptureLayer(
     glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_RENDERBUFFER,
                               capture_renderbuffer.handle);
 
-    blit_applet->DrawScreen(framebuffers, VideoCore::Capture::Layout, true);
+    blit_applet->DrawScreen(capture_layers, VideoCore::Capture::Layout, true);
 
     glBindFramebuffer(GL_READ_FRAMEBUFFER, old_read_fb);
     glBindFramebuffer(GL_DRAW_FRAMEBUFFER, old_draw_fb);

src/video_core/renderer_opengl/renderer_opengl.h

@@ -1,4 +1,4 @@
-// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 // SPDX-FileCopyrightText: 2014 Citra Emulator Project
@@ -61,6 +61,9 @@ private:
     void RenderScreenshot(std::span<const Tegra::FramebufferConfig> framebuffers);
     void RenderAppletCaptureLayer(std::span<const Tegra::FramebufferConfig> framebuffers);
 
+    std::vector<Tegra::FramebufferConfig> applet_capture_layers;
+    std::vector<Tegra::FramebufferConfig> screenshot_layer_scratch;
+
     Core::Frontend::EmuWindow& emu_window;
     Tegra::MaxwellDeviceMemoryManager& device_memory;
     Tegra::GPU& gpu;

src/video_core/renderer_vulkan/renderer_vulkan.cpp

@@ -292,8 +292,11 @@ void RendererVulkan::RenderScreenshot(std::span<const Tegra::FramebufferConfig>
         return;
     }
 
+    const auto screenshot_layers = Tegra::FilterLayerStack(
+        framebuffers, renderer_settings.screenshot_layer_stack, screenshot_layer_scratch);
+
     const auto& layout{renderer_settings.screenshot_framebuffer_layout};
-    const auto dst_buffer = RenderToBuffer(framebuffers, layout, VK_FORMAT_B8G8R8A8_UNORM,
+    const auto dst_buffer = RenderToBuffer(screenshot_layers, layout, VK_FORMAT_B8G8R8A8_UNORM,
                                            layout.width * layout.height * 4);
 
     std::memcpy(renderer_settings.screenshot_bits, dst_buffer.Mapped().data(),
@@ -332,6 +335,12 @@ std::vector<u8> RendererVulkan::GetAppletCaptureBuffer() {
 
 void RendererVulkan::RenderAppletCaptureLayer(
     std::span<const Tegra::FramebufferConfig> framebuffers) {
+    const auto capture_layers = Tegra::FilterLayerStack(
+        framebuffers, Service::Nvnflinger::LayerStackId::LastFrame, applet_capture_layers);
+
+    if (capture_layers.empty())
+        return;
+
     if (!applet_frame.image) {
         applet_frame.image = CreateWrappedImage(memory_allocator, CaptureImageSize, CaptureFormat);
         applet_frame.image_view = CreateWrappedImageView(device, applet_frame.image, CaptureFormat);
@@ -340,7 +349,7 @@ void RendererVulkan::RenderAppletCaptureLayer(
     }
 
     scheduler.RequestOutsideRenderPassOperationContext();
-    blit_applet.DrawToFrame(device, rasterizer, &applet_frame, framebuffers, VideoCore::Capture::Layout, 1,
+    blit_applet.DrawToFrame(device, rasterizer, &applet_frame, capture_layers, VideoCore::Capture::Layout, 1,
                             CaptureFormat);
 }
 

src/video_core/renderer_vulkan/renderer_vulkan.h

@@ -75,6 +75,9 @@ private:
     void RenderScreenshot(std::span<const Tegra::FramebufferConfig> framebuffers);
     void RenderAppletCaptureLayer(std::span<const Tegra::FramebufferConfig> framebuffers);
 
+    std::vector<Tegra::FramebufferConfig> applet_capture_layers;
+    std::vector<Tegra::FramebufferConfig> screenshot_layer_scratch;
+
     Tegra::MaxwellDeviceMemoryManager& device_memory;
     Tegra::GPU& gpu;
 

src/video_core/renderer_vulkan/vk_pipeline_cache.cpp

@@ -592,6 +592,14 @@ void PipelineCache::LoadDiskResources(u64 title_id, std::stop_token stop_loading
     if (title_id == 0) {
         return;
     }
+    if (!pipeline_cache_filename.empty()) {
+        serialization_thread.WaitForRequests();
+        if (use_vulkan_pipeline_cache && !vulkan_pipeline_cache_filename.empty()) {
+            SerializeVulkanPipelineCache(vulkan_pipeline_cache_filename, vulkan_pipeline_cache,
+                                         CACHE_VERSION);
+        }
+    }
+
     const auto shader_dir{Common::FS::GetEdenPath(Common::FS::EdenPath::ShaderDir)};
     const auto base_dir{shader_dir / fmt::format("{:016x}", title_id)};
     if (!Common::FS::CreateDir(shader_dir) || !Common::FS::CreateDir(base_dir)) {