[core, hle, video_core, memory] Improve multi-process, display layers, dynamic shader cache and rework overlay (#4238)

- [x] I have read and followed the [Contribution Guidelines](https://git.eden-emu.dev/eden-emu/eden/src/branch/master/CONTRIBUTING.md#code-contributions).
- [x] I have read and followed the [AI Policy](https://git.eden-emu.dev/eden-emu/eden/src/branch/master/docs/policies/AI.md)
- [x] I have read and followed the [Coding Guidelines](https://git.eden-emu.dev/eden-emu/eden/src/branch/master/docs/policies/Coding.md) to the best of my ability.

-------------------

- Dynamic shader cache reloading capability for qlaunch
- Multi-process improvements (thanks to @frank1734), instead of just using the main application we now respect caller process (also did it for HID devices while I was at it)
- Layer stack masks & shared buffer screenshot for video core, added different masks (screenshot, recording, etc.) this was discovered as an issue due to how in qlaunch the transition was not right between applications. May not be perfect but also fixes screenshots while using qlaunch
- Reworked overlay display management (input and visibility) - instead of random numbers as I've previously did, I decided to add an AppletZIndex enum for better readability. Also split capability of input by touch and gamepad.
- etc.

Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/4238
Reviewed-by: Samuel <lizzie@eden-emu.dev>
Reviewed-by: CamilleLaVey <camillelavey99@gmail.com>
Reviewed-by: MaranBr <maranbr@eden-emu.dev>

src/core/hle/service/nvnflinger/display.h

@@ -1,4 +1,4 @@
-// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 // SPDX-FileCopyrightText: Copyright 2024 yuzu Emulator Project
@@ -15,7 +15,8 @@ struct Layer {
     explicit Layer(std::shared_ptr<android::BufferItemConsumer> buffer_item_consumer_,
                    s32 consumer_id_)
         : buffer_item_consumer(std::move(buffer_item_consumer_)), consumer_id(consumer_id_),
-          blending(LayerBlending::None), visible(true), z_index(0), is_overlay(false) {}
+          blending(LayerBlending::None), visible(true), z_index(0), is_overlay(false),
+          layer_stack_mask(DefaultLayerStackMask) {}
     ~Layer() {
         buffer_item_consumer->Abandon();
     }
@@ -26,6 +27,7 @@ struct Layer {
     bool visible;
     s32 z_index;
     bool is_overlay;
+    u32 layer_stack_mask;
 };
 
 struct LayerStack {

src/core/hle/service/nvnflinger/hardware_composer.cpp

@@ -115,6 +115,7 @@ u32 HardwareComposer::ComposeLocked(f32* out_speed_scale, Display& display,
                 .transform = static_cast<android::BufferTransformFlags>(item.transform),
                 .crop_rect = item.crop,
                 .acquire_fence = item.fence,
+                .layer_stack_mask = layer->layer_stack_mask,
             });
         }
 

src/core/hle/service/nvnflinger/hwc_layer.h

@@ -1,3 +1,6 @@
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
 // SPDX-FileCopyrightText: Copyright 2024 yuzu Emulator Project
 // SPDX-License-Identifier: GPL-3.0-or-later
 
@@ -23,6 +26,25 @@ enum class LayerBlending : u32 {
     Coverage = 0x405,
 };
 
+enum class LayerStackId : u32 {
+    Default = 0,
+    Lcd = 1,
+    Screenshot = 2,
+    Recording = 3,
+    LastFrame = 4,
+    Arbitrary = 5,
+    ApplicationForDebug = 6,
+    Null = 10,
+};
+
+constexpr u32 LayerStackBit(LayerStackId id) {
+    return 1U << static_cast<u32>(id);
+}
+
+constexpr u32 DefaultLayerStackMask =
+    LayerStackBit(LayerStackId::Default) | LayerStackBit(LayerStackId::Screenshot) |
+    LayerStackBit(LayerStackId::Recording) | LayerStackBit(LayerStackId::LastFrame);
+
 struct HwcLayer {
     u32 buffer_handle;
     u32 offset;
@@ -35,6 +57,7 @@ struct HwcLayer {
     android::BufferTransformFlags transform;
     Common::Rectangle<int> crop_rect;
     android::Fence acquire_fence;
+    u32 layer_stack_mask;
 };
 
 } // namespace Service::Nvnflinger

src/core/hle/service/nvnflinger/surface_flinger.cpp

@@ -1,4 +1,4 @@
-// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 // SPDX-FileCopyrightText: Copyright 2024 yuzu Emulator Project
@@ -104,6 +104,14 @@ void SurfaceFlinger::SetLayerBlending(s32 consumer_binder_id, LayerBlending blen
     }
 }
 
+void SurfaceFlinger::SetLayerStackMask(s32 consumer_binder_id, u32 layer_stack_mask) {
+    if (const auto layer = this->FindLayer(consumer_binder_id); layer != nullptr) {
+        layer->layer_stack_mask = layer_stack_mask;
+        LOG_DEBUG(Service_VI, "Layer {} stack mask set to {:#x}", consumer_binder_id,
+                  layer_stack_mask);
+    }
+}
+
 void SurfaceFlinger::SetLayerIsOverlay(s32 consumer_binder_id, bool is_overlay) {
     if (const auto layer = this->FindLayer(consumer_binder_id); layer != nullptr) {
         layer->is_overlay = is_overlay;

src/core/hle/service/nvnflinger/surface_flinger.h

@@ -1,4 +1,4 @@
-// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 // SPDX-FileCopyrightText: Copyright 2024 yuzu Emulator Project
@@ -48,6 +48,7 @@ public:
     void SetLayerVisibility(s32 consumer_binder_id, bool visible);
     void SetLayerBlending(s32 consumer_binder_id, LayerBlending blending);
     void SetLayerIsOverlay(s32 consumer_binder_id, bool is_overlay);
+    void SetLayerStackMask(s32 consumer_binder_id, u32 layer_stack_mask);
 
     std::shared_ptr<Layer> FindLayer(s32 consumer_binder_id);