[am, renderer_vulkan] Fix overlay darkening and SGSR black screen on applets (#4475)

- [x] I have read and followed the [Contribution Guidelines](https://git.eden-emu.dev/eden-emu/eden/src/branch/master/CONTRIBUTING.md#code-contributions).
- [x] I have read and followed the [AI Policy](https://git.eden-emu.dev/eden-emu/eden/src/branch/master/docs/policies/AI.md)
- [x] I have read and followed the [Coding Guidelines](https://git.eden-emu.dev/eden-emu/eden/src/branch/master/docs/policies/Coding.md) to the best of my ability.

-------------------
- Overlay applet: use IsOverlayOpenLocked to pick the Z-index, so the darkening background is only layered above the game while the overlay is open.
- Vulkan: skip the SGSR pass for applet layers to avoid presenting a black frame.
- Partial revert fix crashes in games on UE with the overlay applet enabled.

Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/4475
Reviewed-by: CamilleLaVey <camillelavey99@gmail.com>
Reviewed-by: lizzie <lizzie@eden-emu.dev>

src/core/hle/service/am/window_system.cpp

@@ -517,7 +517,7 @@ void WindowSystem::UpdateAppletStateLocked(Applet* applet, bool is_foreground, b
     // Layer ordering. Composition sorts back-to-front. Now with enums for calrity.
     s32 z_index = Background;
     if (is_overlay) {
-        z_index = Overlay;
+        z_index = this->IsOverlayOpenLocked(*applet) ? Overlay : Background;
     } else if (inherited_foreground) {
         z_index = is_obscured ? Foreground : ForegroundVisible;
     }

src/hid_core/resources/npad/npad.cpp

@@ -45,6 +45,19 @@ NPad::NPad(Core::HID::HIDCore& hid_core_, KernelHelpers::ServiceContext& service
         AbstractPad{hid_core_.kernel},
     }}
 {
+    for (std::size_t aruid_index = 0; aruid_index < AruidIndexMax; ++aruid_index) {
+        for (std::size_t i = 0; i < controller_data[aruid_index].size(); ++i) {
+            auto& controller = controller_data[aruid_index][i];
+            controller.device = hid_core.GetEmulatedControllerByIndex(i);
+            Core::HID::ControllerUpdateCallback engine_callback{
+                .on_change = [this, i, kernel = &hid_core.kernel](Core::HID::ControllerTriggerType type) {
+                    ControllerUpdate(*kernel, type, i);
+                },
+                .is_npad_service = true,
+            };
+            controller.callback_key = controller.device->SetCallback(engine_callback);
+        }
+    }
     for (std::size_t i = 0; i < abstracted_pads.size(); ++i) {
         abstracted_pads[i].SetNpadId(IndexToNpadIdType(i));
     }
@@ -93,16 +106,6 @@ Result NPad::Activate(u64 aruid) {
     for (std::size_t i = 0; i < controller_data[aruid_index].size(); ++i) {
         auto& controller = controller_data[aruid_index][i];
         controller.shared_memory = &data->shared_memory_format->npad.npad_entry[i].internal_state;
-        controller.device = hid_core.GetEmulatedControllerByIndex(i);
-        if (!controller.callback_key) {
-            Core::HID::ControllerUpdateCallback engine_callback{
-                .on_change = [this, i](Core::HID::ControllerTriggerType type) {
-                    ControllerUpdate(hid_core.kernel, type, i);
-                },
-                .is_npad_service = true,
-            };
-            controller.callback_key = controller.device->SetCallback(engine_callback);
-        }
     }
 
     // Prefill controller buffers

src/video_core/renderer_vulkan/present/layer.cpp

@@ -94,13 +94,13 @@ void Layer::ConfigureDraw(const Device& device, PresentPushConstants* out_push_c
     const u32 scaled_width = texture_info ? texture_info->scaled_width : texture_width;
     const u32 scaled_height = texture_info ? texture_info->scaled_height : texture_height;
     const bool use_accelerated = texture_info.has_value();
+    const bool is_applet =
+        (framebuffer.layer_stack_mask & Service::Nvnflinger::LayerStackBit(
+                                            Service::Nvnflinger::LayerStackId::Recording)) == 0;
 
     RefreshResources(device, framebuffer);
     SetAntiAliasPass(device);
 #ifdef HAS_RESHADE
-    const bool is_applet =
-        (framebuffer.layer_stack_mask & Service::Nvnflinger::LayerStackBit(
-                                            Service::Nvnflinger::LayerStackId::Recording)) == 0;
     SetPostProcessPass(device, is_applet);
 #endif
 
@@ -141,8 +141,11 @@ void Layer::ConfigureDraw(const Device& device, PresentPushConstants* out_push_c
         source_image_view = fsr->Draw(device, scheduler, image_index, source_image, source_image_view, render_extent, crop_rect);
         crop_rect = {0, 0, 1, 1};
     } else if (auto* sgsr = std::get_if<SGSR>(&sr_filter)) {
-        source_image_view = sgsr->Draw(device, scheduler, image_index, source_image, source_image_view, render_extent, crop_rect);
-        crop_rect = {0, 0, 1, 1};
+        if (!is_applet) {
+            source_image_view = sgsr->Draw(device, scheduler, image_index, source_image,
+                                           source_image_view, render_extent, crop_rect);
+            crop_rect = {0, 0, 1, 1};
+        }
     }
 
     SetMatrixData(device, *out_push_constants, layout);