diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp
index 99e3f93b57..7b31c6e76d 100644
--- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp
+++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp
@@ -2091,7 +2091,7 @@ void Image::UploadMemory(VkBuffer buffer, VkDeviceSize offset,
                                                 temp_vk_image, info.format, info.num_samples,
                                                 {image_copies.data(), image_copies.size()}, false);
         }
-        initialized = true;
+        InitializationFor(current_image) = true;
         runtime->ReleaseMsaaScratchImage(temp_vk_image);
 
         if (is_rescaled) {
@@ -2112,7 +2112,7 @@ void Image::UploadMemory(VkBuffer buffer, VkDeviceSize offset,
     const VkBuffer src_buffer = buffer;
     const VkImage vk_image = *original_image;
     const VkImageAspectFlags vk_aspect_mask = aspect_mask;
-    const bool was_initialized = std::exchange(initialized, true);
+    const bool was_initialized = std::exchange(InitializationFor(&Image::original_image), true);
 
     scheduler->Record([src_buffer, vk_image, vk_aspect_mask, was_initialized,
                        vk_copies](vk::CommandBuffer cmdbuf) {
@@ -2387,6 +2387,13 @@ std::vector<vk::ImageView>& Image::StorageViewsFor(vk::Image Image::*image) {
     return storage_image_views;
 }
 
+bool& Image::InitializationFor(vk::Image Image::*image) noexcept {
+    if (image == &Image::scaled_image) {
+        return scaled_initialized;
+    }
+    return original_initialized;
+}
+
 VkImageView Image::StorageImageView(s32 level) noexcept {
     const bool astc_decode = WillUseAcceleratedAstcDecode(runtime->device, info);
     const bool unswizzle_upload =
diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.h b/src/video_core/renderer_vulkan/vk_texture_cache.h
index 8eef0e273c..2b3378af61 100644
--- a/src/video_core/renderer_vulkan/vk_texture_cache.h
+++ b/src/video_core/renderer_vulkan/vk_texture_cache.h
@@ -353,7 +353,7 @@ public:
 
     /// Returns true when the image is already initialized and mark it as initialized
     [[nodiscard]] bool ExchangeInitialization() noexcept {
-        return std::exchange(initialized, true);
+        return std::exchange(InitializationFor(current_image), true);
     }
 
     VkImageView StorageImageView(s32 level) noexcept;
@@ -375,6 +375,8 @@ private:
 
     std::vector<vk::ImageView>& StorageViewsFor(vk::Image Image::*image);
 
+    bool& InitializationFor(vk::Image Image::*image) noexcept;
+
     Scheduler* scheduler{};
     TextureCacheRuntime* runtime{};
 
@@ -394,7 +396,8 @@ private:
     std::vector<vk::ImageView> storage_image_views;
     std::vector<vk::ImageView> scaled_storage_image_views;
     VkImageAspectFlags aspect_mask = 0;
-    bool initialized = false;
+    bool original_initialized = false;
+    bool scaled_initialized = false;
 
     std::optional<Framebuffer> scale_framebuffer;
     std::optional<Framebuffer> normal_framebuffer;