diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp
index d9c9a807cc..526293663a 100644
--- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp
+++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp
@@ -2449,6 +2449,7 @@ bool Image::ScaleUp(bool ignore) {
     }
     if (NeedsScaleHelper()) {
         if (!BlitScaleHelper(true)) {
+            flags &= ~ImageFlagBits::Rescaled;
             current_image = &Image::original_image;
             return false;
         }