diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp
index d9c9a807cc..a3babccb3c 100644
--- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp
+++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp
@@ -2440,6 +2440,7 @@ bool Image::ScaleUp(bool ignore) {
                                  runtime->ViewFormats(info.format));
         ignore = false;
     }
+    ignore = true;
     current_image = &Image::scaled_image;
     if (ignore) {
         return true;
@@ -2468,6 +2469,7 @@ bool Image::ScaleDown(bool ignore) {
     }
     ASSERT(info.type != ImageType::Linear);
     flags &= ~ImageFlagBits::Rescaled;
+    ignore = true;
     current_image = &Image::original_image;
     if (ignore) {
         return true;