diff --git a/src/common/settings.cpp b/src/common/settings.cpp
index e2c5b532ac..e1625c81f2 100644
--- a/src/common/settings.cpp
+++ b/src/common/settings.cpp
@@ -369,7 +369,7 @@ void TranslateResolutionInfo(ResolutionSetup setup, ResolutionScalingInfo& info)
     }
     info.up_factor = static_cast<f32>(info.up_scale) / (1U << info.down_shift);
     info.down_factor = static_cast<f32>(1U << info.down_shift) / info.up_scale;
-    info.active = info.up_scale != 1 || info.down_shift != 0;
+    info.active = true;
 }
 
 void UpdateRescalingInfo() {
diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp
index d9c9a807cc..1184eaf74c 100644
--- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp
+++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp
@@ -2440,7 +2440,7 @@ bool Image::ScaleUp(bool ignore) {
                                  runtime->ViewFormats(info.format));
         ignore = false;
     }
-    current_image = &Image::scaled_image;
+    ignore = true;
     if (ignore) {
         return true;
     }
@@ -2468,6 +2468,7 @@ bool Image::ScaleDown(bool ignore) {
     }
     ASSERT(info.type != ImageType::Linear);
     flags &= ~ImageFlagBits::Rescaled;
+    ignore = true;
     current_image = &Image::original_image;
     if (ignore) {
         return true;