diff --git a/src/video_core/renderer_vulkan/vk_scheduler.h b/src/video_core/renderer_vulkan/vk_scheduler.h
index d4bc8d0ec5..7ef28929da 100644
--- a/src/video_core/renderer_vulkan/vk_scheduler.h
+++ b/src/video_core/renderer_vulkan/vk_scheduler.h
@@ -129,6 +129,10 @@ public:
         return master_semaphore->IsFree(tick);
     }
 
+    void RefreshTick() {
+        master_semaphore->Refresh();
+    }
+
     /// Waits for the given GPU tick, optionally pacing frames.
     void Wait(u64 tick, double target_fps = 0.0) {
         if (tick > 0) {
diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp
index a2a04379cd..549eedfe14 100644
--- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp
+++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp
@@ -3235,6 +3235,14 @@ u64 TextureCacheRuntime::CurrentTick() const noexcept {
     return scheduler.CurrentTick();
 }
 
+bool TextureCacheRuntime::IsDirectUploadRetired(u64 tick) {
+    if (scheduler.IsFree(tick)) {
+        return true;
+    }
+    scheduler.RefreshTick();
+    return scheduler.IsFree(tick);
+}
+
 void TextureCacheRuntime::TransitionImageLayout(Image& image) {
     if (!image.ExchangeInitialization()) {
         VkImageMemoryBarrier barrier{
diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.h b/src/video_core/renderer_vulkan/vk_texture_cache.h
index 6ee1c52990..53c4da80bf 100644
--- a/src/video_core/renderer_vulkan/vk_texture_cache.h
+++ b/src/video_core/renderer_vulkan/vk_texture_cache.h
@@ -116,6 +116,8 @@ public:
 
     [[nodiscard]] u64 CurrentTick() const noexcept;
 
+    [[nodiscard]] bool IsDirectUploadRetired(u64 tick);
+
     void InsertUploadMemoryBarrier() {}
 
     void TransitionImageLayout(Image& image);
diff --git a/src/video_core/texture_cache/image_base.h b/src/video_core/texture_cache/image_base.h
index 4fd7eafe7d..6915a4d9af 100644
--- a/src/video_core/texture_cache/image_base.h
+++ b/src/video_core/texture_cache/image_base.h
@@ -95,7 +95,7 @@ struct ImageBase {
     u32 scale_rating = 0;
     u64 scale_tick = 0;
     bool has_scaled = false;
-    bool direct_upload_used = false;
+    u64 direct_upload_tick = 0;
     bool direct_upload_blocked = false;
 
     size_t channel = 0;
diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h
index 75d1f42b31..2e4fb6e6a6 100644
--- a/src/video_core/texture_cache/texture_cache.h
+++ b/src/video_core/texture_cache/texture_cache.h
@@ -119,13 +119,16 @@ void TextureCache<P>::RunGarbageCollector() {
     bool aggressive_mode = false;
     u64 ticks_to_destroy = 0;
     size_t num_iterations = 0;
+    size_t num_downloads = 0;
     const auto Configure = [&](bool allow_aggressive) {
         high_priority_mode = total_used_memory >= expected_memory;
         aggressive_mode = allow_aggressive && total_used_memory >= critical_memory;
         ticks_to_destroy = aggressive_mode ? 10ULL : high_priority_mode ? 25ULL : 50ULL;
         num_iterations = aggressive_mode ? 40 : (high_priority_mode ? 20 : 10);
+        num_downloads = MAX_GC_DOWNLOADS_PER_PASS;
     };
-    const auto Cleanup = [this, &num_iterations, &high_priority_mode, &aggressive_mode](ImageId image_id) {
+    const auto Cleanup = [this, &num_iterations, &num_downloads, &high_priority_mode,
+                          &aggressive_mode](ImageId image_id) {
         if (num_iterations == 0) {
             return true;
         }
@@ -139,6 +142,10 @@ void TextureCache<P>::RunGarbageCollector() {
             return false;
         }
         if (must_download) {
+            if (num_downloads == 0) {
+                return false;
+            }
+            --num_downloads;
             auto map = runtime.DownloadStagingBuffer(image.unswizzled_size_bytes);
             const auto copies = FixSmallVectorADL(FullDownloadCopies(image.info));
             image.DownloadMemory(map, copies);
@@ -585,9 +592,14 @@ FramebufferId TextureCache<P>::GetFramebufferId(const RenderTargets& key) {
 template <class P>
 void TextureCache<P>::WriteMemory(DAddr cpu_addr, size_t size) {
     ForEachImageInRegion(cpu_addr, size, [this](ImageId image_id, Image& image) {
-        if (image.direct_upload_used) {
-            image.direct_upload_used = false;
-            image.direct_upload_blocked = true;
+        if constexpr (USE_UNIFIED_MEMORY) {
+            if (image.direct_upload_tick != 0) {
+                const u64 upload_tick = image.direct_upload_tick;
+                image.direct_upload_tick = 0;
+                if (!runtime.IsDirectUploadRetired(upload_tick)) {
+                    image.direct_upload_blocked = true;
+                }
+            }
         }
         if (True(image.flags & ImageFlagBits::CpuModified)) {
             return;
@@ -1195,7 +1207,7 @@ bool TextureCache<P>::TryUploadFromUnifiedMemory([[maybe_unused]] Image& image)
                                          local_offset, FixSmallVectorADL(swizzles))) {
             return false;
         }
-        image.direct_upload_used = true;
+        image.direct_upload_tick = runtime.CurrentTick();
         return true;
     } else {
         return false;
diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h
index 687c770794..44622a223c 100644
--- a/src/video_core/texture_cache/texture_cache_base.h
+++ b/src/video_core/texture_cache/texture_cache_base.h
@@ -121,6 +121,7 @@ class TextureCache : public VideoCommon::ChannelSetupCaches<TextureCacheChannelI
     static constexpr s64 DEFAULT_EXPECTED_MEMORY = 1_GiB + 125_MiB;
     static constexpr s64 DEFAULT_CRITICAL_MEMORY = 1_GiB + 625_MiB;
     static constexpr size_t GC_EMERGENCY_COUNTS = 2;
+    static constexpr size_t MAX_GC_DOWNLOADS_PER_PASS = 4;
 
     using Runtime = typename P::Runtime;
     using Image = typename P::Image;
diff --git a/src/video_core/vulkan_common/vulkan_device.cpp b/src/video_core/vulkan_common/vulkan_device.cpp
index a289cc6bfb..fc76bb6b65 100644
--- a/src/video_core/vulkan_common/vulkan_device.cpp
+++ b/src/video_core/vulkan_common/vulkan_device.cpp
@@ -1525,7 +1525,11 @@ u64 Device::GetDeviceMemoryUsage() const {
     for (const size_t heap : valid_heap_memory) {
         result += budget.heapUsage[heap];
     }
-    return result;
+    const u64 committed_backing = Common::GetCommittedBackingSize();
+    if (result <= committed_backing) {
+        return result;
+    }
+    return result - committed_backing;
 }
 
 void Device::CollectPhysicalMemoryInfo() {