diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp
index 8d748393fd..49926d01ad 100644
--- a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp
+++ b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp
@@ -26,6 +26,15 @@ namespace Vulkan {
 namespace {
 constexpr u32 COMPACT_VERTEX_BINDINGS = 8;
 
+constexpr VkMemoryBarrier2 HOST_READ_BARRIER{
+    .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2,
+    .pNext = nullptr,
+    .srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
+    .srcAccessMask = VK_ACCESS_2_MEMORY_WRITE_BIT,
+    .dstStageMask = VK_PIPELINE_STAGE_2_HOST_BIT,
+    .dstAccessMask = VK_ACCESS_2_HOST_READ_BIT,
+};
+
 template <u32 N>
 struct VertexBindings {
     std::array<VkBuffer, N> buffers;
@@ -527,9 +536,16 @@ bool BufferCacheRuntime::CanReorderUpload(const Buffer& buffer,
 
 std::span<const u8> BufferCacheRuntime::DirectDownloadSpan(Buffer& buffer) {
     const std::span<u8> mapping = buffer.CoherentMapping();
-    if (!mapping.empty()) {
-        scheduler.Wait((std::max)(buffer.getWriteTick(), buffer.LastUploadTick()));
+    if (mapping.empty()) {
+        return mapping;
     }
+    if ((std::max)(buffer.getWriteTick(), buffer.LastUploadTick()) > host_read_tick) {
+        scheduler.RequestOutsideRenderPassOperationContext();
+        scheduler.Record(
+            [](vk::CommandBuffer cmdbuf) { cmdbuf.PipelineBarrier(HOST_READ_BARRIER); });
+        host_read_tick = scheduler.CurrentTick();
+    }
+    scheduler.Wait(host_read_tick);
     return mapping;
 }
 
diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.h b/src/video_core/renderer_vulkan/vk_buffer_cache.h
index f4c4c53906..3209019bdf 100644
--- a/src/video_core/renderer_vulkan/vk_buffer_cache.h
+++ b/src/video_core/renderer_vulkan/vk_buffer_cache.h
@@ -291,6 +291,7 @@ private:
 
     VkMemoryBarrier2 read_barrier;
     VkMemoryBarrier2 write_barrier;
+    u64 host_read_tick = 0;
 
     bool limit_dynamic_storage_buffers = false;
     u32 max_dynamic_storage_buffers = (std::numeric_limits<u32>::max)();
diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp
index 805c9adf2c..a7bf7aecfd 100644
--- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp
+++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp
@@ -898,7 +898,9 @@ void RasterizerVulkan::TiledCacheBarrier() {
 }
 
 void RasterizerVulkan::FlushCommands() {
-    if (draw_counter == 0) {
+    if (draw_tick != scheduler.CurrentTick() ||
+        (scheduler.IsRenderPassActive() &&
+         !maxwell3d->dirty.flags[VideoCommon::Dirty::RenderTargets])) {
         return;
     }
     draw_counter = 0;
@@ -1008,11 +1010,10 @@ void RasterizerVulkan::FlushWork() {
          maxwell3d->dirty.flags[VideoCommon::Dirty::RenderTargets])) {
         scheduler.Flush();
         draw_counter = 0;
-        return;
-    }
-    if ((draw_counter & CHECK_MASK) == CHECK_MASK && !scheduler.IsRenderPassActive()) {
+    } else if ((draw_counter & CHECK_MASK) == CHECK_MASK && !scheduler.IsRenderPassActive()) {
         scheduler.DispatchWork();
     }
+    draw_tick = scheduler.CurrentTick();
 }
 
 AccelerateDMA::AccelerateDMA(BufferCache& buffer_cache_, TextureCache& texture_cache_,
diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.h b/src/video_core/renderer_vulkan/vk_rasterizer.h
index 57d3aa5cf6..1d54694f2c 100644
--- a/src/video_core/renderer_vulkan/vk_rasterizer.h
+++ b/src/video_core/renderer_vulkan/vk_rasterizer.h
@@ -227,6 +227,7 @@ private:
     boost::container::static_vector<VkSampler, MAX_TEXTURES> sampler_handles;
 
     u32 draw_counter = 0;
+    u64 draw_tick = 0;
     Tegra::DepthFormat depth_bias_zeta_format{};
     std::array<u32, Tegra::Engines::Maxwell3D::Regs::NumVertexArrays> vertex_input_strides{};
     u32 vertex_input_attributes{};
diff --git a/src/video_core/renderer_vulkan/vk_scheduler.cpp b/src/video_core/renderer_vulkan/vk_scheduler.cpp
index 91aac1a51d..1851f0b628 100644
--- a/src/video_core/renderer_vulkan/vk_scheduler.cpp
+++ b/src/video_core/renderer_vulkan/vk_scheduler.cpp
@@ -32,15 +32,6 @@ namespace Vulkan {
 namespace {
 constexpr size_t DEPTH_VIEW = 8;
 
-constexpr VkMemoryBarrier2 HOST_READ_BARRIER{
-    .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2,
-    .pNext = nullptr,
-    .srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
-    .srcAccessMask = VK_ACCESS_2_MEMORY_WRITE_BIT,
-    .dstStageMask = VK_PIPELINE_STAGE_2_HOST_BIT,
-    .dstAccessMask = VK_ACCESS_2_HOST_READ_BIT,
-};
-
 struct BeginRenderingCommand {
     void operator()(vk::CommandBuffer cmdbuf, vk::CommandBuffer) const {
         BeginRendering(cmdbuf, attachments);
@@ -478,6 +469,7 @@ void Scheduler::AllocateWorkerCommandBuffer() {
 }
 
 u64 Scheduler::SubmitExecution(VkSemaphore signal_semaphore, VkSemaphore wait_semaphore) {
+    master_semaphore->Refresh();
     EndPendingOperations();
     InvalidateState();
 
@@ -486,9 +478,6 @@ u64 Scheduler::SubmitExecution(VkSemaphore signal_semaphore, VkSemaphore wait_se
                             this](vk::CommandBuffer cmdbuf, vk::CommandBuffer upload_cmdbuf) {
         upload_cmdbuf.PipelineBarrier(upload_write_barrier);
         upload_cmdbuf.End();
-        if (device.IsUMA()) {
-            cmdbuf.PipelineBarrier(HOST_READ_BARRIER);
-        }
         cmdbuf.End();
 
         if (on_submit) {