1 files changed, 543 insertions, 51 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 1ed8c99..2f453fa 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -38,12 +38,17 @@
 #include <linux/vfio.h>
 #include <linux/jiffies.h>
 #include <linux/mutex.h>
+#include <linux/sort.h>
+#include <linux/bitmap.h>
 
 #include "nvidia-vgpu-vfio.h"
 #include "gsp_sema_service.h"
 
-#define SEMA_MAP_MAX		16384
+#define SEMA_MAP_MAX		65536
+#define SEMA_MAP_SIZE_BITS	16
 #define SEMA_PAGE_SIZE		4096
+#define SEMA_PAGE_SHIFT		12
+#define SEMA_PA_MAX		0x140000000ULL
 #define SEMA_SCAN_PAGES_MAX	0x20000	/* 512 MB worth of pages per pass */
 
 /* Plausible GPU VA window for the client's system-memory objects (Turing). */
@@ -66,11 +71,65 @@
 #define CE_DMA_HDR_ADDR		0xc0
 #define CE_DMA_HDR_COUNT	1
 
+/*
+ * Channel/host class (C46F) semaphore release, written by pushes on a
+ * compute channel as one five-dword run starting at SEM_ADDR_LO:
+ *   [LO][HI][PAYLOAD_LO][PAYLOAD_HI][EXECUTE]
+ * This is the form core CUDA uses for its own progress semaphores; the CE
+ * form above is what UVM pushes on a copy channel.
+ */
+#define HOST_M_SEM_ADDR_LO	0x5c
+#define HOST_SEM_OP_RELEASE	1
+#define HOST_SEM_OP_REDUCTION	6
+
 struct sema_map_entry {
 	u64 gpu_page;	/* GPU VA, page aligned */
 	u64 pa_page;	/* guest PA, page aligned */
 };
 
+/* ------------------------------------------------------------------ */
+/* registered channels (GPFIFO-driven execution)                       */
+/* ------------------------------------------------------------------ */
+
+#define SVC_MAX_CHANNELS	24
+#define SVC_GPFIFO_ENTRY_SIZE	8
+#define SVC_PUSH_MAX		(256 * 1024)
+
+/* USERD control block (NvA06FControl / KeplerAControlGPFifo) */
+#define SVC_USERD_GP_GET	0x88
+#define SVC_USERD_GP_PUT	0x8c
+
+struct svc_channel {
+	bool	valid;
+	u32	h_object;
+	u32	engine_type;
+	u64	gpfifo_va;
+	u32	gpfifo_entries;
+	u64	userd_pa;
+	u32	userd_as;	/* NV_ADDR_SYSMEM / NV_ADDR_FBMEM */
+	u64	instance_pa;
+	u64	notifier_pa;
+	u32	last_get;	/* next GPFIFO entry we have still to run */
+	u64	submissions;	/* GPFIFO entries executed */
+};
+
+/*
+ * Push pages we have actually seen work in. Pushbuffer pools are recycled, so
+ * once a page has carried a push it is a much better scan candidate than the
+ * rest of guest RAM or even the whole VA map.
+ */
+#define SEMA_HOT_PAGES		(SEMA_PA_MAX >> SEMA_PAGE_SHIFT)
+static unsigned long svc_hot[BITS_TO_LONGS(SEMA_HOT_PAGES)];
+static u32 svc_map_hash[SEMA_MAP_MAX * 2];
+
+static void svc_mark_hot(u64 pa)
+{
+	u64 page = pa >> SEMA_PAGE_SHIFT;
+
+	if (page < SEMA_HOT_PAGES)
+		set_bit(page, svc_hot);
+}
+
 struct sema_svc {
 	struct task_struct	*task;
 	bool			stop;
@@ -79,6 +138,11 @@ struct sema_svc {
 	u32			map_n;
 	struct sema_map_entry	map[SEMA_MAP_MAX];
 	u64			last_build;
+	u64			last_scan;
+
+	/* registered guest channels (GPFIFO-driven push execution) */
+	spinlock_t		chan_lock;
+	struct svc_channel	chan[SVC_MAX_CHANNELS];
 
 	/* stats */
 	u64			pairs_seen;
@@ -95,6 +159,9 @@ struct sema_svc {
 	u64			releases_stale;		/* payload <= current */
 	u64			releases_jumped;	/* applied with gap > 1 */
 	u64			builds;
+	u64			gpfifo_entries_done;	/* entries consumed via GPFIFO */
+	u64			gpfifo_nomap;		/* GPFIFO/pushbuf VA not mapped */
+	u64			vidmem_writes;		/* semaphores written in vidmem */
 };
 
 static struct sema_svc svc;
@@ -113,6 +180,164 @@ static int svc_write(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len
 	return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, true);
 }
 
+/* NV_ADDR_SYSMEM / NV_ADDR_FBMEM as carried in the channel alloc params. */
+#define SVC_APERTURE_SYS	1
+#define SVC_APERTURE_FB		2
+
+/*
+ * Aperture-aware access: channels describe their USERD/instance blocks with an
+ * address-space tag, and "frame buffer" addresses are offsets into the
+ * emulated vidmem (BAR1 backing), not guest RAM.
+ */
+static int svc_read_ap(struct nv_vgpu_device *vgpu, u64 addr, u32 aperture,
+		       void *buf, size_t len)
+{
+	if (aperture == SVC_APERTURE_FB) {
+		u8 *base = vgpu->bars[1].vaddr;
+		size_t size = vgpu->bars[1].size;
+
+		if (!base || addr + len > size)
+			return -EINVAL;
+		memcpy(buf, base + addr, len);
+		return 0;
+	}
+	return svc_read(vgpu, addr, buf, len);
+}
+
+static int svc_write_ap(struct nv_vgpu_device *vgpu, u64 addr, u32 aperture,
+			void *buf, size_t len)
+{
+	if (aperture == SVC_APERTURE_FB) {
+		u8 *base = vgpu->bars[1].vaddr;
+		size_t size = vgpu->bars[1].size;
+
+		if (!base || addr + len > size)
+			return -EINVAL;
+		memcpy(base + addr, buf, len);
+		return 0;
+	}
+	return svc_write(vgpu, addr, buf, len);
+}
+
+/*
+ * Register a guest channel. Called from the GSP-RM emulation when the guest
+ * driver allocates a channel (the allocation carries the GPFIFO GPU VA, its
+ * entry count and the guest-physical USERD/instance blocks), and dropped when
+ * the channel is freed.
+ */
+void gsp_sema_service_channel_add(const struct sema_channel_desc *desc)
+{
+	struct svc_channel *ch = NULL;
+	unsigned long flags;
+	int i;
+
+	if (!desc || !desc->h_object || !desc->gpfifo_va ||
+	    desc->gpfifo_entries < 2)
+		return;
+
+	spin_lock_irqsave(&svc.chan_lock, flags);
+	for (i = 0; i < SVC_MAX_CHANNELS; i++) {
+		if (svc.chan[i].valid && svc.chan[i].h_object == desc->h_object) {
+			ch = &svc.chan[i];
+			break;
+		}
+		if (!ch && !svc.chan[i].valid)
+			ch = &svc.chan[i];
+	}
+	if (ch) {
+		memset(ch, 0, sizeof(*ch));
+		ch->valid = true;
+		ch->h_object = desc->h_object;
+		ch->engine_type = desc->engine_type;
+		ch->gpfifo_va = desc->gpfifo_va;
+		ch->gpfifo_entries = desc->gpfifo_entries;
+		ch->userd_pa = desc->userd_pa;
+		ch->userd_as = desc->userd_as ? desc->userd_as : SVC_APERTURE_SYS;
+		ch->instance_pa = desc->instance_pa;
+		ch->notifier_pa = desc->notifier_pa;
+	}
+	spin_unlock_irqrestore(&svc.chan_lock, flags);
+
+	if (ch)
+		pr_info("sema-svc: channel 0x%x registered gpfifo=0x%llx entries=%u userd=0x%llx\n",
+			desc->h_object, desc->gpfifo_va, desc->gpfifo_entries,
+			desc->userd_pa);
+}
+
+void gsp_sema_service_channel_del(u32 h_object)
+{
+	unsigned long flags;
+	int i;
+
+	if (!h_object)
+		return;
+
+	spin_lock_irqsave(&svc.chan_lock, flags);
+	for (i = 0; i < SVC_MAX_CHANNELS; i++) {
+		if (svc.chan[i].valid && svc.chan[i].h_object == h_object) {
+			svc.chan[i].valid = false;
+			pr_info("sema-svc: channel 0x%x unregistered (submissions=%llu)\n",
+				h_object, svc.chan[i].submissions);
+			break;
+		}
+	}
+	spin_unlock_irqrestore(&svc.chan_lock, flags);
+}
+
+/*
+ * The client's vidmem VAs are its window base plus the vidmem offset, and the
+ * backing store is the emulated BAR1 buffer. Everything else that resolves
+ * goes through the discovered guest-RAM map.
+ */
+#define SVC_VIDMEM_VA_BASE	0x200000000ULL
+
+static int svc_map_lookup(u64 gpu_va, u64 *pa);
+
+static int svc_resolve_va(struct nv_vgpu_device *vgpu, u64 va, u64 *addr,
+			  u32 *aperture)
+{
+	if (!svc_map_lookup(va, addr)) {
+		*aperture = 0;		/* guest RAM */
+		return 0;
+	}
+	if (va >= SVC_VIDMEM_VA_BASE &&
+	    va - SVC_VIDMEM_VA_BASE + 4 <= vgpu->bars[1].size) {
+		*addr = va - SVC_VIDMEM_VA_BASE;
+		*aperture = 1;		/* emulated vidmem */
+		return 0;
+	}
+	return -ENOENT;
+}
+
+static int svc_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val)
+{
+	u64 addr = 0;
+	u32 ap = 0;
+
+	if (svc_resolve_va(vgpu, va, &addr, &ap))
+		return -ENOENT;
+	if (ap) {
+		memcpy(val, vgpu->bars[1].vaddr + addr, sizeof(*val));
+		return 0;
+	}
+	return svc_read(vgpu, addr, val, sizeof(*val));
+}
+
+static int svc_sem_write(struct nv_vgpu_device *vgpu, u64 va, u32 val)
+{
+	u64 addr = 0;
+	u32 ap = 0;
+
+	if (svc_resolve_va(vgpu, va, &addr, &ap))
+		return -ENOENT;
+	if (ap) {
+		memcpy(vgpu->bars[1].vaddr + addr, &val, sizeof(val));
+		svc.vidmem_writes++;
+		return 0;
+	}
+	return svc_write(vgpu, addr, &val, sizeof(val));
+}
+
 /* ------------------------------------------------------------------ */
 /* discovery                                                          */
 /* ------------------------------------------------------------------ */
@@ -190,38 +415,82 @@ static void svc_scan_pass(struct nv_vgpu_device *vgpu, u8 *buf,
 
 static void svc_add_map(u64 gpu_va, u64 pa)
 {
-	u64 gpu_page = gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1);
-	u64 pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1);
-	u32 i;
-
-	/*
-	 * First-wins: the guest keeps stale copies of freed allocations around,
-	 * and unrelated adjacent u64 pairs can look like {cpu_va, gpu_va}. Both
-	 * produce bogus mappings for a VA that already has a good one, and
-	 * overwriting it would send every semaphore write to the wrong page.
-	 * The first pair seen for a VA is the live allocation.
-	 */
-	for (i = 0; i < svc.map_n; i++) {
-		if (svc.map[i].gpu_page == gpu_page)
-			return;
-	}
 	if (svc.map_n < SEMA_MAP_MAX) {
-		svc.map[svc.map_n].gpu_page = gpu_page;
-		svc.map[svc.map_n].pa_page = pa_page;
+		svc.map[svc.map_n].gpu_page =
+			gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1);
+		svc.map[svc.map_n].pa_page =
+			pa & ~(u64)(SEMA_PAGE_SIZE - 1);
 		svc.map_n++;
 	}
 }
 
+static int svc_map_cmp(const void *a, const void *b)
+{
+	const struct sema_map_entry *x = a, *y = b;
+
+	if (x->gpu_page < y->gpu_page)
+		return -1;
+	if (x->gpu_page > y->gpu_page)
+		return 1;
+	return 0;
+}
+
+/*
+ * Deduplicate (first-wins: the guest keeps stale copies of freed allocations
+ * around and the first pair seen for a VA is the live one) and sort so lookups
+ * can binary search. The map is large enough that the old linear scan both
+ * missed mappings (it stopped at the cap) and cost too much per release.
+ */
+static void svc_map_finalize(void)
+{
+	u32 i, n = 0;
+	u32 hash_mask = ARRAY_SIZE(svc_map_hash) - 1;
+
+	memset(svc_map_hash, 0, sizeof(svc_map_hash));
+
+	for (i = 0; i < svc.map_n; i++) {
+		u64 page = svc.map[i].gpu_page;
+		u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) &
+			hash_mask;
+		u32 probe = 0;
+
+		for (;;) {
+			u32 slot = (h + probe) & hash_mask;
+			u32 val = svc_map_hash[slot];
+
+			if (!val) {
+				svc_map_hash[slot] = n + 1;
+				svc.map[n++] = svc.map[i];
+				break;
+			}
+			if (svc.map[val - 1].gpu_page == page)
+				break;	/* stale duplicate */
+			if (++probe > 64)
+				break;
+		}
+	}
+	svc.map_n = n;
+
+	sort(svc.map, svc.map_n, sizeof(svc.map[0]), svc_map_cmp, NULL);
+}
+
 static int svc_map_lookup(u64 gpu_va, u64 *pa)
 {
 	u64 gpu_page = gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1);
-	u32 i;
+	int lo = 0, hi = (int)svc.map_n - 1;
 
-	for (i = 0; i < svc.map_n; i++) {
-		if (svc.map[i].gpu_page == gpu_page) {
-			*pa = svc.map[i].pa_page + (gpu_va & (SEMA_PAGE_SIZE - 1));
+	while (lo <= hi) {
+		int mid = (lo + hi) / 2;
+
+		if (svc.map[mid].gpu_page == gpu_page) {
+			*pa = svc.map[mid].pa_page +
+			      (gpu_va & (SEMA_PAGE_SIZE - 1));
 			return 0;
 		}
+		if (svc.map[mid].gpu_page < gpu_page)
+			lo = mid + 1;
+		else
+			hi = mid - 1;
 	}
 	return -ENOENT;
 }
@@ -241,6 +510,7 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu)
 			return;
 	}
 
+	svc.map_n = 0;
 	svc_scan_pass(vgpu, buf, cands, &ncand, pairs, &npairs);
 
 	for (i = 0; i < ncand; i++) {
@@ -261,14 +531,15 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu)
 			pa = cpu - best_base;
 			if (pa >= 0x140000000ULL || (pa & 0xfff))
 				continue;
-			if (pairs[i].gpu_va < 0x100000000ULL ||
-			    pairs[i].gpu_va >= 0x140000000ULL)
+			if (pairs[i].gpu_va < SEMA_VA_LO ||
+			    pairs[i].gpu_va >= SEMA_VA_HI)
 				continue;
 			svc_add_map(pairs[i].gpu_va, pa);
 			svc.pairs_seen++;
 		}
 	}
 
+	svc_map_finalize();
 	svc.builds++;
 	svc.last_build = jiffies;
 
@@ -280,6 +551,9 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu)
 		svc.pages_decoded, svc.pushes_seen, svc.pushes_pending,
 		svc.pushes_nomap, svc.memsets_done, svc.memsets_nomap,
 		svc.memsets_fail);
+	pr_info("sema-svc:   gpfifo done=%llu nomap=%llu vidmem_writes=%llu\n",
+		svc.gpfifo_entries_done, svc.gpfifo_nomap,
+		svc.vidmem_writes);
 	/* show a few entries so mappings can be verified from dmesg */
 	{
 		u32 shown;
@@ -532,7 +806,8 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops,
  * replay old pushes whose semaphore is already satisfied, and those are
  * skipped, so their stale memsets never overwrite newer state.
  */
-static void svc_decode_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page)
+static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
+			      u32 len, u64 page_pa)
 {
 	struct svc_push_state st;
 	struct svc_memset_op ops[SVC_MAX_MEMSETS];
@@ -542,9 +817,9 @@ static void svc_decode_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page)
 	memset(&st, 0, sizeof(st));
 	svc.pages_decoded++;
 
-	while (off + 4 <= SEMA_PAGE_SIZE) {
+	while (off + 4 <= len) {
 		u32 hdr = 0, op, count, subch, addr;
-		u32 v0 = 0, v1 = 0, v2 = 0;
+		u32 v0 = 0, v1 = 0, v2 = 0, v3 = 0, v4 = 0;
 
 		memcpy(&hdr, buf + off, 4);
 		op = (hdr >> 29) & 0x7;
@@ -553,7 +828,7 @@ static void svc_decode_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page)
 		addr = (hdr & 0x1fff) << 2;
 
 		if (op != 1 || !count || count > 64 ||
-		    off + 4 + count * 4 > SEMA_PAGE_SIZE) {
+		    off + 4 + count * 4 > len) {
 			off += 4;	/* resync */
 			continue;
 		}
@@ -564,6 +839,10 @@ static void svc_decode_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page)
 			memcpy(&v1, buf + off + 8, 4);
 		if (count >= 3)
 			memcpy(&v2, buf + off + 12, 4);
+		if (count >= 4 && off + 20 <= len)
+			memcpy(&v3, buf + off + 16, 4);
+		if (count >= 5 && off + 24 <= len)
+			memcpy(&v4, buf + off + 20, 4);
 
 		/*
 		 * OFFSET_IN_UPPER/LOWER and OFFSET_OUT_UPPER/LOWER are
@@ -620,6 +899,32 @@ static void svc_decode_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page)
 			continue;
 		}
 
+		if (addr == HOST_M_SEM_ADDR_LO && count >= 5) {
+			u64 va = ((u64)(v1 & 0xff) << 32) |
+				 (u64)(v0 & 0xfffffffc);
+			u32 payload = v2;
+			u32 op = v4 & 0x7;
+			u32 cur = 0;
+
+			if ((op == HOST_SEM_OP_RELEASE ||
+			     op == HOST_SEM_OP_REDUCTION) && payload &&
+			    va >= SEMA_VA_LO && va < SEMA_VA_HI) {
+				if (page_pa)
+					svc_mark_hot(page_pa);
+				svc.pushes_seen++;
+				if (!svc_sem_read(vgpu, va, &cur) &&
+				    payload > cur) {
+					svc.pushes_pending++;
+					cur = (op == HOST_SEM_OP_REDUCTION) ?
+						cur + payload : payload;
+					if (!svc_sem_write(vgpu, va, cur))
+						svc.releases_applied++;
+				}
+			}
+			off += 4 + count * 4;
+			continue;
+		}
+
 		if (subch != 4) {
 			off += 4 + count * 4;
 			continue;
@@ -710,15 +1015,15 @@ static void svc_decode_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page)
 				u32 cur = 0;
 				int pending = 0;
 
+				if (page_pa)
+					svc_mark_hot(page_pa);
 				svc.pushes_seen++;
 				sem_va = ((u64)st.sem_va_hi << 32) |
 					 st.sem_va_lo;
 
 				if (st.have_sem && st.sem_payload &&
-				    sem_va >= SEMA_VA_LO && sem_va < SEMA_VA_HI &&
-				    !svc_map_lookup(sem_va, &sem_pa)) {
-					if (svc_read(vgpu, sem_pa, &cur,
-						     sizeof(cur)))
+				    sem_va >= SEMA_VA_LO && sem_va < SEMA_VA_HI) {
+					if (svc_sem_read(vgpu, sem_va, &cur))
 						svc.pushes_nomap++;
 					else if (st.sem_payload > cur)
 						pending = 1;
@@ -733,8 +1038,7 @@ static void svc_decode_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page)
 						cur += st.sem_payload;
 					else
 						cur = st.sem_payload;
-					if (!svc_write(vgpu, sem_pa, &cur,
-						       sizeof(cur)))
+					if (!svc_sem_write(vgpu, sem_va, cur))
 						svc.releases_applied++;
 				}
 				/* release ends the push */
@@ -765,19 +1069,17 @@ static void svc_decode_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page)
  * Apply by writing payload to the semaphore slot, but only in order
  * (current == payload - 1) so that UVM's tracker never sees a jump.
  */
-static u32 svc_apply_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page)
+static u32 svc_apply_page_data(struct nv_vgpu_device *vgpu, const u8 *buf,
+			       u64 pa_page)
 {
 	u32 off, applied = 0;
 
-	if (svc_read(vgpu, pa_page, buf, SEMA_PAGE_SIZE))
-		return 0;
-
 	/*
 	 * Decoded execution: walk the page as a method stream and perform the
 	 * pushed work (PTE memsets, semaphore releases) for pushes that are
 	 * still pending.
 	 */
-	svc_decode_page(vgpu, buf, pa_page);
+	svc_decode_stream(vgpu, buf, SEMA_PAGE_SIZE, pa_page);
 
 	for (off = 0; off + 16 <= SEMA_PAGE_SIZE; off += 4) {
 		u32 hdr, hi, lo, payload;
@@ -817,14 +1119,11 @@ static u32 svc_apply_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page)
 
 		svc.releases_seen++;
 
-		if (svc_map_lookup(va, &pa)) {
+		cur32 = 0;
+		if (svc_sem_read(vgpu, va, &cur32)) {
 			svc.releases_missed++;
 			continue;
 		}
-
-		cur32 = 0;
-		if (svc_read(vgpu, pa, &cur32, sizeof(cur32)))
-			continue;
 		cur = cur32;
 
 		/*
@@ -842,7 +1141,7 @@ static u32 svc_apply_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page)
 			continue;
 		}
 
-		if (svc_write(vgpu, pa, &payload, sizeof(payload)) == 0) {
+		if (svc_sem_write(vgpu, va, payload) == 0) {
 			svc.releases_applied++;
 			applied++;
 			if (payload != cur + 1) {
@@ -858,13 +1157,21 @@ static u32 svc_apply_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page)
 	return applied;
 }
 
+static u32 svc_apply_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page)
+{
+	if (svc_read(vgpu, pa_page, buf, SEMA_PAGE_SIZE))
+		return 0;
+	return svc_apply_page_data(vgpu, buf, pa_page);
+}
+
 /*
  * Rolling scan: walk guest RAM in bounded windows (round-robin) looking for
  * semaphore-release groups. Push buffers can live anywhere in the guest's
  * system memory, including pages we have no VA mapping for; only the
  * *semaphore* address inside a release needs to resolve through the map.
  */
-#define SVC_SCAN_WINDOW	(256ULL << 20)	/* 256 MB per pass */
+#define SVC_SCAN_WINDOW	(32ULL << 20)	/* 32 MB per pass */
+#define SVC_SCAN_CHUNK	(64 * 1024)	/* 16 pages per guest read */
 static u64 svc_scan_cursor;
 
 static void svc_apply_pushes(struct nv_vgpu_device *vgpu)
@@ -875,19 +1182,181 @@ static void svc_apply_pushes(struct nv_vgpu_device *vgpu)
 	u64 gpa;
 
 	if (!buf) {
-		buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL);
+		buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL);
 		if (!buf)
 			return;
 	}
 	if (end > 0x140000000ULL)
 		end = 0x140000000ULL;
 
-	for (gpa = start; gpa < end; gpa += SEMA_PAGE_SIZE)
-		svc_apply_page(vgpu, buf, gpa);
+	for (gpa = start; gpa < end; gpa += SVC_SCAN_CHUNK) {
+		u32 o;
+
+		if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK))
+			continue;
+		for (o = 0; o < SVC_SCAN_CHUNK; o += SEMA_PAGE_SIZE)
+			svc_apply_page_data(vgpu, buf + o, gpa + o);
+	}
 
 	svc_scan_cursor = (end >= 0x140000000ULL) ? 0 : end;
 }
 
+/*
+ * Targeted pass: run the work in the pages we have mapped. The map covers the
+ * allocations the guest has described to us - the pushbuffer pools among them
+ * - so new pushes are picked up in milliseconds instead of waiting for the
+ * rolling full-RAM scan to reach their page.
+ */
+#define SVC_SCAN_BUDGET	1024	/* pages per pass: bounds the read traffic */
+
+static void svc_apply_mapped(struct nv_vgpu_device *vgpu)
+{
+	static u8 *buf;
+	static u32 cursor;
+	u32 i, scanned = 0;
+
+	if (!buf) {
+		buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL);
+		if (!buf)
+			return;
+	}
+	if (!svc.map_n)
+		return;
+
+	/* Known push pages first: recycled pushbuffers live there. */
+	for (i = 0; i < svc.map_n && scanned < SVC_SCAN_BUDGET; i++) {
+		u32 k = (cursor + i) % svc.map_n;
+		u64 pa = svc.map[k].pa_page;
+
+		if (test_bit(pa >> SEMA_PAGE_SHIFT, svc_hot)) {
+			svc_apply_page(vgpu, buf, pa);
+			scanned++;
+		}
+	}
+	if (scanned)
+		return;
+
+	/* Bootstrap: walk the map in bounded windows until push pages are
+	 * learned. */
+	for (i = 0; i < svc.map_n && i < SVC_SCAN_BUDGET; i++) {
+		u32 k = (cursor + i) % svc.map_n;
+
+		svc_apply_page(vgpu, buf, svc.map[k].pa_page);
+	}
+	cursor = (cursor + i) % svc.map_n;
+}
+
+/* ------------------------------------------------------------------ */
+/* GPFIFO-driven push execution                                        */
+/* ------------------------------------------------------------------ */
+/* Read `len` bytes at a GPU VA through the discovered VA->PA map. */
+static int svc_read_va(struct nv_vgpu_device *vgpu, u64 va, void *buf,
+		       size_t len)
+{
+	u64 pa;
+
+	if (svc_map_lookup(va, &pa))
+		return -ENOENT;
+	return svc_read(vgpu, pa, buf, len);
+}
+
+/*
+ * Run the work the guest has submitted on each registered channel.
+ *
+ * A submission is visible as GPPut moving ahead of our own read position in
+ * the channel's USERD. For every new GPFIFO entry we fetch the pushbuffer it
+ * points at, decode its method stream and perform the pushed work (PTE
+ * memsets, copies, semaphore releases) - this is what makes the guest's UVM
+ * progress waits complete. The GPU's own GPGet is updated as we go.
+ */
+static void svc_pump_channels(struct nv_vgpu_device *vgpu)
+{
+	/* Only ever called from the service thread. */
+	static struct svc_channel local[SVC_MAX_CHANNELS];
+	unsigned long flags;
+	int i, k, n = 0;
+
+	spin_lock_irqsave(&svc.chan_lock, flags);
+	for (i = 0; i < SVC_MAX_CHANNELS; i++)
+		if (svc.chan[i].valid)
+			local[n++] = svc.chan[i];
+	spin_unlock_irqrestore(&svc.chan_lock, flags);
+
+	for (i = 0; i < n; i++) {
+		struct svc_channel *ch = &local[i];
+		u32 put = 0, get = 0, guard = 0;
+
+		if (!ch->userd_pa)
+			continue;
+
+		if (svc_read_ap(vgpu, ch->userd_pa + SVC_USERD_GP_PUT,
+				ch->userd_as, &put, sizeof(put))) {
+			svc.gpfifo_nomap++;
+			continue;
+		}
+		(void)svc_read_ap(vgpu, ch->userd_pa + SVC_USERD_GP_GET,
+				  ch->userd_as, &get, sizeof(get));
+
+		if (put >= ch->gpfifo_entries)
+			put %= ch->gpfifo_entries;
+
+		while (ch->last_get != put && guard++ < ch->gpfifo_entries) {
+			u8 entry[SVC_GPFIFO_ENTRY_SIZE];
+			u64 va, idx = ch->last_get;
+			u32 len, e0, e1, g;
+
+			if (svc_read_va(vgpu,
+					ch->gpfifo_va +
+					idx * SVC_GPFIFO_ENTRY_SIZE,
+					entry, sizeof(entry))) {
+				svc.gpfifo_nomap++;
+				break;
+			}
+			memcpy(&e0, entry, 4);
+			memcpy(&e1, entry + 4, 4);
+			/* GP_ENTRY0.GET[31:2] | GP_ENTRY1.GET_HI[7:0] */
+			va = ((u64)(e1 & 0xff) << 32) |
+			     (u64)(e0 & 0xfffffffc);
+			/* GP_ENTRY1.LENGTH[30:10], in dwords */
+			len = ((e1 >> 10) & 0x1fffff) << 2;
+
+			if (va && len && len <= SVC_PUSH_MAX) {
+				u8 *pb = kmalloc(len, GFP_KERNEL);
+
+				if (pb) {
+					if (!svc_read_va(vgpu, va, pb, len)) {
+						svc_decode_stream(vgpu, pb,
+								  len, 0);
+						ch->submissions++;
+						svc.gpfifo_entries_done++;
+					} else {
+						svc.gpfifo_nomap++;
+					}
+					kfree(pb);
+				}
+			}
+
+			ch->last_get = (idx + 1) % ch->gpfifo_entries;
+
+			/* Publish progress the way the GPU would. */
+			g = ch->last_get;
+			svc_write_ap(vgpu, ch->userd_pa + SVC_USERD_GP_GET,
+				     ch->userd_as, &g, sizeof(g));
+		}
+
+		spin_lock_irqsave(&svc.chan_lock, flags);
+		for (k = 0; k < SVC_MAX_CHANNELS; k++) {
+			if (svc.chan[k].valid &&
+			    svc.chan[k].h_object == ch->h_object) {
+				svc.chan[k].last_get = ch->last_get;
+				svc.chan[k].submissions = ch->submissions;
+				break;
+			}
+		}
+		spin_unlock_irqrestore(&svc.chan_lock, flags);
+	}
+}
+
 /* ------------------------------------------------------------------ */
 /* thread                                                             */
 /* ------------------------------------------------------------------ */
@@ -985,10 +1454,32 @@ static int svc_thread(void *arg)
 		    time_after(jiffies, svc.last_build + 10 * HZ))
 			svc_rebuild(vgpu);
 
+		/*
+		 * Channel-driven execution: notice submissions as they are made
+		 * and run the pushed work immediately, so the guest's UVM
+		 * progress waits complete in milliseconds rather than in the
+		 * time it takes the fallback scan to find the push.
+		 */
+		svc_pump_channels(vgpu);
+
+		/*
+		 * Fast path for the pushes themselves: every mapped page is a
+		 * candidate pushbuffer page, and there are far fewer of those
+		 * than there are pages of guest RAM.
+		 */
 		if (svc.direct_base && svc.map_n)
-			svc_apply_pushes(vgpu);
+			svc_apply_mapped(vgpu);
+
+		/*
+		 * Rolling RAM scan: the pushbuffers of some clients are not
+		 * covered by the VA map, so sweep guest RAM in bounded windows
+		 * in parallel with the targeted pass. Work per pass is fixed,
+		 * which keeps the emulation responsive while a full sweep
+		 * completes every few seconds.
+		 */
+		svc_apply_pushes(vgpu);
 
-		msleep(100);
+		msleep(10);
 	}
 
 	pr_info("sema-svc: thread stopping (pairs=%llu releases_seen=%llu applied=%llu map=%u)\n",
@@ -1001,6 +1492,7 @@ int gsp_sema_service_start(void)
 	if (svc.task)
 		return 0;
 	svc.stop = false;
+	spin_lock_init(&svc.chan_lock);
 	svc.task = kthread_run(svc_thread, NULL, "gsp-sema-svc");
 	if (IS_ERR(svc.task)) {
 		int ret = PTR_ERR(svc.task);