kernel: track channels and execute host-class pushed work

The guest's CUDA stack was stuck inside context creation because the work it
submits is never executed by the emulated GPU. Two forms of pushed work were
missing, and the completion path had grown far too slow to be useful:

- Channel tracking: on a channel allocation parse the GPFIFO GPU VA and entry
  count, the USERD address/aperture and the instance block, hand them to the
  progress service and drop them again when the channel is freed.

- Host-class (C46F) semaphore releases: pushes on a compute channel signal
  progress with a five-dword method run (SEM_ADDR_LO/HI, PAYLOAD_LO/HI,
  EXECUTE) rather than the copy-engine group the service already understood,
  so those releases were never applied. Decode both forms.

- Emulated vidmem: channel blocks (USERD/instance) and some client semaphores
  live in the emulated frame buffer, whose VAs are a window over the BAR1
  backing rather than guest RAM; access them through the right aperture.

- Latency: the service's full-RAM sweep took seconds per pass, so guests'
  progress waits took whole seconds and the driver tore its own setup back
  down in the meantime. Sweep guest RAM in bounded, chunked windows on every
  pass, remember the pages pushes are actually found in and revisit those
  first, and keep the VA->PA map complete (dedup, sort, binary search) so
  semaphores resolve reliably.

Also add a vidmem poke parameter for bringing up and debugging emulated
frame-buffer state.

5 files changed, 676 insertions, 64 deletions

diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c
index 44de6c0..a21cf28 100644
--- a/kernel/gsp_rm_emulation.c
+++ b/kernel/gsp_rm_emulation.c
@@ -15,6 +15,7 @@
 #include "gsp_emulation.h"
 #include "gsp_rm_emulation.h"
 #include "gsp_rm_tu104.h"
+#include "gsp_sema_service.h"
 
 /* ============================================================
  * Guest-memory access ("virtual DMA" on behalf of the device)
@@ -1925,6 +1926,57 @@ static void gsp_rm_handle_copy_server_pdes(struct gsp_emu_context *ctx,
 static bool gsp_rm_alloc_interesting(u32 class_id);
 
 /*
+ * Channel (GPFIFO) classes across generations. Their allocation parameters
+ * carry the GPFIFO GPU VA/entry count and the guest-physical USERD/instance
+ * blocks, which is what the emulated-GPU progress service needs to run the
+ * work the guest submits on the channel.
+ */
+static bool gsp_rm_is_channel_class(u32 class_id)
+{
+	switch (class_id) {
+	case 0x906f:	/* MAXWELL_CHANNEL_GPFIFO_A */
+	case 0xa06f:	/* KEPLER_CHANNEL_GPFIFO_A */
+	case 0xb06f:	/* PASCAL_CHANNEL_GPFIFO_A */
+	case 0xc06f:	/* VOLTA_CHANNEL_GPFIFO_A */
+	case 0xc46f:	/* TURING_CHANNEL_GPFIFO_A */
+		return true;
+	default:
+		return false;
+	}
+}
+
+/* NV_CHANNEL_ALLOC_PARAMS offsets (alloc/alloc_channel.h) */
+#define GSPRM_CHAN_GPFIFO_OFFSET	8
+#define GSPRM_CHAN_GPFIFO_ENTRIES	16
+#define GSPRM_CHAN_ENGINE_TYPE		128
+#define GSPRM_CHAN_INSTANCE_MEM		144
+#define GSPRM_CHAN_USERD_MEM		168
+#define GSPRM_CHAN_MEM_APERTURE		16	/* offset of addressSpace in the desc */
+#define GSPRM_CHAN_NOTIFIER_MEM		248
+
+static void gsp_rm_register_channel(struct gsp_emu_context *ctx, u32 h_object,
+				    const u8 *params, u32 params_size)
+{
+	struct sema_channel_desc desc;
+
+	if (params_size < GSPRM_CHAN_NOTIFIER_MEM + 8)
+		return;
+
+	memset(&desc, 0, sizeof(desc));
+	desc.h_object = h_object;
+	memcpy(&desc.gpfifo_va, params + GSPRM_CHAN_GPFIFO_OFFSET, 8);
+	memcpy(&desc.gpfifo_entries, params + GSPRM_CHAN_GPFIFO_ENTRIES, 4);
+	memcpy(&desc.engine_type, params + GSPRM_CHAN_ENGINE_TYPE, 4);
+	memcpy(&desc.instance_pa, params + GSPRM_CHAN_INSTANCE_MEM, 8);
+	memcpy(&desc.userd_pa, params + GSPRM_CHAN_USERD_MEM, 8);
+	memcpy(&desc.userd_as, params + GSPRM_CHAN_USERD_MEM +
+	       GSPRM_CHAN_MEM_APERTURE, 4);
+	memcpy(&desc.notifier_pa, params + GSPRM_CHAN_NOTIFIER_MEM, 8);
+
+	gsp_sema_service_channel_add(&desc);
+}
+
+/*
  * GSP_RM_ALLOC (103) - rpc_gsp_rm_alloc_v03_00 (32 byte header) followed by
  * the class-specific allocation parameters.
  *
@@ -1981,6 +2033,11 @@ static void gsp_rm_handle_rm_alloc(struct gsp_emu_context *ctx,
 		}
 	}
 
+	/* Channels: hand the progress service everything it needs to run the
+	 * work submitted on this channel (GPFIFO ring + USERD). */
+	if (gsp_rm_is_channel_class(class_id))
+		gsp_rm_register_channel(ctx, ho, params, params_size);
+
 	/* Echo the fixed alloc header + as many param bytes as are contiguous
 	 * in the first queue element. */
 	echo_len = min(params_size,
@@ -2244,6 +2301,8 @@ static void gsp_rm_handle_free(struct gsp_emu_context *ctx, const u8 *elem,
 	memcpy(&hObjOld, params + 8, 4);
 	memset(params + 12, 0, 4); /* status = NV_OK */
 
+	gsp_sema_service_channel_del(hObjOld);
+
 	nv_vgpu_info(ctx->vgpu, "GSPRM: FREE hObjOld=0x%x seq=%u cmd_read=%u stat_write=%u",
 		     hObjOld, sequence, rm->cmd_read, rm->stat_write);
 
diff --git a/kernel/gsp_rm_tu104.h b/kernel/gsp_rm_tu104.h
index 74d1ba3..3ac01b7 100644
--- a/kernel/gsp_rm_tu104.h
+++ b/kernel/gsp_rm_tu104.h
@@ -293,7 +293,7 @@ static const struct tu104_fifo_device_entry tu104_engine_table[] = {
 			[EI_ENG_DESC]       = TU104_ENG_CE0,
 			[EI_FIFO_TAG]       = TU104_TAG_CE0,
 			[EI_RM_ENGINE_TYPE] = TU104_RM_ENGINE_CE0,
-			[EI_RUNLIST]        = 1,
+			[EI_RUNLIST]        = 0,
 			[EI_MMU_FAULT_ID]   = TU104_FAULT_ID_CE0,
 			[EI_RC_MASK]        = TU104_RC_CE0,
 			[EI_RESET]          = 1,
@@ -318,7 +318,7 @@ static const struct tu104_fifo_device_entry tu104_engine_table[] = {
 			[EI_ENG_DESC]       = TU104_ENG_CE1,
 			[EI_FIFO_TAG]       = TU104_TAG_CE1,
 			[EI_RM_ENGINE_TYPE] = TU104_RM_ENGINE_CE1,
-			[EI_RUNLIST]        = 2,
+			[EI_RUNLIST]        = 1,
 			[EI_MMU_FAULT_ID]   = TU104_FAULT_ID_CE1,
 			[EI_RC_MASK]        = TU104_RC_CE1,
 			[EI_RESET]          = 2,
@@ -332,8 +332,8 @@ static const struct tu104_fifo_device_entry tu104_engine_table[] = {
 			[EI_CHRAM_PRI_BASE]       = 0,
 			[EI_KERNEL_RM_MAX]        = 0,
 		},
-		.pbdmaIds      = {2, 0},
-		.pbdmaFaultIds = {TU104_PBDMA_FAULT_ID_BASE + 2, 0},
+		.pbdmaIds      = {1, 0},
+		.pbdmaFaultIds = {TU104_PBDMA_FAULT_ID_BASE + 1, 0},
 		.numPbdmas     = 1,
 		.engineName    = "CE1",
 	},
@@ -343,7 +343,7 @@ static const struct tu104_fifo_device_entry tu104_engine_table[] = {
 			[EI_ENG_DESC]       = TU104_ENG_CE2,
 			[EI_FIFO_TAG]       = TU104_TAG_CE2,
 			[EI_RM_ENGINE_TYPE] = TU104_RM_ENGINE_CE2,
-			[EI_RUNLIST]        = 3,
+			[EI_RUNLIST]        = 2,
 			[EI_MMU_FAULT_ID]   = TU104_FAULT_ID_CE2,
 			[EI_RC_MASK]        = TU104_RC_CE2,
 			[EI_RESET]          = 3,
@@ -357,8 +357,8 @@ static const struct tu104_fifo_device_entry tu104_engine_table[] = {
 			[EI_CHRAM_PRI_BASE]       = 0,
 			[EI_KERNEL_RM_MAX]        = 0,
 		},
-		.pbdmaIds      = {3, 0},
-		.pbdmaFaultIds = {TU104_PBDMA_FAULT_ID_BASE + 3, 0},
+		.pbdmaIds      = {2, 0},
+		.pbdmaFaultIds = {TU104_PBDMA_FAULT_ID_BASE + 2, 0},
 		.numPbdmas     = 1,
 		.engineName    = "CE2",
 	},
@@ -368,7 +368,7 @@ static const struct tu104_fifo_device_entry tu104_engine_table[] = {
 			[EI_ENG_DESC]       = TU104_ENG_CE3,
 			[EI_FIFO_TAG]       = TU104_TAG_CE3,
 			[EI_RM_ENGINE_TYPE] = TU104_RM_ENGINE_CE3,
-			[EI_RUNLIST]        = 4,
+			[EI_RUNLIST]        = 3,
 			[EI_MMU_FAULT_ID]   = TU104_FAULT_ID_CE3,
 			[EI_RC_MASK]        = TU104_RC_CE3,
 			[EI_RESET]          = 4,
@@ -382,8 +382,8 @@ static const struct tu104_fifo_device_entry tu104_engine_table[] = {
 			[EI_CHRAM_PRI_BASE]       = 0,
 			[EI_KERNEL_RM_MAX]        = 0,
 		},
-		.pbdmaIds      = {4, 0},
-		.pbdmaFaultIds = {TU104_PBDMA_FAULT_ID_BASE + 4, 0},
+		.pbdmaIds      = {3, 0},
+		.pbdmaFaultIds = {TU104_PBDMA_FAULT_ID_BASE + 3, 0},
 		.numPbdmas     = 1,
 		.engineName    = "CE3",
 	},
@@ -393,7 +393,7 @@ static const struct tu104_fifo_device_entry tu104_engine_table[] = {
 			[EI_ENG_DESC]       = TU104_ENG_CE4,
 			[EI_FIFO_TAG]       = TU104_TAG_CE4,
 			[EI_RM_ENGINE_TYPE] = TU104_RM_ENGINE_CE4,
-			[EI_RUNLIST]        = 5,
+			[EI_RUNLIST]        = 4,
 			[EI_MMU_FAULT_ID]   = TU104_FAULT_ID_CE4,
 			[EI_RC_MASK]        = TU104_RC_CE4,
 			[EI_RESET]          = 5,
@@ -407,8 +407,8 @@ static const struct tu104_fifo_device_entry tu104_engine_table[] = {
 			[EI_CHRAM_PRI_BASE]       = 0,
 			[EI_KERNEL_RM_MAX]        = 0,
 		},
-		.pbdmaIds      = {5, 0},
-		.pbdmaFaultIds = {TU104_PBDMA_FAULT_ID_BASE + 5, 0},
+		.pbdmaIds      = {4, 0},
+		.pbdmaFaultIds = {TU104_PBDMA_FAULT_ID_BASE + 4, 0},
 		.numPbdmas     = 1,
 		.engineName    = "CE4",
 	},
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 1ed8c99..2f453fa 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -38,12 +38,17 @@
 #include <linux/vfio.h>
 #include <linux/jiffies.h>
 #include <linux/mutex.h>
+#include <linux/sort.h>
+#include <linux/bitmap.h>
 
 #include "nvidia-vgpu-vfio.h"
 #include "gsp_sema_service.h"
 
-#define SEMA_MAP_MAX		16384
+#define SEMA_MAP_MAX		65536
+#define SEMA_MAP_SIZE_BITS	16
 #define SEMA_PAGE_SIZE		4096
+#define SEMA_PAGE_SHIFT		12
+#define SEMA_PA_MAX		0x140000000ULL
 #define SEMA_SCAN_PAGES_MAX	0x20000	/* 512 MB worth of pages per pass */
 
 /* Plausible GPU VA window for the client's system-memory objects (Turing). */
@@ -66,11 +71,65 @@
 #define CE_DMA_HDR_ADDR		0xc0
 #define CE_DMA_HDR_COUNT	1
 
+/*
+ * Channel/host class (C46F) semaphore release, written by pushes on a
+ * compute channel as one five-dword run starting at SEM_ADDR_LO:
+ *   [LO][HI][PAYLOAD_LO][PAYLOAD_HI][EXECUTE]
+ * This is the form core CUDA uses for its own progress semaphores; the CE
+ * form above is what UVM pushes on a copy channel.
+ */
+#define HOST_M_SEM_ADDR_LO	0x5c
+#define HOST_SEM_OP_RELEASE	1
+#define HOST_SEM_OP_REDUCTION	6
+
 struct sema_map_entry {
 	u64 gpu_page;	/* GPU VA, page aligned */
 	u64 pa_page;	/* guest PA, page aligned */
 };
 
+/* ------------------------------------------------------------------ */
+/* registered channels (GPFIFO-driven execution)                       */
+/* ------------------------------------------------------------------ */
+
+#define SVC_MAX_CHANNELS	24
+#define SVC_GPFIFO_ENTRY_SIZE	8
+#define SVC_PUSH_MAX		(256 * 1024)
+
+/* USERD control block (NvA06FControl / KeplerAControlGPFifo) */
+#define SVC_USERD_GP_GET	0x88
+#define SVC_USERD_GP_PUT	0x8c
+
+struct svc_channel {
+	bool	valid;
+	u32	h_object;
+	u32	engine_type;
+	u64	gpfifo_va;
+	u32	gpfifo_entries;
+	u64	userd_pa;
+	u32	userd_as;	/* NV_ADDR_SYSMEM / NV_ADDR_FBMEM */
+	u64	instance_pa;
+	u64	notifier_pa;
+	u32	last_get;	/* next GPFIFO entry we have still to run */
+	u64	submissions;	/* GPFIFO entries executed */
+};
+
+/*
+ * Push pages we have actually seen work in. Pushbuffer pools are recycled, so
+ * once a page has carried a push it is a much better scan candidate than the
+ * rest of guest RAM or even the whole VA map.
+ */
+#define SEMA_HOT_PAGES		(SEMA_PA_MAX >> SEMA_PAGE_SHIFT)
+static unsigned long svc_hot[BITS_TO_LONGS(SEMA_HOT_PAGES)];
+static u32 svc_map_hash[SEMA_MAP_MAX * 2];
+
+static void svc_mark_hot(u64 pa)
+{
+	u64 page = pa >> SEMA_PAGE_SHIFT;
+
+	if (page < SEMA_HOT_PAGES)
+		set_bit(page, svc_hot);
+}
+
 struct sema_svc {
 	struct task_struct	*task;
 	bool			stop;
@@ -79,6 +138,11 @@ struct sema_svc {
 	u32			map_n;
 	struct sema_map_entry	map[SEMA_MAP_MAX];
 	u64			last_build;
+	u64			last_scan;
+
+	/* registered guest channels (GPFIFO-driven push execution) */
+	spinlock_t		chan_lock;
+	struct svc_channel	chan[SVC_MAX_CHANNELS];
 
 	/* stats */
 	u64			pairs_seen;
@@ -95,6 +159,9 @@ struct sema_svc {
 	u64			releases_stale;		/* payload <= current */
 	u64			releases_jumped;	/* applied with gap > 1 */
 	u64			builds;
+	u64			gpfifo_entries_done;	/* entries consumed via GPFIFO */
+	u64			gpfifo_nomap;		/* GPFIFO/pushbuf VA not mapped */
+	u64			vidmem_writes;		/* semaphores written in vidmem */
 };
 
 static struct sema_svc svc;
@@ -113,6 +180,164 @@ static int svc_write(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len
 	return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, true);
 }
 
+/* NV_ADDR_SYSMEM / NV_ADDR_FBMEM as carried in the channel alloc params. */
+#define SVC_APERTURE_SYS	1
+#define SVC_APERTURE_FB		2
+
+/*
+ * Aperture-aware access: channels describe their USERD/instance blocks with an
+ * address-space tag, and "frame buffer" addresses are offsets into the
+ * emulated vidmem (BAR1 backing), not guest RAM.
+ */
+static int svc_read_ap(struct nv_vgpu_device *vgpu, u64 addr, u32 aperture,
+		       void *buf, size_t len)
+{
+	if (aperture == SVC_APERTURE_FB) {
+		u8 *base = vgpu->bars[1].vaddr;
+		size_t size = vgpu->bars[1].size;
+
+		if (!base || addr + len > size)
+			return -EINVAL;
+		memcpy(buf, base + addr, len);
+		return 0;
+	}
+	return svc_read(vgpu, addr, buf, len);
+}
+
+static int svc_write_ap(struct nv_vgpu_device *vgpu, u64 addr, u32 aperture,
+			void *buf, size_t len)
+{
+	if (aperture == SVC_APERTURE_FB) {
+		u8 *base = vgpu->bars[1].vaddr;
+		size_t size = vgpu->bars[1].size;
+
+		if (!base || addr + len > size)
+			return -EINVAL;
+		memcpy(base + addr, buf, len);
+		return 0;
+	}
+	return svc_write(vgpu, addr, buf, len);
+}
+
+/*
+ * Register a guest channel. Called from the GSP-RM emulation when the guest
+ * driver allocates a channel (the allocation carries the GPFIFO GPU VA, its
+ * entry count and the guest-physical USERD/instance blocks), and dropped when
+ * the channel is freed.
+ */
+void gsp_sema_service_channel_add(const struct sema_channel_desc *desc)
+{
+	struct svc_channel *ch = NULL;
+	unsigned long flags;
+	int i;
+
+	if (!desc || !desc->h_object || !desc->gpfifo_va ||
+	    desc->gpfifo_entries < 2)
+		return;
+
+	spin_lock_irqsave(&svc.chan_lock, flags);
+	for (i = 0; i < SVC_MAX_CHANNELS; i++) {
+		if (svc.chan[i].valid && svc.chan[i].h_object == desc->h_object) {
+			ch = &svc.chan[i];
+			break;
+		}
+		if (!ch && !svc.chan[i].valid)
+			ch = &svc.chan[i];
+	}
+	if (ch) {
+		memset(ch, 0, sizeof(*ch));
+		ch->valid = true;
+		ch->h_object = desc->h_object;
+		ch->engine_type = desc->engine_type;
+		ch->gpfifo_va = desc->gpfifo_va;
+		ch->gpfifo_entries = desc->gpfifo_entries;
+		ch->userd_pa = desc->userd_pa;
+		ch->userd_as = desc->userd_as ? desc->userd_as : SVC_APERTURE_SYS;
+		ch->instance_pa = desc->instance_pa;
+		ch->notifier_pa = desc->notifier_pa;
+	}
+	spin_unlock_irqrestore(&svc.chan_lock, flags);
+
+	if (ch)
+		pr_info("sema-svc: channel 0x%x registered gpfifo=0x%llx entries=%u userd=0x%llx\n",
+			desc->h_object, desc->gpfifo_va, desc->gpfifo_entries,
+			desc->userd_pa);
+}
+
+void gsp_sema_service_channel_del(u32 h_object)
+{
+	unsigned long flags;
+	int i;
+
+	if (!h_object)
+		return;
+
+	spin_lock_irqsave(&svc.chan_lock, flags);
+	for (i = 0; i < SVC_MAX_CHANNELS; i++) {
+		if (svc.chan[i].valid && svc.chan[i].h_object == h_object) {
+			svc.chan[i].valid = false;
+			pr_info("sema-svc: channel 0x%x unregistered (submissions=%llu)\n",
+				h_object, svc.chan[i].submissions);
+			break;
+		}
+	}
+	spin_unlock_irqrestore(&svc.chan_lock, flags);
+}
+
+/*
+ * The client's vidmem VAs are its window base plus the vidmem offset, and the
+ * backing store is the emulated BAR1 buffer. Everything else that resolves
+ * goes through the discovered guest-RAM map.
+ */
+#define SVC_VIDMEM_VA_BASE	0x200000000ULL
+
+static int svc_map_lookup(u64 gpu_va, u64 *pa);
+
+static int svc_resolve_va(struct nv_vgpu_device *vgpu, u64 va, u64 *addr,
+			  u32 *aperture)
+{
+	if (!svc_map_lookup(va, addr)) {
+		*aperture = 0;		/* guest RAM */
+		return 0;
+	}
+	if (va >= SVC_VIDMEM_VA_BASE &&
+	    va - SVC_VIDMEM_VA_BASE + 4 <= vgpu->bars[1].size) {
+		*addr = va - SVC_VIDMEM_VA_BASE;
+		*aperture = 1;		/* emulated vidmem */
+		return 0;
+	}
+	return -ENOENT;
+}
+
+static int svc_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val)
+{
+	u64 addr = 0;
+	u32 ap = 0;
+
+	if (svc_resolve_va(vgpu, va, &addr, &ap))
+		return -ENOENT;
+	if (ap) {
+		memcpy(val, vgpu->bars[1].vaddr + addr, sizeof(*val));
+		return 0;
+	}
+	return svc_read(vgpu, addr, val, sizeof(*val));
+}
+
+static int svc_sem_write(struct nv_vgpu_device *vgpu, u64 va, u32 val)
+{
+	u64 addr = 0;
+	u32 ap = 0;
+
+	if (svc_resolve_va(vgpu, va, &addr, &ap))
+		return -ENOENT;
+	if (ap) {
+		memcpy(vgpu->bars[1].vaddr + addr, &val, sizeof(val));
+		svc.vidmem_writes++;
+		return 0;
+	}
+	return svc_write(vgpu, addr, &val, sizeof(val));
+}
+
 /* ------------------------------------------------------------------ */
 /* discovery                                                          */
 /* ------------------------------------------------------------------ */
@@ -190,38 +415,82 @@ static void svc_scan_pass(struct nv_vgpu_device *vgpu, u8 *buf,
 
 static void svc_add_map(u64 gpu_va, u64 pa)
 {
-	u64 gpu_page = gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1);
-	u64 pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1);
-	u32 i;
-
-	/*
-	 * First-wins: the guest keeps stale copies of freed allocations around,
-	 * and unrelated adjacent u64 pairs can look like {cpu_va, gpu_va}. Both
-	 * produce bogus mappings for a VA that already has a good one, and
-	 * overwriting it would send every semaphore write to the wrong page.
-	 * The first pair seen for a VA is the live allocation.
-	 */
-	for (i = 0; i < svc.map_n; i++) {
-		if (svc.map[i].gpu_page == gpu_page)
-			return;
-	}
 	if (svc.map_n < SEMA_MAP_MAX) {
-		svc.map[svc.map_n].gpu_page = gpu_page;
-		svc.map[svc.map_n].pa_page = pa_page;
+		svc.map[svc.map_n].gpu_page =
+			gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1);
+		svc.map[svc.map_n].pa_page =
+			pa & ~(u64)(SEMA_PAGE_SIZE - 1);
 		svc.map_n++;
 	}
 }
 
+static int svc_map_cmp(const void *a, const void *b)
+{
+	const struct sema_map_entry *x = a, *y = b;
+
+	if (x->gpu_page < y->gpu_page)
+		return -1;
+	if (x->gpu_page > y->gpu_page)
+		return 1;
+	return 0;
+}
+
+/*
+ * Deduplicate (first-wins: the guest keeps stale copies of freed allocations
+ * around and the first pair seen for a VA is the live one) and sort so lookups
+ * can binary search. The map is large enough that the old linear scan both
+ * missed mappings (it stopped at the cap) and cost too much per release.
+ */
+static void svc_map_finalize(void)
+{
+	u32 i, n = 0;
+	u32 hash_mask = ARRAY_SIZE(svc_map_hash) - 1;
+
+	memset(svc_map_hash, 0, sizeof(svc_map_hash));
+
+	for (i = 0; i < svc.map_n; i++) {
+		u64 page = svc.map[i].gpu_page;
+		u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) &
+			hash_mask;
+		u32 probe = 0;
+
+		for (;;) {
+			u32 slot = (h + probe) & hash_mask;
+			u32 val = svc_map_hash[slot];
+
+			if (!val) {
+				svc_map_hash[slot] = n + 1;
+				svc.map[n++] = svc.map[i];
+				break;
+			}
+			if (svc.map[val - 1].gpu_page == page)
+				break;	/* stale duplicate */
+			if (++probe > 64)
+				break;
+		}
+	}
+	svc.map_n = n;
+
+	sort(svc.map, svc.map_n, sizeof(svc.map[0]), svc_map_cmp, NULL);
+}
+
 static int svc_map_lookup(u64 gpu_va, u64 *pa)
 {
 	u64 gpu_page = gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1);
-	u32 i;
+	int lo = 0, hi = (int)svc.map_n - 1;
 
-	for (i = 0; i < svc.map_n; i++) {
-		if (svc.map[i].gpu_page == gpu_page) {
-			*pa = svc.map[i].pa_page + (gpu_va & (SEMA_PAGE_SIZE - 1));
+	while (lo <= hi) {
+		int mid = (lo + hi) / 2;
+
+		if (svc.map[mid].gpu_page == gpu_page) {
+			*pa = svc.map[mid].pa_page +
+			      (gpu_va & (SEMA_PAGE_SIZE - 1));
 			return 0;
 		}
+		if (svc.map[mid].gpu_page < gpu_page)
+			lo = mid + 1;
+		else
+			hi = mid - 1;
 	}
 	return -ENOENT;
 }
@@ -241,6 +510,7 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu)
 			return;
 	}
 
+	svc.map_n = 0;
 	svc_scan_pass(vgpu, buf, cands, &ncand, pairs, &npairs);
 
 	for (i = 0; i < ncand; i++) {
@@ -261,14 +531,15 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu)
 			pa = cpu - best_base;
 			if (pa >= 0x140000000ULL || (pa & 0xfff))
 				continue;
-			if (pairs[i].gpu_va < 0x100000000ULL ||
-			    pairs[i].gpu_va >= 0x140000000ULL)
+			if (pairs[i].gpu_va < SEMA_VA_LO ||
+			    pairs[i].gpu_va >= SEMA_VA_HI)
 				continue;
 			svc_add_map(pairs[i].gpu_va, pa);
 			svc.pairs_seen++;
 		}
 	}
 
+	svc_map_finalize();
 	svc.builds++;
 	svc.last_build = jiffies;
 
@@ -280,6 +551,9 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu)
 		svc.pages_decoded, svc.pushes_seen, svc.pushes_pending,
 		svc.pushes_nomap, svc.memsets_done, svc.memsets_nomap,
 		svc.memsets_fail);
+	pr_info("sema-svc:   gpfifo done=%llu nomap=%llu vidmem_writes=%llu\n",
+		svc.gpfifo_entries_done, svc.gpfifo_nomap,
+		svc.vidmem_writes);
 	/* show a few entries so mappings can be verified from dmesg */
 	{
 		u32 shown;
@@ -532,7 +806,8 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops,
  * replay old pushes whose semaphore is already satisfied, and those are
  * skipped, so their stale memsets never overwrite newer state.
  */
-static void svc_decode_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page)
+static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
+			      u32 len, u64 page_pa)
 {
 	struct svc_push_state st;
 	struct svc_memset_op ops[SVC_MAX_MEMSETS];
@@ -542,9 +817,9 @@ static void svc_decode_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page)
 	memset(&st, 0, sizeof(st));
 	svc.pages_decoded++;
 
-	while (off + 4 <= SEMA_PAGE_SIZE) {
+	while (off + 4 <= len) {
 		u32 hdr = 0, op, count, subch, addr;
-		u32 v0 = 0, v1 = 0, v2 = 0;
+		u32 v0 = 0, v1 = 0, v2 = 0, v3 = 0, v4 = 0;
 
 		memcpy(&hdr, buf + off, 4);
 		op = (hdr >> 29) & 0x7;
@@ -553,7 +828,7 @@ static void svc_decode_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page)
 		addr = (hdr & 0x1fff) << 2;
 
 		if (op != 1 || !count || count > 64 ||
-		    off + 4 + count * 4 > SEMA_PAGE_SIZE) {
+		    off + 4 + count * 4 > len) {
 			off += 4;	/* resync */
 			continue;
 		}
@@ -564,6 +839,10 @@ static void svc_decode_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page)
 			memcpy(&v1, buf + off + 8, 4);
 		if (count >= 3)
 			memcpy(&v2, buf + off + 12, 4);
+		if (count >= 4 && off + 20 <= len)
+			memcpy(&v3, buf + off + 16, 4);
+		if (count >= 5 && off + 24 <= len)
+			memcpy(&v4, buf + off + 20, 4);
 
 		/*
 		 * OFFSET_IN_UPPER/LOWER and OFFSET_OUT_UPPER/LOWER are
@@ -620,6 +899,32 @@ static void svc_decode_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page)
 			continue;
 		}
 
+		if (addr == HOST_M_SEM_ADDR_LO && count >= 5) {
+			u64 va = ((u64)(v1 & 0xff) << 32) |
+				 (u64)(v0 & 0xfffffffc);
+			u32 payload = v2;
+			u32 op = v4 & 0x7;
+			u32 cur = 0;
+
+			if ((op == HOST_SEM_OP_RELEASE ||
+			     op == HOST_SEM_OP_REDUCTION) && payload &&
+			    va >= SEMA_VA_LO && va < SEMA_VA_HI) {
+				if (page_pa)
+					svc_mark_hot(page_pa);
+				svc.pushes_seen++;
+				if (!svc_sem_read(vgpu, va, &cur) &&
+				    payload > cur) {
+					svc.pushes_pending++;
+					cur = (op == HOST_SEM_OP_REDUCTION) ?
+						cur + payload : payload;
+					if (!svc_sem_write(vgpu, va, cur))
+						svc.releases_applied++;
+				}
+			}
+			off += 4 + count * 4;
+			continue;
+		}
+
 		if (subch != 4) {
 			off += 4 + count * 4;
 			continue;
@@ -710,15 +1015,15 @@ static void svc_decode_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page)
 				u32 cur = 0;
 				int pending = 0;
 
+				if (page_pa)
+					svc_mark_hot(page_pa);
 				svc.pushes_seen++;
 				sem_va = ((u64)st.sem_va_hi << 32) |
 					 st.sem_va_lo;
 
 				if (st.have_sem && st.sem_payload &&
-				    sem_va >= SEMA_VA_LO && sem_va < SEMA_VA_HI &&
-				    !svc_map_lookup(sem_va, &sem_pa)) {
-					if (svc_read(vgpu, sem_pa, &cur,
-						     sizeof(cur)))
+				    sem_va >= SEMA_VA_LO && sem_va < SEMA_VA_HI) {
+					if (svc_sem_read(vgpu, sem_va, &cur))
 						svc.pushes_nomap++;
 					else if (st.sem_payload > cur)
 						pending = 1;
@@ -733,8 +1038,7 @@ static void svc_decode_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page)
 						cur += st.sem_payload;
 					else
 						cur = st.sem_payload;
-					if (!svc_write(vgpu, sem_pa, &cur,
-						       sizeof(cur)))
+					if (!svc_sem_write(vgpu, sem_va, cur))
 						svc.releases_applied++;
 				}
 				/* release ends the push */
@@ -765,19 +1069,17 @@ static void svc_decode_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page)
  * Apply by writing payload to the semaphore slot, but only in order
  * (current == payload - 1) so that UVM's tracker never sees a jump.
  */
-static u32 svc_apply_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page)
+static u32 svc_apply_page_data(struct nv_vgpu_device *vgpu, const u8 *buf,
+			       u64 pa_page)
 {
 	u32 off, applied = 0;
 
-	if (svc_read(vgpu, pa_page, buf, SEMA_PAGE_SIZE))
-		return 0;
-
 	/*
 	 * Decoded execution: walk the page as a method stream and perform the
 	 * pushed work (PTE memsets, semaphore releases) for pushes that are
 	 * still pending.
 	 */
-	svc_decode_page(vgpu, buf, pa_page);
+	svc_decode_stream(vgpu, buf, SEMA_PAGE_SIZE, pa_page);
 
 	for (off = 0; off + 16 <= SEMA_PAGE_SIZE; off += 4) {
 		u32 hdr, hi, lo, payload;
@@ -817,14 +1119,11 @@ static u32 svc_apply_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page)
 
 		svc.releases_seen++;
 
-		if (svc_map_lookup(va, &pa)) {
+		cur32 = 0;
+		if (svc_sem_read(vgpu, va, &cur32)) {
 			svc.releases_missed++;
 			continue;
 		}
-
-		cur32 = 0;
-		if (svc_read(vgpu, pa, &cur32, sizeof(cur32)))
-			continue;
 		cur = cur32;
 
 		/*
@@ -842,7 +1141,7 @@ static u32 svc_apply_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page)
 			continue;
 		}
 
-		if (svc_write(vgpu, pa, &payload, sizeof(payload)) == 0) {
+		if (svc_sem_write(vgpu, va, payload) == 0) {
 			svc.releases_applied++;
 			applied++;
 			if (payload != cur + 1) {
@@ -858,13 +1157,21 @@ static u32 svc_apply_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page)
 	return applied;
 }
 
+static u32 svc_apply_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page)
+{
+	if (svc_read(vgpu, pa_page, buf, SEMA_PAGE_SIZE))
+		return 0;
+	return svc_apply_page_data(vgpu, buf, pa_page);
+}
+
 /*
  * Rolling scan: walk guest RAM in bounded windows (round-robin) looking for
  * semaphore-release groups. Push buffers can live anywhere in the guest's
  * system memory, including pages we have no VA mapping for; only the
  * *semaphore* address inside a release needs to resolve through the map.
  */
-#define SVC_SCAN_WINDOW	(256ULL << 20)	/* 256 MB per pass */
+#define SVC_SCAN_WINDOW	(32ULL << 20)	/* 32 MB per pass */
+#define SVC_SCAN_CHUNK	(64 * 1024)	/* 16 pages per guest read */
 static u64 svc_scan_cursor;
 
 static void svc_apply_pushes(struct nv_vgpu_device *vgpu)
@@ -875,19 +1182,181 @@ static void svc_apply_pushes(struct nv_vgpu_device *vgpu)
 	u64 gpa;
 
 	if (!buf) {
-		buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL);
+		buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL);
 		if (!buf)
 			return;
 	}
 	if (end > 0x140000000ULL)
 		end = 0x140000000ULL;
 
-	for (gpa = start; gpa < end; gpa += SEMA_PAGE_SIZE)
-		svc_apply_page(vgpu, buf, gpa);
+	for (gpa = start; gpa < end; gpa += SVC_SCAN_CHUNK) {
+		u32 o;
+
+		if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK))
+			continue;
+		for (o = 0; o < SVC_SCAN_CHUNK; o += SEMA_PAGE_SIZE)
+			svc_apply_page_data(vgpu, buf + o, gpa + o);
+	}
 
 	svc_scan_cursor = (end >= 0x140000000ULL) ? 0 : end;
 }
 
+/*
+ * Targeted pass: run the work in the pages we have mapped. The map covers the
+ * allocations the guest has described to us - the pushbuffer pools among them
+ * - so new pushes are picked up in milliseconds instead of waiting for the
+ * rolling full-RAM scan to reach their page.
+ */
+#define SVC_SCAN_BUDGET	1024	/* pages per pass: bounds the read traffic */
+
+static void svc_apply_mapped(struct nv_vgpu_device *vgpu)
+{
+	static u8 *buf;
+	static u32 cursor;
+	u32 i, scanned = 0;
+
+	if (!buf) {
+		buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL);
+		if (!buf)
+			return;
+	}
+	if (!svc.map_n)
+		return;
+
+	/* Known push pages first: recycled pushbuffers live there. */
+	for (i = 0; i < svc.map_n && scanned < SVC_SCAN_BUDGET; i++) {
+		u32 k = (cursor + i) % svc.map_n;
+		u64 pa = svc.map[k].pa_page;
+
+		if (test_bit(pa >> SEMA_PAGE_SHIFT, svc_hot)) {
+			svc_apply_page(vgpu, buf, pa);
+			scanned++;
+		}
+	}
+	if (scanned)
+		return;
+
+	/* Bootstrap: walk the map in bounded windows until push pages are
+	 * learned. */
+	for (i = 0; i < svc.map_n && i < SVC_SCAN_BUDGET; i++) {
+		u32 k = (cursor + i) % svc.map_n;
+
+		svc_apply_page(vgpu, buf, svc.map[k].pa_page);
+	}
+	cursor = (cursor + i) % svc.map_n;
+}
+
+/* ------------------------------------------------------------------ */
+/* GPFIFO-driven push execution                                        */
+/* ------------------------------------------------------------------ */
+/* Read `len` bytes at a GPU VA through the discovered VA->PA map. */
+static int svc_read_va(struct nv_vgpu_device *vgpu, u64 va, void *buf,
+		       size_t len)
+{
+	u64 pa;
+
+	if (svc_map_lookup(va, &pa))
+		return -ENOENT;
+	return svc_read(vgpu, pa, buf, len);
+}
+
+/*
+ * Run the work the guest has submitted on each registered channel.
+ *
+ * A submission is visible as GPPut moving ahead of our own read position in
+ * the channel's USERD. For every new GPFIFO entry we fetch the pushbuffer it
+ * points at, decode its method stream and perform the pushed work (PTE
+ * memsets, copies, semaphore releases) - this is what makes the guest's UVM
+ * progress waits complete. The GPU's own GPGet is updated as we go.
+ */
+static void svc_pump_channels(struct nv_vgpu_device *vgpu)
+{
+	/* Only ever called from the service thread. */
+	static struct svc_channel local[SVC_MAX_CHANNELS];
+	unsigned long flags;
+	int i, k, n = 0;
+
+	spin_lock_irqsave(&svc.chan_lock, flags);
+	for (i = 0; i < SVC_MAX_CHANNELS; i++)
+		if (svc.chan[i].valid)
+			local[n++] = svc.chan[i];
+	spin_unlock_irqrestore(&svc.chan_lock, flags);
+
+	for (i = 0; i < n; i++) {
+		struct svc_channel *ch = &local[i];
+		u32 put = 0, get = 0, guard = 0;
+
+		if (!ch->userd_pa)
+			continue;
+
+		if (svc_read_ap(vgpu, ch->userd_pa + SVC_USERD_GP_PUT,
+				ch->userd_as, &put, sizeof(put))) {
+			svc.gpfifo_nomap++;
+			continue;
+		}
+		(void)svc_read_ap(vgpu, ch->userd_pa + SVC_USERD_GP_GET,
+				  ch->userd_as, &get, sizeof(get));
+
+		if (put >= ch->gpfifo_entries)
+			put %= ch->gpfifo_entries;
+
+		while (ch->last_get != put && guard++ < ch->gpfifo_entries) {
+			u8 entry[SVC_GPFIFO_ENTRY_SIZE];
+			u64 va, idx = ch->last_get;
+			u32 len, e0, e1, g;
+
+			if (svc_read_va(vgpu,
+					ch->gpfifo_va +
+					idx * SVC_GPFIFO_ENTRY_SIZE,
+					entry, sizeof(entry))) {
+				svc.gpfifo_nomap++;
+				break;
+			}
+			memcpy(&e0, entry, 4);
+			memcpy(&e1, entry + 4, 4);
+			/* GP_ENTRY0.GET[31:2] | GP_ENTRY1.GET_HI[7:0] */
+			va = ((u64)(e1 & 0xff) << 32) |
+			     (u64)(e0 & 0xfffffffc);
+			/* GP_ENTRY1.LENGTH[30:10], in dwords */
+			len = ((e1 >> 10) & 0x1fffff) << 2;
+
+			if (va && len && len <= SVC_PUSH_MAX) {
+				u8 *pb = kmalloc(len, GFP_KERNEL);
+
+				if (pb) {
+					if (!svc_read_va(vgpu, va, pb, len)) {
+						svc_decode_stream(vgpu, pb,
+								  len, 0);
+						ch->submissions++;
+						svc.gpfifo_entries_done++;
+					} else {
+						svc.gpfifo_nomap++;
+					}
+					kfree(pb);
+				}
+			}
+
+			ch->last_get = (idx + 1) % ch->gpfifo_entries;
+
+			/* Publish progress the way the GPU would. */
+			g = ch->last_get;
+			svc_write_ap(vgpu, ch->userd_pa + SVC_USERD_GP_GET,
+				     ch->userd_as, &g, sizeof(g));
+		}
+
+		spin_lock_irqsave(&svc.chan_lock, flags);
+		for (k = 0; k < SVC_MAX_CHANNELS; k++) {
+			if (svc.chan[k].valid &&
+			    svc.chan[k].h_object == ch->h_object) {
+				svc.chan[k].last_get = ch->last_get;
+				svc.chan[k].submissions = ch->submissions;
+				break;
+			}
+		}
+		spin_unlock_irqrestore(&svc.chan_lock, flags);
+	}
+}
+
 /* ------------------------------------------------------------------ */
 /* thread                                                             */
 /* ------------------------------------------------------------------ */
@@ -985,10 +1454,32 @@ static int svc_thread(void *arg)
 		    time_after(jiffies, svc.last_build + 10 * HZ))
 			svc_rebuild(vgpu);
 
+		/*
+		 * Channel-driven execution: notice submissions as they are made
+		 * and run the pushed work immediately, so the guest's UVM
+		 * progress waits complete in milliseconds rather than in the
+		 * time it takes the fallback scan to find the push.
+		 */
+		svc_pump_channels(vgpu);
+
+		/*
+		 * Fast path for the pushes themselves: every mapped page is a
+		 * candidate pushbuffer page, and there are far fewer of those
+		 * than there are pages of guest RAM.
+		 */
 		if (svc.direct_base && svc.map_n)
-			svc_apply_pushes(vgpu);
+			svc_apply_mapped(vgpu);
+
+		/*
+		 * Rolling RAM scan: the pushbuffers of some clients are not
+		 * covered by the VA map, so sweep guest RAM in bounded windows
+		 * in parallel with the targeted pass. Work per pass is fixed,
+		 * which keeps the emulation responsive while a full sweep
+		 * completes every few seconds.
+		 */
+		svc_apply_pushes(vgpu);
 
-		msleep(100);
+		msleep(10);
 	}
 
 	pr_info("sema-svc: thread stopping (pairs=%llu releases_seen=%llu applied=%llu map=%u)\n",
@@ -1001,6 +1492,7 @@ int gsp_sema_service_start(void)
 	if (svc.task)
 		return 0;
 	svc.stop = false;
+	spin_lock_init(&svc.chan_lock);
 	svc.task = kthread_run(svc_thread, NULL, "gsp-sema-svc");
 	if (IS_ERR(svc.task)) {
 		int ret = PTR_ERR(svc.task);
diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h
index f6ec3fc..16a045f 100644
--- a/kernel/gsp_sema_service.h
+++ b/kernel/gsp_sema_service.h
@@ -10,6 +10,29 @@
  * See gsp_sema_service.c for the full description.
  */
 
+/*
+ * A guest channel, registered when the driver allocates it. The service uses
+ * the channel's GPFIFO + USERD to notice submissions as they happen (instead
+ * of discovering the pushed work by scanning guest RAM) and to run the pushed
+ * work promptly.
+ *
+ * All addresses are guest-physical except gpfifo_va, which is the GPU VA of
+ * the channel's GPFIFO ring (translated through the discovered VA->PA map).
+ */
+struct sema_channel_desc {
+	u32 h_object;
+	u32 engine_type;
+	u64 gpfifo_va;
+	u32 gpfifo_entries;
+	u64 userd_pa;		/* NvA06FControl: GPGet @0x88, GPPut @0x8c */
+	u32 userd_as;		/* NV_ADDR_SYSMEM(1) / NV_ADDR_FBMEM(2) */
+	u64 instance_pa;
+	u64 notifier_pa;
+};
+
+void gsp_sema_service_channel_add(const struct sema_channel_desc *desc);
+void gsp_sema_service_channel_del(u32 h_object);
+
 int  gsp_sema_service_start(void);
 void gsp_sema_service_stop(void);
 int  gsp_sema_service_probe(u64 gpu_va, u32 count);
diff --git a/kernel/main.c b/kernel/main.c
index 2ec7ab9..84a05ca 100644
--- a/kernel/main.c
+++ b/kernel/main.c
@@ -2628,6 +2628,44 @@ module_param_cb(dump_vidmem, &nv_vgpu_dump_vidmem_ops, NULL, 0200);
 MODULE_PARM_DESC(dump_vidmem, "Dump emulated vidmem range (hex offset,hex len)");
 
 /*
+ * Debug: write a u32 into the emulated vidmem (BAR1 backing) at an offset:
+ *   echo 0x<offset>,0x<value> > .../parameters/poke_vidmem
+ */
+static int nv_vgpu_poke_vidmem_set(const char *val, const struct kernel_param *kp)
+{
+	struct nv_vgpu_device *vgpu;
+	u64 off = 0, value = 0;
+	char *end;
+	int ret = 0;
+
+	if (sscanf(val, "%lli,%lli", &off, &value) != 2)
+		return -EINVAL;
+
+	mutex_lock(&nv_vgpu_driver.lock);
+	list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) {
+		if (atomic_read(&vgpu->vfio_dev_open_count) <= 0)
+			continue;
+		if (!vgpu->bars[1].vaddr || off + 4 > vgpu->bars[1].size) {
+			ret = -EINVAL;
+			break;
+		}
+		memcpy(vgpu->bars[1].vaddr + off, &value, 4);
+		pr_info("poke_vidmem: [0x%llx] = 0x%llx\n", off, value);
+		break;
+	}
+	mutex_unlock(&nv_vgpu_driver.lock);
+	(void)end;
+	return ret;
+}
+
+static const struct kernel_param_ops nv_vgpu_poke_vidmem_ops = {
+	.set = nv_vgpu_poke_vidmem_set,
+};
+
+module_param_cb(poke_vidmem, &nv_vgpu_poke_vidmem_ops, NULL, 0200);
+MODULE_PARM_DESC(poke_vidmem, "Write u32 to emulated vidmem (hex offset,hex value)");
+
+/*
  * Debug: hex-dump a guest physical range (kthread: guest-DMA needs it):
  *   echo 0x<gpa>,0x<len> > .../parameters/dump_guest
  */