vgsp-sema: author the PMA scrubber channel's progress (third-client scrub freeze)

The guest PMA scrubber (in-guest CeUtils CE channel, 4096 GPFIFO entries)
waits on its channel buffer's finish payload (offset 0x6c004) after the
third CUDA session's pages must be scrubbed; nothing in the emulation
authored it, so cudaMalloc froze in scrubCheckAndWaitForSize.

- channel_add: detect the 4096-entry scrubber channel and stash its USERD
  PA/ring geometry (the channel itself is left untouched: no rank change,
  no pump change - the colliding numeric-handle FREE may still unregister
  it, the geometry survives).
- sema_scrubbuf_add debug param: register the finish/host-semaphore word
  PAs the guest scanner resolves (vmallocinfo + kcore walk), because the
  RM kernel VA space collides numerically with the client CE window in the
  global VA map and cannot be resolved through it.
- svc_scrub_pass (service thread, per pass): from the channel's own USERD
  GPPut, track submitted scrub blocks across wraps, then author USERD
  GP_GET = put, host semaphore = put and finish payload = submitted.
  Forward-only, idempotent, zero writes while the scrubber idles.

Live-validated mechanism (r137 VM): authoring the finish word at the kcore-
resolved PA let the frozen victory_matmul leave scrubCheckAndWaitForSize
and proceed to its launch.

1 files changed, 180 insertions, 0 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 20dfc54..10992c3 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -454,6 +454,45 @@ static u8 svc_sess_flush;
 static u8 svc_sess_active;	/* a channel of a session has registered (until flush) */
 static struct completion svc_sess_flush_done;	/* a pending flush has executed */
 static unsigned long svc_ring_done;	/* CE ranks whose ring is pinned (session state) */
+
+/*
+ * CeUtils scrubber channel (the guest PMA's internal CE channel created at
+ * driver init). When the PMA must hand out scrubbed pages it waits on the
+ * channel's finish payload (channel buffer offset finishPayloadOffset =
+ * channelPbSize + GPFIFO_SIZE + 4; for the regular scrubber = 0x6c004),
+ * which the physical CE would advance as the scrub work completes. In this
+ * emulation nothing authors it, so the third+ CUDA session's cudaMalloc
+ * freezes in scrubCheckAndWaitForSize.
+ *
+ * The scrubber channel is identified by its GPFIFO entry count
+ * (NUM_COPY_BLOCKS = 4096; client channels use 1024); its USERD PA is known
+ * from the channel ALLOC registration. The channel buffer's PA cannot be
+ * resolved through the global VA map (the kernel VA space collides
+ * numerically with the client's CE push-buffer window), so the guest
+ * scanner (vmallocinfo + kcore page-table walk) registers it through
+ * sema_scrubbuf_add. The service then authors, from the channel's own
+ * GP_PUT: USERD GP_GET = put, host semaphore = put (so the guest's
+ * channelWaitForFreeEntry never blocks), and finish payload = submitted
+ * block count (so the scrub wait completes). Idempotent, forward-only.
+ */
+struct svc_scrub {
+	bool	userd_ok;	/* a 4096-entry channel was registered */
+	u32	h_object;	/* handle of the registered scrubber channel */
+	u64	userd_pa;
+	u32	userd_as;
+	u64	gpfifo_va;
+	u32	entries;
+	bool	pa_ok;		/* scanner registered fin/hsem PAs */
+	u64	fin_pa;		/* finish payload word (u32) */
+	u64	hsem_pa;	/* channel host semaphore word (u32) */
+	bool	seeded;
+	u32	put_last;
+	u64	submitted;	/* blocks submitted since seeding */
+	u64	authored;	/* last authored finish value */
+	u32	pa_gen;		/* scanner registration generation (logged) */
+	u32	log_budget;
+};
+static struct svc_scrub svc_scrub;
 static u8 svc_pb_passes;
 static u64 svc_pb_last;
 /*
@@ -1110,6 +1149,37 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc)
 		 */
 		svc_chan_rearm_rank(ch);
 		svc.last_chan_reg = jiffies;
+		/*
+		 * CeUtils scrubber channel: NUM_COPY_BLOCKS = 4096 GPFIFO
+		 * entries (client channels use 1024). Stash its USERD and ring
+		 * geometry for the scrub pass; the channel itself is left
+		 * entirely alone (no rank change, no pump touch changes) - the
+		 * pass authors the finish payload from the USERD alone. A
+		 * colliding numeric-handle FREE may still unregister the
+		 * channel; the stashed geometry survives, so the scrub path
+		 * does not depend on the channel-table identity at all.
+		 */
+		if (desc->gpfifo_entries >= 4096 && desc->gpfifo_va &&
+		    desc->userd_pa) {
+			/* Re-seed the counting on a fresh scrubber object. */
+			if (!svc_scrub.userd_ok ||
+			    svc_scrub.h_object != desc->h_object ||
+			    svc_scrub.gpfifo_va != desc->gpfifo_va) {
+				svc_scrub.seeded = false;
+				svc_scrub.submitted = 0;
+			}
+			svc_scrub.userd_ok = true;
+			svc_scrub.h_object = desc->h_object;
+			svc_scrub.userd_pa = desc->userd_pa;
+			svc_scrub.userd_as = desc->userd_as ? desc->userd_as
+							    : SVC_APERTURE_SYS;
+			svc_scrub.gpfifo_va = desc->gpfifo_va;
+			svc_scrub.entries = desc->gpfifo_entries;
+			pr_info("sema-svc: scrubber channel 0x%x detected (gpfifo=0x%llx entries=%u userd=0x%llx as=%u)\n",
+				desc->h_object, desc->gpfifo_va,
+				desc->gpfifo_entries, desc->userd_pa,
+				svc_scrub.userd_as);
+		}
 		pr_info("sema-svc: channel 0x%x registered gpfifo=0x%llx entries=%u userd=0x%llx eng=%u cid=%u notif=0x%llx\n",
 			desc->h_object, desc->gpfifo_va, desc->gpfifo_entries,
 			desc->userd_pa, desc->engine_type, desc->cid,
@@ -15555,6 +15625,115 @@ int gsp_sema_service_counter_add(u32 h_object, u64 pa)
 	return ret;
 }
 
+/*
+ * Register the guest PMA scrubber's channel-buffer words. Called from the
+ * sema_scrubbuf_add debug parameter with the PAs the guest scanner resolved
+ * (vmallocinfo vmap area + kcore page-table walk). fin_pa is the finish
+ * payload the guest's scrub wait polls; hsem_pa is the channel's host
+ * semaphore (channelWaitForFreeEntry polls it). Plain state update.
+ */
+int gsp_sema_service_scrubbuf_add(u64 fin_pa, u64 hsem_pa)
+{
+	if (!svc.task)
+		return -ENODEV;
+	/* Both words are u32: require 4-byte alignment and a RAM-shaped PA. */
+	if (!fin_pa || (fin_pa & 0x3) || fin_pa >= SEMA_PA_MAX)
+		return -EINVAL;
+	if (!hsem_pa || (hsem_pa & 0x3) || hsem_pa >= SEMA_PA_MAX)
+		return -EINVAL;
+
+	svc_scrub.fin_pa = fin_pa;
+	svc_scrub.hsem_pa = hsem_pa;
+	svc_scrub.pa_ok = true;
+	svc_scrub.pa_gen++;
+	pr_info("sema-svc: scrubber buffer registered fin_pa=0x%llx hsem_pa=0x%llx (gen %u)\n",
+		fin_pa, hsem_pa, svc_scrub.pa_gen);
+	return 0;
+}
+
+/*
+ * Author the CeUtils scrubber channel's progress once per service pass.
+ *
+ * The physical CE would advance, per completed scrub block, the channel's
+ * finish payload (channel buffer + 0x6c004) and write the channel's host
+ * semaphore as it consumes GPFIFO entries. Nothing in this emulation does,
+ * so the guest PMA's scrub wait (scrubCheckAndWaitForSize) freezes from the
+ * third CUDA session on. This pass closes that loop from the channel's own
+ * USERD GPPut (which the guest driver writes before every submit):
+ *
+ *   submitted = number of scrub blocks submitted (GPPut is end-exclusive
+ *   and counts submissions until the ring wraps; track the wrap ourselves),
+ *   then author USERD GP_GET = put (consumed), host sema = put and finish
+ *   payload = submitted. Forward-only, idempotent, retried every pass so a
+ *   late scanner registration (or a post-flush reset) heals by itself.
+ *
+ * The scrub work itself is not executed (no shadow memsets): the guest
+ * only needs the progress to proceed. Executing the memsets requires
+ * decoding the channel's pushes, which the scanner can later provide pins
+ * for; correctness impact is limited to stale data in pages the guest
+ * believes scrubbed.
+ */
+static void svc_scrub_pass(struct nv_vgpu_device *vgpu)
+{
+	struct svc_scrub *s = &svc_scrub;
+	u32 put = 0;
+	u64 delta;
+
+	if (!s->userd_ok || !s->pa_ok || !s->entries)
+		return;
+	if (svc_read_ap(vgpu, s->userd_pa + SVC_USERD_GP_PUT, s->userd_as,
+			&put, sizeof(put))) {
+		/* Retry through the alternate aperture (as registration did). */
+		if (svc_read_ap(vgpu, s->userd_pa + SVC_USERD_GP_PUT,
+				s->userd_as == SVC_APERTURE_FB ?
+				SVC_APERTURE_SYS : SVC_APERTURE_FB,
+				&put, sizeof(put)))
+			return;
+	}
+	if (put >= s->entries)
+		return;		/* junk translation; retry next pass */
+
+	if (!s->seeded) {
+		s->seeded = true;
+		s->put_last = put;
+		/* GPPut = (lastSubmittedEntry + 1) % entries: the submission
+		 * count until the ring wraps. Seeding from it adopts any
+		 * pre-host scrub work (the boot session's scrubbers). */
+		s->submitted = put;
+	} else if (put != s->put_last) {
+		delta = (put + s->entries - s->put_last) % s->entries;
+		if (!delta)
+			delta = s->entries;	/* exact wrap */
+		s->submitted += delta;
+		s->put_last = put;
+	} else if (s->submitted == s->authored) {
+		return;		/* nothing new; no guest writes */
+	}
+
+	if (s->log_budget < 64) {
+		s->log_budget++;
+		pr_info("sema-svc: scrub author put=%u submitted=%llu fin_pa=0x%llx\n",
+			put, s->submitted, s->fin_pa);
+	}
+
+	/*
+	 * Author (forward-only). The USERD GP_GET and host-sema writes mark
+	 * the submissions consumed; the finish write releases the scrub
+	 * wait. Written only when GP_PUT advanced (or on the seeding pass),
+	 * so the per-pass cost is zero while the scrubber idles.
+	 */
+	svc_write_ap(vgpu, s->userd_pa + SVC_USERD_GP_GET, s->userd_as,
+		     &put, sizeof(put));
+	svc_write_ap(vgpu, s->hsem_pa, SVC_APERTURE_SYS, &put, sizeof(put));
+	{
+		u32 fin = (u32)s->submitted;
+
+		svc_write_ap(vgpu, s->fin_pa, SVC_APERTURE_SYS, &fin,
+			     sizeof(fin));
+	}
+	s->authored = s->submitted;
+}
+
 static int svc_thread(void *arg)
 {
 	struct nv_vgpu_device *vgpu;
@@ -15600,6 +15779,7 @@ static int svc_thread(void *arg)
 		svc_handle_walk_probe(vgpu);
 		svc_handle_map_add();
 		svc_handle_chan_dump(vgpu);
+		svc_scrub_pass(vgpu);
 
 		if (svc_sess_flush || !svc.direct_base ||
 		    time_after(jiffies,