vgsp-sema: author the PMA scrubber channel's progress (third-client scrub freeze)

The guest PMA scrubber (in-guest CeUtils CE channel, 4096 GPFIFO entries)
waits on its channel buffer's finish payload (offset 0x6c004) after the
third CUDA session's pages must be scrubbed; nothing in the emulation
authored it, so cudaMalloc froze in scrubCheckAndWaitForSize.

- channel_add: detect the 4096-entry scrubber channel and stash its USERD
  PA/ring geometry (the channel itself is left untouched: no rank change,
  no pump change - the colliding numeric-handle FREE may still unregister
  it, the geometry survives).
- sema_scrubbuf_add debug param: register the finish/host-semaphore word
  PAs the guest scanner resolves (vmallocinfo + kcore walk), because the
  RM kernel VA space collides numerically with the client CE window in the
  global VA map and cannot be resolved through it.
- svc_scrub_pass (service thread, per pass): from the channel's own USERD
  GPPut, track submitted scrub blocks across wraps, then author USERD
  GP_GET = put, host semaphore = put and finish payload = submitted.
  Forward-only, idempotent, zero writes while the scrubber idles.

Live-validated mechanism (r137 VM): authoring the finish word at the kcore-
resolved PA let the frozen victory_matmul leave scrubCheckAndWaitForSize
and proceed to its launch.

4 files changed, 243 insertions, 0 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 20dfc54..10992c3 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -454,6 +454,45 @@ static u8 svc_sess_flush;
 static u8 svc_sess_active;	/* a channel of a session has registered (until flush) */
 static struct completion svc_sess_flush_done;	/* a pending flush has executed */
 static unsigned long svc_ring_done;	/* CE ranks whose ring is pinned (session state) */
+
+/*
+ * CeUtils scrubber channel (the guest PMA's internal CE channel created at
+ * driver init). When the PMA must hand out scrubbed pages it waits on the
+ * channel's finish payload (channel buffer offset finishPayloadOffset =
+ * channelPbSize + GPFIFO_SIZE + 4; for the regular scrubber = 0x6c004),
+ * which the physical CE would advance as the scrub work completes. In this
+ * emulation nothing authors it, so the third+ CUDA session's cudaMalloc
+ * freezes in scrubCheckAndWaitForSize.
+ *
+ * The scrubber channel is identified by its GPFIFO entry count
+ * (NUM_COPY_BLOCKS = 4096; client channels use 1024); its USERD PA is known
+ * from the channel ALLOC registration. The channel buffer's PA cannot be
+ * resolved through the global VA map (the kernel VA space collides
+ * numerically with the client's CE push-buffer window), so the guest
+ * scanner (vmallocinfo + kcore page-table walk) registers it through
+ * sema_scrubbuf_add. The service then authors, from the channel's own
+ * GP_PUT: USERD GP_GET = put, host semaphore = put (so the guest's
+ * channelWaitForFreeEntry never blocks), and finish payload = submitted
+ * block count (so the scrub wait completes). Idempotent, forward-only.
+ */
+struct svc_scrub {
+	bool	userd_ok;	/* a 4096-entry channel was registered */
+	u32	h_object;	/* handle of the registered scrubber channel */
+	u64	userd_pa;
+	u32	userd_as;
+	u64	gpfifo_va;
+	u32	entries;
+	bool	pa_ok;		/* scanner registered fin/hsem PAs */
+	u64	fin_pa;		/* finish payload word (u32) */
+	u64	hsem_pa;	/* channel host semaphore word (u32) */
+	bool	seeded;
+	u32	put_last;
+	u64	submitted;	/* blocks submitted since seeding */
+	u64	authored;	/* last authored finish value */
+	u32	pa_gen;		/* scanner registration generation (logged) */
+	u32	log_budget;
+};
+static struct svc_scrub svc_scrub;
 static u8 svc_pb_passes;
 static u64 svc_pb_last;
 /*
@@ -1110,6 +1149,37 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc)
 		 */
 		svc_chan_rearm_rank(ch);
 		svc.last_chan_reg = jiffies;
+		/*
+		 * CeUtils scrubber channel: NUM_COPY_BLOCKS = 4096 GPFIFO
+		 * entries (client channels use 1024). Stash its USERD and ring
+		 * geometry for the scrub pass; the channel itself is left
+		 * entirely alone (no rank change, no pump touch changes) - the
+		 * pass authors the finish payload from the USERD alone. A
+		 * colliding numeric-handle FREE may still unregister the
+		 * channel; the stashed geometry survives, so the scrub path
+		 * does not depend on the channel-table identity at all.
+		 */
+		if (desc->gpfifo_entries >= 4096 && desc->gpfifo_va &&
+		    desc->userd_pa) {
+			/* Re-seed the counting on a fresh scrubber object. */
+			if (!svc_scrub.userd_ok ||
+			    svc_scrub.h_object != desc->h_object ||
+			    svc_scrub.gpfifo_va != desc->gpfifo_va) {
+				svc_scrub.seeded = false;
+				svc_scrub.submitted = 0;
+			}
+			svc_scrub.userd_ok = true;
+			svc_scrub.h_object = desc->h_object;
+			svc_scrub.userd_pa = desc->userd_pa;
+			svc_scrub.userd_as = desc->userd_as ? desc->userd_as
+							    : SVC_APERTURE_SYS;
+			svc_scrub.gpfifo_va = desc->gpfifo_va;
+			svc_scrub.entries = desc->gpfifo_entries;
+			pr_info("sema-svc: scrubber channel 0x%x detected (gpfifo=0x%llx entries=%u userd=0x%llx as=%u)\n",
+				desc->h_object, desc->gpfifo_va,
+				desc->gpfifo_entries, desc->userd_pa,
+				svc_scrub.userd_as);
+		}
 		pr_info("sema-svc: channel 0x%x registered gpfifo=0x%llx entries=%u userd=0x%llx eng=%u cid=%u notif=0x%llx\n",
 			desc->h_object, desc->gpfifo_va, desc->gpfifo_entries,
 			desc->userd_pa, desc->engine_type, desc->cid,
@@ -15555,6 +15625,115 @@ int gsp_sema_service_counter_add(u32 h_object, u64 pa)
 	return ret;
 }
 
+/*
+ * Register the guest PMA scrubber's channel-buffer words. Called from the
+ * sema_scrubbuf_add debug parameter with the PAs the guest scanner resolved
+ * (vmallocinfo vmap area + kcore page-table walk). fin_pa is the finish
+ * payload the guest's scrub wait polls; hsem_pa is the channel's host
+ * semaphore (channelWaitForFreeEntry polls it). Plain state update.
+ */
+int gsp_sema_service_scrubbuf_add(u64 fin_pa, u64 hsem_pa)
+{
+	if (!svc.task)
+		return -ENODEV;
+	/* Both words are u32: require 4-byte alignment and a RAM-shaped PA. */
+	if (!fin_pa || (fin_pa & 0x3) || fin_pa >= SEMA_PA_MAX)
+		return -EINVAL;
+	if (!hsem_pa || (hsem_pa & 0x3) || hsem_pa >= SEMA_PA_MAX)
+		return -EINVAL;
+
+	svc_scrub.fin_pa = fin_pa;
+	svc_scrub.hsem_pa = hsem_pa;
+	svc_scrub.pa_ok = true;
+	svc_scrub.pa_gen++;
+	pr_info("sema-svc: scrubber buffer registered fin_pa=0x%llx hsem_pa=0x%llx (gen %u)\n",
+		fin_pa, hsem_pa, svc_scrub.pa_gen);
+	return 0;
+}
+
+/*
+ * Author the CeUtils scrubber channel's progress once per service pass.
+ *
+ * The physical CE would advance, per completed scrub block, the channel's
+ * finish payload (channel buffer + 0x6c004) and write the channel's host
+ * semaphore as it consumes GPFIFO entries. Nothing in this emulation does,
+ * so the guest PMA's scrub wait (scrubCheckAndWaitForSize) freezes from the
+ * third CUDA session on. This pass closes that loop from the channel's own
+ * USERD GPPut (which the guest driver writes before every submit):
+ *
+ *   submitted = number of scrub blocks submitted (GPPut is end-exclusive
+ *   and counts submissions until the ring wraps; track the wrap ourselves),
+ *   then author USERD GP_GET = put (consumed), host sema = put and finish
+ *   payload = submitted. Forward-only, idempotent, retried every pass so a
+ *   late scanner registration (or a post-flush reset) heals by itself.
+ *
+ * The scrub work itself is not executed (no shadow memsets): the guest
+ * only needs the progress to proceed. Executing the memsets requires
+ * decoding the channel's pushes, which the scanner can later provide pins
+ * for; correctness impact is limited to stale data in pages the guest
+ * believes scrubbed.
+ */
+static void svc_scrub_pass(struct nv_vgpu_device *vgpu)
+{
+	struct svc_scrub *s = &svc_scrub;
+	u32 put = 0;
+	u64 delta;
+
+	if (!s->userd_ok || !s->pa_ok || !s->entries)
+		return;
+	if (svc_read_ap(vgpu, s->userd_pa + SVC_USERD_GP_PUT, s->userd_as,
+			&put, sizeof(put))) {
+		/* Retry through the alternate aperture (as registration did). */
+		if (svc_read_ap(vgpu, s->userd_pa + SVC_USERD_GP_PUT,
+				s->userd_as == SVC_APERTURE_FB ?
+				SVC_APERTURE_SYS : SVC_APERTURE_FB,
+				&put, sizeof(put)))
+			return;
+	}
+	if (put >= s->entries)
+		return;		/* junk translation; retry next pass */
+
+	if (!s->seeded) {
+		s->seeded = true;
+		s->put_last = put;
+		/* GPPut = (lastSubmittedEntry + 1) % entries: the submission
+		 * count until the ring wraps. Seeding from it adopts any
+		 * pre-host scrub work (the boot session's scrubbers). */
+		s->submitted = put;
+	} else if (put != s->put_last) {
+		delta = (put + s->entries - s->put_last) % s->entries;
+		if (!delta)
+			delta = s->entries;	/* exact wrap */
+		s->submitted += delta;
+		s->put_last = put;
+	} else if (s->submitted == s->authored) {
+		return;		/* nothing new; no guest writes */
+	}
+
+	if (s->log_budget < 64) {
+		s->log_budget++;
+		pr_info("sema-svc: scrub author put=%u submitted=%llu fin_pa=0x%llx\n",
+			put, s->submitted, s->fin_pa);
+	}
+
+	/*
+	 * Author (forward-only). The USERD GP_GET and host-sema writes mark
+	 * the submissions consumed; the finish write releases the scrub
+	 * wait. Written only when GP_PUT advanced (or on the seeding pass),
+	 * so the per-pass cost is zero while the scrubber idles.
+	 */
+	svc_write_ap(vgpu, s->userd_pa + SVC_USERD_GP_GET, s->userd_as,
+		     &put, sizeof(put));
+	svc_write_ap(vgpu, s->hsem_pa, SVC_APERTURE_SYS, &put, sizeof(put));
+	{
+		u32 fin = (u32)s->submitted;
+
+		svc_write_ap(vgpu, s->fin_pa, SVC_APERTURE_SYS, &fin,
+			     sizeof(fin));
+	}
+	s->authored = s->submitted;
+}
+
 static int svc_thread(void *arg)
 {
 	struct nv_vgpu_device *vgpu;
@@ -15600,6 +15779,7 @@ static int svc_thread(void *arg)
 		svc_handle_walk_probe(vgpu);
 		svc_handle_map_add();
 		svc_handle_chan_dump(vgpu);
+		svc_scrub_pass(vgpu);
 
 		if (svc_sess_flush || !svc.direct_base ||
 		    time_after(jiffies,
diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h
index 4b04892..6822bd5 100644
--- a/kernel/gsp_sema_service.h
+++ b/kernel/gsp_sema_service.h
@@ -36,6 +36,16 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc);
 void gsp_sema_service_channel_del(u32 h_object);
 
 /*
+ * Register the guest PMA scrubber's channel-buffer words (the CeUtils
+ * finish payload the guest's scrub wait polls, and the channel's host
+ * semaphore). The PAs come from the guest scanner (vmallocinfo + kcore):
+ * the guest RM kernel channel lives in a GPU VA space that collides
+ * numerically with the client's CE window in the global translation map,
+ * so it cannot be resolved through that map.
+ */
+int gsp_sema_service_scrubbuf_add(u64 fin_pa, u64 hsem_pa);
+
+/*
  * A GPU VA space whose top-level page directory the driver has published (the
  * driver builds its own page tables for the address spaces it owns, then hands
  * the directory's physical address to the GPU through SET_PAGE_DIRECTORY).
diff --git a/kernel/main.c b/kernel/main.c
index ac970f1..d0308c4 100644
--- a/kernel/main.c
+++ b/kernel/main.c
@@ -2544,6 +2544,37 @@ module_param_cb(sema_probe, &nv_vgpu_sema_probe_ops, NULL, 0200);
 MODULE_PARM_DESC(sema_probe, "Dump sema-service map + slot values for a GPU VA");
 
 /*
+ * Register the guest PMA scrubber channel buffer words (finish payload PA,
+ * host-semaphore PA) the guest scanner resolved:
+ *   echo 0x11a8ff004,0x11a8ff000 > .../parameters/sema_scrubbuf_add
+ */
+static int nv_vgpu_sema_scrubbuf_set(const char *val,
+				     const struct kernel_param *kp)
+{
+	char buf[64];
+	char *comma;
+	u64 fin_pa = 0, hsem_pa = 0;
+
+	strscpy(buf, val, sizeof(buf));
+	comma = strchr(buf, ',');
+	if (!comma)
+		return -EINVAL;
+	*comma = '\0';
+	if (kstrtoull(buf, 0, &fin_pa) || kstrtoull(comma + 1, 0, &hsem_pa))
+		return -EINVAL;
+
+	return gsp_sema_service_scrubbuf_add(fin_pa, hsem_pa);
+}
+
+static const struct kernel_param_ops nv_vgpu_sema_scrubbuf_ops = {
+	.set = nv_vgpu_sema_scrubbuf_set,
+};
+
+module_param_cb(sema_scrubbuf_add, &nv_vgpu_sema_scrubbuf_ops, NULL, 0200);
+MODULE_PARM_DESC(sema_scrubbuf_add,
+		 "Register the PMA scrubber finish/host-sema PAs (hex fin,hex hsem)");
+
+/*
  * Debug: run the page-table walks for a GPU VA with per-level tracing,
  * bypassing the pin/lat/pair map (see gsp_sema_service_walk_probe):
  *   echo 0x20400f000 > .../parameters/walk_probe
diff --git a/tools/cuda-ab/push_battery.sh b/tools/cuda-ab/push_battery.sh
new file mode 100755
index 0000000..644beba
--- /dev/null
+++ b/tools/cuda-ab/push_battery.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+# push_battery.sh - push every CUDA battery binary from tools/cuda-ab/run/
+# to the running guest via scp over the SSH hostfwd (port 2222).
+# Usage: push_battery.sh [guest_dir]  (default /tmp)
+set -u
+HERE="$(cd "$(dirname "$0")" && pwd)"
+RUN="$HERE/run"
+GUEST_DIR="${1:-/tmp}"
+CLIENTS="victory_mail smiley vecadd_loop streams2 memcpy_roundtrip multikernel victory_saxpy victory_reduce victory_matmul victory_strided victory_memset2d victory_atomics victory_streams_events victory_math vector_add"
+export SSHPASS="${VGPU_GUEST_PASS:-ubuntu}"
+FILES=""
+for c in $CLIENTS; do [ -x "$RUN/$c" ] && FILES="$FILES $RUN/$c"; done
+[ -n "$FILES" ] || { echo "no binaries in $RUN"; exit 1; }
+# shellcheck disable=SC2086
+sshpass -e scp -P "${VGPU_GUEST_PORT:-2222}" \
+  -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+  -o PubkeyAuthentication=no -o PreferredAuthentications=password \
+  -o ConnectTimeout=15 $FILES "ubuntu@127.0.0.1:$GUEST_DIR/" && \
+sshpass -e ssh -p "${VGPU_GUEST_PORT:-2222}" \
+  -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+  -o PubkeyAuthentication=no -o PreferredAuthentications=password \
+  "ubuntu@127.0.0.1" "chmod +x $GUEST_DIR/* 2>/dev/null; ls $GUEST_DIR | wc -l; echo BATTERY_PUSHED"