kernel: semaphore service forward-jump completion, stats, sema_probe

Relax the in-order rule to forward progress (payload > current), with the
jump bounded and counted: UVM tolerates forward jumps up to 2M
(2 * UVM_CHANNEL_NUM_GPFIFO_ENTRIES_MAX), and push buffers are recycled so
intermediate pushes can be missed. Add miss/stale/jump counters to the
rebuild log and a sema_probe module parameter to dump the service's map
entry and live slot values for a GPU VA (diagnostics for tracker stalls).

1 files changed, 136 insertions, 14 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index ad6c6cb..68a890c 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -48,15 +48,23 @@
 
 /* Plausible GPU VA window for the client's system-memory objects (Turing). */
 #define SEMA_VA_LO		0x100000000ULL
-#define SEMA_VA_HI		0x160000000ULL
+#define SEMA_VA_HI		0x2000000000ULL
 
 /* Canonical kernel virtual address range (x86_64). */
 #define KVA_LO			0xffff800000000000ULL
 #define KVA_HI			0xffffc00000000000ULL
 
-/* INC_METHOD header for the CE semaphore group: addr>>2 == 0x240, count == 3 */
+/*
+ * INC_METHOD header for the CE semaphore group. Observed value 0x20038090:
+ *   bits [31:28] opcode/const, [23:16] count = 3, lower bits encode
+ *   (address >> 2) == 0x240 plus subchannel flags (0x8000 in the observed
+ *   header). Match on count == 3 and (address >> 2) low byte == 0x90.
+ */
 #define CE_SEMA_HDR_ADDR	0x90
 #define CE_SEMA_HDR_COUNT	3
+/* LAUNCH_DMA group that follows: observed 0x200180c0 (count 1, low byte 0xc0) */
+#define CE_DMA_HDR_ADDR		0xc0
+#define CE_DMA_HDR_COUNT	1
 
 struct sema_map_entry {
 	u64 gpu_page;	/* GPU VA, page aligned */
@@ -76,6 +84,9 @@ struct sema_svc {
 	u64			pairs_seen;
 	u64			releases_seen;
 	u64			releases_applied;
+	u64			releases_missed;	/* no VA->PA entry */
+	u64			releases_stale;		/* payload <= current */
+	u64			releases_jumped;	/* applied with gap > 1 */
 	u64			builds;
 };
 
@@ -249,9 +260,10 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu)
 	svc.builds++;
 	svc.last_build = jiffies;
 
-	pr_info("sema-svc: rebuild #%llu base=0x%llx (count=%u) pairs=%u map=%u seen=%llu applied=%llu\n",
+	pr_info("sema-svc: rebuild #%llu base=0x%llx (count=%u) pairs=%u map=%u seen=%llu applied=%llu miss=%llu stale=%llu jump=%llu\n",
 		svc.builds, best_base, best_count, npairs, svc.map_n,
-		svc.releases_seen, svc.releases_applied);
+		svc.releases_seen, svc.releases_applied, svc.releases_missed,
+		svc.releases_stale, svc.releases_jumped);
 	/* show a few entries so mappings can be verified from dmesg */
 	{
 		u32 shown;
@@ -292,15 +304,29 @@ static u32 svc_apply_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page)
 		u32 cur32;
 
 		memcpy(&hdr, buf + off, 4);
-		if ((hdr & 0xffff) != CE_SEMA_HDR_ADDR)
-			continue;
+		/* observed: 0x20038090 -> count 3, addr>>2 low bits 0x90, subch 0x8000 */
 		if (((hdr >> 16) & 0xff) != CE_SEMA_HDR_COUNT)
 			continue;
+		if ((hdr & 0xff) != CE_SEMA_HDR_ADDR)
+			continue;
 
 		memcpy(&hi, buf + off + 4, 4);
 		memcpy(&lo, buf + off + 8, 4);
 		memcpy(&payload, buf + off + 12, 4);
 
+		/* Require the trailing LAUNCH_DMA group: dramatically reduces
+		 * false positives when scanning unrelated memory. */
+		if (off + 20 <= SEMA_PAGE_SIZE) {
+			u32 dma;
+
+			memcpy(&dma, buf + off + 16, 4);
+			if (((dma >> 16) & 0xff) != CE_DMA_HDR_COUNT ||
+			    (dma & 0xff) != CE_DMA_HDR_ADDR)
+				continue;
+		} else {
+			continue;
+		}
+
 		if (hi > 0xffff || payload == 0 || payload > 0x10000000)
 			continue;
 
@@ -310,48 +336,142 @@ static u32 svc_apply_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page)
 
 		svc.releases_seen++;
 
-		if (svc_map_lookup(va, &pa))
+		if (svc_map_lookup(va, &pa)) {
+			svc.releases_missed++;
 			continue;
+		}
 
-		cur = 0;
+		cur32 = 0;
 		if (svc_read(vgpu, pa, &cur32, sizeof(cur32)))
 			continue;
 		cur = cur32;
 
-		/* in-order completion only */
-		if (cur + 1 != payload)
+		/*
+		 * Forward progress only. A payload at or below the slot's
+		 * current value is a stale copy of an already-completed push.
+		 * Any payload ahead of the slot is applied even across a gap:
+		 * UVM tolerates forward jumps up to
+		 * 2 * UVM_CHANNEL_NUM_GPFIFO_ENTRIES_MAX (2M) when polling, and
+		 * a gap only means we missed seeing an intermediate push (push
+		 * buffers are recycled). Bound the jump to keep false
+		 * positives harmless.
+		 */
+		if (payload <= cur || payload - cur > 0x100000) {
+			svc.releases_stale++;
 			continue;
+		}
 
 		if (svc_write(vgpu, pa, &payload, sizeof(payload)) == 0) {
 			svc.releases_applied++;
 			applied++;
-			if (svc.releases_applied <= 16)
+			if (payload != cur + 1) {
+				svc.releases_jumped++;
+				pr_info("sema-svc: applied va=0x%llx pa=0x%llx payload=%u (gap: cur=%u)\n",
+					va, pa, payload, cur32);
+			} else if (svc.releases_applied <= 16) {
 				pr_info("sema-svc: applied va=0x%llx pa=0x%llx payload=%u\n",
 					va, pa, payload);
+			}
 		}
 	}
 	return applied;
 }
 
+/*
+ * Rolling scan: walk guest RAM in bounded windows (round-robin) looking for
+ * semaphore-release groups. Push buffers can live anywhere in the guest's
+ * system memory, including pages we have no VA mapping for; only the
+ * *semaphore* address inside a release needs to resolve through the map.
+ */
+#define SVC_SCAN_WINDOW	(256ULL << 20)	/* 256 MB per pass */
+static u64 svc_scan_cursor;
+
 static void svc_apply_pushes(struct nv_vgpu_device *vgpu)
 {
 	static u8 *buf;
-	u32 i;
+	u64 start = svc_scan_cursor;
+	u64 end = start + SVC_SCAN_WINDOW;
+	u64 gpa;
 
 	if (!buf) {
 		buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL);
 		if (!buf)
 			return;
 	}
+	if (end > 0x140000000ULL)
+		end = 0x140000000ULL;
+
+	for (gpa = start; gpa < end; gpa += SEMA_PAGE_SIZE)
+		svc_apply_page(vgpu, buf, gpa);
 
-	for (i = 0; i < svc.map_n; i++)
-		svc_apply_page(vgpu, buf, svc.map[i].pa_page);
+	svc_scan_cursor = (end >= 0x140000000ULL) ? 0 : end;
 }
 
 /* ------------------------------------------------------------------ */
 /* thread                                                             */
 /* ------------------------------------------------------------------ */
 
+/*
+ * Diagnostic probe: dump the service's view of one GPU VA (map entry and
+ * the current slot values). Runs in the service thread because guest-DMA
+ * reads require a kthread context; the sysfs writer waits for completion.
+ */
+static struct completion probe_done;
+static u64 probe_va;
+static u32 probe_n;
+static bool probe_pending;
+
+static void svc_handle_probe(struct nv_vgpu_device *vgpu)
+{
+	u64 pa = 0;
+	u32 i;
+
+	if (!probe_pending)
+		return;
+
+	pr_info("sema-probe: va=0x%llx page=0x%llx base=0x%llx map_n=%u\n",
+		probe_va, probe_va & ~(u64)(SEMA_PAGE_SIZE - 1),
+		svc.direct_base, svc.map_n);
+
+	if (svc_map_lookup(probe_va, &pa)) {
+		pr_info("sema-probe: NO MAP ENTRY for page 0x%llx\n",
+			probe_va & ~(u64)(SEMA_PAGE_SIZE - 1));
+	} else {
+		pr_info("sema-probe: pa=0x%llx\n", pa);
+		for (i = 0; i < probe_n; i++) {
+			u32 v = 0;
+
+			if (svc_read(vgpu, pa + 4ULL * i, &v, sizeof(v))) {
+				pr_info("sema-probe:   [%u] read failed\n", i);
+				break;
+			}
+			pr_info("sema-probe:   [%u] 0x%08x\n", i, v);
+		}
+	}
+
+	probe_pending = false;
+	complete(&probe_done);
+}
+
+int gsp_sema_service_probe(u64 gpu_va, u32 count)
+{
+	if (!svc.task)
+		return -ENODEV;
+	if (!count || count > 64)
+		count = 8;
+
+	init_completion(&probe_done);
+	probe_va = gpu_va;
+	probe_n = count;
+	probe_pending = true;
+
+	if (!wait_for_completion_timeout(&probe_done, 3 * HZ)) {
+		probe_pending = false;
+		return -ETIMEDOUT;
+	}
+	return 0;
+}
+
 static int svc_thread(void *arg)
 {
 	struct nv_vgpu_device *vgpu;
@@ -378,6 +498,8 @@ static int svc_thread(void *arg)
 			continue;
 		}
 
+		svc_handle_probe(vgpu);
+
 		if (!svc.direct_base ||
 		    time_after(jiffies, svc.last_build + 10 * HZ))
 			svc_rebuild(vgpu);