sem: run GR channels - accept CE methods on subchannel 0, discover GR rings

- The CE semaphore/memset method groups also arrive on subchannel 0 (CUDA's
  push library binds the copy class on compute channels to carry the
  work-submission releases), so the stream decoder accepts both 0 and 4.
- GR (compute) channel rings live in the same video-memory allocation as the
  CE rings, 0x3000 bytes per channel after them: ring (0x2000) at
  ce_end + 0x3000*rank, USERD at +0x2000. Discover and pin them once the CE
  ranks are known; verified against ring entry 0 (1 MB aligned push VA in
  the client window, sane length) and the USERD's put/get, so a layout
  change costs a failed pin, not a wrong one. Gated by sema_gr_rings
  (default on).
- memprobe: require the sem-VA triple in the sem-object scanner (a bare
  qword in the window matches thousands of unrelated pointers once the
  ring/pushbuffer windows are part of the range).

Live evidence: ring entries read through the guest's /proc/pid/mem decode to
the GR channels' first pushes; pinning the ring (fb 0x140000+0x3000*rank),
USERD (ring+0x2000) and first-push pages made all 20 GR channels execute
their entry 0 for the first time (get=1 sub=1).

3 files changed, 117 insertions, 2 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 4cfed0c..7f39209 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -125,6 +125,7 @@ struct svc_channel {
 	u32	engine_type;
 	u32	cid;		/* RM channel id (doorbell token vector) */
 	u8	ce_index;	/* rank among the live CE channels (creation order) */
+	u8	gr_index;	/* rank among the live GR channels (creation order) */
 	u64	gpfifo_va;
 	u32	gpfifo_entries;
 	u64	userd_pa;
@@ -462,6 +463,14 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc)
 				    svc.chan[k].h_object < desc->h_object)
 					rank++;
 			ch->ce_index = rank;
+		} else {
+			u8 rank = 0;
+
+			for (k = 0; k < SVC_MAX_CHANNELS; k++)
+				if (svc.chan[k].valid && !svc.chan[k].engine_type &&
+				    svc.chan[k].h_object < desc->h_object)
+					rank++;
+			ch->gr_index = rank;
 		}
 		ch->valid = true;
 		ch->h_object = desc->h_object;
@@ -1893,7 +1902,14 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 			continue;
 		}
 
-		if (subch != 4) {
+		/*
+		 * The CE semaphore/memset methods arrive on the copy subchannel
+		 * (4) on a CE channel, but CUDA's push library also binds the
+		 * copy class (0xc5b5) on subchannel 0 of a compute channel to
+		 * carry the same method groups (the work-submission pushes that
+		 * release the client tracker semaphores). Accept both.
+		 */
+		if (subch != 0 && subch != 4) {
 			off += 4 + count * 4;
 			continue;
 		}
@@ -2730,6 +2746,8 @@ static void svc_notify_complete(struct nv_vgpu_device *vgpu,
 #define SVC_CE_RING_FB_STRIDE	0x20000ULL
 
 static unsigned long svc_ring_done;	/* CE ranks whose ring is pinned */
+static unsigned long svc_gr_ring_done;	/* GR ranks whose ring is pinned */
+static u64 svc_gr_ring_base;		/* FB offset where GR chunks start */
 
 /* Read and decode the channel's first GPFIFO entry (its first push). */
 static int svc_chan_entry0(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
@@ -2747,6 +2765,84 @@ static int svc_chan_entry0(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 	return 0;
 }
 
+/*
+ * GR (compute) channel rings live in the same video-memory allocation as the
+ * CE rings, immediately after them: the CE region is 0x80000 + 0x20000 * (CE
+ * channel count) and each GR channel then occupies a 0x3000 chunk in
+ * registration order - the ring (0x2000, 1024 entries) followed by the USERD
+ * at +0x2000. The chunk is only accepted when entry 0 decodes to a 1 MB
+ * aligned push in the client VA window *and* the USERD reads a coherent
+ * put/get pair, so a layout change costs a failed pin, not a wrong one.
+ */
+#define SVC_GR_RING_STRIDE	0x3000ULL
+#define SVC_GR_USERD_OFF	0x2000ULL
+
+bool gsp_sema_gr_rings = true;
+
+static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu,
+				struct svc_channel *ch)
+{
+	u64 pa;
+	int ap = SVC_APERTURE_SYS;
+	u8 e[SVC_GPFIFO_ENTRY_SIZE];
+	u32 e0, e1, va, len, put = 0, get = 0;
+	unsigned long flags;
+	int k;
+
+	if (!gsp_sema_gr_rings)
+		return;
+	if (ch->gr_index >= 32)
+		return;
+	if (test_bit(ch->gr_index, &svc_gr_ring_done))
+		return;
+	if (!svc_pin_lookup(ch->gpfifo_va, &pa, &ap)) {
+		/* Already translated (bridge/manual pin): nothing to do. */
+		set_bit(ch->gr_index, &svc_gr_ring_done);
+		svc_ring_n++;
+		return;
+	}
+
+	if (!svc_gr_ring_base) {
+		u32 max_ce = 0;
+
+		spin_lock_irqsave(&svc.chan_lock, flags);
+		for (k = 0; k < SVC_MAX_CHANNELS; k++)
+			if (svc.chan[k].valid && svc.chan[k].engine_type &&
+			    svc.chan[k].ce_index + 1 > max_ce)
+				max_ce = svc.chan[k].ce_index + 1;
+		spin_unlock_irqrestore(&svc.chan_lock, flags);
+		if (!max_ce)
+			return;	/* wait until the CE channels are known */
+		svc_gr_ring_base = SVC_CE_RING_FB_BASE +
+				   SVC_CE_RING_FB_STRIDE * max_ce;
+	}
+
+	pa = svc_gr_ring_base + SVC_GR_RING_STRIDE * ch->gr_index;
+	if (svc_read_ap(vgpu, pa, SVC_APERTURE_FB, e, sizeof(e)))
+		return;
+	memcpy(&e0, e, 4);
+	memcpy(&e1, e + 4, 4);
+	va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc);
+	len = ((e1 >> 10) & 0x1fffff) << 2;
+	if ((va & 0xfffff) || va < SEMA_VA_LO || va >= SEMA_VA_HI ||
+	    len < 4 || len > 0x10000)
+		return;
+	if (svc_userd_probe(vgpu, pa + SVC_GR_USERD_OFF, SVC_APERTURE_FB,
+			    ch->gpfifo_entries, &put, &get))
+		return;
+
+	svc_pin_store(ch->gpfifo_va, pa, SVC_APERTURE_FB);
+	svc_lat_store(ch->gpfifo_va, pa, SVC_APERTURE_FB);
+	ch->userd_pa = pa + SVC_GR_USERD_OFF;
+	ch->userd_as = SVC_APERTURE_FB;
+	ch->userd_ok = true;
+	set_bit(ch->gr_index, &svc_gr_ring_done);
+	svc_ring_n++;
+	pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> fb 0x%llx (entry0 va=0x%llx len=%u put=%u get=%u)\n",
+		ch->h_object, ch->gr_index, ch->gpfifo_va, pa, va, len, put,
+		get);
+}
+
 static void svc_ring_autopin(struct nv_vgpu_device *vgpu,
 			     struct svc_channel *ch)
 {
@@ -2755,7 +2851,11 @@ static void svc_ring_autopin(struct nv_vgpu_device *vgpu,
 	u8 e[SVC_GPFIFO_ENTRY_SIZE];
 	u32 e0, e1, len;
 
-	if (!ch->engine_type || ch->ce_index >= 32)
+	if (!ch->engine_type) {
+		svc_gr_ring_autopin(vgpu, ch);
+		return;
+	}
+	if (ch->ce_index >= 32)
 		return;
 	if (test_bit(ch->ce_index, &svc_ring_done))
 		return;
diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h
index d5e41a6..f652ffd 100644
--- a/kernel/gsp_sema_service.h
+++ b/kernel/gsp_sema_service.h
@@ -61,4 +61,11 @@ int  gsp_sema_service_dump_channels(void);
 int  gsp_sema_service_add_map(u64 gpu_va, u64 pa, u64 len, u32 ap);
 void gsp_sema_service_doorbell(u32 token);
 
+/*
+ * Discover and pin GR (compute) channel rings after the CE region of the
+ * channel video-memory allocation. Set false to leave GR rings untranslated
+ * (the pump then only runs the CE channels).
+ */
+extern bool gsp_sema_gr_rings;
+
 #endif /* _GSP_SEMA_SERVICE_H_ */
diff --git a/kernel/main.c b/kernel/main.c
index f23be27..4f7f9ee 100644
--- a/kernel/main.c
+++ b/kernel/main.c
@@ -2265,6 +2265,14 @@ module_param_cb(chan_dump, &nv_vgpu_chan_dump_ops, NULL, 0200);
 MODULE_PARM_DESC(chan_dump, "Dump the sema service channel table (any value)");
 
 /*
+ * GR ring discovery: pin the compute channels' rings/USERDs, which follow the
+ * CE rings in the channel video-memory allocation. Default on; set 0 to leave
+ * GR channels untranslated when debugging push-execution side effects.
+ */
+module_param_named(sema_gr_rings, gsp_sema_gr_rings, bool, 0644);
+MODULE_PARM_DESC(sema_gr_rings, "Auto-discover+pin GR channel rings (default 1)");
+
+/*
  * Debug: inject a verified {GPU VA -> guest PA} translation into the
  * semaphore service (used when the client's mapping cannot be discovered
  * from guest structures):