kernel: self-configure CE rings and push buffers in the semaphore service

The service pins the CE channel rings and push buffers without any
guest-supplied addresses. Rings are derived from the channel rank among
the live CE channels (ring at FB 0x80000 + 0x20000*rank, USERD at
ring + 0x10000) and verified against the ring's first GPFIFO entry before
the pin is stored. Push-buffer base pages are found by their first-push
signature (SET_OBJECT on subchannel 0, SET_OBJECT on the compute
subchannel, tracking-semaphore release at page offset 0x10 with payload
1); the release's in-page slot offset names the channel and the ring's
first entry names the push-buffer VA, then the whole 1 MB region is
pinned and marked hot.

Pinning a pair also re-arms the page-directory discovery and feeds it the
verified VA/PA pairs as samples, so the walk can translate the parts of
the pool no base page describes.

With rings and push buffers pinned the pump executes the submitted work
(gpfifo done grows, semaphore releases apply, released kept in vidmem)
and the guest kernel UVM waits complete.

1 files changed, 316 insertions, 11 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 3b009ef..a67107b 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -124,6 +124,7 @@ struct svc_channel {
 	u32	h_object;
 	u32	engine_type;
 	u32	cid;		/* RM channel id (doorbell token vector) */
+	u8	ce_index;	/* rank among the live CE channels (creation order) */
 	u64	gpfifo_va;
 	u32	gpfifo_entries;
 	u64	userd_pa;
@@ -156,7 +157,7 @@ static u32 svc_map_hash[SEMA_MAP_MAX * 2];
  * (debug injection) and that must not be evicted by the learned cache, which
  * is content-addressed and gets thrashed by the lookup stream.
  */
-#define SVC_PIN_SLOTS		4096
+#define SVC_PIN_SLOTS		8192
 
 struct svc_pin {
 	u64 va_page;
@@ -187,6 +188,63 @@ static void svc_mark_hot(u64 pa)
 		set_bit(page, svc_hot);
 }
 
+/*
+ * Push-buffer base pages found by content.
+ *
+ * Every channel's first push starts at its push-buffer base and is
+ * structurally identical across channels: a SET_OBJECT on subchannel 0, a
+ * SET_OBJECT on the compute subchannel (4), then the channel's tracking
+ * semaphore release at page offset 0x10 with payload 1. The release target's
+ * in-page offset selects the channel (0x00, 0x04, ... one slot per channel),
+ * so finding that group in a page names both the page (the channel's
+ * push-buffer base) and the channel (its slot).
+ */
+#define SVC_PB_MAX	32
+struct svc_pb_hit {
+	u64	pa_page;
+	u64	va;		/* push-buffer base VA, once paired with a channel */
+	u32	slot;
+};
+
+static struct svc_pb_hit svc_pb[SVC_PB_MAX];
+static u32 svc_pb_n;
+static u32 svc_ring_n;
+static u8 svc_pb_passes;
+static u64 svc_pb_last;
+
+static bool svc_pb_first_push(const u8 *buf)
+{
+	u32 h0, h1;
+
+	memcpy(&h0, buf, 4);
+	memcpy(&h1, buf + 8, 4);
+	if (((h0 >> 29) & 0x7) != 1 || ((h0 >> 16) & 0xfff) != 1 ||
+	    ((h0 >> 13) & 0x7) != 0 || (h0 & 0x1fff))
+		return false;
+	/* The second group is the compute class' SET_OBJECT: one method word
+	 * to offset 0 of a non-zero subchannel. */
+	if (((h1 >> 29) & 0x7) != 1 || ((h1 >> 16) & 0xfff) != 1 ||
+	    ((h1 >> 13) & 0x7) == 0 || (h1 & 0x1fff))
+		return false;
+	return true;
+}
+
+static void svc_pb_add(u64 pa_page, u32 slot)
+{
+	u32 i;
+
+	for (i = 0; i < svc_pb_n; i++)
+		if (svc_pb[i].slot == slot || svc_pb[i].pa_page == pa_page)
+			return;
+	if (svc_pb_n >= SVC_PB_MAX)
+		return;
+	svc_pb[svc_pb_n].pa_page = pa_page;
+	svc_pb[svc_pb_n].slot = slot;
+	svc_pb_n++;
+	pr_info("sema-svc: push-buffer base pa=0x%llx slot=%u (n=%u)\n",
+		pa_page, slot, svc_pb_n);
+}
+
 struct sema_svc {
 	struct task_struct	*task;
 	bool			stop;
@@ -333,7 +391,26 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc)
 			ch = &svc.chan[i];
 	}
 	if (ch) {
+		int k;
+
 		memset(ch, 0, sizeof(*ch));
+		/*
+		 * Rank among the live CE channels. The RM hands out object
+		 * handles in creation order and the guest's UVM allocates one
+		 * push buffer, ring and tracking semaphore per channel in that
+		 * same order, so the rank is what ties a channel to its ring
+		 * offset and its push-buffer base (see svc_ring_autopin /
+		 * svc_pb_pair).
+		 */
+		if (desc->engine_type) {
+			u8 rank = 0;
+
+			for (k = 0; k < SVC_MAX_CHANNELS; k++)
+				if (svc.chan[k].valid && svc.chan[k].engine_type &&
+				    svc.chan[k].h_object < desc->h_object)
+					rank++;
+			ch->ce_index = rank;
+		}
 		ch->valid = true;
 		ch->h_object = desc->h_object;
 		ch->engine_type = desc->engine_type;
@@ -770,16 +847,34 @@ static void svc_pd_discover(struct nv_vgpu_device *vgpu)
 		return;
 	svc_pd_tried = 1;
 
-	if (svc.map_n < SVC_PD_SAMPLES)
-		return;
-	step = svc.map_n / SVC_PD_SAMPLES;
-	for (i = 0; i < SVC_PD_SAMPLES; i++) {
-		sva[i] = svc.map[i * step].gpu_page;
-		spa[i] = svc.map[i * step].pa_page;
-		if (sva[i] >> SVC_MMU_VA_BITS)
-			return;	/* outside the walkable VA range */
+	/*
+	 * Pinned push-buffer pairs are the strongest samples available: they
+	 * are UVM-internal VAs whose translation was verified by content, and
+	 * the directory that maps them is the one that translates the rest of
+	 * the push-buffer pool. Fall back to the discovered VA->PA map while
+	 * fewer than SVC_PD_MIN_HITS of them exist: that map also holds pairs
+	 * from other VA spaces and no single directory reproduces a mixed set.
+	 */
+	for (i = 0; i < svc_pb_n && n < SVC_PD_SAMPLES; i++) {
+		if (!svc_pb[i].va)
+			continue;
+		sva[n] = svc_pb[i].va;
+		spa[n] = svc_pb[i].pa_page;
 		n++;
 	}
+	if (n < SVC_PD_MIN_HITS) {
+		n = 0;
+		if (svc.map_n < SVC_PD_SAMPLES)
+			return;
+		step = svc.map_n / SVC_PD_SAMPLES;
+		for (i = 0; i < SVC_PD_SAMPLES; i++) {
+			sva[i] = svc.map[i * step].gpu_page;
+			spa[i] = svc.map[i * step].pa_page;
+			if (sva[i] >> SVC_MMU_VA_BITS)
+				return;	/* outside the walkable VA range */
+			n++;
+		}
+	}
 	if (!n)
 		return;
 
@@ -1296,9 +1391,9 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu)
 		svc.pages_decoded, svc.pushes_seen, svc.pushes_pending,
 		svc.pushes_nomap, svc.memsets_done, svc.memsets_nomap,
 		svc.memsets_fail);
-	pr_info("sema-svc:   gpfifo done=%llu nomap=%llu vidmem_writes=%llu\n",
+	pr_info("sema-svc:   gpfifo done=%llu nomap=%llu vidmem_writes=%llu ring_auto=%u pb=%u/%u\n",
 		svc.gpfifo_entries_done, svc.gpfifo_nomap,
-		svc.vidmem_writes);
+		svc.vidmem_writes, svc_ring_n, svc_pb_n, svc_pb_passes);
 	/* show a few entries so mappings can be verified from dmesg */
 	{
 		u32 shown;
@@ -1960,6 +2055,15 @@ static u32 svc_apply_page_data(struct nv_vgpu_device *vgpu, const u8 *buf,
 			continue;
 		}
 
+		/*
+		 * A release at page offset 0x10 with payload 1 in front of a
+		 * channel's first push identifies the page as that channel's
+		 * push-buffer base (see svc_pb_add).
+		 */
+		if (off == 0x10 && hi == 1 && payload == 1 &&
+		    (lo & 0xfff) < 0x20 && svc_pb_first_push(buf))
+			svc_pb_add(pa_page, lo & 0xfff);
+
 		if (svc_sem_write(vgpu, va, payload) == 0) {
 			svc.releases_applied++;
 			svc.post_hint_until = jiffies + 10 * HZ;
@@ -2062,6 +2166,75 @@ static void svc_apply_pushes(struct nv_vgpu_device *vgpu)
 	svc_scan_cursor = (end >= SEMA_PA_MAX) ? 0 : end;
 }
 
+static void svc_pb_note(struct nv_vgpu_device *vgpu, const u8 *buf, u64 pa_page)
+{
+	u32 hdr, hi, lo, payload;
+
+	(void)vgpu;
+	memcpy(&hdr, buf + 0x10, 4);
+	if (((hdr >> 16) & 0xff) != CE_SEMA_HDR_COUNT ||
+	    (hdr & 0xff) != CE_SEMA_HDR_ADDR)
+		return;
+	memcpy(&hi, buf + 0x14, 4);
+	memcpy(&lo, buf + 0x18, 4);
+	memcpy(&payload, buf + 0x1c, 4);
+	if (hi != 1 || payload != 1 || (lo & 0xfff) >= 0x20)
+		return;
+	if (!svc_pb_first_push(buf))
+		return;
+	svc_pb_add(pa_page, lo & 0xfff);
+	svc_mark_hot(pa_page);
+}
+
+/*
+ * Dedicated pass for the push-buffer base pages: a chunked walk of guest RAM
+ * completes in a couple of seconds, where the rolling sweep visits only 32 MB
+ * per pass. It runs while CE channels exist and stops once enough bases are
+ * known, so a page that appears while a client waits on it is pinned and
+ * pumped immediately.
+ */
+static void svc_pb_discover(struct nv_vgpu_device *vgpu)
+{
+	unsigned long flags;
+	u8 *buf;
+	u64 gpa;
+	bool ce = false;
+	int i;
+
+	if (svc_pb_passes >= 12 || svc_pb_n >= SVC_PB_MAX)
+		return;
+	if (time_before(jiffies, (unsigned long)svc_pb_last + 3 * HZ))
+		return;
+
+	spin_lock_irqsave(&svc.chan_lock, flags);
+	for (i = 0; i < SVC_MAX_CHANNELS; i++)
+		if (svc.chan[i].valid && svc.chan[i].engine_type) {
+			ce = true;
+			break;
+		}
+	spin_unlock_irqrestore(&svc.chan_lock, flags);
+	if (!ce)
+		return;
+
+	buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL);
+	if (!buf)
+		return;
+
+	svc_pb_passes++;
+	svc_pb_last = jiffies;
+	for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SVC_SCAN_CHUNK) {
+		u32 o;
+
+		if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK))
+			continue;
+		for (o = 0; o < SVC_SCAN_CHUNK; o += SEMA_PAGE_SIZE)
+			svc_pb_note(vgpu, buf + o, gpa + o);
+	}
+	kfree(buf);
+	pr_info("sema-svc: push-buffer discovery pass %u done (%u bases, ring %u)\n",
+		svc_pb_passes, svc_pb_n, svc_ring_n);
+}
+
 /*
  * Targeted pass: run the work in the pages we have mapped. The map covers the
  * allocations the guest has described to us - the pushbuffer pools among them
@@ -2315,6 +2488,125 @@ static void svc_notify_complete(struct nv_vgpu_device *vgpu,
  * memsets, copies, semaphore releases) - this is what makes the guest's UVM
  * progress waits complete. The GPU's own GPGet is updated as we go.
  */
+
+/*
+ * The guest's UVM allocates each CE channel's ring and USERD in one
+ * contiguous video-memory block, in channel creation order, and the emulation
+ * sees the ring only as a GPU VA. The block's layout is a function of the
+ * channel's rank (ring at 0x80000 + 0x20000 * rank, USERD at ring + 0x10000;
+ * observed for every CE channel across boots). The derived offset is verified
+ * against the ring's first GPFIFO entry, so a layout change costs a failed
+ * pin, not a wrong one.
+ */
+#define SVC_CE_RING_FB_BASE	0x80000ULL
+#define SVC_CE_RING_FB_STRIDE	0x20000ULL
+
+static unsigned long svc_ring_done;	/* CE ranks whose ring is pinned */
+
+/* Read and decode the channel's first GPFIFO entry (its first push). */
+static int svc_chan_entry0(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
+			   u64 *va, u32 *len)
+{
+	u8 e[SVC_GPFIFO_ENTRY_SIZE];
+	u32 e0, e1;
+
+	if (svc_read_va(vgpu, ch->gpfifo_va, e, sizeof(e)))
+		return -ENOENT;
+	memcpy(&e0, e, 4);
+	memcpy(&e1, e + 4, 4);
+	*va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc);
+	*len = ((e1 >> 10) & 0x1fffff) << 2;
+	return 0;
+}
+
+static void svc_ring_autopin(struct nv_vgpu_device *vgpu,
+			     struct svc_channel *ch)
+{
+	u64 pa = 0, va = 0;
+	int ap = SVC_APERTURE_SYS;
+	u8 e[SVC_GPFIFO_ENTRY_SIZE];
+	u32 e0, e1, len;
+
+	if (!ch->engine_type || ch->ce_index >= 32)
+		return;
+	if (test_bit(ch->ce_index, &svc_ring_done))
+		return;
+	if (!svc_pin_lookup(ch->gpfifo_va, &pa, &ap)) {
+		set_bit(ch->ce_index, &svc_ring_done);
+		svc_ring_n++;
+		return;
+	}
+
+	pa = SVC_CE_RING_FB_BASE + SVC_CE_RING_FB_STRIDE * ch->ce_index;
+	if (svc_read_ap(vgpu, pa, SVC_APERTURE_FB, e, sizeof(e)))
+		return;
+	memcpy(&e0, e, 4);
+	memcpy(&e1, e + 4, 4);
+	va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc);
+	len = ((e1 >> 10) & 0x1fffff) << 2;
+	if ((va & 0xfffff) || va < SEMA_VA_LO || va >= SEMA_VA_HI ||
+	    len < 4 || len > 0x10000)
+		return;
+
+	svc_pin_store(ch->gpfifo_va, pa, SVC_APERTURE_FB);
+	svc_lat_store(ch->gpfifo_va, pa, SVC_APERTURE_FB);
+	set_bit(ch->ce_index, &svc_ring_done);
+	svc_ring_n++;
+	pr_info("sema-chan: auto-pinned ring h=0x%x rank=%u va=0x%llx -> fb 0x%llx (entry0 va=0x%llx len=%u)\n",
+		ch->h_object, ch->ce_index, ch->gpfifo_va, pa, va, len);
+}
+
+/*
+ * Pair a channel's push buffer with its base page: the ring's first entry
+ * carries the push-buffer VA, the base page was found by content and its
+ * release names the channel's slot (4 bytes per rank).
+ */
+static unsigned long svc_pb_done;	/* CE ranks whose push buffer is pinned */
+
+static void svc_pb_pair(struct nv_vgpu_device *vgpu, struct svc_channel *ch)
+{
+	u64 hit_pa = 0, va = 0;
+	u32 len = 0, i, hit_i = 0, want, off;
+
+	if (!ch->engine_type || ch->ce_index >= 32)
+		return;
+	if (test_bit(ch->ce_index, &svc_pb_done))
+		return;
+	want = 4 * ch->ce_index;
+	for (i = 0; i < svc_pb_n; i++)
+		if (svc_pb[i].slot == want) {
+			hit_pa = svc_pb[i].pa_page;
+			hit_i = i;
+			break;
+		}
+	if (!hit_pa)
+		return;
+	if (svc_chan_entry0(vgpu, ch, &va, &len))
+		return;
+	if ((va & 0xfffff) || va < SEMA_VA_LO || va >= SEMA_VA_HI ||
+	    len < 4 || len > 0x10000)
+		return;
+
+	for (off = 0; off < 0x100000; off += SEMA_PAGE_SIZE) {
+		svc_pin_store(va + off, hit_pa + off, SVC_APERTURE_SYS);
+		svc_lat_store(va + off, hit_pa + off, SVC_APERTURE_SYS);
+		svc_mark_hot(hit_pa + off);
+	}
+	svc_pb[hit_i].va = va;
+	set_bit(ch->ce_index, &svc_pb_done);
+
+	/*
+	 * The pinned pairs are verified samples of UVM's own page directory;
+	 * let the directory discovery run again with them, so the walk can
+	 * translate the parts of the pool no base page describes.
+	 */
+	if (!svc_pd_found)
+		svc_pd_tried = 0;
+
+	pr_info("sema-svc: push-buffer pinned h=0x%x rank=%u slot=%u va=0x%llx -> pa=0x%llx (first push len=%u)\n",
+		ch->h_object, ch->ce_index, want, va, hit_pa, len);
+}
+
 static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 {
 	/* Only ever called from the service thread. */
@@ -2340,8 +2632,13 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 		if (!ch->gpfifo_va || !ch->gpfifo_entries)
 			continue;
 
+		svc_ring_autopin(vgpu, ch);
+
 		rrc = svc_lookup(vgpu, ch->gpfifo_va, &ring_pa, &ring_ap);
 
+		if (!rrc)
+			svc_pb_pair(vgpu, ch);
+
 		if (!ch->userd_ok) {
 			if (svc_chan_userd(vgpu, ch, rrc ? 0 : ring_pa,
 					   rrc ? -1 : ring_ap, &put, &get))
@@ -2620,6 +2917,14 @@ static int svc_thread(void *arg)
 		svc_pump_channels(vgpu);
 
 		/*
+		 * Push-buffer base discovery: a chunked walk of guest RAM
+		 * finds the pages whose first push names a channel, so a
+		 * channel that appears with work already waiting is pinned and
+		 * pumped without waiting for the rolling sweep.
+		 */
+		svc_pb_discover(vgpu);
+
+		/*
 		 * Completion hint: re-post the events for a few seconds after a
 		 * completion, so a client that starts waiting after the
 		 * edge-triggered wake-up still observes it.