1 files changed, 316 insertions, 11 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 3b009ef..a67107b 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -124,6 +124,7 @@ struct svc_channel {
 	u32	h_object;
 	u32	engine_type;
 	u32	cid;		/* RM channel id (doorbell token vector) */
+	u8	ce_index;	/* rank among the live CE channels (creation order) */
 	u64	gpfifo_va;
 	u32	gpfifo_entries;
 	u64	userd_pa;
@@ -156,7 +157,7 @@ static u32 svc_map_hash[SEMA_MAP_MAX * 2];
  * (debug injection) and that must not be evicted by the learned cache, which
  * is content-addressed and gets thrashed by the lookup stream.
  */
-#define SVC_PIN_SLOTS		4096
+#define SVC_PIN_SLOTS		8192
 
 struct svc_pin {
 	u64 va_page;
@@ -187,6 +188,63 @@ static void svc_mark_hot(u64 pa)
 		set_bit(page, svc_hot);
 }
 
+/*
+ * Push-buffer base pages found by content.
+ *
+ * Every channel's first push starts at its push-buffer base and is
+ * structurally identical across channels: a SET_OBJECT on subchannel 0, a
+ * SET_OBJECT on the compute subchannel (4), then the channel's tracking
+ * semaphore release at page offset 0x10 with payload 1. The release target's
+ * in-page offset selects the channel (0x00, 0x04, ... one slot per channel),
+ * so finding that group in a page names both the page (the channel's
+ * push-buffer base) and the channel (its slot).
+ */
+#define SVC_PB_MAX	32
+struct svc_pb_hit {
+	u64	pa_page;
+	u64	va;		/* push-buffer base VA, once paired with a channel */
+	u32	slot;
+};
+
+static struct svc_pb_hit svc_pb[SVC_PB_MAX];
+static u32 svc_pb_n;
+static u32 svc_ring_n;
+static u8 svc_pb_passes;
+static u64 svc_pb_last;
+
+static bool svc_pb_first_push(const u8 *buf)
+{
+	u32 h0, h1;
+
+	memcpy(&h0, buf, 4);
+	memcpy(&h1, buf + 8, 4);
+	if (((h0 >> 29) & 0x7) != 1 || ((h0 >> 16) & 0xfff) != 1 ||
+	    ((h0 >> 13) & 0x7) != 0 || (h0 & 0x1fff))
+		return false;
+	/* The second group is the compute class' SET_OBJECT: one method word
+	 * to offset 0 of a non-zero subchannel. */
+	if (((h1 >> 29) & 0x7) != 1 || ((h1 >> 16) & 0xfff) != 1 ||
+	    ((h1 >> 13) & 0x7) == 0 || (h1 & 0x1fff))
+		return false;
+	return true;
+}
+
+static void svc_pb_add(u64 pa_page, u32 slot)
+{
+	u32 i;
+
+	for (i = 0; i < svc_pb_n; i++)
+		if (svc_pb[i].slot == slot || svc_pb[i].pa_page == pa_page)
+			return;
+	if (svc_pb_n >= SVC_PB_MAX)
+		return;
+	svc_pb[svc_pb_n].pa_page = pa_page;
+	svc_pb[svc_pb_n].slot = slot;
+	svc_pb_n++;
+	pr_info("sema-svc: push-buffer base pa=0x%llx slot=%u (n=%u)\n",
+		pa_page, slot, svc_pb_n);
+}
+
 struct sema_svc {
 	struct task_struct	*task;
 	bool			stop;
@@ -333,7 +391,26 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc)
 			ch = &svc.chan[i];
 	}
 	if (ch) {
+		int k;
+
 		memset(ch, 0, sizeof(*ch));
+		/*
+		 * Rank among the live CE channels. The RM hands out object
+		 * handles in creation order and the guest's UVM allocates one
+		 * push buffer, ring and tracking semaphore per channel in that
+		 * same order, so the rank is what ties a channel to its ring
+		 * offset and its push-buffer base (see svc_ring_autopin /
+		 * svc_pb_pair).
+		 */
+		if (desc->engine_type) {
+			u8 rank = 0;
+
+			for (k = 0; k < SVC_MAX_CHANNELS; k++)
+				if (svc.chan[k].valid && svc.chan[k].engine_type &&
+				    svc.chan[k].h_object < desc->h_object)
+					rank++;
+			ch->ce_index = rank;
+		}
 		ch->valid = true;
 		ch->h_object = desc->h_object;
 		ch->engine_type = desc->engine_type;
@@ -770,16 +847,34 @@ static void svc_pd_discover(struct nv_vgpu_device *vgpu)
 		return;
 	svc_pd_tried = 1;
 
-	if (svc.map_n < SVC_PD_SAMPLES)
-		return;
-	step = svc.map_n / SVC_PD_SAMPLES;
-	for (i = 0; i < SVC_PD_SAMPLES; i++) {
-		sva[i] = svc.map[i * step].gpu_page;
-		spa[i] = svc.map[i * step].pa_page;
-		if (sva[i] >> SVC_MMU_VA_BITS)
-			return;	/* outside the walkable VA range */
+	/*
+	 * Pinned push-buffer pairs are the strongest samples available: they
+	 * are UVM-internal VAs whose translation was verified by content, and
+	 * the directory that maps them is the one that translates the rest of
+	 * the push-buffer pool. Fall back to the discovered VA->PA map while
+	 * fewer than SVC_PD_MIN_HITS of them exist: that map also holds pairs
+	 * from other VA spaces and no single directory reproduces a mixed set.
+	 */
+	for (i = 0; i < svc_pb_n && n < SVC_PD_SAMPLES; i++) {
+		if (!svc_pb[i].va)
+			continue;
+		sva[n] = svc_pb[i].va;
+		spa[n] = svc_pb[i].pa_page;
 		n++;
 	}
+	if (n < SVC_PD_MIN_HITS) {
+		n = 0;
+		if (svc.map_n < SVC_PD_SAMPLES)
+			return;
+		step = svc.map_n / SVC_PD_SAMPLES;
+		for (i = 0; i < SVC_PD_SAMPLES; i++) {
+			sva[i] = svc.map[i * step].gpu_page;
+			spa[i] = svc.map[i * step].pa_page;
+			if (sva[i] >> SVC_MMU_VA_BITS)
+				return;	/* outside the walkable VA range */
+			n++;
+		}
+	}
 	if (!n)
 		return;
 
@@ -1296,9 +1391,9 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu)
 		svc.pages_decoded, svc.pushes_seen, svc.pushes_pending,
 		svc.pushes_nomap, svc.memsets_done, svc.memsets_nomap,
 		svc.memsets_fail);
-	pr_info("sema-svc:   gpfifo done=%llu nomap=%llu vidmem_writes=%llu\n",
+	pr_info("sema-svc:   gpfifo done=%llu nomap=%llu vidmem_writes=%llu ring_auto=%u pb=%u/%u\n",
 		svc.gpfifo_entries_done, svc.gpfifo_nomap,
-		svc.vidmem_writes);
+		svc.vidmem_writes, svc_ring_n, svc_pb_n, svc_pb_passes);
 	/* show a few entries so mappings can be verified from dmesg */
 	{
 		u32 shown;
@@ -1960,6 +2055,15 @@ static u32 svc_apply_page_data(struct nv_vgpu_device *vgpu, const u8 *buf,
 			continue;
 		}
 
+		/*
+		 * A release at page offset 0x10 with payload 1 in front of a
+		 * channel's first push identifies the page as that channel's
+		 * push-buffer base (see svc_pb_add).
+		 */
+		if (off == 0x10 && hi == 1 && payload == 1 &&
+		    (lo & 0xfff) < 0x20 && svc_pb_first_push(buf))
+			svc_pb_add(pa_page, lo & 0xfff);
+
 		if (svc_sem_write(vgpu, va, payload) == 0) {
 			svc.releases_applied++;
 			svc.post_hint_until = jiffies + 10 * HZ;
@@ -2062,6 +2166,75 @@ static void svc_apply_pushes(struct nv_vgpu_device *vgpu)
 	svc_scan_cursor = (end >= SEMA_PA_MAX) ? 0 : end;
 }
 
+static void svc_pb_note(struct nv_vgpu_device *vgpu, const u8 *buf, u64 pa_page)
+{
+	u32 hdr, hi, lo, payload;
+
+	(void)vgpu;
+	memcpy(&hdr, buf + 0x10, 4);
+	if (((hdr >> 16) & 0xff) != CE_SEMA_HDR_COUNT ||
+	    (hdr & 0xff) != CE_SEMA_HDR_ADDR)
+		return;
+	memcpy(&hi, buf + 0x14, 4);
+	memcpy(&lo, buf + 0x18, 4);
+	memcpy(&payload, buf + 0x1c, 4);
+	if (hi != 1 || payload != 1 || (lo & 0xfff) >= 0x20)
+		return;
+	if (!svc_pb_first_push(buf))
+		return;
+	svc_pb_add(pa_page, lo & 0xfff);
+	svc_mark_hot(pa_page);
+}
+
+/*
+ * Dedicated pass for the push-buffer base pages: a chunked walk of guest RAM
+ * completes in a couple of seconds, where the rolling sweep visits only 32 MB
+ * per pass. It runs while CE channels exist and stops once enough bases are
+ * known, so a page that appears while a client waits on it is pinned and
+ * pumped immediately.
+ */
+static void svc_pb_discover(struct nv_vgpu_device *vgpu)
+{
+	unsigned long flags;
+	u8 *buf;
+	u64 gpa;
+	bool ce = false;
+	int i;
+
+	if (svc_pb_passes >= 12 || svc_pb_n >= SVC_PB_MAX)
+		return;
+	if (time_before(jiffies, (unsigned long)svc_pb_last + 3 * HZ))
+		return;
+
+	spin_lock_irqsave(&svc.chan_lock, flags);
+	for (i = 0; i < SVC_MAX_CHANNELS; i++)
+		if (svc.chan[i].valid && svc.chan[i].engine_type) {
+			ce = true;
+			break;
+		}
+	spin_unlock_irqrestore(&svc.chan_lock, flags);
+	if (!ce)
+		return;
+
+	buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL);
+	if (!buf)
+		return;
+
+	svc_pb_passes++;
+	svc_pb_last = jiffies;
+	for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SVC_SCAN_CHUNK) {
+		u32 o;
+
+		if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK))
+			continue;
+		for (o = 0; o < SVC_SCAN_CHUNK; o += SEMA_PAGE_SIZE)
+			svc_pb_note(vgpu, buf + o, gpa + o);
+	}
+	kfree(buf);
+	pr_info("sema-svc: push-buffer discovery pass %u done (%u bases, ring %u)\n",
+		svc_pb_passes, svc_pb_n, svc_ring_n);
+}
+
 /*
  * Targeted pass: run the work in the pages we have mapped. The map covers the
  * allocations the guest has described to us - the pushbuffer pools among them
@@ -2315,6 +2488,125 @@ static void svc_notify_complete(struct nv_vgpu_device *vgpu,
  * memsets, copies, semaphore releases) - this is what makes the guest's UVM
  * progress waits complete. The GPU's own GPGet is updated as we go.
  */
+
+/*
+ * The guest's UVM allocates each CE channel's ring and USERD in one
+ * contiguous video-memory block, in channel creation order, and the emulation
+ * sees the ring only as a GPU VA. The block's layout is a function of the
+ * channel's rank (ring at 0x80000 + 0x20000 * rank, USERD at ring + 0x10000;
+ * observed for every CE channel across boots). The derived offset is verified
+ * against the ring's first GPFIFO entry, so a layout change costs a failed
+ * pin, not a wrong one.
+ */
+#define SVC_CE_RING_FB_BASE	0x80000ULL
+#define SVC_CE_RING_FB_STRIDE	0x20000ULL
+
+static unsigned long svc_ring_done;	/* CE ranks whose ring is pinned */
+
+/* Read and decode the channel's first GPFIFO entry (its first push). */
+static int svc_chan_entry0(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
+			   u64 *va, u32 *len)
+{
+	u8 e[SVC_GPFIFO_ENTRY_SIZE];
+	u32 e0, e1;
+
+	if (svc_read_va(vgpu, ch->gpfifo_va, e, sizeof(e)))
+		return -ENOENT;
+	memcpy(&e0, e, 4);
+	memcpy(&e1, e + 4, 4);
+	*va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc);
+	*len = ((e1 >> 10) & 0x1fffff) << 2;
+	return 0;
+}
+
+static void svc_ring_autopin(struct nv_vgpu_device *vgpu,
+			     struct svc_channel *ch)
+{
+	u64 pa = 0, va = 0;
+	int ap = SVC_APERTURE_SYS;
+	u8 e[SVC_GPFIFO_ENTRY_SIZE];
+	u32 e0, e1, len;
+
+	if (!ch->engine_type || ch->ce_index >= 32)
+		return;
+	if (test_bit(ch->ce_index, &svc_ring_done))
+		return;
+	if (!svc_pin_lookup(ch->gpfifo_va, &pa, &ap)) {
+		set_bit(ch->ce_index, &svc_ring_done);
+		svc_ring_n++;
+		return;
+	}
+
+	pa = SVC_CE_RING_FB_BASE + SVC_CE_RING_FB_STRIDE * ch->ce_index;
+	if (svc_read_ap(vgpu, pa, SVC_APERTURE_FB, e, sizeof(e)))
+		return;
+	memcpy(&e0, e, 4);
+	memcpy(&e1, e + 4, 4);
+	va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc);
+	len = ((e1 >> 10) & 0x1fffff) << 2;
+	if ((va & 0xfffff) || va < SEMA_VA_LO || va >= SEMA_VA_HI ||
+	    len < 4 || len > 0x10000)
+		return;
+
+	svc_pin_store(ch->gpfifo_va, pa, SVC_APERTURE_FB);
+	svc_lat_store(ch->gpfifo_va, pa, SVC_APERTURE_FB);
+	set_bit(ch->ce_index, &svc_ring_done);
+	svc_ring_n++;
+	pr_info("sema-chan: auto-pinned ring h=0x%x rank=%u va=0x%llx -> fb 0x%llx (entry0 va=0x%llx len=%u)\n",
+		ch->h_object, ch->ce_index, ch->gpfifo_va, pa, va, len);
+}
+
+/*
+ * Pair a channel's push buffer with its base page: the ring's first entry
+ * carries the push-buffer VA, the base page was found by content and its
+ * release names the channel's slot (4 bytes per rank).
+ */
+static unsigned long svc_pb_done;	/* CE ranks whose push buffer is pinned */
+
+static void svc_pb_pair(struct nv_vgpu_device *vgpu, struct svc_channel *ch)
+{
+	u64 hit_pa = 0, va = 0;
+	u32 len = 0, i, hit_i = 0, want, off;
+
+	if (!ch->engine_type || ch->ce_index >= 32)
+		return;
+	if (test_bit(ch->ce_index, &svc_pb_done))
+		return;
+	want = 4 * ch->ce_index;
+	for (i = 0; i < svc_pb_n; i++)
+		if (svc_pb[i].slot == want) {
+			hit_pa = svc_pb[i].pa_page;
+			hit_i = i;
+			break;
+		}
+	if (!hit_pa)
+		return;
+	if (svc_chan_entry0(vgpu, ch, &va, &len))
+		return;
+	if ((va & 0xfffff) || va < SEMA_VA_LO || va >= SEMA_VA_HI ||
+	    len < 4 || len > 0x10000)
+		return;
+
+	for (off = 0; off < 0x100000; off += SEMA_PAGE_SIZE) {
+		svc_pin_store(va + off, hit_pa + off, SVC_APERTURE_SYS);
+		svc_lat_store(va + off, hit_pa + off, SVC_APERTURE_SYS);
+		svc_mark_hot(hit_pa + off);
+	}
+	svc_pb[hit_i].va = va;
+	set_bit(ch->ce_index, &svc_pb_done);
+
+	/*
+	 * The pinned pairs are verified samples of UVM's own page directory;
+	 * let the directory discovery run again with them, so the walk can
+	 * translate the parts of the pool no base page describes.
+	 */
+	if (!svc_pd_found)
+		svc_pd_tried = 0;
+
+	pr_info("sema-svc: push-buffer pinned h=0x%x rank=%u slot=%u va=0x%llx -> pa=0x%llx (first push len=%u)\n",
+		ch->h_object, ch->ce_index, want, va, hit_pa, len);
+}
+
 static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 {
 	/* Only ever called from the service thread. */
@@ -2340,8 +2632,13 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 		if (!ch->gpfifo_va || !ch->gpfifo_entries)
 			continue;
 
+		svc_ring_autopin(vgpu, ch);
+
 		rrc = svc_lookup(vgpu, ch->gpfifo_va, &ring_pa, &ring_ap);
 
+		if (!rrc)
+			svc_pb_pair(vgpu, ch);
+
 		if (!ch->userd_ok) {
 			if (svc_chan_userd(vgpu, ch, rrc ? 0 : ring_pa,
 					   rrc ? -1 : ring_ap, &put, &get))
@@ -2620,6 +2917,14 @@ static int svc_thread(void *arg)
 		svc_pump_channels(vgpu);
 
 		/*
+		 * Push-buffer base discovery: a chunked walk of guest RAM
+		 * finds the pages whose first push names a channel, so a
+		 * channel that appears with work already waiting is pinned and
+		 * pumped without waiting for the rolling sweep.
+		 */
+		svc_pb_discover(vgpu);
+
+		/*
 		 * Completion hint: re-post the events for a few seconds after a
 		 * completion, so a client that starts waiting after the
 		 * edge-triggered wake-up still observes it.