gsp_sema_service: add content rescue for unmapped push-buffer pool VAs

- Add expected_sem_payload tracking to svc_channel for CE channels
- Implement svc_rescue_push_page() to scan guest RAM for push with
  expected semaphore release when VA translation fails
- Add svc_init_expected_payload() to initialize expected payload from pool
- Add rate-limited logging of distinct stalled VAs (svc_log_stalled_va)
- Advance expected_sem_payload on successful push decode
- Pushes SEMA_MAP_MAX to 65536, SVC_PIN_SLOTS to 32768 for scale

1 files changed, 179 insertions, 6 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 168e017..a203a66 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -137,6 +137,7 @@ struct svc_channel {
 	u64	notified;	/* work-submit-token notifications completed */
 	u32	last_get;	/* next GPFIFO entry we have still to run */
 	u64	submissions;	/* GPFIFO entries executed */
+	u32	expected_sem_payload;	/* next semaphore payload we expect for this channel's tracking sem */
 };
 
 /*
@@ -312,6 +313,11 @@ struct sema_svc {
 	u32			doorbell_token;		/* last token written */
 	u32			doorbell_log;		/* token log budget */
 	bool			doorbell_pending;	/* pump on next service pass */
+
+	/* Distinct stalled VA tracking for rate-limited logging. */
+#define SVC_STALLED_MAX		256
+	u64			stalled_vas[SVC_STALLED_MAX];
+	u32			stalled_va_count;
 };
 
 static struct sema_svc svc;
@@ -319,6 +325,31 @@ static struct sema_svc svc;
 /* Rolling RAM scan position (defined with the scan below; logged by rebuild) */
 static u64 svc_scan_cursor;
 
+/*
+ * Rate-limited logging of distinct stalled entry VAs.
+ * Tracks up to SVC_STALLED_MAX distinct VAs that fail translation.
+ */
+static void svc_log_stalled_va(u64 va)
+{
+	u32 i;
+
+	if (!va)
+		return;
+
+	/* Check if already logged. */
+	for (i = 0; i < svc.stalled_va_count; i++) {
+		if (svc.stalled_vas[i] == va)
+			return;
+	}
+
+	/* Add to tracked set. */
+	if (svc.stalled_va_count < SVC_STALLED_MAX) {
+		svc.stalled_vas[svc.stalled_va_count++] = va;
+		pr_info("sema-svc: stalled VA #%u: 0x%llx (total distinct: %u)\n",
+			svc.stalled_va_count, va, svc.stalled_va_count);
+	}
+}
+
 /* ------------------------------------------------------------------ */
 /* guest memory helpers                                                */
 /* ------------------------------------------------------------------ */
@@ -2623,6 +2654,121 @@ static void svc_pb_pair(struct nv_vgpu_device *vgpu, struct svc_channel *ch)
 		ch->h_object, ch->ce_index, want, va, hit_pa, len);
 }
 
+/*
+ * Content rescue for unmapped push-buffer pool VAs.
+ * When the pump encounters a GPFIFO entry whose push-buffer VA is not in the
+ * VA->PA map, we scan guest RAM for a page containing a push whose semaphore
+ * release matches this channel's tracking semaphore (slot = 4 * ce_index) and
+ * the expected next payload. If found, we pin the entire 1 MB push-buffer slot
+ * at the entry VA so the pump can execute the push.
+ */
+static int svc_rescue_push_page(struct nv_vgpu_device *vgpu,
+				struct svc_channel *ch, u64 entry_va, u32 entry_len)
+{
+	u8 *buf;
+	u64 gpa;
+	u32 slot = 4 * ch->ce_index;
+	u32 want_payload = ch->expected_sem_payload;
+	int found = 0;
+	u64 hit_pa = 0;
+
+	if (!ch->engine_type || ch->ce_index >= 32 || !want_payload)
+		return -ENOENT;
+
+	buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	/*
+	 * Scan guest RAM for a push with the expected semaphore release.
+	 * The push format at the semaphore release (CE path) has:
+	 *   offset 0x10: hdr (count=3, addr_lo=0x90)
+	 *   offset 0x14: hi (== 1)
+	 *   offset 0x18: lo (slot in low 12 bits)
+	 *   offset 0x1c: payload
+	 */
+	for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SVC_SCAN_CHUNK) {
+		u32 o;
+
+		if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK))
+			continue;
+		for (o = 0; o < SVC_SCAN_CHUNK; o += SEMA_PAGE_SIZE) {
+			u32 hdr, hi, lo, payload;
+
+			memcpy(&hdr, buf + o + 0x10, 4);
+			if ((((hdr >> 16) & 0xff) != CE_SEMA_HDR_COUNT) ||
+			    ((hdr & 0xff) != CE_SEMA_HDR_ADDR))
+				continue;
+			memcpy(&hi, buf + o + 0x14, 4);
+			memcpy(&lo, buf + o + 0x18, 4);
+			memcpy(&payload, buf + o + 0x1c, 4);
+			if (hi != 1)
+				continue;
+			if ((lo & 0xfff) != slot)
+				continue;
+			if (payload != want_payload)
+				continue;
+
+			hit_pa = (gpa + o) & ~(SEMA_PAGE_SIZE - 1);
+			found = 1;
+			break;
+		}
+		if (found)
+			break;
+	}
+	kfree(buf);
+
+	if (!found)
+		return -ENOENT;
+
+	/*
+	 * Pin the entire 1 MB push-buffer slot (16 pages) at the entry VA.
+	 * The entry_va is 1 MB aligned (push-buffer pool VAs are 1 MB aligned).
+	 */
+	if ((entry_va & 0xfffff) || entry_va < SEMA_VA_LO || entry_va >= SEMA_VA_HI)
+		return -EINVAL;
+
+	for (u32 off = 0; off < 0x100000; off += SEMA_PAGE_SIZE) {
+		svc_pin_store(entry_va + off, hit_pa + off, SVC_APERTURE_SYS);
+		svc_lat_store(entry_va + off, hit_pa + off, SVC_APERTURE_SYS);
+		svc_mark_hot(hit_pa + off);
+	}
+
+	pr_info("sema-svc: content rescue ch=0x%x rank=%u slot=%u entry_va=0x%llx len=%u -> pa=0x%llx (payload=%u)\n",
+		ch->h_object, ch->ce_index, slot, entry_va, entry_len, hit_pa, want_payload);
+
+	return 0;
+}
+
+/*
+ * Try to read the current semaphore value from the pool for this channel,
+ * so we can initialize expected_sem_payload to current + 1.
+ */
+static void svc_init_expected_payload(struct nv_vgpu_device *vgpu,
+				      struct svc_channel *ch)
+{
+	u64 pool_va = 0x121000000ULL + 4ULL * ch->ce_index;
+	u64 pa = 0;
+	int ap = SVC_APERTURE_SYS;
+	u32 cur = 0;
+
+	if (ch->expected_sem_payload)
+		return; /* already initialized */
+
+	if (svc_lookup_trusted(vgpu, pool_va, &pa, &ap))
+		return; /* pool page not mapped yet */
+
+	if (svc_read_ap(vgpu, pa, ap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS,
+			&cur, sizeof(cur)))
+		return;
+
+	if (cur > 0 && cur < 0x100000) {
+		ch->expected_sem_payload = cur + 1;
+		pr_info("sema-svc: ch=0x%x rank=%u pool sem @0x%llx cur=%u -> expected=%u\n",
+			ch->h_object, ch->ce_index, pool_va, cur, ch->expected_sem_payload);
+	}
+}
+
 static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 {
 	/* Only ever called from the service thread. */
@@ -2677,6 +2823,9 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 			ch->logged = true;
 		}
 
+		/* Initialize expected semaphore payload from the pool (if mapped). */
+		svc_init_expected_payload(vgpu, ch);
+
 		if (put >= ch->gpfifo_entries)
 			put %= ch->gpfifo_entries;
 
@@ -2719,13 +2868,37 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 								  len, 0);
 						ch->submissions++;
 						svc.gpfifo_entries_done++;
+						/* Advance expected payload for this channel's tracking semaphore. */
+						if (ch->engine_type && ch->expected_sem_payload)
+							ch->expected_sem_payload++;
 					} else {
-						/* Keep the entry pending: the
-						 * mapping may resolve once the
-						 * page tables are filled. */
-						svc.gpfifo_nomap++;
-						kfree(pb);
-						break;
+						/*
+						 * Content rescue: the push-buffer VA is not
+						 * in our map. Scan guest RAM for a push with
+						 * the expected semaphore release for this
+						 * channel's tracking semaphore, and pin it.
+						 */
+						svc_log_stalled_va(va);
+						if (!svc_rescue_push_page(vgpu, ch, va, len)) {
+							/* Rescue succeeded, retry the read. */
+							if (!svc_read_va_trusted(vgpu, va, pb,
+										 len)) {
+								svc_decode_stream(vgpu, pb,
+											  len, 0);
+								ch->submissions++;
+								svc.gpfifo_entries_done++;
+								if (ch->engine_type && ch->expected_sem_payload)
+									ch->expected_sem_payload++;
+							} else {
+								svc.gpfifo_nomap++;
+								kfree(pb);
+								break;
+							}
+						} else {
+							svc.gpfifo_nomap++;
+							kfree(pb);
+							break;
+						}
 					}
 					kfree(pb);
 				}