sema: strong rescue signature; rescue on false translations; slot pin

The content rescue matched only the release group bytes and had matched
GSP firmware pages. It now requires the full known push shape around the
release (two SET_OBJECT class binds, the 3-word SET_SEMAPHORE_A run with
this channel's next payload, and the LAUNCH_DMA semaphore-release
follower), and it is also invoked when a trusted read resolves but the
buffer does not parse as a push (false content-scanned pair; observed
firmware/kernel-data pages), not only when the VA is unmapped. The rescue
pins the whole 1 MB push-buffer slot, which is what the UVM MEMOPS
channels' pushbuffer pool (0x1200_0000 + n*1MB) needs. New false= stat.

1 files changed, 78 insertions, 29 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 5cdca31..e6f4a62 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -315,6 +315,7 @@ struct sema_svc {
 	u64			lda_log;		/* LAUNCH_DMA inventory budget */
 	u64			entry_log;		/* per-push summary budget */
 	u64			ops_denied;		/* work refused: implausible stream */
+	u64			false_trans;		/* resolvable VA mapped to non-push */
 	u64			releases_seen;
 	u64			releases_applied;
 	u64			releases_missed;	/* no VA->PA entry */
@@ -1461,10 +1462,11 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu)
 		svc.releases_stale, svc.releases_jumped,
 		svc.walks_ok, svc.walks_ok + svc.walks_miss, svc.lat_hits,
 		svc_scan_cursor);
-	pr_info("sema-svc:   pages=%llu pushes=%llu pending=%llu nomap=%llu memset=%llu/%llu/%llu ringops=%llu denied=%llu\n",
+	pr_info("sema-svc:   pages=%llu pushes=%llu pending=%llu nomap=%llu memset=%llu/%llu/%llu ringops=%llu denied=%llu false=%llu\n",
 		svc.pages_decoded, svc.pushes_seen, svc.pushes_pending,
 		svc.pushes_nomap, svc.memsets_done, svc.memsets_nomap,
-		svc.memsets_fail, svc.ring_ops, svc.ops_denied);
+		svc.memsets_fail, svc.ring_ops, svc.ops_denied,
+		svc.false_trans);
 	pr_info("sema-svc:   gpfifo done=%llu nomap=%llu vidmem_writes=%llu ring_auto=%u pb=%u/%u\n",
 		svc.gpfifo_entries_done, svc.gpfifo_nomap,
 		svc.vidmem_writes, svc_ring_n, svc_pb_n, svc_pb_passes);
@@ -1714,6 +1716,26 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops,
 }
 
 /*
+ * Does this buffer start like a push? The discovered VA->PA map is
+ * content-scanned and can hand a push VA an unrelated page (observed: GSP
+ * firmware and kernel data pages). A real push starts with a method at
+ * address 0 (SET_OBJECT / INC_METHOD class bind) with a count that fits
+ * the buffer.
+ */
+static bool svc_method_stream_plausible(const u8 *buf, u32 len)
+{
+	u32 hdr = 0, op, count;
+
+	if (len < 8)
+		return false;
+	memcpy(&hdr, buf, 4);
+	op = (hdr >> 29) & 0x7;
+	count = (hdr >> 16) & 0xfff;
+	return op == 1 && count && count <= 64 &&
+	       4 + count * 4 <= len && ((hdr & 0x1fff) << 2) == 0;
+}
+
+/*
  * Decode one page as a method stream and execute the pushes it contains.
  *
  * Push buffers are written back to back and pushes can span page
@@ -1742,18 +1764,9 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 	 * addr 0, sane count). Releases from such a page still apply, so a
 	 * genuinely shifted stream is not lost.
 	 */
-	if (allow_ops) {
-		u32 hdr = 0, op = 0, count = 0;
-
-		if (len >= 8)
-			memcpy(&hdr, buf, 4);
-		op = (hdr >> 29) & 0x7;
-		count = (hdr >> 16) & 0xfff;
-		if (op != 1 || !count || count > 64 ||
-		    4 + count * 4 > len || ((hdr & 0x1fff) << 2) != 0) {
-			svc.ops_denied++;
-			allow_ops = false;
-		}
+	if (allow_ops && !svc_method_stream_plausible(buf, len)) {
+		svc.ops_denied++;
+		allow_ops = false;
 	}
 
 	memset(&st, 0, sizeof(st));
@@ -3098,12 +3111,15 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu,
 		return -ENOMEM;
 
 	/*
-	 * Scan guest RAM for a push with the expected semaphore release.
-	 * The push format at the semaphore release (CE path) has:
-	 *   offset 0x10: hdr (count=3, addr_lo=0x90)
-	 *   offset 0x14: hi (== 1)
-	 *   offset 0x18: lo (slot in low 12 bits)
-	 *   offset 0x1c: payload
+	 * Scan guest RAM for a page that starts with the known 40-byte release
+	 * push shape:
+	 *   +0x00 SET_OBJECT (class bind)
+	 *   +0x08 SET_OBJECT (class bind, address 0)
+	 *   +0x10 SET_SEMAPHORE_A run of 3 (hdr, hi==1, lo=slot)
+	 *   +0x1c payload == this channel's next expected value
+	 *   +0x20 LAUNCH_DMA with semaphore-release transfer type
+	 * The old matcher checked only the release group and matched GSP
+	 * firmware pages; requiring the full push shape around it does not.
 	 */
 	for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SVC_SCAN_CHUNK) {
 		u32 o;
@@ -3111,21 +3127,33 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu,
 		if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK))
 			continue;
 		for (o = 0; o < SVC_SCAN_CHUNK; o += SEMA_PAGE_SIZE) {
-			u32 hdr, hi, lo, payload;
+			u32 w0, w2, hdr, hi, lo, payload, w5, v5;
 
+			memcpy(&w0, buf + o, 4);
+			memcpy(&w2, buf + o + 8, 4);
 			memcpy(&hdr, buf + o + 0x10, 4);
-			if ((((hdr >> 16) & 0xff) != CE_SEMA_HDR_COUNT) ||
-			    ((hdr & 0xff) != CE_SEMA_HDR_ADDR))
-				continue;
 			memcpy(&hi, buf + o + 0x14, 4);
 			memcpy(&lo, buf + o + 0x18, 4);
 			memcpy(&payload, buf + o + 0x1c, 4);
+			memcpy(&w5, buf + o + 0x20, 4);
+			memcpy(&v5, buf + o + 0x24, 4);
+			if (w0 != 0x20010000u)
+				continue;
+			if ((w2 >> 29) != 1 || ((w2 >> 16) & 0xfff) != 1 ||
+			    ((w2 & 0x1fff) << 2) != 0)
+				continue;
+			if (hdr != 0x20038090u)
+				continue;
 			if (hi != 1)
 				continue;
 			if ((lo & 0xfff) != slot)
 				continue;
 			if (payload != want_payload)
 				continue;
+			if ((w5 >> 29) != 1 || ((w5 >> 13) & 0x7) != 4 ||
+			    ((w5 & 0x1fff) << 2) != 0x300 ||
+			    (v5 & 0x3) != 0)
+				continue;
 
 			hit_pa = (gpa + o) & ~(SEMA_PAGE_SIZE - 1);
 			found = 1;
@@ -3301,8 +3329,25 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 				u8 *pb = kmalloc(len, GFP_KERNEL);
 
 				if (pb) {
-					if (!svc_read_va_trusted(vgpu, va, pb,
-								 len)) {
+					bool unreadable =
+						svc_read_va_trusted(vgpu, va,
+								    pb, len) != 0;
+
+					/*
+					 * A resolved buffer that does not start
+					 * like a push is a false VA->PA pairing
+					 * (content-scanned map; observed firmware
+					 * and kernel data pages). Handle it like
+					 * an unreadable push so the content rescue
+					 * can find the real page.
+					 */
+					if (!unreadable &&
+					    !svc_method_stream_plausible(pb, len)) {
+						svc.false_trans++;
+						unreadable = true;
+					}
+
+					if (!unreadable) {
 						/*
 						 * Fills/copies execute only for CE
 						 * channels: the push was written for the
@@ -3326,11 +3371,15 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 					} else {
 						/*
 						 * Content rescue: the push-buffer VA is not
-						 * in our map. Scan guest RAM for a push with
-						 * the expected semaphore release for this
-						 * channel's tracking semaphore, and pin it.
+						 * in our map. Scan guest RAM for a page
+						 * that parses as the release push with this
+						 * channel's next expected payload, and pin
+						 * its whole slot.
 						 */
 						svc_log_stalled_va(va);
+						if (svc.false_trans && svc.false_trans < 8)
+							pr_info("sema-svc: false translation for push va=0x%llx (h=0x%x len=%u) - rescuing\n",
+								va, ch->h_object, len);
 						if (!svc_rescue_push_page(vgpu, ch, va, len)) {
 							/* Rescue succeeded, retry the read. */
 							if (!svc_read_va_trusted(vgpu, va, pb,