vgsp: fail4 r259-r262 - capture ring (4 slots) + diagnostic budgets; CE auto-advance tried and reverted

1. svc_gr_cand is now a 4-slot ring addressed by an accessor macro
   (svc_gr_cand -> ring[pos]); captures that would overwrite a
   DIFFERENT launch's unconsumed candidate move to a free slot, and
   svc_gr_request selects this channel's oldest unconsumed candidate
   before consuming; the defer-arm gate checks any unconsumed
   candidate. Struct tag renamed svc_gr_cand_rec to keep the macro
   safe. (The streams2 cross survives this - it is not (only) the
   single slot; see the new evidence in the handoff.)

2. Diagnostic budgets raised for the triage (opadd/op_trace 2048,
   memsets rc-log 2048). These exposed the real memset shape: the
   8 MB cudaMemset is SIXTEEN grid=1024/block=512 kernel launches
   whose win0 param bank is fully correct (dst patched to the reloc,
   w0x168=0x5a5a5a5a, launches OK) - yet the D2H readback returns
   zeros, so the loss is between the engine's stores and the
   readback pages (the rec hpa/landing class, GRVAUDIT unlanded
   mass), not the params.

3. A CE-style per-op dst auto-advance was implemented, evaluated
   (no memset improvement) and reverted - re-sets are decoded and
   the risk to the green clients was not justified.

Verified on r262 (boot 1 of the acceptance round): ctaid_probe RAWA
0 1 2 3 4 5 6 7 + PROBE2 bad=0; transformer STATUS OK; streams2
bad=65536 (red); memcpy_roundtrip wedged the guest (flaky: green on
r254, wedges on r260/r262).

1 files changed, 80 insertions, 18 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index b1ad7ba..587fb96 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -4710,7 +4710,7 @@ struct svc_gr_data_win {
 	u64 reloc_base;		/* relocated window base */
 };
 
-struct svc_gr_cand {
+struct svc_gr_cand_rec {
 	bool valid;
 	bool offered;		/* consumed by a request or a fallback */
 	/*
@@ -4746,7 +4746,19 @@ struct svc_gr_cand {
 	u32 pointer_word[SVC_GR_POINTER_MAX];	/* QMD word of the pair */
 };
 
-static struct svc_gr_cand svc_gr_cand;
+/*
+ * fail4: the capture was a single slot; two concurrent launches (streams2's
+ * vadd pair on two channels) overwrote each other's candidate while the
+ * first was kept for a retry (the candidate-overwrite witness) and the
+ * engines ran crossed parameters (c1 == a2+b2). Keep up to four candidates
+ * side by side in a ring; the accessor macro addresses the current slot,
+ * so every existing use keeps working on the selected candidate, and
+ * capture/consume select the slot instead of clobbering it.
+ */
+#define SVC_GR_CAND_RING	4
+static struct svc_gr_cand_rec svc_gr_cand_ring[SVC_GR_CAND_RING];
+static u32 svc_gr_cand_pos;
+#define svc_gr_cand (svc_gr_cand_ring[svc_gr_cand_pos])
 
 static_assert(SVC_GR_QMD_WORDS == GSP_PHYS_GR_QMD_WORDS,
 	      "the sema GR QMD word count must match the backend ABI");
@@ -4782,7 +4794,7 @@ static_assert(sizeof(struct svc_gr_page) == 24,
 static u64 svc_push_hash(const u8 *p, u32 len);
 
 /* QMD field decode: the exact shape the host CUDA bridge uses. */
-static void svc_gr_qmd_decode(struct svc_gr_cand *c)
+static void svc_gr_qmd_decode(struct svc_gr_cand_rec *c)
 {
 	const u32 *q = c->qmd;
 	u32 i;
@@ -4827,6 +4839,25 @@ static void svc_gr_cand_capture(struct svc_channel *ch,
 	 * overwrite hazard. Bounded, unconditional (gsp_sema_dbg is defined
 	 * later in the file and this site may not reference it).
 	 */
+	/*
+	 * fail4: never overwrite a DIFFERENT launch's unconsumed candidate -
+	 * move to a free ring slot instead (same-push re-captures still
+	 * refresh in place).
+	 */
+	if (svc_gr_cand.valid && !svc_gr_cand.offered &&
+	    svc_gr_cand.push_va != push_va) {
+		u32 ri;
+
+		for (ri = 1; ri < SVC_GR_CAND_RING; ri++) {
+			u32 rp = (svc_gr_cand_pos + ri) % SVC_GR_CAND_RING;
+
+			if (!svc_gr_cand_ring[rp].valid ||
+			    svc_gr_cand_ring[rp].offered) {
+				svc_gr_cand_pos = rp;
+				break;
+			}
+		}
+	}
 	if (svc_gr_cand.valid && svc_cand_ovw_log < 64) {
 		svc_cand_ovw_log++;
 		pr_info("sema-gr: CAND-OVERWRITE old_h=0x%x old_push=0x%llx new_push=0x%llx\n",
@@ -4932,8 +4963,14 @@ static bool svc_fwd_defer_arm(void)
 {
 	if (!gsp_phys_guest_mem_enabled())
 		return false;
-	if (svc_gr_cand.valid && gsp_phys_gr_forward_enabled())
-		return true;
+	if (gsp_phys_gr_forward_enabled()) {
+		u32 ri;
+
+		for (ri = 0; ri < SVC_GR_CAND_RING; ri++)
+			if (svc_gr_cand_ring[ri].valid &&
+			    !svc_gr_cand_ring[ri].offered)
+				return true;
+	}
 	return svc_fwd_cand.valid &&
 	       gsp_phys_work_defer_release_enabled();
 }
@@ -6186,7 +6223,7 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops,
 		}
 		if (gsp_sema_dbg &&
 		    ((rc == 0 && svc.memsets_done <= gsp_sema_ops_log) ||
-		     (rc != 0 && svc.memsets_fail + svc.memsets_nomap <= 64)))
+		     (rc != 0 && svc.memsets_fail + svc.memsets_nomap <= 2048)))
 			pr_info("sema-svc: %s dst=0x%llx value=0x%llx elem=%u count=%u bytes=%llu phys=%u ap=%u src=0x%llx srcphys=%u srcap=%u rc=%d h=0x%x\n",
 				m->type == 1 ? "memcopy" : "memset",
 				m->dst, m->value, m->elem, m->count,
@@ -6204,7 +6241,7 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops,
 		 * read-back signature: always keep room for it past the
 		 * general trace budget so the DtoH's src/dst/rc is on record.
 		 */
-		if ((svc.op_trace_log < 160 &&
+		if ((svc.op_trace_log < 2048 &&
 		     (time_before(jiffies, svc.op_trace_until) ||
 		      m->dst >= SVC_VAS_FLOOR ||
 		      (m->type == 1 && !m->src_phys && m->src >= SVC_VAS_FLOOR))) ||
@@ -7424,7 +7461,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 	svc_fwd_defer_armed = false;
 	svc.pages_decoded++;
 	/* Diagnostic: name every push decoded in the post-completion window. */
-	if (svc.op_trace_log < 160 &&
+	if (svc.op_trace_log < 2048 &&
 	    time_before(jiffies, svc.op_trace_until)) {
 		svc.op_trace_log++;
 		pr_info("sema-pushtrace: h=0x%x eng=%u va=0x%llx len=%u allow=%d trusted=%d\n",
@@ -8178,7 +8215,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 			}
 			/* Diagnostic: full operand state in the post-completion
 			 * window (or for high device VAs), bounded. */
-			if (svc.op_trace_log < 160 && ch->h_object &&
+			if (svc.op_trace_log < 2048 && ch->h_object &&
 			    (time_before(jiffies, svc.op_trace_until) ||
 			     st.offset_out >= SVC_VAS_FLOOR ||
 			     st.offset_in >= SVC_VAS_FLOOR)) {
@@ -8300,16 +8337,26 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 			 * executor's population is on record even when the
 			 * 64 KB copy witness below never fires for it.
 			 */
-			if (n_ops != n_ops_before && svc.opadd_log < 256) {
+			if (n_ops != n_ops_before) {
 				struct svc_memset_op *lm = &ops[n_ops - 1];
 
-				svc.opadd_log++;
-				pr_info("sema-opadd: h=0x%x type=%u dst=0x%llx src=0x%llx bytes=%llu elem=%u count=%u ap=%u src_ap=%u phys=%u src_phys=%u\n",
-					ch->h_object, lm->type, lm->dst,
-					lm->src, lm->bytes, lm->elem,
-					lm->count, lm->aperture,
-					lm->src_aperture, lm->phys,
-					lm->src_phys);
+				/*
+				 * fail4 A/B note: a CE-style auto-advance of
+				 * st.offset_out per op was tried here and
+				 * reverted - it did not change the memset
+				 * coverage and the driver's per-op re-sets are
+				 * already decoded, so the executed dsts match
+				 * the stream.
+				 */
+				if (svc.opadd_log < 2048) {
+					svc.opadd_log++;
+					pr_info("sema-opadd: h=0x%x type=%u dst=0x%llx src=0x%llx bytes=%llu elem=%u count=%u ap=%u src_ap=%u phys=%u src_phys=%u\n",
+						ch->h_object, lm->type,
+						lm->dst, lm->src, lm->bytes,
+						lm->elem, lm->count,
+						lm->aperture, lm->src_aperture,
+						lm->phys, lm->src_phys);
+				}
 			}
 
 			/*
@@ -16329,8 +16376,23 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu,
 	 * The candidate is per-channel: a capture left unconsumed by an
 	 * earlier pass (e.g. the entry was held before its request) must never
 	 * be queued against a different channel's handle by that channel's
-	 * own end-of-pass request call.
+	 * own end-of-pass request call. fail4: with several reserved ring
+	 * slots, select THIS channel's oldest unconsumed candidate first.
 	 */
+	if (gsp_phys_gr_forward_enabled()) {
+		u32 ri;
+
+		for (ri = 0; ri < SVC_GR_CAND_RING; ri++) {
+			u32 rp = (svc_gr_cand_pos + ri) % SVC_GR_CAND_RING;
+
+			if (svc_gr_cand_ring[rp].valid &&
+			    !svc_gr_cand_ring[rp].offered &&
+			    svc_gr_cand_ring[rp].h_object == ch->h_object) {
+				svc_gr_cand_pos = rp;
+				break;
+			}
+		}
+	}
 	if (!gsp_phys_gr_forward_enabled() || !svc_gr_cand.valid ||
 	    svc_gr_cand.h_object != ch->h_object)
 		return false;