r506w kill the r505/r506 post-DEVICE grind: bounded op-failure latch + exact release routing

Two independent defects turned one unresolvable virtual op into a
client-wide hang (r505 multikernel, r506 matmul: 'client at DEVICE
forever' while the pump ground its stream at ~10 s per push):

- an op that keeps failing -ENOENT re-runs on every decode pass and
  re-set svc.op_tr_pending each time, so its push held for the full
  SVC_OPDEF_HOLD_MS opdef bound every pass (r506 witness: the RM's
  channel-progress zeroing at 0x2000c0xxx, 1913 failed attempts, 65 ring
  entries in ~90 s). New per-VA failure latch: after 8 failures the op
  still attempts its write but stops forcing the hold; the entry consumes
  and its withheld release applies - exactly the outcome the expired
  bound already sanctioned, minus the 10 s tax.

- svc_rel_apply_launch routed EVERY release through the exact-slot
  counter fallback (release_pa ?: counter_pa), which writes (and
  forward-only-skips against) the channel's registered completion slot -
  a different word. The tracker releases to 0x20480fef0 were skipped
  against the polluted PUT counter (cur=96) and lost: rel-apply
  h=0x5c000059 pa=0x10809def0 pay=1 cur=96 -> skip. The fallback is now
  restricted to counter VAs; generic releases are forward-only by value
  (payload > fresh cur) so a decode blinded by a stale landing can no
  longer drop them.

1 files changed, 75 insertions, 6 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 35abb15..83bae47 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -7527,6 +7527,54 @@ static struct svc_wait_hold *svc_wait_hold_get(u32 h, bool create)
 	return free_slot;
 }
 
+/*
+ * r506w: bounded per-VA failure latch for virtual ops. A virtual op that
+ * keeps failing -ENOENT (an unresolvable VA) re-runs on every decode pass
+ * and sets svc.op_tr_pending each time, so its push holds for the full
+ * SVC_OPDEF_HOLD_MS opdef bound - a client whose stream carries such an op
+ * (r505/r506 witness: the RM's channel-progress zeroing at 0x2000c0xxx
+ * after the first client, dst never translated) ground its whole stream
+ * at ~10 s per push ("hang after DEVICE" for matmul/multikernel).
+ * After SVC_OPFAIL_LATCH failures of the same VA the op still attempts
+ * the write but no longer forces the hold: the entry consumes promptly and
+ * its withheld release is applied, exactly the outcome the expired bound
+ * already sanctioned (op stays skipped, release delivered), minus the
+ * 10 s tax per pass. Service-thread only, like every other op path.
+ */
+#define SVC_OPFAIL_LATCH	8
+#define SVC_OPFAIL_MAX		16
+static struct {
+	u64 va;
+	u32 n;
+} svc_opfail[SVC_OPFAIL_MAX];
+static u32 svc_opfail_log;
+
+static bool svc_opfail_latched(u64 va)
+{
+	u32 i;
+	u32 free_i = SVC_OPFAIL_MAX;
+
+	for (i = 0; i < SVC_OPFAIL_MAX; i++) {
+		if (svc_opfail[i].va == va) {
+			svc_opfail[i].n++;
+			if (svc_opfail[i].n == SVC_OPFAIL_LATCH &&
+			    svc_opfail_log < 32) {
+				svc_opfail_log++;
+				pr_info("sema-svc: OPFAIL-LATCH va=0x%llx n=%u (op no longer holds its entry)\n",
+					va, svc_opfail[i].n);
+			}
+			return svc_opfail[i].n >= SVC_OPFAIL_LATCH;
+		}
+		if (!svc_opfail[i].va && free_i == SVC_OPFAIL_MAX)
+			free_i = i;
+	}
+	if (free_i < SVC_OPFAIL_MAX) {
+		svc_opfail[free_i].va = va;
+		svc_opfail[free_i].n = 1;
+	}
+	return false;
+}
+
 static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops,
 			u32 n_ops, u32 ch_h, u64 *ops_done, u32 ops_base,
 			u32 entry_idx, u64 pass_hash)
@@ -7817,7 +7865,8 @@ svc_wait_done:
 		if (rc != 0 && rc != SVC_COPY_FORWARDED &&
 		    (m->bytes >= (256u << 10) ||
 		     (m->type == 1 && m->dst >= SVC_VAS_FLOOR) ||
-		     (m->type == 0 && !m->phys && m->dst >= SVC_VAS_FLOOR)))
+		     (m->type == 0 && !m->phys && m->dst >= SVC_VAS_FLOOR)) &&
+		    !svc_opfail_latched(m->dst))
 			svc.op_tr_pending = 1;
 		if (gsp_sema_dbg && m->bytes == 256 && svc.sem256_log < 64) {
 			svc.sem256_log++;
@@ -8777,7 +8826,17 @@ static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu,
 					  inc && inc_apply);
 		return false;
 	}
-	if (pending || inc_apply) {
+	/*
+	 * r506w: generic absolute releases are forward-only by VALUE, whichever
+	 * pass discovers the gap. The recorded pending bit came from the
+	 * decode's read of the release word, and that read can be blinded by a
+	 * stale/crossed landing (r506 witness: a withheld tracker release went
+	 * to the flush with pending=0 while the fresh read still showed 0, and
+	 * the release was then dropped by the counter fallback below - the
+	 * client's tracker never advanced and its stream ground at the 10 s
+	 * bound per push). Increments keep the one-shot claim gate (`inc_apply`).
+	 */
+	if (pending || inc_apply || (!inc && payload > cur)) {
 		svc.pushes_pending++;
 		if (inc)
 			cur += payload;
@@ -8799,10 +8858,20 @@ static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu,
 			return true;
 		}
 	}
-	if (ch->engine_type == 0)
-		svc_counter_learn(vgpu, ch->h_object, sem_va);
-	svc_counter_release_apply(vgpu, ch->h_object, sem_va, payload,
-				  inc && inc_apply);
+	/*
+	 * r506w: the exact-slot counter fallback is for COUNTER VAs only. Its
+	 * `release_pa ?: counter_pa` fallback writes (and forward-only-skips
+	 * against) the channel's REGISTERED slot, which belongs to a different
+	 * word - the r506 witness: releases to the tracker VA 0x20480fef0 were
+	 * skipped against the polluted PUT counter (cur=96) and lost. For a
+	 * non-counter VA the generic write above is the delivery path.
+	 */
+	if (svc_counter_va(sem_va)) {
+		if (ch->engine_type == 0)
+			svc_counter_learn(vgpu, ch->h_object, sem_va);
+		svc_counter_release_apply(vgpu, ch->h_object, sem_va, payload,
+					  inc && inc_apply);
+	}
 	/*
 	 * Stall-class fix (R2): when the decode said this absolute release is
 	 * genuinely pending and every write attempt above failed without