vgsp-gr r607: the stream-order watermark must watch EVERY ring, not engine_type!=0 (final form of r590b, on the r606 red-draw capture)

The r606 fresh-boot first mail client drew the classic red VADD
(bad=32735 csum=8139264, c = a + 0) with the peek armed, and the capture
names the blindness exactly:

  362177.927 GR-DEC push=0x20043c394 dropped=9 partial=1 dstpend=0
             copypend=0 ceback=0 defer_exp=0
  362178.001 peek-backlog h=0x5c000059 eng=0 put=95 last=92
  362182.145 peek-backlog h=0x5c000059 eng=0 put=95 last=92
  362182.556 GR LAUNCH submitted (one-shot arg-defer retry)
  362182.882 ce-soft-copy va=d_b executed - 326 ms AFTER the forward
  -> VADD bad=32735 csum=8139264

The driver's H2D relocation copy sat three entries behind on the
COMPUTE-class channel h=0x5c000059 (eng=0) for 4.7 s, held by
'sema-pend: hold h=0x5c000059 idx=90 ... (operand pending)' - the wait
group (0x20480fff0 pay=84) passed every pass, so the only guard blindness
was the engine_type filter: ceback=0 while the copy channel was provably
behind. The r590 watermark now considers every other channel of the
device (the launching channel itself is excluded: its ring legitimately
runs ahead with future entries). Same pass snapshot, same bounded
busy-defer; this guard never refuses - the r586 fail-closed branch is
separate and fires only when argument pages never resolve. r595's
GR-class ops admission has since cut hold resolution to 0.5-4.7 s (the
captured 4.7 s), well inside the 15 s cap, so the deferral ends by
forwarding with the copy executed.

Companion to r605 (newest-registered-first walk within the class): r605
drains consumable published entries before the launch decision in the
same pass; r607 defers the decision while any other ring still holds
unconsumed entries (held chains included).

1 files changed, 39 insertions, 11 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index c5fafa1..f90ed2c 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -21152,15 +21152,42 @@ static struct svc_channel *svc_pass_local;
 static u32 svc_pass_local_n;
 
 /*
- * r590: does any CE ring of this device still hold entries this pass's pump
- * has not consumed? Reads the ring's live GP_PUT from its own USERD and
- * compares it (modulo the ring size) with the snapshot's last_get. A
- * mismatch means the client submitted copies/fills the pump has not decoded
- * yet - exactly the stream-order backlog the r541/r568 guards cannot see.
+ * r590: does any ring of this device still hold entries this pass's pump has
+ * not consumed? Reads the ring's live GP_PUT from its own USERD and compares
+ * it (modulo the ring size) with the snapshot's last_get. A mismatch means
+ * the client submitted copies/fills the pump has not decoded yet - exactly
+ * the stream-order backlog the r541/r568 guards cannot see.
+ *
+ * r607 (r606 red-draw capture, final form of r590b): the engine_type==0
+ * filter skipped the channel the guard exists for. The driver's H2D
+ * relocation copies ride a COMPUTE-class channel (h=0x5c000059, eng=0 -
+ * the r595 finding), and the captured red draw shows the launch forwarding
+ * while that channel held three published, unconsumed entries behind an
+ * opdef operand hold:
+ *   362177.927 GR-DEC push=0x20043c394 dropped=9 partial=1 dstpend=0
+ *              copypend=0 ceback=0
+ *   362178.001 peek-backlog h=0x5c000059 eng=0 put=95 last=92
+ *   362182.145 peek-backlog h=0x5c000059 eng=0 put=95 last=92
+ *   362182.556 GR LAUNCH submitted (forwarded at the one-shot arg-defer
+ *              retry)  -> VADD bad=32735 csum=8139264 (c = a + 0)
+ *   362182.882 ce-soft-copy va=d_b executed, 326 ms AFTER the forward
+ * The chain was held by `sema-pend: hold h=0x5c000059 idx=90 ... (operand
+ * pending)`; the wait group (0x20480fff0 pay=84) passed every pass, so the
+ * only blindness was the class filter - ceback=0 while the copy channel
+ * was three entries behind. Consider every other channel of the device;
+ * the launching channel itself is excluded (its own ring legitimately runs
+ * ahead with future entries). r590's validated part is unchanged: same
+ * pass snapshot, same bounded busy-defer, historical forward at the
+ * absolute cap (this guard itself never refuses; the r586 fail-closed
+ * path is a separate branch that fires only when argument pages never
+ * resolve). r595's GR-class ops admission has since cut the hold
+ * resolution to 0.5-4.7 s (the captured 4.7 s), well inside the 15 s cap.
+ *
  * Channels whose USERD is unreadable are skipped (their work cannot be
  * ordered by this mechanism anyway).
  */
-static bool svc_ce_stream_backlog(struct nv_vgpu_device *vgpu)
+static bool svc_ce_stream_backlog(struct nv_vgpu_device *vgpu,
+				  const struct svc_channel *self)
 {
 	u32 k;
 
@@ -21170,8 +21197,9 @@ static bool svc_ce_stream_backlog(struct nv_vgpu_device *vgpu)
 		const struct svc_channel *c = &svc_pass_local[k];
 		u32 put = 0;
 
-		if (!c->valid || (vgpu && c->vgpu != vgpu) ||
-		    !c->engine_type)
+		if (!c->valid || (vgpu && c->vgpu != vgpu))
+			continue;
+		if (self && c->h_object == self->h_object)
 			continue;
 		if (!c->gpfifo_entries || !c->userd_pa)
 			continue;
@@ -21297,7 +21325,7 @@ static void svc_gr_dec_dump(const struct svc_channel *ch)
 
 	dstpend = svc_gr_dstpend_overlap();
 	copypend = svc_copy_pending();
-	ceback = svc_ce_stream_backlog(NULL);
+	ceback = svc_ce_stream_backlog(NULL, ch);
 
 	pr_info("vgsp-gr: GR-DEC h=0x%x push=0x%llx n_data=%u dropped=%u partial=%d stalled=%d dstpend=%d copypend=%d ceback=%d defer_exp=%d\n",
 		ch->h_object, svc_gr_cand.push_va, svc_fwd_req.gr_n_data,
@@ -21443,7 +21471,7 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu,
 	if (gsp_phys_gr_dec_log_max &&
 	    (svc_fwd_req.gr_dropped > 0 || svc_fwd_req.gr_arg_partial ||
 	     svc_gr_data_stalled || svc_gr_dstpend_overlap() ||
-	     svc_copy_pending() || svc_ce_stream_backlog(vgpu)))
+	     svc_copy_pending() || svc_ce_stream_backlog(vgpu, ch)))
 		svc_gr_dec_dump(ch);
 	/*
 	 * r541: launch-ordering guard. The fill's data pages are what the
@@ -21527,7 +21555,7 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu,
 	 * it never refuses one, so it cannot recreate r588's mail refusal.
 	 */
 	svc_gr_peek_lookahead(vgpu, ch);
-	if (svc_ce_stream_backlog(vgpu)) {
+	if (svc_ce_stream_backlog(vgpu, ch)) {
 		if (!svc_gr_defer_expired(svc_gr_cand.push_va)) {
 			clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state);
 			mutex_unlock(&svc_fwd_lock);