vgsp-sema r588: copy-pending sees a CE ring's undecoded backlog (stream-order)

The r568 copy-pending guard in svc_gr_request could only see decoded
state: an active entry hold (svc_copy_pending) or a recorded device-write
target (svc_dstpend_overlap). A CE copy entry sitting BEHIND a
translation-held entry in the ring has neither - it was never decoded -
so copypend read 0 while the ring still owed that copy. A GR launch
decoded in that window captured the copy's destination before its data
landed: r587 mail VADD computed c = a + 0 (h_c == h_a exactly: bad =
32735 = 32768 - 33, the count of i where b[i] == 0; csum = sum(a) =
8139264), because the d_b H2D bigcopy executed after the launch's
copy-in.

Extend svc_copy_pending() (now per-device) to also report a CE channel
whose ring has submitted-but-unconsumed entries (last_get != put_prev -
the same outstanding relation the completion publish uses). With the
r507o CE-before-GR pass order the CE channel's write-back for the pass
has already run when the GR decision reads it, so the guard defers
(bounded by the existing defer/expiry machinery) until the ring drains
and the re-fill captures the post-copy pages.

1 files changed, 40 insertions, 4 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index e86c463..23d456e 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -21073,7 +21073,26 @@ static bool svc_gr_dstpend_overlap(void)
  * copy delays a launch by the cap and no more. GR channels are excluded
  * (a launch's own withheld release must not defer itself).
  */
-static bool svc_copy_pending(void)
+/*
+ * True while any CE channel of this device still OWES work: either an
+ * entry hold is active on it (the original opdef check, r568) or its
+ * ring has submitted-but-unconsumed entries (last_get != put_prev).
+ *
+ * The r568 guard in svc_gr_request was built for the mail b-copy class
+ * but could not see its live instance (r587): the h_b copy entry sat
+ * BEHIND a translation-held CE entry, had never been decoded, so no
+ * opdef hold existed at the decision instant and copypend read 0; the
+ * launch's copy-in then captured d_b's pre-copy zeros and the engine
+ * computed c = a + 0 (h_c == h_a exactly: bad = 32735 = 32768 - 33,
+ * the count of i where b[i] == 0; csum = sum(a) = 8139264). A CE
+ * channel with last_get != put_prev has entries the pump has not
+ * consumed yet - the same "outstanding" relation the completion
+ * publish uses (svc_counter_publish) - and with the r507o CE-before-GR
+ * pass order its write-back for this pass has already run when the GR
+ * decision reads it. Bounded by the same defer/expiry machinery as
+ * every other copy-pending refusal.
+ */
+static bool svc_copy_pending(struct nv_vgpu_device *vgpu)
 {
 	unsigned long flags;
 	bool pending = false;
@@ -21095,6 +21114,23 @@ static bool svc_copy_pending(void)
 			}
 		spin_unlock_irqrestore(&svc.chan_lock, flags);
 	}
+	if (pending)
+		return true;
+
+	spin_lock_irqsave(&svc.chan_lock, flags);
+	for (c = 0; c < SVC_MAX_CHANNELS; c++) {
+		struct svc_channel *chan = &svc.chan[c];
+
+		if (!chan->valid || chan->vgpu != vgpu ||
+		    !chan->engine_type)
+			continue;
+		if (chan->last_get != chan->put_prev) {
+			pending = true;
+			break;
+		}
+	}
+	spin_unlock_irqrestore(&svc.chan_lock, flags);
+
 	return pending;
 }
 
@@ -21121,7 +21157,7 @@ static void svc_gr_dec_dump(const struct svc_channel *ch)
 	svc_gr_dec_log++;
 
 	dstpend = svc_gr_dstpend_overlap();
-	copypend = svc_copy_pending();
+	copypend = svc_copy_pending(ch->vgpu);
 
 	pr_info("vgsp-gr: GR-DEC h=0x%x push=0x%llx n_data=%u dropped=%u partial=%d stalled=%d dstpend=%d copypend=%d defer_exp=%d\n",
 		ch->h_object, svc_gr_cand.push_va, svc_fwd_req.gr_n_data,
@@ -21266,7 +21302,7 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu,
 	if (gsp_phys_gr_dec_log_max &&
 	    (svc_fwd_req.gr_dropped > 0 || svc_fwd_req.gr_arg_partial ||
 	     svc_gr_data_stalled || svc_gr_dstpend_overlap() ||
-	     svc_copy_pending()))
+	     svc_copy_pending(vgpu)))
 		svc_gr_dec_dump(ch);
 	/*
 	 * r541: launch-ordering guard. The fill's data pages are what the
@@ -21331,7 +21367,7 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu,
 	 * (see svc_copy_pending). Covers the dropped-argument-base case the
 	 * page-level guard cannot see (the r571 mail b-copy).
 	 */
-	if (svc_copy_pending() &&
+	if (svc_copy_pending(vgpu) &&
 	    (svc_fwd_req.gr_arg_partial || svc_fwd_req.gr_dropped > 0) &&
 	    !svc_gr_defer_expired(svc_gr_cand.push_va)) {
 		clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state);