vgsp-gr r590: CE submission watermark - read the ring's live GP_PUT against this pass's consumption before forwarding a launch

The r589 mail VADD drew c = a + 0: the h_b H2D entries sat behind a
translation-held CE entry while the pump's last read of the ring's
GP_PUT was 92, so the GR launch (processed later in the same pass)
forwarded against the still-zero b window. The r541 guard needs a
decoded copy and the r568 opdef gate needs a decoded hold; an entry the
pump has not reached yet is invisible to both.

Record this pass's channel snapshot (CE channels first), read each CE
ring's live USERD GP_PUT at the GR decision, and defer while it differs
from the snapshot's consumed position. Both sides of the comparison come
from the same channel and the same pass, so it cannot reproduce r588's
stale-epoch over-fire; the guard only defers (bounded, witnessed) and at
the absolute cap the historical forward stands, never a refusal.

1 files changed, 111 insertions, 3 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index e86c463..a0cdb2b 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -1117,6 +1117,11 @@ struct sema_svc {
 	u64			fwd_gr_argdefer;	/* r543: held for a partial
 						 * argument set (unresolved arg) */
 	u32			fwd_gr_argdefer_log;
+	u64			fwd_gr_cewm;	/* r590: held on the CE ring's
+						 * live submission watermark */
+	u64			fwd_gr_cewm_cap;	/* r590: watermark still
+						 * backlogged at the defer cap */
+	u32			fwd_gr_cewm_log;
 	u64			fwd_gr_timeout;
 	u64			fwd_gr_fault;
 	u64			fwd_gr_bridge;
@@ -21099,6 +21104,51 @@ static bool svc_copy_pending(void)
 }
 
 /*
+ * r590: this pass's channel snapshot, so the CE watermark guard can read the
+ * pump's CURRENT consumption position (local.last_get) instead of the live
+ * table's previous-pass merge. The pump is single-threaded and walks CE
+ * channels before GR ones, so a GR decision always sees every CE channel's
+ * consumed position for this pass.
+ */
+static struct svc_channel *svc_pass_local;
+static u32 svc_pass_local_n;
+
+/*
+ * r590: does any CE ring of this device still hold entries this pass's pump
+ * has not consumed? Reads the ring's live GP_PUT from its own USERD and
+ * compares it (modulo the ring size) with the snapshot's last_get. A
+ * mismatch means the client submitted copies/fills the pump has not decoded
+ * yet - exactly the stream-order backlog the r541/r568 guards cannot see.
+ * Channels whose USERD is unreadable are skipped (their work cannot be
+ * ordered by this mechanism anyway).
+ */
+static bool svc_ce_stream_backlog(struct nv_vgpu_device *vgpu)
+{
+	u32 k;
+
+	if (!svc_pass_local || !svc_pass_local_n)
+		return false;
+	for (k = 0; k < svc_pass_local_n; k++) {
+		const struct svc_channel *c = &svc_pass_local[k];
+		u32 put = 0;
+
+		if (!c->valid || (vgpu && c->vgpu != vgpu) ||
+		    !c->engine_type)
+			continue;
+		if (!c->gpfifo_entries || !c->userd_pa)
+			continue;
+		if (svc_read_ap(vgpu ? vgpu : c->vgpu,
+				c->userd_pa + SVC_USERD_GP_PUT,
+				c->userd_as, &put, sizeof(put)))
+			continue;
+		put %= c->gpfifo_entries;
+		if (put != ((u32)c->last_get % c->gpfifo_entries))
+			return true;
+	}
+	return false;
+}
+
+/*
  * r576 DIAG: the GR launch-decision witness (gsp_phys_gr_dec_log=N budget
  * per boot). Logged from svc_gr_request right after the fill, for launches
  * whose fill saw dropped pages / a partial argument set or whose decision
@@ -21113,7 +21163,7 @@ static unsigned svc_gr_dec_log;
 
 static void svc_gr_dec_dump(const struct svc_channel *ch)
 {
-	bool dstpend, copypend;
+	bool dstpend, copypend, ceback;
 	u32 i, n = 0;
 
 	if (svc_gr_dec_log >= gsp_phys_gr_dec_log_max)
@@ -21122,13 +21172,15 @@ static void svc_gr_dec_dump(const struct svc_channel *ch)
 
 	dstpend = svc_gr_dstpend_overlap();
 	copypend = svc_copy_pending();
+	ceback = svc_ce_stream_backlog(NULL);
 
-	pr_info("vgsp-gr: GR-DEC h=0x%x push=0x%llx n_data=%u dropped=%u partial=%d stalled=%d dstpend=%d copypend=%d defer_exp=%d\n",
+	pr_info("vgsp-gr: GR-DEC h=0x%x push=0x%llx n_data=%u dropped=%u partial=%d stalled=%d dstpend=%d copypend=%d ceback=%d defer_exp=%d\n",
 		ch->h_object, svc_gr_cand.push_va, svc_fwd_req.gr_n_data,
 		svc_fwd_req.gr_dropped,
 		svc_fwd_req.gr_arg_partial ? 1 : 0,
 		svc_gr_data_stalled ? 1 : 0,
 		dstpend ? 1 : 0, copypend ? 1 : 0,
+		ceback ? 1 : 0,
 		svc_gr_defer_expired(svc_gr_cand.push_va) ? 1 : 0);
 
 	/*
@@ -21266,7 +21318,7 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu,
 	if (gsp_phys_gr_dec_log_max &&
 	    (svc_fwd_req.gr_dropped > 0 || svc_fwd_req.gr_arg_partial ||
 	     svc_gr_data_stalled || svc_gr_dstpend_overlap() ||
-	     svc_copy_pending()))
+	     svc_copy_pending() || svc_ce_stream_backlog(vgpu)))
 		svc_gr_dec_dump(ch);
 	/*
 	 * r541: launch-ordering guard. The fill's data pages are what the
@@ -21327,6 +21379,50 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu,
 				 * candidate is owned and sw-completed */
 	}
 	/*
+	 * r590: the CE submission watermark. A launch must not be forwarded
+	 * while a CE ring still holds entries the pump has not consumed yet:
+	 * those are the client's stream-ordered copies, and an entry the
+	 * pump has not decoded cannot be seen by the r541 dstpend guard
+	 * (decoded copies only) nor by the r568 opdef gate (decoded holds
+	 * only). The r589 mail VADD drew exactly this: the h_b H2D entries
+	 * sat behind a translation-held CE entry, the pump's last read of the
+	 * ring's GP_PUT was 92 while the guest had already submitted 95, the
+	 * GR channel was processed later in the same pass and the launch was
+	 * forwarded with the b page still zero (c = a + 0).
+	 *
+	 * The watermark is read LIVE (USERD GP_PUT) against THIS pass's
+	 * consumption position (the pump runs CE channels before GR ones, so
+	 * local.last_get already includes everything this pass consumed).
+	 * r588's proxy failed because it compared the live table's stale
+	 * last_get against a put sampled in a different epoch; both sides of
+	 * this comparison come from the same channel and the same pass.
+	 *
+	 * Bounded by the normal busy-defer window; at the absolute cap the
+	 * historical forward stands (witnessed) - the guard delays a launch,
+	 * it never refuses one, so it cannot recreate r588's mail refusal.
+	 */
+	if (svc_ce_stream_backlog(vgpu)) {
+		if (!svc_gr_defer_expired(svc_gr_cand.push_va)) {
+			clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state);
+			mutex_unlock(&svc_fwd_lock);
+			svc_gr_busy_defer = true;
+			svc.fwd_gr_refused++;
+			svc.fwd_gr_dstpend++;
+			if (svc.fwd_gr_dstpend_log < 32) {
+				svc.fwd_gr_dstpend_log++;
+				pr_info("vgsp-work: GR forward deferred h=0x%x push=0x%llx (CE ring has unconsumed entries; stream-order watermark)\n",
+					ch->h_object, svc_gr_cand.push_va);
+			}
+			return false;
+		}
+		svc.fwd_gr_cewm_cap++;
+		if (svc.fwd_gr_cewm_log < 32) {
+			svc.fwd_gr_cewm_log++;
+			pr_warn("vgsp-work: GR forward at defer cap h=0x%x push=0x%llx (CE ring still has unconsumed entries; historical forward)\n",
+				ch->h_object, svc_gr_cand.push_va);
+		}
+	}
+	/*
 	 * r568: the coarse companion gate - any unexecuted CE write at all
 	 * (see svc_copy_pending). Covers the dropped-argument-base case the
 	 * page-level guard cannot see (the r571 mail b-copy).
@@ -22161,6 +22257,14 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 	 */
 	svc_opdef_drop_gone(vgpu, local, n);
 
+	/*
+	 * r590: publish this pass's snapshot for the CE submission-watermark
+	 * guard (svc_ce_stream_backlog). CE channels were copied first, so a
+	 * GR launch's decision sees the pass-current consumed positions.
+	 */
+	svc_pass_local = local;
+	svc_pass_local_n = (u32)n;
+
 	for (i = 0; i < n; i++) {
 		struct svc_channel *ch = &local[i];
 		u64 ring_pa = 0;
@@ -24745,6 +24849,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 	 */
 	if (stalled)
 		svc_apply_sweep_slice(vgpu);
+
+	/* r590: the pass snapshot is only valid while the loop runs. */
+	svc_pass_local = NULL;
+	svc_pass_local_n = 0;
 }
 
 /* ------------------------------------------------------------------ */