r484 the staging-drain completion edge: when the acquire gate holds an entry (its op may have already run on an earlier gate-less decode), force exactly one op re-run on the first pass whose gate finally passes - the pass at which the dependency just became installed. Without it the once-only done bitmap suppressed the op and the release applied while the staging still held the pre-dependency bytes (streams_events s=2: the D2H read dc2 before the vadd_into wrote it, ran early, was skipped when the wait passed, and the guest drained a2+b2 instead of a2+b2+c1)

1 files changed, 34 insertions, 2 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index fe68f60..b328eea 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -4136,6 +4136,23 @@ struct svc_wait_op {
 };
 static struct svc_wait_op svc_waits[SVC_WAIT_MAX];
 static u32 svc_wait_n;			/* acquires captured this entry */
+/*
+ * r484 THE STAGING-DRAIN COMPLETION EDGE: when the acquire gate holds an
+ * entry, the op can still have executed on an earlier decode whose gate did
+ * not trigger (svc_wait_n was 0 on that pass), and the once-only done
+ * bitmap then suppresses it on the first pass whose gate finally passes -
+ * the pass at which the dependency just became installed and the op's
+ * inputs changed. The release then applies and the guest drains a staging
+ * buffer holding the PRE-dependency bytes (streams_events s=2: the D2H
+ * copy read dc2 before the vadd_into wrote it, ran early on gate-less
+ * passes, was once-only-skipped on the passing pass, and the client
+ * drained a2+b2 instead of a2+b2+c1). Remember the last held (channel,
+ * entry) and force exactly ONE re-run of its ops on the first passing
+ * pass.
+ */
+static u32 svc_wait_hold_h;
+static u32 svc_wait_hold_idx = 0xffffffffu;
+static bool svc_wait_repass;
 static u32 svc_wait_ovf;		/* acquires dropped past the array */
 static u64 svc_wait_held;
 static u64 svc_wait_pass;
@@ -7394,6 +7411,8 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops,
 	static u64 bigop_log;
 
 	svc.cur_ch = ch_h;
+	/* r484: one-shot per call; set only by the hold->pass transition. */
+	svc_wait_repass = false;
 	/*
 	 * m163: apply every queued scanner injection before resolving this
 	 * pass's copy operands. Without this, a CE copy decoded in the window
@@ -7434,7 +7453,6 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops,
 		struct svc_wait_hold *gh;
 		bool hold = false;
 		u32 wi;
-
 		/*
 		 * Kill switch: a wait that never satisfies (a counter our
 		 * emulation does not advance the way the driver expects)
@@ -7519,9 +7537,22 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops,
 		}
 		if (hold) {
 			svc_wait_held++;
+			svc_wait_hold_h = ch_h;
+			svc_wait_hold_idx = entry_idx;
 			svc.op_tr_pending = 1;
 			return;
 		}
+		/*
+		 * r484: the gate just PASSED for an entry it previously held -
+		 * the dependency is installed now, so this pass's ops must
+		 * re-run even if the once-only bitmap says they already did
+		 * (they ran against the pre-dependency state).
+		 */
+		if (ch_h == svc_wait_hold_h &&
+		    entry_idx == svc_wait_hold_idx) {
+			svc_wait_repass = true;
+			svc_wait_hold_idx = 0xffffffffu;
+		}
 		svc_wait_pass++;
 		if (gh)
 			gh->active = false;
@@ -7539,7 +7570,8 @@ svc_wait_done:
 		 * ran; re-running a CE copy would read a staging buffer the
 		 * client has since refilled and overwrite finished data.
 		 */
-		if (ops_done && seq < 64 && ((*ops_done >> seq) & 1)) {
+		if (!svc_wait_repass && ops_done && seq < 64 &&
+		    ((*ops_done >> seq) & 1)) {
 			svc.ops_once_skipped++;
 			/*
 			 * r456: this op already ran in an earlier pass. Any