sema: r401 acquire-wait hold bound + session-wide exactly-once capture memory

Two fixes for the streams2 / multi-stream ordering class, both verified on
fresh boot r401 with the acquire gate enabled:

1. sema_wait_hold_ms (new module param, default 4000 ms) replaces the
   hardcoded 100 ms acquire-wait bound in svc_run_ops. The old bound is
   shorter than one emulated GR launch completion (0.5-1.6 s observed), so
   a D2H readback's ACQ_CIRC_GEQ on the producer channel's progress
   counter (op=3 pay=4 on 0x20400ffd0) gave up and ran BEFORE the scale
   launch wrote d_c2; the client then verified pre-scale data
   (bad=32767). r401 log: the same entry now holds ~1.2 s across
   cur=1->4 and runs after GUEST GR LAUNCH OK (copy-back optrace
   92315.594824 with dst16_after = the post-scale bytes). Fail-open is
   preserved: a genuinely unsatisfiable acquire still expires, just late
   enough that the dependency's work has landed (late-but-correct over
   early-and-wrong).

2. svc_gr_seen: a session-scoped open-addressed (push_va, hash) table
   behind svc_gr_cand_push_seen(). The 4-slot candidate ring + 8-VA
   history evicted a real capture within ~12 later captures, so a true
   replay of an already-forwarded launch was re-captured and re-executed
   (r399: the same nine launch pushes re-submitted every ~10-11 s for
   minutes). Cleared in svc_session_flush because a new client's
   identical-content pushes restart their release counters - a
   boot-scoped table would silently drop them (LAUNCH-SKIP).

Also adds tools/cuda-ab/streams2d.cu: streams2 with per-array/per-4KB-block
diff reporting plus the device pointer dump, which localised the remaining
failure to the D2H staging write path (our staging-page resolution vs the
guest's view).

3 files changed, 130 insertions, 16 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index f803309..add8b29 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -4048,7 +4048,19 @@ static u32 svc_d2hmap_log;		/* D2H-MAP per-page destination budget */
  *   5 ACQ_NOR = ~(v | p) != 0.
  */
 #define SVC_WAIT_MAX		8
-#define SVC_WAIT_HOLD_MS	100u	/* bounded acquire wait (fail-open) */
+/*
+ * r401: bounded acquire wait (fail-open), now a module parameter. The
+ * original 100 ms bound was shorter than a single GR launch's completion
+ * latency in this emulation (observed 0.5-1.6 s per launch), so a copy
+ * that waited on a launch's tracker (streams2: the D2H readback's
+ * ACQ_CIRC_GEQ on the scale channel's progress counter) gave up and ran
+ * BEFORE the launch wrote its output - the host read pre-scale data
+ * (bad=32767, c2 all wrong). 4 s covers several launch latencies; the
+ * bound still guarantees the entry is never parked forever (a genuinely
+ * unsatisfiable acquire fails open late, after the dependency's own work
+ * would have landed - late-but-correct instead of early-and-wrong).
+ */
+unsigned gsp_sema_wait_hold_ms = 4000;
 struct svc_wait_op {
 	u64	va;
 	u32	payload;
@@ -5285,6 +5297,82 @@ static u32 svc_gr_cand_pos;
 static u64 svc_gr_cap_hist[SVC_GR_CAP_HIST];
 static u32 svc_gr_cap_hist_pos;
 
+/*
+ * r401: robust session-scoped "already captured" memory for the r383
+ * replay-capture path. The 4-slot candidate ring + 8-VA history evicted a
+ * real capture within ~12 later captures, so a true replay of an
+ * already-forwarded launch (identical push VA and content) was
+ * re-captured and re-executed: the r399 soak log shows the same nine
+ * launch pushes re-submitted every ~10-11 s for minutes (idempotent
+ * kernels survive it; accumulators would not). The open-addressed table
+ * keeps the (push_va, push_hash) identity of every capture of the
+ * session, so a true replay always classifies as "seen" (LAUNCH-SKIP,
+ * exactly once) while a genuinely new push - even at the same VA with
+ * the same entry words - is captured through the normal
+ * `trusted && !replay` path and never consults this table. Cleared by
+ * svc_session_flush (a new client's identical-payload pushes restart
+ * their release counters, so a boot-scoped table would wrongly suppress
+ * them). Service-thread only.
+ */
+#define SVC_GR_SEEN_SLOTS	2048
+#define SVC_GR_SEEN_PROBE	8
+static struct {
+	u64	push_va;
+	u64	hash;
+} svc_gr_seen[SVC_GR_SEEN_SLOTS];
+static u64 svc_gr_seen_stale;	/* probe-chain evictions (telemetry) */
+
+static u32 svc_gr_seen_mix(u64 push_va, u64 hash)
+{
+	u64 k = push_va * 0x9E3779B97F4A7C15ULL;
+
+	k ^= hash + 0x9E3779B97F4A7C15ULL + (k << 6) + (k >> 2);
+	return (u32)(k % SVC_GR_SEEN_SLOTS);
+}
+
+static bool svc_gr_seen_lookup(u64 push_va, u64 hash)
+{
+	u32 i, base = svc_gr_seen_mix(push_va, hash);
+
+	if (!push_va)
+		return false;
+	for (i = 0; i < SVC_GR_SEEN_PROBE; i++) {
+		u32 k = (base + i) % SVC_GR_SEEN_SLOTS;
+
+		if (!svc_gr_seen[k].push_va)
+			return false;
+		if (svc_gr_seen[k].push_va == push_va &&
+		    svc_gr_seen[k].hash == hash)
+			return true;
+	}
+	return false;
+}
+
+static void svc_gr_seen_store(u64 push_va, u64 hash)
+{
+	u32 i, base = svc_gr_seen_mix(push_va, hash);
+
+	if (!push_va)
+		return;
+	for (i = 0; i < SVC_GR_SEEN_PROBE; i++) {
+		u32 k = (base + i) % SVC_GR_SEEN_SLOTS;
+
+		if (!svc_gr_seen[k].push_va ||
+		    (svc_gr_seen[k].push_va == push_va &&
+		     svc_gr_seen[k].hash == hash)) {
+			svc_gr_seen[k].push_va = push_va;
+			svc_gr_seen[k].hash = hash;
+			return;
+		}
+	}
+	/* Full probe chain: replace the tail (oldest-ish) entry. */
+	svc_gr_seen_stale++;
+	svc_gr_seen[(base + SVC_GR_SEEN_PROBE - 1) % SVC_GR_SEEN_SLOTS].push_va =
+		push_va;
+	svc_gr_seen[(base + SVC_GR_SEEN_PROBE - 1) % SVC_GR_SEEN_SLOTS].hash =
+		hash;
+}
+
 static_assert(SVC_GR_QMD_WORDS == GSP_PHYS_GR_QMD_WORDS,
 	      "the sema GR QMD word count must match the backend ABI");
 static_assert(SVC_GR_PUSH_WORDS == GSP_PHYS_GR_RAW_WORDS,
@@ -5399,6 +5487,9 @@ static void svc_gr_cand_capture(struct svc_channel *ch,
 	svc_gr_cap_hist_pos++;
 	if (push && push_len)
 		svc_gr_cand.push_hash = svc_push_hash(push, push_len);
+	/* r401: the session-wide exactly-once memory (survives ring rotation;
+	 * see svc_gr_seen_lookup). */
+	svc_gr_seen_store(push_va, svc_gr_cand.push_hash);
 	if (push && push_len >= 4) {
 		u32 total = push_len / 4;
 		u32 n = total < SVC_GR_PUSH_WORDS ? total : SVC_GR_PUSH_WORDS;
@@ -7052,12 +7143,12 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops,
 	 *   3. drain the pending-translation release queue (releases OUR
 	 *      machinery is merely late about) and re-read once;
 	 *   4. still unsatisfied -> hold the entry (release withheld through
-	 *      the op-pending machinery) for at most SVC_WAIT_HOLD_MS per
-	 *      episode, then fail open loudly ('give-up': the value cannot
-	 *      be installed by firmware alone - it sits behind an upstream
-	 *      opdef/operand hold - so the old pass-through behaviour is
-	 *      safer than parking the channel forever). Unreadable
-	 *      semaphore: fail-open (the pre-r336 behaviour).
+	 *      the op-pending machinery) for at most gsp_sema_wait_hold_ms
+	 *      per episode, then fail open loudly ('give-up': the value
+	 *      cannot be installed by firmware alone - it sits behind an
+	 *      upstream opdef/operand hold - so the old pass-through
+	 *      behaviour is safer than parking the channel forever).
+	 *      Unreadable semaphore: fail-open (the pre-r336 behaviour).
 	 * The pass/hold/give-up decisions are logged as 'sema-wait' rows
 	 * (op, payload, observed value, decision).
 	 */
@@ -7116,7 +7207,7 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops,
 				continue;
 			}
 			pend = svc_wait_pend_release(w->va);
-			if (waited_ms >= SVC_WAIT_HOLD_MS) {
+			if (waited_ms >= gsp_sema_wait_hold_ms) {
 				/*
 				 * Bounded episode expired: fail open for the
 				 * WHOLE entry (one row, then the ops run). A
@@ -8388,12 +8479,16 @@ static u64 svc_gr_nocap_log;
 
 /*
  * r383: has a launch with this push VA ever been captured? Scans the live
- * candidate ring and the recent-capture history. Used by the replayed-QMD
- * dispatch below: a replay whose push was never captured is a launch that
- * would otherwise be silently dropped (the streams2 second-vadd class); a
- * replay of a captured/offered push is the exactly-once no-op.
+ * candidate ring, the recent-capture history and the r401 session-wide
+ * (push_va, hash) seen table. Used by the replayed-QMD dispatch below: a
+ * replay whose push was never captured is a launch that would otherwise be
+ * silently dropped (the streams2 second-vadd class); a replay of a
+ * captured/offered push is the exactly-once no-op. The hash is required
+ * for the seen table so a later, DIFFERENT push at the same VA (a loop
+ * iteration: same entry words, new content) is never misclassified as
+ * seen - the caller only consults this on a true replay anyway.
  */
-static bool svc_gr_cand_push_seen(u64 push_va)
+static bool svc_gr_cand_push_seen(u64 push_va, u64 hash)
 {
 	u32 i;
 
@@ -8406,7 +8501,7 @@ static bool svc_gr_cand_push_seen(u64 push_va)
 	for (i = 0; i < SVC_GR_CAP_HIST; i++)
 		if (svc_gr_cap_hist[i] == push_va)
 			return true;
-	return false;
+	return svc_gr_seen_lookup(push_va, hash);
 }
 
 /*
@@ -8543,11 +8638,15 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 				n, dup);
 		}
 		if (gsp_phys_gr_forward_enabled()) {
+			u64 push_hash = (push && push_len) ?
+				svc_push_hash(push, push_len) : 0;
+
 			if (trusted && !replay) {
 				svc_gr_cand_capture(ch, st, recs, n, push,
 						    push_len, st->push_va);
 			} else if (trusted && replay &&
-				   !svc_gr_cand_push_seen(st->push_va)) {
+				   !svc_gr_cand_push_seen(st->push_va,
+							  push_hash)) {
 				/*
 				 * r383 (streams2 second-vadd class): a
 				 * complete QMD first seen on a replay pass
@@ -8570,7 +8669,8 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 				svc_gr_nocap_log++;
 				pr_info("sema-gr: LAUNCH-SKIP h=0x%x push=0x%llx (replay; already captured=%d trusted=%d)\n",
 					ch->h_object, st->push_va,
-					svc_gr_cand_push_seen(st->push_va),
+					svc_gr_cand_push_seen(st->push_va,
+							      push_hash),
 					trusted);
 			} else if (!replay) {
 				gsp_cuda_bridge_launch(vgpu, ch->h_object,
@@ -10958,6 +11058,15 @@ static void svc_session_flush(void)
 	svc_hold_fb_log = 0;
 	svc_cand_ovw_log = 0;
 	svc_inline_recs_log = 0;
+	/*
+	 * r401: the exactly-once capture memory is session-scoped. A new
+	 * client's pushes restart their release counters, so their content
+	 * (and hence hash) can collide with the previous client's - keeping
+	 * the table would wrongly classify a fresh launch of the new
+	 * session as already captured (LAUNCH-SKIP, silent drop).
+	 */
+	memset(svc_gr_seen, 0, sizeof(svc_gr_seen));
+	svc_gr_seen_stale = 0;
 	/* m176: new session generation - channels and scrubbers re-stamp on
 	 * registration and never inherit per-session learned state. */
 	svc_sess_gen++;
diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h
index ea025ca..3765cd2 100644
--- a/kernel/gsp_sema_service.h
+++ b/kernel/gsp_sema_service.h
@@ -310,6 +310,9 @@ extern bool gsp_sema_pool_hold_fresh;
  */
 extern bool gsp_sema_copy_dst_pin_guard;
 extern bool gsp_sema_wait_gate;
+/* r401: bounded acquire-wait hold (fail-open) in ms; must exceed the
+ * emulated launch completion latency for copy-after-launch ordering. */
+extern unsigned gsp_sema_wait_hold_ms;
 
 /*
  * r384 (guest boot-corruption lottery): fill-destination guards (default
diff --git a/kernel/main.c b/kernel/main.c
index 92754d1..d2a44d6 100644
--- a/kernel/main.c
+++ b/kernel/main.c
@@ -2829,6 +2829,8 @@ MODULE_PARM_DESC(sema_pool_hold_fresh, "apply pool releases at fresh-pin time in
  */
 module_param_named(sema_copy_dst_pin_guard, gsp_sema_copy_dst_pin_guard, bool, 0644);
 module_param_named(sema_wait_gate, gsp_sema_wait_gate, bool, 0644);
+module_param_named(sema_wait_hold_ms, gsp_sema_wait_hold_ms, uint, 0644);
+MODULE_PARM_DESC(sema_wait_hold_ms, "bounded acquire-wait hold before fail-open, ms (r401, default 4000; must exceed the emulated launch latency)");
 MODULE_PARM_DESC(sema_copy_dst_pin_guard, "hold device-dst copies without a pin-resolved translation (default 1)");
 
 /*