sema: r391f take 2 - once-only GRCE inline writes keyed by (push_va, push hash, dst, bytes)

The first take gated on !replay, which broke pushes whose first pass was
untrusted (their bytes are only authored on the later trusted pass) and
hung the second client of the boot. Key the inline write by
(push_va, push content hash, dst, bytes): a replay of the same push is a
genuine duplicate and is skipped, a new push (its release payload makes the
hash unique) always runs. Table is session-scoped (cleared by
svc_session_flush), 64 slots.

Root cause retained: victory_reduce's cudaMemset(d_sum,0,8) is a GRCE
LOAD_INLINE_DATA write; every re-decode of that push re-ran it and
re-zeroed the accumulator after the launch's copy-out (the sum=0 class).

1 files changed, 140 insertions, 48 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 41a5c58..ef89a52 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -8409,6 +8409,51 @@ static bool svc_gr_cand_push_seen(u64 push_va)
 	return false;
 }
 
+/*
+ * r391f: once-only identity for GRCE inline writes. A push's
+ * LOAD_INLINE_DATA runs author bytes into the vidmem shadow; a re-decode of
+ * the same push (replay/rescue/rehold) must not re-run them - the target
+ * page may have been superseded in between (victory_reduce: the client's
+ * cudaMemset inline write re-zeroed the accumulator after the launch's
+ * copy-out, the sum=0 class). Keyed by (push_va, push content hash, dst,
+ * bytes): the release payload inside every push makes the hash unique per
+ * push, so a new client's identical-looking write never collides. Session
+ * scoped (cleared by svc_session_flush). Service-thread only.
+ */
+#define SVC_INLINE_ONCE_SLOTS 64
+static struct {
+	u64 push_va;
+	u64 hash;
+	u64 dst;
+	u32 bytes;
+} svc_inline_once[SVC_INLINE_ONCE_SLOTS];
+static u32 svc_inline_once_next;
+
+static bool svc_inline_once_seen(u64 push_va, u64 hash, u64 dst, u32 bytes)
+{
+	u32 i;
+
+	if (!push_va)
+		return false;
+	for (i = 0; i < SVC_INLINE_ONCE_SLOTS; i++)
+		if (svc_inline_once[i].push_va == push_va &&
+		    svc_inline_once[i].hash == hash &&
+		    svc_inline_once[i].dst == dst &&
+		    svc_inline_once[i].bytes == bytes)
+			return true;
+	return false;
+}
+
+static void svc_inline_once_store(u64 push_va, u64 hash, u64 dst, u32 bytes)
+{
+	u32 i = svc_inline_once_next++ % SVC_INLINE_ONCE_SLOTS;
+
+	svc_inline_once[i].push_va = push_va;
+	svc_inline_once[i].hash = hash;
+	svc_inline_once[i].dst = dst;
+	svc_inline_once[i].bytes = bytes;
+}
+
 static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 			  struct svc_push_state *st, u32 addr, u32 count,
 			  const u8 *data, const u8 *push, u32 push_len,
@@ -8540,6 +8585,8 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 	}
 }
 
+static u64 svc_pass_hash;	/* r391f: this decode pass's push content hash */
+
 static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 			      u32 len, u64 page_pa, bool allow_ops,
 			      struct svc_channel *ch, u32 entry_idx,
@@ -8548,6 +8595,9 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 {
 	struct svc_push_state st;
 
+	/* r391f: the inline-write once-only key needs the push's identity. */
+	svc_pass_hash = svc_push_hash(buf, len);
+
 	/*
 	 * r316: pending releases record the channel they were decoded for;
 	 * the verified pool resolver is channel-keyed (see svc_pending_add
@@ -8777,27 +8827,28 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 					slot_bytes = 0x8000 - 4;
 			}
 			/*
-			 * r391f once-only (the sum=0 class): a REPLAYED
-			 * entry must not re-run its inline writes. The ops
-			 * are already gated by allow_ops (entry_replay
-			 * clears it), but the inline-write gate used
-			 * `trusted || allow_ops` - and a replay is trusted
-			 * by construction - so every re-decode of an
-			 * already-processed push re-wrote its inline data
-			 * into the vidmem shadow. victory_reduce's
+			 * r391f once-only (the sum=0 class): the same
+			 * inline write must not run twice for one push
+			 * identity. The ops are gated by allow_ops
+			 * (entry_replay clears it), but the inline-write
+			 * gate `trusted || allow_ops` re-ran the write on
+			 * every re-decode (a replay is trusted by
+			 * construction): victory_reduce's
 			 * cudaMemset(d_sum,0,8) is exactly such an inline
 			 * write (observed: `inline write
 			 * dst=0x7845ef000000 -> pa=0x7c00000 bytes=8
 			 * data8=0` firing 86 us before the D2H readback,
 			 * zeroing the accumulator the launch had just
-			 * written - the readback then reads zeros). The
-			 * r383 replay-capture path only needs to CAPTURE
-			 * the launch; the inline bytes were authored on the
-			 * first (non-replay) pass.
+			 * written - the client then read zeros). A coarse
+			 * !replay gate is wrong (a push whose first pass
+			 * was untrusted must still author its bytes on the
+			 * later trusted pass); key the write by
+			 * (push_va, push content hash, dst, bytes) - a new
+			 * push's release payload makes the hash unique -
+			 * and skip only a genuine duplicate. The table is
+			 * session-scoped (a new client starts clean).
 			 */
-			if ((trusted || allow_ops) && !replay &&
-			    !gsp_sema_skip_ops &&
-			    st.have_out && st.line_length) {
+			{
 				u32 bytes = slot_run ? slot_bytes :
 					st.line_length *
 					(st.gr_line_count ?
@@ -8806,42 +8857,81 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 
 				if (bytes > have)
 					bytes = have;
-				svc_grce_inline_write(vgpu, st.offset_out,
-						      buf + off + 4, bytes,
-						      ch->h_object);
-				/*
-				 * fail4: hardware semantics - a LOAD_INLINE_DATA
-				 * run advances the destination cursor by the
-				 * bytes it wrote. Without this, the consecutive
-				 * runs of a split 64 KB upload all land at the
-				 * first destination and the page ends up with
-				 * only the last run's words (the RAWA junk).
-				 */
-				st.offset_out += bytes;
 				/*
-				 * fail4: name the header that follows each big
-				 * LID run - the split 64 KB upload's other
-				 * payload carriers sit exactly there.
+				 * r391f once-only (the sum=0 class): the same
+				 * inline write must not run twice for one push
+				 * identity. The ops are gated by allow_ops
+				 * (entry_replay clears it), but the
+				 * inline-write gate `trusted || allow_ops`
+				 * re-ran the write on every re-decode (a replay
+				 * is trusted by construction): victory_reduce's
+				 * cudaMemset(d_sum,0,8) is exactly such an
+				 * inline write (observed: `inline write
+				 * dst=0x7845ef000000 -> pa=0x7c00000 bytes=8
+				 * data8=0` firing 86 us before the D2H
+				 * readback, zeroing the accumulator the launch
+				 * had just written - the client then read
+				 * zeros). A coarse !replay gate is wrong (a push
+				 * whose first pass was untrusted must still
+				 * author its bytes on the later trusted pass);
+				 * key the write by (push_va, push content hash,
+				 * dst, bytes) - the release payload inside every
+				 * push makes the hash unique per push - and skip
+				 * only a genuine duplicate. Session-scoped: a new
+				 * client starts clean.
 				 */
-				if (gsp_sema_dbg && len >= 4096 &&
-				    svc.next_hdr_log < 24) {
-					u32 noff = off + 4 + count * 4;
-					u32 nh = 0, nw = 0;
-
-					if (noff + 4 <= len)
-						memcpy(&nh, buf + noff, 4);
-					if (noff + 8 <= len)
-						memcpy(&nw, buf + noff + 4, 4);
-					svc.next_hdr_log++;
-					pr_info("sema-next-hdr: h=0x%x push=0x%llx after=0x%x cnt=%u nxt_hdr=0x%08x nxt_w0=%08x cursor=0x%llx\n",
-						ch->h_object, push_va, off,
-						count, nh, nw, st.offset_out);
+				if ((trusted || allow_ops) && !gsp_sema_skip_ops &&
+				    st.have_out && st.line_length && bytes &&
+				    !svc_inline_once_seen(push_va, svc_pass_hash,
+							  st.offset_out,
+							  bytes)) {
+					svc_inline_once_store(push_va,
+							      svc_pass_hash,
+							      st.offset_out,
+							      bytes);
+					svc_grce_inline_write(vgpu, st.offset_out,
+							      buf + off + 4,
+							      bytes,
+							      ch->h_object);
+					/*
+					 * fail4: hardware semantics - a
+					 * LOAD_INLINE_DATA run advances the
+					 * destination cursor by the bytes it
+					 * wrote. Without this, the consecutive
+					 * runs of a split 64 KB upload all land
+					 * at the first destination and the page
+					 * ends up with only the last run's
+					 * words (the RAWA junk).
+					 */
+					st.offset_out += bytes;
+					/*
+					 * fail4: name the header that follows
+					 * each big LID run - the split 64 KB
+					 * upload's other payload carriers sit
+					 * exactly there.
+					 */
+					if (gsp_sema_dbg && len >= 4096 &&
+					    svc.next_hdr_log < 24) {
+						u32 noff = off + 4 + count * 4;
+						u32 nh = 0, nw = 0;
+
+						if (noff + 4 <= len)
+							memcpy(&nh, buf + noff, 4);
+						if (noff + 8 <= len)
+							memcpy(&nw, buf + noff + 4,
+							       4);
+						svc.next_hdr_log++;
+						pr_info("sema-next-hdr: h=0x%x push=0x%llx after=0x%x cnt=%u nxt_hdr=0x%08x nxt_w0=%08x cursor=0x%llx\n",
+							ch->h_object, push_va,
+							off, count, nh, nw,
+							st.offset_out);
+					}
+				} else if (gsp_sema_dbg && svc.grce_log < 64) {
+					svc.grce_log++;
+					pr_info("sema-grce: inline data not run (trusted=%d allow=%d have_out=%d line=%u) h=0x%x\n",
+						trusted, allow_ops, st.have_out,
+						st.line_length, ch->h_object);
 				}
-			} else if (gsp_sema_dbg && svc.grce_log < 64) {
-				svc.grce_log++;
-				pr_info("sema-grce: inline data not run (trusted=%d allow=%d have_out=%d line=%u) h=0x%x\n",
-					trusted, allow_ops, st.have_out,
-					st.line_length, ch->h_object);
 			}
 			/*
 			 * Record the destination and the payload bytes for
@@ -10819,6 +10909,8 @@ static void svc_session_flush(void)
 	memset(svc_retry_q, 0, sizeof(svc_retry_q));
 	memset(svc_pool_pa, 0, sizeof(svc_pool_pa));
 	memset(svc_fwd_rel, 0, sizeof(svc_fwd_rel));
+	memset(svc_inline_once, 0, sizeof(svc_inline_once));
+	svc_inline_once_next = 0;
 	memset(svc_grce_dst_ring, 0, sizeof(svc_grce_dst_ring));
 	for (i = 0; i < SVC_VAS_SLOTS; i++) {
 		if (svc_vas[i].data)