sema: r402 pitched (2D) fills/copies + copy-dst divergence witness

Two additions for the memset2d/memcpy_roundtrip and readback classes:

1. Pitched transfers. The CE decoder now captures LINE_COUNT (0x41c),
   PITCH_IN (0x410) and PITCH_OUT (0x414); a fill/copy whose LINE_COUNT is
   > 1 and whose destination pitch exceeds the row length builds a row-wise
   op (svc_fill/svc_copy are thin wrappers around the existing 1D bodies,
   advancing dst by dpitch and src by spitch per row). Each row keeps the
   whole per-page resolution/ledger/guard machinery, and a failed row's
   retry re-runs already-written rows with the same pattern (idempotent).
   A flat transfer carries pitch == row length (or no pitch method) and
   keeps the historical single-run path unchanged. Bounded 'sema-2d' row
   names every 2D op build.

   Note: the current victory_memset2d traffic reaches the engine as GR
   kernels plus per-row 4 KB CE copies, so this capability does not yet
   trigger for that client - it is the correct decoder for the CE-class 2D
   shape (driver channel_utils.c memset2d emits LINE_COUNT/PITCH).

2. DST-DIVERGE witness: when a copy destination resolves from the write
   ledger, a read-only resolver probe cross-checks the answer and logs
   (bounded, gsp_sema_dbg) when the writer's page differs from the live
   resolver's - the 'writer follows the ledger while the reader holds
   another page' shape behind the failed D2H staging class (streams2 c1
   zeros, smiley pass failures). Zero rows fired on r403, so the ledger is
   not the active divergence for that boot; the witness stays for the next
   failing capture.

1 files changed, 190 insertions, 4 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index add8b29..a7f4b4e 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -4076,6 +4076,7 @@ static u64 svc_wait_giveup;
 static u32 svc_wait_log;		/* per-session held-row print budget */
 static u32 svc_wait_pass_log;		/* per-session resolved-row budget */
 static u32 svc_remapconst_log;		/* r343 FILL: SET_REMAP_CONST_A rows */
+static u32 svc_pitch_log;		/* r402: 2D (pitched) op builds */
 static u32 svc_fill_raw_log;		/* r365 FILL-RAW: raw dropped-fill dumps */
 static u32 svc_lowfill_log;		/* r365b LOWFILL-WIT: below-floor fills */
 static u32 svc_tabcopy_log;		/* r367 TABLE-COPY: table-page copies */
@@ -4098,6 +4099,7 @@ struct svc_wait_hold {
 };
 #define SVC_WAIT_HOLD_MAX 16
 static struct svc_wait_hold svc_wait_holds[SVC_WAIT_HOLD_MAX];
+static u32 svc_dst_div_log;		/* r402: copy-dst ledger/probe divergence */
 static u64 svc_ledger_stores;
 static u64 svc_ledger_evictions;
 static u64 svc_ledger_evict_since_reset;
@@ -4312,6 +4314,9 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu)
 #define CE_M_OFFSET_OUT_UPPER	0x408
 #define CE_M_OFFSET_OUT_LOWER	0x40c
 #define CE_M_LINE_LENGTH_IN	0x418
+#define CE_M_LINE_COUNT		0x41c
+#define CE_M_PITCH_IN		0x410
+#define CE_M_PITCH_OUT		0x414
 #define CE_M_SET_REMAP_CONST_A	0x700
 #define CE_M_SET_REMAP_CONST_B	0x704
 #define CE_M_SET_REMAP_COMPONENTS 0x708
@@ -4386,6 +4391,19 @@ struct svc_memset_op {
 	u64	bytes;	/* copy length */
 	u32	src_phys;
 	u32	src_aperture;
+	/*
+	 * r402: 2D (pitched) transfers. rows > 1 selects row-wise mode:
+	 * the op runs `rows` times with the destination advancing by
+	 * dpitch (and the source by spitch) per row, each row carrying
+	 * `bytes` (copy) or elem*count (fill) bytes. cudaMemset2D and
+	 * cudaMemcpy2D issue exactly this shape (LINE_COUNT rows,
+	 * PITCH_OUT/PITCH_IN in bytes); without it the fill/copy ran flat
+	 * and the partial-width rows corrupted the neighbouring columns
+	 * (victory_memset2d bad>12M, memcpy_roundtrip pitched class).
+	 */
+	u32	rows;
+	u32	dpitch;
+	u32	spitch;
 };
 
 /* Decoder state carried across the methods of one push. */
@@ -4397,6 +4415,15 @@ struct svc_push_state {
 	u32	have_in, have_out;
 	u32	line_length;
 	u32	gr_line_count;	/* GRCE LINE_COUNT */
+	/*
+	 * r402: 2D transfer geometry (CE class): PITCH_IN/PITCH_OUT in
+	 * bytes and LINE_COUNT rows. A pitched fill/copy whose pitch
+	 * exceeds the row length must run row-wise; a flat transfer leaves
+	 * pitch == line_length (or no pitch method at all).
+	 */
+	u32	pitch_in, pitch_out;
+	u32	have_pitch_in, have_pitch_out;
+	u32	line_count;
 	u32	sem_va_hi, sem_va_lo, sem_payload;
 	u32	have_sem;
 	u32	have_remap;
@@ -4788,8 +4815,49 @@ static int svc_fill_dst_page_pa(struct nv_vgpu_device *vgpu, u64 va,
 	return -ENOENT;
 }
 
+static int svc_fill_1d(struct nv_vgpu_device *vgpu, u64 addr, u64 value,
+		       u32 elem, u32 count, u32 phys, u32 aperture,
+		       u64 *done_out);
+static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src,
+		       size_t total, u32 dst_phys, u32 dst_aperture,
+		       u32 src_phys, u32 src_aperture, u64 *done_out);
+
+/*
+ * r402 row-wise wrapper: a 2D (pitched) fill runs the 1D fill once per
+ * row with the destination advancing by dpitch. Each row keeps the whole
+ * per-page resolution, ledger-store and guard machinery of the 1D path
+ * (no new resolution semantics), and a failed row's retry simply re-runs
+ * the already-written rows with the same pattern - idempotent for fills.
+ */
 static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem,
-		    u32 count, u32 phys, u32 aperture, u64 *done_out)
+		    u32 count, u32 phys, u32 aperture, u64 *done_out,
+		    u32 rows, u32 dpitch)
+{
+	if (rows > 1 && dpitch) {
+		u64 done = 0;
+		int rc = 0;
+		u32 r;
+
+		for (r = 0; r < rows; r++) {
+			u64 row_done = 0;
+
+			rc = svc_fill_1d(vgpu, addr + (u64)r * dpitch,
+					 value, elem, count, phys, aperture,
+					 &row_done);
+			done += row_done;
+			if (rc)
+				break;
+		}
+		if (done_out)
+			*done_out = done;
+		return rc;
+	}
+	return svc_fill_1d(vgpu, addr, value, elem, count, phys, aperture,
+			   done_out);
+}
+
+static int svc_fill_1d(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem,
+		       u32 count, u32 phys, u32 aperture, u64 *done_out)
 {
 	u8 pat[8];
 	u8 bounce[256];
@@ -5965,9 +6033,41 @@ static int svc_fwd_operand_hpa(struct nv_vgpu_device *vgpu, u8 kind,
 			       u64 pa, u32 off, u32 bytes, u64 *hpa,
 			       u32 *out_off);
 
+/* r402 row-wise wrapper: see svc_fill. The source advances by spitch. */
+/* r402: read-only resolver probe (defined with the GR fill diagnostics). */
+static const char *svc_gr_src_probe(struct nv_vgpu_device *vgpu, u64 va,
+				    u64 *pa_out, u32 *ap_out);
 static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
 		    u32 dst_phys, u32 dst_aperture, u32 src_phys, u32 src_aperture,
-		    u64 *done_out)
+		    u64 *done_out, u32 rows, u32 dpitch, u32 spitch)
+{
+	if (rows > 1 && dpitch) {
+		u64 done = 0;
+		int rc = 0;
+		u32 r;
+
+		for (r = 0; r < rows; r++) {
+			u64 row_done = 0;
+
+			rc = svc_copy_1d(vgpu, dst + (u64)r * dpitch,
+					 src + (u64)r * spitch, total,
+					 dst_phys, dst_aperture, src_phys,
+					 src_aperture, &row_done);
+			done += row_done;
+			if (rc)
+				break;
+		}
+		if (done_out)
+			*done_out = done;
+		return rc;
+	}
+	return svc_copy_1d(vgpu, dst, src, total, dst_phys, dst_aperture,
+			   src_phys, src_aperture, done_out);
+}
+
+static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
+		       u32 dst_phys, u32 dst_aperture, u32 src_phys, u32 src_aperture,
+		       u64 *done_out)
 {
 	u8 bounce[256];
 	size_t o;
@@ -6008,6 +6108,34 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
 			dst_shadow = true;
 		} else if (!svc_ledger_lookup(dst, &pa, &aper)) {
 			/*
+			 * r402 diagnostic (bounded): name the case where the
+			 * ledger's landing page for a copy DESTINATION
+			 * disagrees with the live resolver (pin/walk). The
+			 * D2H staging writes failed on the class this
+			 * witness detects: the writer follows the ledger
+			 * while the guest's reader holds a different page,
+			 * so the client reads stale data (streams2 c1
+			 * zeros / smiley 3-of-4 passes). Read-only probe
+			 * (svc_gr_src_probe stores nothing).
+			 */
+			if (gsp_sema_dbg && svc_dst_div_log < 64 &&
+			    dst >= SVC_VAS_FLOOR) {
+				u64 wpa = 0;
+				u32 wap = 0;
+				const char *src =
+					svc_gr_src_probe(vgpu, dst, &wpa, &wap);
+
+				if (wpa &&
+				    (((wpa & ~(u64)(SEMA_PAGE_SIZE - 1)) !=
+				      (pa & ~(u64)(SEMA_PAGE_SIZE - 1))) ||
+				     ((wap == SVC_APERTURE_FB) !=
+				      (aper == SVC_APERTURE_FB)))) {
+					svc_dst_div_log++;
+					pr_err_ratelimited("vgsp-sema: DST-DIVERGE va=0x%llx ledger=0x%llx/%u probe=%s 0x%llx/%u (write follows ledger)\n",
+						dst, pa, aper, src, wpa, wap);
+				}
+			}
+			/*
 			 * m180 unification: a previous write of this VA is
 			 * the authority for where the content lives (and so
 			 * for where a new write lands); it also spares the
@@ -7275,11 +7403,12 @@ svc_wait_done:
 				rc = svc_copy(vgpu, m->dst, m->src, m->bytes,
 					      m->phys, m->aperture,
 					      m->src_phys, m->src_aperture,
-					      &ce_done);
+					      &ce_done, m->rows, m->dpitch,
+					      m->spitch);
 			else
 				rc = svc_fill(vgpu, m->dst, m->value, m->elem,
 					      m->count, m->phys, m->aperture,
-					      &ce_done);
+					      &ce_done, m->rows, m->dpitch);
 			/*
 			 * CEAUDIT (2026-10-18 r220): per-op audit of the CE
 			 * data path - the software executor of every guest
@@ -9681,6 +9810,18 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 		case CE_M_LINE_LENGTH_IN:
 			st.line_length = v0;
 			break;
+		/* r402: 2D geometry (cudaMemset2D / cudaMemcpy2D). */
+		case CE_M_LINE_COUNT:
+			st.line_count = v0;
+			break;
+		case CE_M_PITCH_IN:
+			st.pitch_in = v0 & 0xffffff;
+			st.have_pitch_in = 1;
+			break;
+		case CE_M_PITCH_OUT:
+			st.pitch_out = v0 & 0xffffff;
+			st.have_pitch_out = 1;
+			break;
 		case CE_M_LAUNCH_DMA: {
 			u32 dma = v0;
 			u32 transfer = DMA_DATA_TRANSFER_TYPE(dma);
@@ -9855,6 +9996,30 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 						m->count = st.line_length /
 							m->elem;
 					}
+					/*
+					 * r402: a 2D fill (cudaMemset2D):
+					 * LINE_COUNT rows of
+					 * LINE_LENGTH_IN bytes at
+					 * PITCH_OUT stride. Only a pitch
+					 * WIDER than the row selects row
+					 * mode; a flat fill carries
+					 * pitch == row length (or no
+					 * pitch method) and keeps the
+					 * historical single-run path.
+					 */
+					if (st.have_pitch_out &&
+					    st.line_count > 1 &&
+					    st.pitch_out > st.line_length) {
+						m->rows = st.line_count;
+						m->dpitch = st.pitch_out;
+					}
+					if (m->rows > 1 && svc_pitch_log < 32) {
+						svc_pitch_log++;
+						pr_info("sema-2d: fill rows=%u dpitch=%u line=%u dst=0x%llx h=0x%x push=0x%llx\n",
+							m->rows, m->dpitch,
+							st.line_length, m->dst,
+							ch->h_object, push_va);
+					}
 					m->phys = DMA_DST_TYPE_PHYSICAL(dma);
 					m->aperture = st.have_dst_mode ?
 						st.dst_phys_mode : CE_APERTURE_VID;
@@ -9883,6 +10048,26 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 						DMA_SRC_TYPE_PHYSICAL(dma);
 					m->src_aperture = st.have_src_mode ?
 						st.src_phys_mode : CE_APERTURE_VID;
+					/*
+					 * r402: a 2D copy (cudaMemcpy2D):
+					 * LINE_COUNT rows of
+					 * LINE_LENGTH_IN bytes with
+					 * PITCH_IN/PITCH_OUT strides.
+					 * Row mode only when a pitch is
+					 * wider than the row (a flat 1D
+					 * copy keeps pitch == length).
+					 */
+					if (st.line_count > 1 &&
+					    st.have_pitch_out &&
+					    st.pitch_out > st.line_length) {
+						m->rows = st.line_count;
+						m->dpitch = st.pitch_out;
+						m->spitch = st.have_pitch_in ?
+							(st.pitch_in ?
+							 st.pitch_in :
+							 st.pitch_out) :
+							st.pitch_out;
+					}
 					n_ops++;
 					ops_seq++;
 				} else if (gsp_sema_dbg && svc.dma_skip_log < 32) {
@@ -10967,6 +11152,7 @@ static void svc_session_flush(void)
 	svc_wait_log = 0;
 	svc_wait_pass_log = 0;
 	svc_remapconst_log = 0;
+	svc_pitch_log = 0;
 	svc_fill_raw_log = 0;
 	svc_lowfill_log = 0;
 	svc_tabcopy_log = 0;