vgsp: CE copy completeness + CEAUDIT instrumentation

Trace the CE (copy-engine) data path that services every guest
cudaMemcpy: the guest driver chunks large copies into <=1 MB
LAUNCH_DMA pushes; svc_decode_stream makes one op each and the host
software copy svc_copy executes it (per-256 B bounce, per-page
translation, page-clipped chunks). The hardware-forwarding path only
carries <=4 KB single-page candidates, so it cannot move these.

Completeness verdict: the CE executor is complete-or-error at every
size - the loop copies exactly 'total' or returns an errno (never a
partial success); failures hold the entry (bounded opdef hold, loud
EXPIRED) or are dropped loudly at fwd-ownership. r219 confirms: all 96
audited big copies done==len, zero COPY-MISMATCH, and every CE failure
was a pre-loop resolution miss (rc=-2, no byte moved) - invisible to
the copy-level audit until now.

New instrumentation (all additive, no new locks/waits):

- CEAUDIT per >=32 KB CE op (copy and fill): op/src/dst/len/done/rc,
  successes unconditional (4096/boot budget), rc!=0 repeats rate
  limited; done=0x0 exposes the pre-loop resolution-miss retries.
- CEAUDIT-MISMATCH: the same true start/end 16-byte destination
  readback as COPYAUDIT (moved to the real end sample), emitted next to
  COPY-MISMATCH for ce-soft-copy.
- D2H authority visibility: ledger counters + table-fill diag line
  (sema-svc: ledger n=/stores/evict/diverges/rewrites) and a
  sema-ledger: EVICT warn on the silent home-replacement path.

D2H source verdict (documented in /tmp/opencode/agent-COPY.md): for
pages the launch owns the D2H follows the submit-time ledger landing =
the page the copy-out wrote (correct); it can silently return prior
content for unowned pages (fill-span cuts, work-item drops), on ledger
eviction, and after a failed launch whose copy-out never ran.

make kernel green; only the pre-existing %x warning at
gsp_phys_booter.c:14088.

1 files changed, 131 insertions, 32 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 91a128f..8ae11ec 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -3866,6 +3866,23 @@ static int svc_map_lookup_range(u64 gpu_va, u32 from, u32 to, u64 *pa)
 
 static void svc_session_flush(void);	/* defined next to the pending queue */
 
+/*
+ * Ledger sizing + counters hoisted above svc_rebuild so the periodic diag
+ * can report the table's fill state: a full table degrades the write/read
+ * authority silently (home replacement evicts another VA's landing), which
+ * is exactly the D2H "reads back prior content" window.
+ */
+#define SVC_LEDGER_MAX		(1u << 14)
+#define SVC_LEDGER_MASK		(SVC_LEDGER_MAX - 1u)
+#define SVC_LEDGER_PROBE	16u
+static u32 svc_ledger_n;
+static u64 svc_ledger_diverges;
+static u64 svc_ledger_stores;
+static u64 svc_ledger_evictions;
+static u64 svc_ledger_rewrites;
+static u64 svc_ledger_reloc_diverges;
+static u32 svc_ledger_store_log;
+
 static void svc_rebuild(struct nv_vgpu_device *vgpu)
 {
 	static u8 *buf;
@@ -3990,6 +4007,17 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu)
 		svc_vas_n, SVC_VAS_SLOTS, svc.vas_writes, svc.vas_reads,
 		svc.vas_shadowed);
 	/*
+	 * D2H-source authority visibility (r220): a full ledger degrades to
+	 * home-slot replacement, which silently evicts another VA's landing
+	 * and lets readers fall back to the resolver - the "reads back prior
+	 * content" window. n approaching SVC_LEDGER_MAX or evict > 0 in a
+	 * client's window is the tell.
+	 */
+	pr_info("sema-svc:   ledger n=%u/%u stores=%llu evict=%llu diverges=%llu reloc_div=%llu rewrites=%llu\n",
+		svc_ledger_n, SVC_LEDGER_MAX, svc_ledger_stores,
+		svc_ledger_evictions, svc_ledger_diverges,
+		svc_ledger_reloc_diverges, svc_ledger_rewrites);
+	/*
 	 * P6b wave 1: guest-forwarding summary. Printed whenever any counter
 	 * moved (not only on requests) so a run that captured no candidate is
 	 * still diagnosable from the log.
@@ -4298,7 +4326,9 @@ static bool svc_dst_watched(u64 dst)
  */
 #define SVC_COPY_AUDIT_MIN	(32u << 10)	/* >32 KB copies audited */
 #define SVC_COPY_AUDIT_MAX	2048u		/* per-boot line budget */
+#define SVC_CEAUDIT_MAX		4096u		/* per-boot CEAUDIT budget */
 static u64 svc_copy_audit_n;
+static u64 svc_ceaudit_n;
 
 /* COPYAUDIT for a finished fill: log want/done and check the first 16
  * destination bytes against the repeated pattern. The readback uses the
@@ -4346,7 +4376,7 @@ static void svc_ledger_store(u64 va, u64 pa, u32 ap);	/* R2 ledger, defined belo
 static int svc_ledger_lookup(u64 va, u64 *pa, u32 *ap);	/* R2 ledger, defined below */
 
 static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem,
-		    u32 count, u32 phys, u32 aperture)
+		    u32 count, u32 phys, u32 aperture, u64 *done_out)
 {
 	u8 pat[8];
 	u8 bounce[256];
@@ -4355,6 +4385,8 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem,
 	bool shadow = false;
 	u64 va0 = addr;		/* original destination VA (fills, C2) */
 
+	if (done_out)
+		*done_out = 0;
 	if (elem != 1 && elem != 2 && elem != 4 && elem != 8)
 		return -EINVAL;
 	if (!count || total > (256ULL << 20))
@@ -4401,8 +4433,13 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem,
 			svc.op_tr_src, elem, pb, svc.cur_ch);
 	}
 
-	if (shadow)
-		return svc_vas_fill(addr, pat, elem, total);
+	if (shadow) {
+		int rc = svc_vas_fill(addr, pat, elem, total);
+
+		if (!rc && done_out)
+			*done_out = (u64)total;
+		return rc;
+	}
 
 	/*
 	 * C2 (data class) + m180 unification: a large virtual fill is a
@@ -4430,6 +4467,8 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem,
 			memcpy(base + addr + o, pat, 8);
 		if (o < total)
 			memcpy(base + addr + o, pat, total - o);
+		if (done_out)
+			*done_out = (u64)total;
 		svc_fill_audit(vgpu, va0, addr, aperture, total, pat, elem);
 		return 0;
 	}
@@ -4446,6 +4485,8 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem,
 		if (i < chunk)
 			memcpy(bounce + i, pat, chunk - i);
 		if (svc_write(vgpu, addr + o, bounce, chunk)) {
+			if (done_out)
+				*done_out = (u64)o;
 			if (total >= SVC_COPY_AUDIT_MIN)
 				pr_err_ratelimited("vgsp: COPY-TRUNC va=0x%llx want=0x%llx done=0x%llx path=sema-fill rc=%d\n",
 						   va0, (u64)total, (u64)o, -EIO);
@@ -4453,6 +4494,8 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem,
 		}
 		o += chunk;
 	}
+	if (done_out)
+		*done_out = (u64)total;
 	svc_fill_audit(vgpu, va0, addr, aperture, total, pat, elem);
 	return 0;
 }
@@ -4874,17 +4917,12 @@ struct svc_ledger_ent {
  * (8 MB in + 8 MB out + engine records ~ 12 K pages); hash collisions
  * only occur between VA pages 64 MiB apart or across clients whose pages
  * do not co-reside in time.
+ *
+ * COPYAUDIT r220: SVC_LEDGER_MAX/MASK/PROBE and the counters live above
+ * svc_rebuild (hoisted for the diag line); only the table and the home
+ * helper stay here.
  */
-#define SVC_LEDGER_MAX		(1u << 14)
-#define SVC_LEDGER_MASK		(SVC_LEDGER_MAX - 1u)
-#define SVC_LEDGER_PROBE	16u
 static struct svc_ledger_ent svc_ledger[SVC_LEDGER_MAX];
-static u32 svc_ledger_n;
-static u64 svc_ledger_diverges;
-static u64 svc_ledger_stores;
-static u64 svc_ledger_rewrites;
-static u64 svc_ledger_reloc_diverges;
-static u32 svc_ledger_store_log;
 
 static inline u32 svc_ledger_home(u64 page)
 {
@@ -4937,8 +4975,15 @@ static void svc_ledger_store(u64 va, u64 pa, u32 ap)
 	/*
 	 * Cluster full (16 same-home VA pages co-resident): replace the home
 	 * slot. Bounded, rare, and only affects VAs whose pages hash to the
-	 * identical home slot.
+	 * identical home slot. r220: this is a SILENT loss of the evicted
+	 * VA's landing authority (its later readers fall back to the
+	 * resolver, possibly stale), so the first occurrences are logged and
+	 * counted into the diag line.
 	 */
+	svc_ledger_evictions++;
+	if (svc_ledger_evictions <= 32)
+		pr_warn("sema-ledger: EVICT home=%u old_va=0x%llx new_va=0x%llx (cluster full; old landing authority lost until rewritten)\n",
+			home, (u64)svc_ledger[home].va_page, page);
 	svc_ledger[home].va_page = page;
 	svc_ledger[home].pa_page = pa_page;
 	svc_ledger[home].ap = ap;
@@ -4986,30 +5031,38 @@ static void svc_ledger_clear(void)
  */
 static void svc_copy_audit_chunk(struct nv_vgpu_device *vgpu, u64 dst_va,
 				 u64 o, const u8 *bounce, size_t chunk,
-				 bool shadow, u64 pa_eff, u32 aper)
+				 bool shadow, u64 pa_eff, u32 aper, bool tail)
 {
 	size_t n = chunk < 16 ? chunk : 16;
+	size_t so = tail ? chunk - n : 0;	/* true end sample */
 	u8 rb[16] = {0};
 
 	if (shadow) {
-		if (svc_vas_read(dst_va + o, rb, n))
+		if (svc_vas_read(dst_va + o + so, rb, n))
 			return;
 	} else if (aper == CE_APERTURE_VID) {
 		u8 *base = vgpu->bars[1].vaddr;
 
-		if (!base || pa_eff + n > svc_fb_shadow_size(vgpu))
+		if (!base || pa_eff + so + n > svc_fb_shadow_size(vgpu))
 			return;
-		memcpy(rb, base + pa_eff, n);
-	} else if (svc_read(vgpu, pa_eff, rb, n)) {
+		memcpy(rb, base + pa_eff + so, n);
+	} else if (svc_read(vgpu, pa_eff + so, rb, n)) {
 		return;
 	}
-	if (memcmp(rb, bounce, n))
+	if (memcmp(rb, bounce + so, n)) {
 		pr_info_ratelimited("vgsp: COPY-MISMATCH va=0x%llx off=0x%llx path=ce-soft-copy want=%*phN got=%*phN\n",
-				    dst_va + o, o, (int)n, bounce, (int)n, rb);
+				    dst_va + o + so, o + so, (int)n,
+				    bounce + so, (int)n, rb);
+		pr_info_ratelimited("vgsp: CEAUDIT-MISMATCH op=copy va=0x%llx off=0x%llx path=ce-soft-copy want=%*phN got=%*phN\n",
+				    dst_va + o + so, o + so, (int)n,
+				    bounce + so, (int)n, rb);
+	}
 }
 
 #define SVC_COPY_TRUNC_RET(rc_)						\
 	do {								\
+		if (done_out)						\
+			*done_out = (u64)o;				\
 		if (total >= SVC_COPY_AUDIT_MIN)			\
 			pr_err_ratelimited("vgsp: COPY-TRUNC va=0x%llx want=0x%llx done=0x%llx path=ce-soft-copy rc=%d\n",	\
 					   dst_orig, (u64)total, (u64)o, (rc_));	\
@@ -5017,7 +5070,8 @@ static void svc_copy_audit_chunk(struct nv_vgpu_device *vgpu, u64 dst_va,
 	} while (0)
 
 static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
-		    u32 dst_phys, u32 dst_aperture, u32 src_phys, u32 src_aperture)
+		    u32 dst_phys, u32 dst_aperture, u32 src_phys, u32 src_aperture,
+		    u64 *done_out)
 {
 	u8 bounce[256];
 	size_t o;
@@ -5030,6 +5084,8 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
 	bool audit = total >= SVC_COPY_AUDIT_MIN;
 	static u64 bigcopy_log;
 
+	if (done_out)
+		*done_out = 0;	/* 0 until the first byte lands */
 	if (!total || total > (256ULL << 20))
 		return -EINVAL;
 
@@ -5439,7 +5495,8 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
 				SVC_COPY_TRUNC_RET(-ENOMEM);
 			if (audit && (!o || o + chunk == total))
 				svc_copy_audit_chunk(vgpu, dst_orig, o, bounce,
-						     chunk, true, 0, 0);
+						     chunk, true, 0, 0,
+						     (o != 0));
 		} else if (dst_virt) {
 			u64 pa = 0;
 			u32 aper = 0;
@@ -5489,7 +5546,8 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
 			}
 			if (audit && (!o || o + chunk == total))
 				svc_copy_audit_chunk(vgpu, dst_orig, o, bounce,
-						     chunk, false, pa, aper);
+						     chunk, false, pa, aper,
+						     (o != 0));
 			/*
 			 * m171/m180: a large device-dest write records its
 			 * landing page - SYS and VID alike, because the
@@ -5514,14 +5572,16 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
 			if (audit && (!o || o + chunk == total))
 				svc_copy_audit_chunk(vgpu, dst_orig, o, bounce,
 						     chunk, false, dst + o,
-						     CE_APERTURE_VID);
+						     CE_APERTURE_VID,
+						     (o != 0));
 		} else {
 			if (svc_write(vgpu, dst + o, bounce, chunk))
 				SVC_COPY_TRUNC_RET(-EIO);
 			if (audit && (!o || o + chunk == total))
 				svc_copy_audit_chunk(vgpu, dst_orig, o, bounce,
 						     chunk, false, dst + o,
-						     CE_APERTURE_SYS);
+						     CE_APERTURE_SYS,
+						     (o != 0));
 		}
 		(void)i;
 		o += chunk;
@@ -5588,6 +5648,8 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
 		pr_info("vgsp: COPYAUDIT path=ce-soft-copy va=0x%llx len=0x%llx done=0x%llx\n",
 			dst_orig, (u64)total, (u64)o);
 	}
+	if (done_out)
+		*done_out = (u64)o;	/* o == total on the success path */
 	return 0;
 }
 
@@ -5765,13 +5827,50 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops,
 			continue;
 		}
 
-		if (m->type == 1)
-			rc = svc_copy(vgpu, m->dst, m->src, m->bytes,
-				      m->phys, m->aperture,
-				      m->src_phys, m->src_aperture);
-		else
-			rc = svc_fill(vgpu, m->dst, m->value, m->elem,
-				      m->count, m->phys, m->aperture);
+		{
+			u64 ce_len = (m->type == 1) ? m->bytes :
+				     (u64)m->elem * m->count;
+			u64 ce_done = 0;
+
+			if (m->type == 1)
+				rc = svc_copy(vgpu, m->dst, m->src, m->bytes,
+					      m->phys, m->aperture,
+					      m->src_phys, m->src_aperture,
+					      &ce_done);
+			else
+				rc = svc_fill(vgpu, m->dst, m->value, m->elem,
+					      m->count, m->phys, m->aperture,
+					      &ce_done);
+			/*
+			 * CEAUDIT (2026-10-18 r220): per-op audit of the CE
+			 * data path - the software executor of every guest
+			 * cudaMemcpy - at/above SVC_COPY_AUDIT_MIN, including
+			 * the rc != 0 attempts the copy-level COPYAUDIT cannot
+			 * see (the pre-loop resolution misses return before
+			 * any byte moves: done=0x0). src/dst are the guest
+			 * VAs of the op; done is the byte count that landed.
+			 * Bounded per boot.
+			 */
+			if (ce_len >= SVC_COPY_AUDIT_MIN &&
+			    svc_ceaudit_n < SVC_CEAUDIT_MAX) {
+				svc_ceaudit_n++;
+				/*
+				 * A failing op is retried by the hold and would
+				 * spam: throttle the rc != 0 repeats; successful
+				 * big ops log unconditionally (budget-bounded).
+				 */
+				if (rc)
+					pr_info_ratelimited("vgsp: CEAUDIT op=%s src=0x%llx dst=0x%llx len=0x%llx done=0x%llx rc=%d\n",
+						m->type == 1 ? "copy" : "fill",
+						m->type == 1 ? m->src : 0ull,
+						m->dst, ce_len, ce_done, rc);
+				else
+					pr_info("vgsp: CEAUDIT op=%s src=0x%llx dst=0x%llx len=0x%llx done=0x%llx rc=%d\n",
+						m->type == 1 ? "copy" : "fill",
+						m->type == 1 ? m->src : 0ull,
+						m->dst, ce_len, ce_done, rc);
+			}
+		}
 		if ((rc == 0 || rc == SVC_COPY_FORWARDED) &&
 		    ops_done && seq < 64)
 			*ops_done |= 1ULL << seq;