vgsp: fail4 - pipeline-order source snapshot for held staged copies

The m169 device-dst guard defers a staged H2D up to 3 s; the driver's
ping-pong staging buffers are overwritten meanwhile, so a deferred read
returned a later chunk's content (the +2 MB vector_add phase = the 2x1
MB staging stride). Held copies now snapshot their source at hold time
(bounded: 8 slots x 4 MB, 30 s TTL) and the retried execution copies
from the snapshot, matching the hardware's in-order staging read.

r296 A/B: the snapshot never fires for vector_add - its H2D dsts
resolve with pins promptly and execute without the hold; the +2 MB
persists identically. So the staging overrun happens WITHOUT the
guard: the driver's 2-buffer staging discipline relies on the copy's
completion fence (a semaphore applied when the copy executes) to pace
the restaging; if our completion for chunk i arrives before/independent
of the staging read, the driver restages and our later decode reads
chunk i+2. Next lead: the staging fence's increment timing vs the
copy's execution (the release-before-work class extended to the H2D
staging fences), not the m169 hold.

1 files changed, 110 insertions, 0 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 6c75a9e..ce15524 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -5235,6 +5235,105 @@ static void svc_copy_audit_chunk(struct nv_vgpu_device *vgpu, u64 dst_va,
 		return (rc_);						\
 	} while (0)
 
+/*
+ * fail4: pipeline-order source snapshot for held copies. The m169
+ * device-dst guard defers a staged H2D (host->device) for up to 3 s;
+ * the driver's ping-pong staging buffers are overwritten by the next
+ * chunk meanwhile, so the deferred read returns the content of a later
+ * chunk (vector_add: device a[i] pages held a[i+2 MB] - exactly the
+ * 2x1 MB staging stride). A held copy snapshots its source at hold
+ * time and the retried execution copies from the snapshot, matching
+ * the hardware's in-order read of the staging.
+ */
+#define SVC_COPY_SNAP_N		8
+#define SVC_COPY_SNAP_MAX	(4u << 20)
+#define SVC_COPY_SNAP_TTL_MS	30000u
+struct svc_copy_snap {
+	u64 dst, src;
+	size_t total;
+	u8 *buf;
+	unsigned long stamp;
+};
+static struct svc_copy_snap svc_copy_snaps[SVC_COPY_SNAP_N];
+
+static u8 *svc_copy_snap_lookup(u64 dst, u64 src, size_t total)
+{
+	u32 i;
+
+	for (i = 0; i < SVC_COPY_SNAP_N; i++)
+		if (svc_copy_snaps[i].buf &&
+		    svc_copy_snaps[i].dst == dst &&
+		    svc_copy_snaps[i].src == src &&
+		    svc_copy_snaps[i].total == total)
+			return svc_copy_snaps[i].buf;
+	return NULL;
+}
+
+static void svc_copy_snap_free(u64 dst, u64 src, size_t total)
+{
+	u32 i;
+
+	for (i = 0; i < SVC_COPY_SNAP_N; i++)
+		if (svc_copy_snaps[i].buf &&
+		    svc_copy_snaps[i].dst == dst &&
+		    svc_copy_snaps[i].src == src &&
+		    svc_copy_snaps[i].total == total) {
+			kfree(svc_copy_snaps[i].buf);
+			svc_copy_snaps[i].buf = NULL;
+			return;
+		}
+}
+
+/* Capture the source of a virtual-operand copy about to be held. */
+static void svc_copy_snap_capture(struct nv_vgpu_device *vgpu, u64 dst,
+				  u64 src, size_t total)
+{
+	struct svc_copy_snap *s = NULL;
+	u8 *buf;
+	size_t o;
+	u32 i;
+
+	if (!total || total > SVC_COPY_SNAP_MAX)
+		return;
+	if (svc_copy_snap_lookup(dst, src, total))
+		return;	/* already captured */
+	for (i = 0; i < SVC_COPY_SNAP_N; i++) {
+		if (!svc_copy_snaps[i].buf) {
+			s = &svc_copy_snaps[i];
+			break;
+		}
+		if (time_after(jiffies, svc_copy_snaps[i].stamp +
+			       msecs_to_jiffies(SVC_COPY_SNAP_TTL_MS))) {
+			kfree(svc_copy_snaps[i].buf);
+			svc_copy_snaps[i].buf = NULL;
+			s = &svc_copy_snaps[i];
+			break;
+		}
+	}
+	if (!s)
+		return;	/* all slots live: keep the historical behaviour */
+	buf = kmalloc(total, GFP_KERNEL);
+	if (!buf)
+		return;
+	for (o = 0; o < total; ) {
+		size_t chunk = min_t(size_t, 0x1000 - ((src + o) & 0xfff),
+				     total - o);
+
+		if (svc_read(vgpu, src + o, buf + o, chunk)) {
+			kfree(buf);
+			return;
+		}
+		o += chunk;
+	}
+	s->dst = dst;
+	s->src = src;
+	s->total = total;
+	s->stamp = jiffies;
+	s->buf = buf;
+	pr_info_ratelimited("vgsp-sema: copy snapshot captured dst=0x%llx src=0x%llx total=%zu (held staged copy)\n",
+			    dst, src, total);
+}
+
 static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
 		    u32 dst_phys, u32 dst_aperture, u32 src_phys, u32 src_aperture,
 		    u64 *done_out)
@@ -5248,6 +5347,7 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
 	u32 dst_prov = 0, src_prov = 0;
 	bool bigcopy_this = false;
 	bool audit = total >= SVC_COPY_AUDIT_MIN;
+	u8 *snap = NULL;
 	static u64 bigcopy_log;
 	bool trace64 = (total == 65536u);
 	const char *dir = "?";
@@ -5267,6 +5367,7 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
 			svc.cur_ch, dst, src, total, dst_phys, dst_aperture,
 			src_phys, src_aperture);
 	}
+	snap = svc_copy_snap_lookup(dst_orig, src_orig, total);
 
 	if (!dst_phys) {
 		u64 pa;
@@ -5338,6 +5439,13 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
 		if (svc.copy_dst_hold <= 16)
 			pr_info("sema-svc: device-dst copy held: dst=0x%llx prov=%u (no pin yet; stale-table guard)\n",
 				dst_orig, dst_prov);
+		/*
+		 * fail4: the staging source is valid NOW; snapshot it so
+		 * the deferred execution copies the pipeline-order bytes.
+		 */
+		if (!src_phys && src_orig >= SVC_VAS_FLOOR)
+			svc_copy_snap_capture(vgpu, dst_orig, src_orig,
+					      total);
 		return -ENOENT;
 	}
 	/*
@@ -5930,6 +6038,8 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
 		for (po = 0; po < total; po += SEMA_PAGE_SIZE)
 			svc_ledger_store(dst_orig + po, dst + po, cap);
 	}
+	if (snap)
+		svc_copy_snap_free(dst_orig, src_orig, total);
 	if (done_out)
 		*done_out = (u64)o;	/* o == total on the success path */
 	return 0;