vgsp: end-to-end CUDA acceptance in the guest

The no-claim emulation path now runs a stock CUDA program end to end:
device properties, cudaMalloc, a GPU-authored string kernel read back
byte-for-byte, two 4 MB host-to-device copies, a 1 M-element vectorAdd
executed on the physical GPU through the host bridge, and the
device-to-host copy-back verified element by element. Three consecutive
supervised cycles report MAIL + VADD bad=0 checksum=392344090.500000 +
STATUS OK with identical input/output hashes.

Two data-path correctness fixes make the transfers sound:

- Once-only op execution per GPFIFO entry. The pump replays recently
  consumed ring entries when translation batches are injected so missed
  releases can land; re-executing a CE copy is not idempotent because
  its source is a client staging buffer that may have been refilled.
  Slots now record their raw entry words and a replayed entry matching
  that identity suppresses operations while still processing releases.

- Per-page translation of virtual operands. svc_copy resolved only the
  first page of a VA range and walked the PA linearly; client staging
  ranges are frequently split (observed 195 + 61 pages), so the tail of
  alternating chunks was read from unrelated memory. Multi-page copies
  now re-resolve each 4 KB page through the trusted resolver.

Also carries the earlier uncommitted emulation fixes that got the client
to the launch (QMD address-window gate, counter registration
persistence, 512 MiB emulated FB, bridge slot hygiene) and bounded
diagnostics (sema-bigop, sema-bigcopy, bridge-in/out hashes, daemon
INCHECK/OUTCHECK/dumps).

1 files changed, 389 insertions, 106 deletions

diff --git a/kernel/gsp_cuda_bridge.c b/kernel/gsp_cuda_bridge.c
index aaad5ee..2fac2b2 100644
--- a/kernel/gsp_cuda_bridge.c
+++ b/kernel/gsp_cuda_bridge.c
@@ -36,6 +36,7 @@
 #include <linux/string.h>
 
 #include "nvidia-vgpu-vfio.h"
+#include "gsp_cuda_bridge.h"
 
 /* Records and limits (see tools/cuda-bridge/PROTOCOL.md, v2). */
 #define CUBB2_JOB_MAGIC		0x32424243u	/* 'CBB2' */
@@ -111,14 +112,7 @@ static_assert(offsetof(struct cubb2_job, prog_va) == 0x60,
 static_assert(offsetof(struct cubb2_done, host_ns) == 0x70,
 	      "cubb2_done natural layout changed");
 
-/* Entry points (also declared in gsp_sema_service.c). */
-bool gsp_cuda_bridge_enabled(void);
-void gsp_cuda_bridge_launch(struct nv_vgpu_device *vgpu, u32 h_object,
-			    const u32 qmd[64], u64 qmd_dev_va, u64 push_va,
-			    const u64 *recent_dsts, u32 n_recent_dsts);
-void gsp_cuda_bridge_service(struct nv_vgpu_device *vgpu);
-int gsp_cuda_bridge_init(void);
-void gsp_cuda_bridge_fini(void);
+/* Entry points are declared in gsp_cuda_bridge.h. */
 
 /* Helpers exported by gsp_sema_service.c (declared here, not in the header,
  * so the bridge stays a leaf of the sema service). */
@@ -128,6 +122,10 @@ int gsp_sema_service_va_write(struct nv_vgpu_device *vgpu, u64 va,
 			      const void *buf, size_t len);
 int gsp_sema_service_va_resolve(struct nv_vgpu_device *vgpu, u64 va, u64 *pa,
 				u32 *ap);
+int gsp_sema_service_va_translate(struct nv_vgpu_device *vgpu, u64 va,
+				  u64 *pa, u32 *ap);
+void gsp_sema_service_va_explain(struct nv_vgpu_device *vgpu, u64 va);
+void gsp_sema_service_trace_window(u32 ms);
 int gsp_sema_service_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val);
 int gsp_sema_service_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va,
 			       u32 val);
@@ -148,8 +146,8 @@ struct cubb_req {
 	u64 push_va;
 	u32 qmd[64];
 	u32 attempts;
-	u32 n_recent_dsts;
-	u64 recent_dsts[CUBB2_INLINE_MAX];
+	u32 n_inl;
+	struct gsp_bridge_inline inl[GSP_BRIDGE_INLINE_MAX];
 };
 
 static struct cubb_req cubb_req[CUBB2_REQ_MAX];
@@ -190,6 +188,18 @@ struct cubb_pend {
 	u32 sem_flags;
 	struct cubb2_job job;
 	u8 *in;			/* vmalloc'd input blob */
+	/*
+	 * Deferred completion: a done record whose output write could not
+	 * resolve yet (the UVM device VA's translation is learned
+	 * asynchronously) is staged here and retried on later service passes
+	 * for a bounded time instead of failing the job immediately.
+	 */
+	bool have_done;
+	u32 done_tries;
+	unsigned long done_at;
+	u64 done_out_len;
+	u8 *done_out;		/* vmalloc'd output blob */
+	struct cubb2_done done;
 };
 
 static struct cubb_pend cubb_pend[CUBB2_PEND_MAX];
@@ -215,6 +225,7 @@ static u64 cubb_dropped;	/* request ring/marshal/allocation drops */
 static u64 cubb_stale;		/* done records for unknown seq */
 static u64 cubb_malformed;	/* done records with bad magic/version */
 static u64 cubb_last_seq;
+static bool cubb_explained;	/* one-shot walk explainer fired */
 static int cubb_last_err;
 static u64 cubb_launch_log;	/* verbose header budget */
 
@@ -387,17 +398,21 @@ static int cubb_shape_kind(const u8 *p, size_t len)
 /*
  * Locate the launch's argument block inside the QMD parameter bank. CUDA
  * writes the arguments with GRCE LOAD_INLINE_DATA at an arbitrary bank
- * offset, so never assume +0: first try the recorded recent inline-write
- * destinations that fall inside [bank_va, bank_va + bank_size), newest
- * first, then scan the bank for a mail/vadd argument shape. `want` names the
- * shape implied by grid/block (0 = unknown). On a match returns 0 and fills
- * off/kind/params/src; -ENODATA means the bank was readable but no shape was
- * found (caller may fall back to bank+0); -ENOENT means a read failed (the
- * page resolve may still be in progress) and nothing matched, so the caller
- * should retry.
+ * offset, so never assume +0: first match the payload bytes the sema service
+ * cached for the recent inline-write destinations that fall inside
+ * [bank_va, bank_va + bank_size), newest first (no VA read needed: these
+ * banks are UVM-managed VAs with no reliable translation). If the writes
+ * were split, VA-adjacent cached records are merged and shape-scanned. Then
+ * scan the bank itself for a mail/vadd argument shape as a fallback. `want`
+ * names the shape implied by grid/block (0 = unknown). On a match returns 0
+ * and fills off/kind/params/src; -ENODATA means nothing matched and no read
+ * failed (caller may fall back to bank+0); -ENOENT means a bank read failed
+ * (the page resolve may still be in progress) and nothing matched, so the
+ * caller should retry.
  */
 static int cubb_find_params(struct nv_vgpu_device *vgpu, u64 bank_va,
-			    u32 bank_size, const u64 *dsts, u32 n_dsts,
+			    u32 bank_size,
+			    const struct gsp_bridge_inline *recs, u32 n_recs,
 			    u32 want, u64 *off, u32 *kind, u8 *params,
 			    size_t plen, const char **src)
 {
@@ -412,31 +427,72 @@ static int cubb_find_params(struct nv_vgpu_device *vgpu, u64 bank_va,
 	if (bank_size < 16)
 		return -ENODATA;
 
-	/* (a) most recently written inline destinations inside the bank */
-	for (i = 0; i < n_dsts; i++) {
-		u64 va = dsts[i];
+	/*
+	 * (a) cached inline-write payloads, newest first. The bytes are in
+	 * hand from the push decode, so a shape match needs no translation.
+	 */
+	for (i = 0; i < n_recs; i++) {
+		const struct gsp_bridge_inline *r = &recs[i];
 		size_t need = want == 2 ? 32 : 12;
-		size_t got = min_t(size_t, plen, sizeof(cand));
+		size_t got;
 		int k;
 
-		if (!va || va < bank_va || va + need > hi)
-			continue;
-		if (va + got > hi)
-			got = hi - va;
-		if (gsp_sema_service_va_read(vgpu, va, cand, got)) {
-			read_fail = true;
+		if (!r->dst_va || !r->len || r->dst_va < bank_va ||
+		    r->dst_va + need > hi)
 			continue;
-		}
-		k = cubb_shape_kind(cand, got);
+		got = min_t(size_t, r->len, sizeof(cand));
+		if (r->dst_va + got > hi)
+			got = hi - r->dst_va;
+		k = cubb_shape_kind(r->data, got);
 		if (!k || (want && k != want))
 			continue;
-		*off = va - bank_va;
+		*off = r->dst_va - bank_va;
 		*kind = k;
-		memcpy(params, cand, min_t(size_t, got, plen));
-		*src = "inline";
+		memcpy(params, r->data, min_t(size_t, got, plen));
+		*src = "cache";
 		return 0;
 	}
 
+	/*
+	 * (a2) a large argument block can arrive as several consecutive
+	 * LOAD_INLINE_DATA runs: merge VA-adjacent cached records and scan the
+	 * merged bytes for a shape.
+	 */
+	for (i = 0; i < n_recs; i++) {
+		u8 merged[CUBB2_PARAM_BLOB];
+		u32 mlen, j;
+		u64 base;
+		int k;
+
+		base = recs[i].dst_va;
+		if (!base || !recs[i].len || base < bank_va || base + 12 > hi)
+			continue;
+		mlen = min_t(u32, recs[i].len, sizeof(merged));
+		memcpy(merged, recs[i].data, mlen);
+		for (j = 0; j < n_recs && mlen < sizeof(merged); j++) {
+			u32 take;
+
+			if (!recs[j].len ||
+			    recs[j].dst_va != base + mlen)
+				continue;
+			take = min_t(u32, recs[j].len,
+				     sizeof(merged) - mlen);
+			memcpy(merged + mlen, recs[j].data, take);
+			mlen += take;
+		}
+		for (o = 0; o + 12 <= mlen; o += 4) {
+			k = cubb_shape_kind(merged + o, mlen - o);
+			if (!k || (want && k != want))
+				continue;
+			*off = base + o - bank_va;
+			*kind = k;
+			memcpy(params, merged + o,
+			       min_t(u32, mlen - o, plen));
+			*src = "cache-merge";
+			return 0;
+		}
+	}
+
 	/* (b) scan the bank for an argument shape (straddle-safe per page) */
 	scan = bank_size > CUBB2_PARAM_SCAN_MAX ? CUBB2_PARAM_SCAN_MAX :
 						 bank_size;
@@ -512,7 +568,7 @@ bool gsp_cuda_bridge_enabled(void)
  */
 void gsp_cuda_bridge_launch(struct nv_vgpu_device *vgpu, u32 h_object,
 			    const u32 qmd[64], u64 qmd_dev_va, u64 push_va,
-			    const u64 *recent_dsts, u32 n_recent_dsts)
+			    const struct gsp_bridge_inline *recs, u32 n_recs)
 {
 	unsigned long flags;
 	u64 hash = 1469598103934665603ULL;
@@ -561,13 +617,13 @@ void gsp_cuda_bridge_launch(struct nv_vgpu_device *vgpu, u32 h_object,
 		r->push_va = push_va;
 		r->attempts = 0;
 		memcpy(r->qmd, qmd, sizeof(r->qmd));
-		r->n_recent_dsts = 0;
-		if (recent_dsts && n_recent_dsts) {
-			if (n_recent_dsts > CUBB2_INLINE_MAX)
-				n_recent_dsts = CUBB2_INLINE_MAX;
-			memcpy(r->recent_dsts, recent_dsts,
-			       n_recent_dsts * sizeof(u64));
-			r->n_recent_dsts = n_recent_dsts;
+		r->n_inl = 0;
+		if (recs && n_recs) {
+			if (n_recs > GSP_BRIDGE_INLINE_MAX)
+				n_recs = GSP_BRIDGE_INLINE_MAX;
+			memcpy(r->inl, recs,
+			       n_recs * sizeof(struct gsp_bridge_inline));
+			r->n_inl = n_recs;
 		}
 	}
 	cubb_req_head++;
@@ -626,9 +682,41 @@ static void cubb_log_raw_qmd(const u32 *qmd)
 			qmd[i], qmd[i + 1], qmd[i + 2], qmd[i + 3]);
 }
 
+/*
+ * One-shot evidence dump when a launch's argument block cannot be located:
+ * the cached inline-write records the search had to work with (destination,
+ * length, first bytes) and the QMD's declared parameter-bank window. Decides
+ * on the next run whether the cache missed the write or the shape check was
+ * too strict.
+ */
+static u64 cubb_inline_log;
+
+static void cubb_dump_inline(const struct cubb_req *r, u64 param_va,
+			     u32 param_size)
+{
+	u32 i;
+
+	if (cubb_inline_log >= 8)
+		return;
+	cubb_inline_log++;
+	pr_info("param miss: bank=0x%llx size=0x%x push=0x%llx h=0x%x records=%u\n",
+		param_va, param_size, r->push_va, r->h_object, r->n_inl);
+	for (i = 0; i < r->n_inl; i++) {
+		const struct gsp_bridge_inline *rec = &r->inl[i];
+		u32 show = min_t(u32, rec->len, 16);
+
+		pr_info("param miss: [%u] dst=0x%llx len=%u data16=%*phN\n",
+			i, rec->dst_va, rec->len, show, rec->data);
+	}
+}
+
 static int cubb_marshal_one(struct nv_vgpu_device *vgpu)
 {
-	struct cubb_req r;
+	/*
+	 * Static: the request carries 16 inline-payload records (~1.2 KB) and
+	 * this runs on the single service thread only (frame-size audit).
+	 */
+	static struct cubb_req r;
 	struct cubb2_job job;
 	unsigned long flags;
 	u32 gx, gy, gz, bx, by, bz, shared;
@@ -687,12 +775,13 @@ static int cubb_marshal_one(struct nv_vgpu_device *vgpu)
 		 bz == 1)
 		hint = 2;
 
-	rc = cubb_find_params(vgpu, param_va, param_size, r.recent_dsts,
-			      r.n_recent_dsts, hint, &params_off, &found_kind,
+	rc = cubb_find_params(vgpu, param_va, param_size, r.inl,
+			      r.n_inl, hint, &params_off, &found_kind,
 			      params, plen, &params_src);
 	if (rc) {
 		if (cubb_req_retry(&r, rc))
 			return 0;
+		cubb_dump_inline(&r, param_va, param_size);
 		if (rc != -ENODATA) {
 			cubb_req_drop_head();
 			cubb_failed++;
@@ -796,6 +885,36 @@ static int cubb_marshal_one(struct nv_vgpu_device *vgpu)
 					    a, b, rc);
 			return 0;
 		}
+		/*
+		 * Input-path proof: hash exactly what the bridge read from the
+		 * guest's device VAs (FNV-1a over the raw bytes) plus a float
+		 * NaN count per half, so the guest-side data and the daemon's
+		 * received data can be compared without guessing.
+		 */
+		{
+			uint64_t h = 1469598103934665603ULL;
+			u32 k, nan_a = 0, nan_b = 0;
+			const u32 *w = (const u32 *)in;
+
+			for (k = 0; k < in_bytes / 4; k++) {
+				h ^= w[k];
+				h *= 1099511628211ULL;
+			}
+			for (k = 0; k < in_bytes / 8; k++) {
+				u32 wa = w[k];
+				u32 wb = w[in_bytes / 8 + k];
+
+				if ((wa & 0x7f800000u) == 0x7f800000u &&
+				    (wa & 0x007fffffu))
+					nan_a++;
+				if ((wb & 0x7f800000u) == 0x7f800000u &&
+				    (wb & 0x007fffffu))
+					nan_b++;
+			}
+			pr_info("bridge-in: a=0x%llx b=0x%llx bytes=%llu hash=%016llx nan_a=%u nan_b=%u\n",
+				a, b, in_bytes, (unsigned long long)h,
+				nan_a, nan_b);
+		}
 	}
 
 	if (!n || !out_bytes || out_bytes > CUBB2_OUT_MAX) {
@@ -816,6 +935,10 @@ static int cubb_marshal_one(struct nv_vgpu_device *vgpu)
 				    params_off, dst_va, n);
 
 	rc = gsp_sema_service_va_resolve(vgpu, dst_va, &dst_pa, &dst_ap);
+	if (rc && !cubb_explained) {
+		cubb_explained = true;
+		gsp_sema_service_va_explain(vgpu, dst_va);
+	}
 
 	memset(&job, 0, sizeof(job));
 	job.magic = CUBB2_JOB_MAGIC;
@@ -859,6 +982,12 @@ static int cubb_marshal_one(struct nv_vgpu_device *vgpu)
 	cubb_pend[slot].sem_flags = sem_flags;
 	cubb_pend[slot].job = job;
 	cubb_pend[slot].in = in;
+	/* A reused slot must never inherit the previous job's staged done. */
+	cubb_pend[slot].have_done = false;
+	cubb_pend[slot].done_out = NULL;
+	cubb_pend[slot].done_out_len = 0;
+	cubb_pend[slot].done_tries = 0;
+	memset(&cubb_pend[slot].done, 0, sizeof(cubb_pend[slot].done));
 	cubb_jobs++;
 	cubb_last_seq = job.seq;
 	mutex_unlock(&cubb_lock);
@@ -905,20 +1034,65 @@ static bool cubb_done_pop(struct cubb2_done *out)
 	return have;
 }
 
-static void cubb_finish_one(struct nv_vgpu_device *vgpu,
-			    const struct cubb2_done *d)
+/* How long a staged done record waits for its destination translation (and a
+ * consistent read-back) before the job is declared failed. The service thread
+ * passes frequently, so a 300 s bound is tens of thousands of attempts. */
+#define CUBB2_FINISH_RETRY_MS	300000
+
+/* How long completion waits for a VA translation to appear (the guest's
+ * sysmem-backed device buffers are published asynchronously by the pin
+ * stream) before falling back to shadow-only delivery. */
+#define CUBB2_XLATE_GRACE_MS	5000
+
+/* Release a pending slot and compact the younger entries down one slot. */
+static void cubb_pend_release(struct cubb_pend *p, int idx)
+{
+	vfree(p->in);
+	p->in = NULL;
+	vfree(p->done_out);
+	p->done_out = NULL;
+	/*
+	 * Clear the deferred-completion state together with the blob. Leaving
+	 * have_done/done_out_len set on a freed slot is fatal: the next job
+	 * staged into the same slot then looks like it already has a staged
+	 * done record and the completion path copies from a NULL blob
+	 * (observed: svc_vas_write NULL deref on the second launch, killing
+	 * the service thread while it held cubb_lock).
+	 */
+	p->have_done = false;
+	p->done_out_len = 0;
+	p->done_tries = 0;
+	p->valid = false;
+	cubb_out_len = 0;
+	if (idx >= 0) {
+		int j = idx;
+
+		while (j + 1 < CUBB2_PEND_MAX && cubb_pend[j + 1].valid) {
+			cubb_pend[j] = cubb_pend[j + 1];
+			j++;
+		}
+		cubb_pend[j].in = NULL;
+		cubb_pend[j].done_out = NULL;
+		cubb_pend[j].have_done = false;
+		cubb_pend[j].done_out_len = 0;
+		cubb_pend[j].done_tries = 0;
+		cubb_pend[j].valid = false;
+	}
+}
+
+/*
+ * Stage a daemon done record: validate it against the pending job and copy the
+ * output blob out of the shared accumulation buffer (so the next job's writes
+ * cannot overwrite it while completion retries). Called under cubb_lock.
+ */
+static void cubb_stage_done(const struct cubb2_done *d)
 {
 	struct cubb_pend *p;
 	int idx = -1;
-	int rc = 0;
-	u32 sem_val = 0;
-	u32 sem_rc = 0;
 	u64 wrote;
 
-	mutex_lock(&cubb_lock);
 	p = cubb_pend_lookup(d->seq, &idx);
 	if (!p) {
-		mutex_unlock(&cubb_lock);
 		cubb_stale++;
 		pr_info_ratelimited("stale done seq=%llu (no pending job)\n",
 				    d->seq);
@@ -930,7 +1104,8 @@ static void cubb_finish_one(struct nv_vgpu_device *vgpu,
 		cubb_last_err = (int)d->err_code;
 		pr_info("failed seq=%llu err=%u status=%u op=%u\n",
 			d->seq, d->err_code, d->status, p->opcode);
-		goto out;
+		cubb_pend_release(p, idx);
+		return;
 	}
 
 	wrote = min_t(u64, p->out_bytes, cubb_out_len);
@@ -939,45 +1114,134 @@ static void cubb_finish_one(struct nv_vgpu_device *vgpu,
 		cubb_last_err = -ENODATA;
 		pr_info("failed seq=%llu err=%d (daemon wrote %llu of %llu out bytes)\n",
 			d->seq, -ENODATA, wrote, p->out_bytes);
-		goto out;
+		cubb_pend_release(p, idx);
+		return;
 	}
 
-	rc = gsp_sema_service_va_write(vgpu, p->dst_va, cubb_out_buf, wrote);
-	if (rc) {
+	p->done_out = vmalloc(wrote);
+	if (!p->done_out) {
 		cubb_failed++;
-		cubb_last_err = rc;
-		pr_info("failed seq=%llu err=%d (output write va=0x%llx len=%llu)\n",
-			d->seq, rc, p->dst_va, wrote);
-		goto out;
+		cubb_last_err = -ENOMEM;
+		pr_info("failed seq=%llu err=%d (done out alloc %llu bytes)\n",
+			d->seq, -ENOMEM, wrote);
+		cubb_pend_release(p, idx);
+		return;
 	}
+	memcpy(p->done_out, cubb_out_buf, wrote);
+	p->done_out_len = wrote;
 	/*
-	 * Read-back check over the whole output: proves the bytes landed
-	 * through the translation the guest will read (vidmem shadow or
-	 * guest RAM). A missing/mismatching copy must NOT be signalled
-	 * complete, or the guest reads stale data and runs on.
+	 * Output-path proof: hash the blob the daemon produced exactly as it
+	 * arrives (before any delivery translation) so the daemon's OUTCHECK
+	 * hash and the guest's read-back can be compared end to end.
+	 */
+	if (wrote >= (256u << 10)) {
+		uint64_t h = 1469598103934665603ULL;
+		u64 k;
+		const u32 *w = (const u32 *)p->done_out;
+
+		for (k = 0; k < wrote / 4; k++) {
+			h ^= w[k];
+			h *= 1099511628211ULL;
+		}
+		pr_info("bridge-out: seq=%llu bytes=%llu hash=%016llx\n",
+			d->seq, wrote, (unsigned long long)h);
+	}
+	p->done = *d;
+	p->have_done = true;
+	p->done_tries = 0;
+	p->done_at = jiffies;
+	cubb_out_len = 0;
+}
+
+/*
+ * Try to complete one staged job: write the output through the VA
+ * translation, read it back, then apply the release semaphore. A translation
+ * miss (-ENOENT/-EAGAIN) or a read-back mismatch (-EIO) is retried until the
+ * bounded deadline (the UVM device VA's translation is learned
+ * asynchronously); every other error fails immediately. Called under
+ * cubb_lock; returns true when the job was released.
+ */
+static bool cubb_finish_try(struct nv_vgpu_device *vgpu, struct cubb_pend *p,
+			    int idx)
+{
+	const struct cubb2_done *d = &p->done;
+	u32 sem_val = 0;
+	u32 sem_rc = 0;
+	bool retryable;
+	int rc;
+
+	/*
+	 * Defensive: a staged done record with output bytes but no blob is
+	 * invalid state (a slot reused before this was fixed). Never copy
+	 * from a NULL source - release the job instead of oopsing the
+	 * service thread while it holds cubb_lock.
+	 */
+	if (!p->done_out && p->done_out_len) {
+		cubb_failed++;
+		cubb_last_err = -EINVAL;
+		pr_info("seq=%llu invalid staged done (out=%llu, no blob), dropping\n",
+			d->seq, p->done_out_len);
+		cubb_pend_release(p, idx);
+		return true;
+	}
+
+	/*
+	 * The guest's read-back may be a CPU read of a guest page named by the
+	 * VA's translation (UVM sysmem-backed device buffers), so prefer a
+	 * translated delivery. While no translation exists, stage the bytes in
+	 * the VA shadow and retry for a bounded grace window (the pin stream
+	 * publishes guest mappings asynchronously); after the window fall back
+	 * to shadow-only delivery so a shadow-served read still works and the
+	 * failure stays visible.
 	 */
 	{
+		u64 xpa = 0;
+		u32 xap = 0;
+
+		if (gsp_sema_service_va_translate(vgpu, p->dst_va, &xpa, &xap)) {
+			(void)gsp_sema_service_va_write(vgpu, p->dst_va,
+							p->done_out,
+							p->done_out_len);
+			if (time_before(jiffies,
+					p->done_at +
+					msecs_to_jiffies(CUBB2_XLATE_GRACE_MS))) {
+				p->done_tries++;
+				if ((p->done_tries & 0x7f) == 1)
+					pr_info("seq=%llu no translation yet, staged in shadow (retry=%u)\n",
+						d->seq, p->done_tries);
+				return false;
+			}
+		}
+	}
+
+	rc = gsp_sema_service_va_write(vgpu, p->dst_va, p->done_out,
+				       p->done_out_len);
+	if (!rc) {
+		/*
+		 * Read-back check over the whole output: proves the bytes
+		 * landed through the translation the guest will read (vidmem
+		 * shadow or guest RAM). A missing/mismatching copy must NOT
+		 * be signalled complete, or the guest reads stale data and
+		 * runs on.
+		 */
 		u8 check[256];
 		size_t off = 0;
 
-		while (off < wrote) {
-			size_t clen = min_t(size_t, wrote - off,
+		while (off < p->done_out_len) {
+			size_t clen = min_t(size_t,
+					    p->done_out_len - off,
 					    sizeof(check));
 
 			if (gsp_sema_service_va_read(vgpu, p->dst_va + off,
 						     check, clen) ||
-			    memcmp(check, cubb_out_buf + off, clen)) {
-				cubb_failed++;
-				cubb_last_err = -EIO;
-				pr_info("failed seq=%llu err=%d (output read-back mismatch va=0x%llx off=%zu len=%zu)\n",
-					d->seq, -EIO, p->dst_va, off, clen);
-				goto out;
+			    memcmp(check, p->done_out + off, clen)) {
+				rc = -EIO;
+				break;
 			}
 			off += clen;
 		}
 	}
-
-	if ((p->sem_flags & 4u) && p->sem_va) {
+	if (!rc && (p->sem_flags & 4u) && p->sem_va) {
 		sem_val = p->sem_payload;
 		if (p->sem_flags & 1u) {
 			u32 cur = 0;
@@ -987,16 +1251,36 @@ static void cubb_finish_one(struct nv_vgpu_device *vgpu,
 		}
 		sem_rc = gsp_sema_service_sem_write(vgpu, p->h_object,
 						    p->sem_va, sem_val);
-		if (sem_rc) {
-			cubb_failed++;
-			cubb_last_err = sem_rc;
-			pr_info("failed seq=%llu err=%d sem_va=0x%llx pay=%u\n",
-				d->seq, sem_rc, p->sem_va, sem_val);
-			goto out;
+		if (sem_rc)
+			rc = sem_rc;
+	}
+
+	retryable = (rc == -ENOENT || rc == -EAGAIN || rc == -EIO);
+	p->done_tries++;
+	if (rc && retryable &&
+	    time_before(jiffies,
+			p->done_at + msecs_to_jiffies(CUBB2_FINISH_RETRY_MS))) {
+		/* Periodic resolve diagnostic: proves whether the
+		 * translation ever becomes available while we retry. */
+		if ((p->done_tries & 0x7f) == 1) {
+			u64 dpa = 0;
+			u32 dap = 0;
+			int rrc = gsp_sema_service_va_resolve(vgpu,
+							      p->dst_va,
+							      &dpa, &dap);
+
+			pr_info("seq=%llu dst resolve retry=%u rc=%d pa=0x%llx ap=%u\n",
+				d->seq, p->done_tries, rrc, dpa, dap);
 		}
+		return false;
 	}
 
-	{
+	if (rc) {
+		cubb_failed++;
+		cubb_last_err = rc;
+		pr_info("failed seq=%llu err=%d (completion va=0x%llx len=%llu tries=%u)\n",
+			d->seq, rc, p->dst_va, p->done_out_len, p->done_tries);
+	} else {
 		u64 dpa = 0;
 		u32 dap = 0;
 		u64 sempa = 0;
@@ -1007,40 +1291,27 @@ static void cubb_finish_one(struct nv_vgpu_device *vgpu,
 			(void)gsp_sema_service_va_resolve(vgpu, p->sem_va,
 							  &sempa, &semap);
 		cubb_completed++;
-		pr_info("completed seq=%llu op=%u dst=0x%llx dstpa=0x%llx dstap=%u out_bytes=%llu/%llu sem_va=0x%llx sempa=0x%llx pay=%u bad=%u kernel_ns=%llu\n",
-			d->seq, p->opcode, p->dst_va, dpa, dap, wrote,
-			p->out_bytes, p->sem_va, sempa, sem_val, d->bad,
-			d->kernel_ns);
-	}
-
-out:
-	vfree(p->in);
-	p->in = NULL;
-	p->valid = false;
-	cubb_out_len = 0;
-	if (idx >= 0) {
-		/* compact: shift the younger entries down one slot */
-		int j = idx;
-
-		while (j + 1 < CUBB2_PEND_MAX && cubb_pend[j + 1].valid) {
-			cubb_pend[j] = cubb_pend[j + 1];
-			j++;
-		}
-		cubb_pend[j].in = NULL;
-		cubb_pend[j].valid = false;
+		pr_info("completed seq=%llu op=%u dst=0x%llx dstpa=0x%llx dstap=%u out_bytes=%llu/%llu sem_va=0x%llx sempa=0x%llx pay=%u bad=%u kernel_ns=%llu tries=%u\n",
+			d->seq, p->opcode, p->dst_va, dpa, dap,
+			p->done_out_len, p->out_bytes, p->sem_va, sempa,
+			sem_val, d->bad, d->kernel_ns, p->done_tries);
+		/* Record how the guest reads the delivered buffer back. */
+		gsp_sema_service_trace_window(10000);
 	}
-	mutex_unlock(&cubb_lock);
+	cubb_pend_release(p, idx);
+	return true;
 }
 
 /*
  * Service-thread hook: marshal at most one queued launch into a daemon-visible
- * job, then resolve any completed jobs. Called every service pass; a no-op
- * when no launch has been seen.
+ * job, stage any returned done records, then drive staged completions. Called
+ * every service pass; a no-op when no launch has been seen.
  */
 void gsp_cuda_bridge_service(struct nv_vgpu_device *vgpu)
 {
 	struct cubb2_done d;
 	int guard = 0;
+	int i;
 
 	if (!gsp_cuda_bridge_enabled())
 		return;
@@ -1052,8 +1323,18 @@ void gsp_cuda_bridge_service(struct nv_vgpu_device *vgpu)
 			cubb_malformed++;
 			continue;
 		}
-		cubb_finish_one(vgpu, &d);
+		mutex_lock(&cubb_lock);
+		cubb_stage_done(&d);
+		mutex_unlock(&cubb_lock);
+	}
+
+	mutex_lock(&cubb_lock);
+	for (i = 0; i < CUBB2_PEND_MAX; i++) {
+		if (cubb_pend[i].valid && cubb_pend[i].have_done &&
+		    cubb_finish_try(vgpu, &cubb_pend[i], i))
+			i = -1;	/* compaction shifted the entries; rescan */
 	}
+	mutex_unlock(&cubb_lock);
 }
 
 /* ------------------------------------------------------------------ */
@@ -1319,6 +1600,8 @@ void gsp_cuda_bridge_fini(void)
 	for (i = 0; i < CUBB2_PEND_MAX; i++) {
 		vfree(cubb_pend[i].in);
 		cubb_pend[i].in = NULL;
+		vfree(cubb_pend[i].done_out);
+		cubb_pend[i].done_out = NULL;
 		cubb_pend[i].valid = false;
 	}
 	mutex_unlock(&cubb_lock);