vgsp: host CUDA bridge + sema release recovery; greeneye simulator wave

- kernel: gsp_cuda_bridge (QMD launch capture, debugfs job/out/done, param+output resolution cached from inline writes), sema drain fixes (never hold/starve a channel), GR ring autopin + counter autoreg, per-page push reads, counter-derive gate; module builds clean
- tools/cuda-bridge: CUDA driver-API daemon executing mail/vadd kernels on the physical GPU, CUBB2/CBD2 v2 protocol, layout check + self-tests
- tools/guest-diag + tools/cuda-ab: scanner/pin fixes, victory push-only mode
- debug/risc-v-gsp-rm-vm: Phase S simulator wave (GSP device model, boot ladder, triage/re-demo harnesses, docs 30-41), GPGPU-sim scaffolding
- STATE-CURRENT/docs updates

8 files changed, 3044 insertions, 269 deletions

diff --git a/kernel/Makefile b/kernel/Makefile
index c035e41..da8f240 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -8,6 +8,7 @@ vgsp-y := \
 	gsp_emulation.o \
 	gsp_rm_emulation.o \
 	gsp_sema_service.o \
+	gsp_cuda_bridge.o \
 	gsp_rpc_client.o \
 	gsp_rpc_integration.o \
 	memory.o \
diff --git a/kernel/gsp_cuda_bridge.c b/kernel/gsp_cuda_bridge.c
new file mode 100644
index 0000000..aaad5ee
--- /dev/null
+++ b/kernel/gsp_cuda_bridge.c
@@ -0,0 +1,1328 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Host CUDA bridge transport (v2): module side of the debugfs protocol under
+ * /sys/kernel/debug/vgsp/. The guest's queued kernel launches (inline-QMD
+ * method runs on the GR compute channel, see gsp_sema_service.c) are turned
+ * into CUBB2 job records; a userspace daemon executes them on the physical
+ * GPU and returns a CBD2 done record plus the output payload. Because the
+ * guest's CUDA buffers live in this module's emulated vidmem shadow, the
+ * daemon cannot reach them: the module marshals the inputs out and copies
+ * the daemon's output back into the resolved destination, then writes the
+ * QMD release semaphore so the guest's cudaDeviceSynchronize completes.
+ *
+ * Files (all under the existing vgsp debugfs directory):
+ *   cuda_bridge_job    0444  module serves the oldest pending 256-byte job
+ *   cuda_bridge_in     0444  module serves that job's input blob (ppos)
+ *   cuda_bridge_out    0200  daemon writes its output blob (ppos)
+ *   cuda_bridge_done   0200  daemon writes a 256-byte CBD2 record
+ *   cuda_bridge_status 0444  text counters
+ *
+ * Everything is inert until a launch is detected; the whole path can be
+ * switched off with the cuda_bridge module parameter.
+ */
+
+#define pr_fmt(fmt) "cuda-bridge: " fmt
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/debugfs.h>
+#include <linux/fs.h>
+#include <linux/mutex.h>
+#include <linux/spinlock.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <linux/uaccess.h>
+#include <linux/jiffies.h>
+#include <linux/string.h>
+
+#include "nvidia-vgpu-vfio.h"
+
+/* Records and limits (see tools/cuda-bridge/PROTOCOL.md, v2). */
+#define CUBB2_JOB_MAGIC		0x32424243u	/* 'CBB2' */
+#define CUBB2_DONE_MAGIC	0x32444243u	/* 'CBD2' */
+#define CUBB2_VERSION		2
+#define CUBB2_RECORD_SIZE	256
+#define CUBB2_IN_MAX		(32u * 1024u * 1024u)
+#define CUBB2_OUT_MAX		(32u * 1024u * 1024u)
+#define CUBB2_REQ_MAX		16
+#define CUBB2_PEND_MAX		4
+#define CUBB2_DONE_RING		16
+#define CUBB2_NAME_MAX		64
+
+/*
+ * Parameter-bank location. CUDA writes the launch arguments into the QMD's
+ * parameter bank with GRCE LOAD_INLINE_DATA at an arbitrary offset (observed:
+ * mail args at bank+0x160 while bank+0 held unrelated constants), so the
+ * bridge searches the recorded recent inline-write destinations inside the
+ * bank, then scans the bank itself for a mail/vadd argument shape.
+ */
+#define CUBB2_INLINE_MAX	16
+#define CUBB2_PARAM_BLOB	40
+#define CUBB2_PARAM_SCAN_MAX	(128u * 1024u)
+#define CUBB2_PAGE		4096
+
+/*
+ * Natural-alignment layout (x86_64 kernel and daemon agree):
+ * in_bytes at 0x38, sem_flags 0x58, prog_va 0x60, rsv at 0x70.
+ */
+struct cubb2_job {
+	u32 magic;
+	u32 version;
+	u64 seq;
+	u32 opcode;
+	u32 grid[3];
+	u32 block[3];
+	u32 shared;
+	u32 n;
+	u64 in_bytes;
+	u64 out_bytes;
+	u64 sem_va;
+	u64 sem_payload;
+	u32 sem_flags;
+	u32 pad;
+	u64 prog_va;
+	u32 param_count;
+	u32 pad2;
+	u8  rsv[0x100 - 0x70];
+};
+
+/* status 0 = ok, 1 = bad job, 2 = CUDA error, 3 = RAM/mapping error. */
+struct cubb2_done {
+	u32 magic;
+	u32 version;
+	u64 seq;
+	u32 status;
+	u32 bad;
+	u64 checksum;
+	u64 kernel_ns;
+	u32 out_bytes_written;
+	u32 err_code;
+	char device_name[CUBB2_NAME_MAX];
+	u64 host_ns;
+	u8  rsv[0x100 - 0x78];
+};
+
+static_assert(sizeof(struct cubb2_job) == CUBB2_RECORD_SIZE,
+	      "cubb2_job must be 256 bytes");
+static_assert(sizeof(struct cubb2_done) == CUBB2_RECORD_SIZE,
+	      "cubb2_done must be 256 bytes");
+static_assert(offsetof(struct cubb2_job, prog_va) == 0x60,
+	      "cubb2_job natural layout changed");
+static_assert(offsetof(struct cubb2_done, host_ns) == 0x70,
+	      "cubb2_done natural layout changed");
+
+/* Entry points (also declared in gsp_sema_service.c). */
+bool gsp_cuda_bridge_enabled(void);
+void gsp_cuda_bridge_launch(struct nv_vgpu_device *vgpu, u32 h_object,
+			    const u32 qmd[64], u64 qmd_dev_va, u64 push_va,
+			    const u64 *recent_dsts, u32 n_recent_dsts);
+void gsp_cuda_bridge_service(struct nv_vgpu_device *vgpu);
+int gsp_cuda_bridge_init(void);
+void gsp_cuda_bridge_fini(void);
+
+/* Helpers exported by gsp_sema_service.c (declared here, not in the header,
+ * so the bridge stays a leaf of the sema service). */
+int gsp_sema_service_va_read(struct nv_vgpu_device *vgpu, u64 va, void *buf,
+			     size_t len);
+int gsp_sema_service_va_write(struct nv_vgpu_device *vgpu, u64 va,
+			      const void *buf, size_t len);
+int gsp_sema_service_va_resolve(struct nv_vgpu_device *vgpu, u64 va, u64 *pa,
+				u32 *ap);
+int gsp_sema_service_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val);
+int gsp_sema_service_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va,
+			       u32 val);
+extern bool gsp_sema_dbg;
+
+static bool cubb_enable = true;
+module_param_named(cuda_bridge, cubb_enable, bool, 0644);
+MODULE_PARM_DESC(cuda_bridge,
+		 "Enable the host CUDA bridge launch transport (default 1)");
+
+/* ------------------------------------------------------------------ */
+/* launch requests: decode-thread producer, service-thread consumer     */
+/* ------------------------------------------------------------------ */
+
+struct cubb_req {
+	u32 h_object;
+	u64 qmd_dev_va;
+	u64 push_va;
+	u32 qmd[64];
+	u32 attempts;
+	u32 n_recent_dsts;
+	u64 recent_dsts[CUBB2_INLINE_MAX];
+};
+
+static struct cubb_req cubb_req[CUBB2_REQ_MAX];
+static u32 cubb_req_head;	/* producer */
+static u32 cubb_req_tail;	/* consumer */
+static DEFINE_SPINLOCK(cubb_req_lock);
+
+/*
+ * A pin-batch rewind replays recently consumed GPFIFO entries, so the same
+ * launch push is decoded again and would queue a duplicate job (victory24
+ * showed the same push three times). Dedupe on a content hash of the QMD:
+ * replaying one push hashes identically, a genuine relaunch has a different
+ * completion semaphore/payload inside the QMD.
+ */
+#define CUBB2_DEDUP		8
+#define CUBB2_DEDUP_MS		5000
+struct cubb_dedup {
+	u64 hash;
+	unsigned long when;
+};
+static struct cubb_dedup cubb_dedup[CUBB2_DEDUP];
+static u32 cubb_dedup_next;
+static u64 cubb_dedup_hits;
+
+/* ------------------------------------------------------------------ */
+/* pending jobs (daemon-visible) and daemon write-back state            */
+/* ------------------------------------------------------------------ */
+
+struct cubb_pend {
+	bool valid;
+	u64 seq;
+	u32 h_object;
+	u32 opcode;
+	u64 dst_va;
+	u64 out_bytes;
+	u64 sem_va;
+	u32 sem_payload;
+	u32 sem_flags;
+	struct cubb2_job job;
+	u8 *in;			/* vmalloc'd input blob */
+};
+
+static struct cubb_pend cubb_pend[CUBB2_PEND_MAX];
+static u64 cubb_seq;
+static DEFINE_MUTEX(cubb_lock);
+
+/* daemon out blob, accumulated by cuda_bridge_out writes */
+static u8 *cubb_out_buf;
+static unsigned int cubb_out_alloc;
+static size_t cubb_out_len;
+
+/* done ring, producer = daemon write handler, consumer = service thread */
+static struct cubb2_done cubb_done_ring[CUBB2_DONE_RING];
+static u32 cubb_done_head;
+static u32 cubb_done_tail;
+static DEFINE_SPINLOCK(cubb_done_lock);
+
+/* counters (plain reads from the status file) */
+static u64 cubb_jobs;
+static u64 cubb_completed;
+static u64 cubb_failed;
+static u64 cubb_dropped;	/* request ring/marshal/allocation drops */
+static u64 cubb_stale;		/* done records for unknown seq */
+static u64 cubb_malformed;	/* done records with bad magic/version */
+static u64 cubb_last_seq;
+static int cubb_last_err;
+static u64 cubb_launch_log;	/* verbose header budget */
+
+/* ------------------------------------------------------------------ */
+/* debugfs                                                             */
+/* ------------------------------------------------------------------ */
+
+static bool cubb_root_owned;
+static bool cubb_debugfs_ok;
+
+static void cubb_debugfs_fini(void);
+static const struct file_operations cubb_job_fops;
+static const struct file_operations cubb_in_fops;
+static const struct file_operations cubb_out_fops;
+static const struct file_operations cubb_done_fops;
+static const struct file_operations cubb_status_fops;
+
+static struct dentry *cubb_find_root(void)
+{
+	struct dentry *root = debugfs_lookup("vgsp", NULL);
+
+	if (IS_ERR(root))
+		return NULL;
+	return root;
+}
+
+static void cubb_debugfs_init(void)
+{
+	struct dentry *root, *f;
+
+	cubb_root_owned = false;
+	root = cubb_find_root();
+	if (!root) {
+		root = debugfs_create_dir("vgsp", NULL);
+		if (IS_ERR_OR_NULL(root)) {
+			pr_warn("debugfs dir vgsp unavailable\n");
+			return;
+		}
+		cubb_root_owned = true;
+	}
+
+	f = debugfs_create_file("cuda_bridge_job", 0444, root, NULL,
+				&cubb_job_fops);
+	if (IS_ERR_OR_NULL(f))
+		goto fail;
+	f = debugfs_create_file("cuda_bridge_in", 0444, root, NULL,
+				&cubb_in_fops);
+	if (IS_ERR_OR_NULL(f))
+		goto fail;
+	f = debugfs_create_file("cuda_bridge_out", 0200, root, NULL,
+				&cubb_out_fops);
+	if (IS_ERR_OR_NULL(f))
+		goto fail;
+	f = debugfs_create_file("cuda_bridge_done", 0200, root, NULL,
+				&cubb_done_fops);
+	if (IS_ERR_OR_NULL(f))
+		goto fail;
+	f = debugfs_create_file("cuda_bridge_status", 0444, root, NULL,
+				&cubb_status_fops);
+	if (IS_ERR_OR_NULL(f))
+		goto fail;
+
+	cubb_debugfs_ok = true;
+	dput(root);
+	pr_info("debugfs ready (/sys/kernel/debug/vgsp/cuda_bridge_*)\n");
+	return;
+
+fail:
+	dput(root);
+	cubb_debugfs_fini();
+	pr_warn("debugfs file creation failed\n");
+}
+
+static void cubb_debugfs_fini(void)
+{
+	struct dentry *root;
+
+	cubb_debugfs_ok = false;
+	root = cubb_find_root();
+	if (!root)
+		return;
+	debugfs_lookup_and_remove("cuda_bridge_job", root);
+	debugfs_lookup_and_remove("cuda_bridge_in", root);
+	debugfs_lookup_and_remove("cuda_bridge_out", root);
+	debugfs_lookup_and_remove("cuda_bridge_done", root);
+	debugfs_lookup_and_remove("cuda_bridge_status", root);
+	dput(root);
+	if (cubb_root_owned) {
+		debugfs_lookup_and_remove("vgsp", NULL);
+		cubb_root_owned = false;
+	}
+}
+
+/* ------------------------------------------------------------------ */
+/* staging helpers                                                     */
+/* ------------------------------------------------------------------ */
+
+static u32 cubb_rd32(const u8 *p)
+{
+	u32 v;
+
+	memcpy(&v, p, 4);
+	return v;
+}
+
+static u64 cubb_rd64(const u8 *p)
+{
+	u64 v;
+
+	memcpy(&v, p, 8);
+	return v;
+}
+
+/* A pointer the guest could have given the GPU (user/device VA range). */
+static bool cubb_va_plausible(u64 v)
+{
+	return v >= 0x10000 && v < (1ULL << 49);
+}
+
+/* mail arguments: {u64 output ptr, u32 len}, len 1..4096. */
+static bool cubb_mail_shape(const u8 *p, size_t len, u64 *ptr, u32 *n)
+{
+	u64 v;
+	u32 l;
+
+	if (len < 12)
+		return false;
+	v = cubb_rd64(p);
+	l = cubb_rd32(p + 8);
+	if (!cubb_va_plausible(v) || !l || l > 4096)
+		return false;
+	if (ptr)
+		*ptr = v;
+	if (n)
+		*n = l;
+	return true;
+}
+
+/* vadd arguments: {u64 a, b, c, u32 n}, n == 1 MiB (frozen launch shape). */
+static bool cubb_vadd_shape(const u8 *p, size_t len, u64 *out, u32 *n)
+{
+	u64 a, b, c;
+	u32 k;
+
+	if (len < 32)
+		return false;
+	a = cubb_rd64(p);
+	b = cubb_rd64(p + 8);
+	c = cubb_rd64(p + 16);
+	k = cubb_rd32(p + 24);
+	if (!cubb_va_plausible(a) || !cubb_va_plausible(b) ||
+	    !cubb_va_plausible(c) || k != (1u << 20))
+		return false;
+	if (out)
+		*out = c;
+	if (n)
+		*n = k;
+	return true;
+}
+
+static int cubb_shape_kind(const u8 *p, size_t len)
+{
+	if (cubb_mail_shape(p, len, NULL, NULL))
+		return 1;
+	if (cubb_vadd_shape(p, len, NULL, NULL))
+		return 2;
+	return 0;
+}
+
+/*
+ * Locate the launch's argument block inside the QMD parameter bank. CUDA
+ * writes the arguments with GRCE LOAD_INLINE_DATA at an arbitrary bank
+ * offset, so never assume +0: first try the recorded recent inline-write
+ * destinations that fall inside [bank_va, bank_va + bank_size), newest
+ * first, then scan the bank for a mail/vadd argument shape. `want` names the
+ * shape implied by grid/block (0 = unknown). On a match returns 0 and fills
+ * off/kind/params/src; -ENODATA means the bank was readable but no shape was
+ * found (caller may fall back to bank+0); -ENOENT means a read failed (the
+ * page resolve may still be in progress) and nothing matched, so the caller
+ * should retry.
+ */
+static int cubb_find_params(struct nv_vgpu_device *vgpu, u64 bank_va,
+			    u32 bank_size, const u64 *dsts, u32 n_dsts,
+			    u32 want, u64 *off, u32 *kind, u8 *params,
+			    size_t plen, const char **src)
+{
+	u8 cand[CUBB2_PARAM_BLOB];
+	u8 *page;
+	u64 hi = bank_va + bank_size;
+	u32 i, pgoff, o, scan;
+	bool read_fail = false;
+
+	*off = 0;
+	*kind = 0;
+	if (bank_size < 16)
+		return -ENODATA;
+
+	/* (a) most recently written inline destinations inside the bank */
+	for (i = 0; i < n_dsts; i++) {
+		u64 va = dsts[i];
+		size_t need = want == 2 ? 32 : 12;
+		size_t got = min_t(size_t, plen, sizeof(cand));
+		int k;
+
+		if (!va || va < bank_va || va + need > hi)
+			continue;
+		if (va + got > hi)
+			got = hi - va;
+		if (gsp_sema_service_va_read(vgpu, va, cand, got)) {
+			read_fail = true;
+			continue;
+		}
+		k = cubb_shape_kind(cand, got);
+		if (!k || (want && k != want))
+			continue;
+		*off = va - bank_va;
+		*kind = k;
+		memcpy(params, cand, min_t(size_t, got, plen));
+		*src = "inline";
+		return 0;
+	}
+
+	/* (b) scan the bank for an argument shape (straddle-safe per page) */
+	scan = bank_size > CUBB2_PARAM_SCAN_MAX ? CUBB2_PARAM_SCAN_MAX :
+						 bank_size;
+	page = vmalloc(CUBB2_PAGE + CUBB2_PARAM_BLOB);
+	if (!page)
+		return read_fail ? -ENOENT : -ENODATA;
+	for (pgoff = 0; pgoff + 32 <= scan; pgoff += CUBB2_PAGE) {
+		u32 available = scan - pgoff;
+		u32 read_len = available > CUBB2_PAGE + CUBB2_PARAM_BLOB ?
+			       CUBB2_PAGE + CUBB2_PARAM_BLOB : available;
+
+		if (gsp_sema_service_va_read(vgpu, bank_va + pgoff, page,
+					     read_len)) {
+			read_fail = true;
+			continue;
+		}
+		for (o = 0; o < available && o < CUBB2_PAGE &&
+			    o + 32 <= read_len; o += 8) {
+			int k = cubb_shape_kind(page + o, read_len - o);
+
+			if (!k || (want && k != want))
+				continue;
+			*off = pgoff + o;
+			*kind = k;
+			memcpy(params, page + o,
+			       min_t(size_t, read_len - o, plen));
+			*src = "scan";
+			vfree(page);
+			return 0;
+		}
+	}
+	vfree(page);
+	return read_fail ? -ENOENT : -ENODATA;
+}
+
+static struct cubb_pend *cubb_pend_lookup(u64 seq, int *idx)
+{
+	int i;
+
+	for (i = 0; i < CUBB2_PEND_MAX; i++) {
+		if (cubb_pend[i].valid && cubb_pend[i].seq == seq) {
+			if (idx)
+				*idx = i;
+			return &cubb_pend[i];
+		}
+	}
+	return NULL;
+}
+
+static int cubb_pend_slot(void)
+{
+	int i;
+
+	for (i = 0; i < CUBB2_PEND_MAX; i++)
+		if (!cubb_pend[i].valid)
+			return i;
+	return -1;
+}
+
+/* ------------------------------------------------------------------ */
+/* launch decode -> request queue                                      */
+/* ------------------------------------------------------------------ */
+
+bool gsp_cuda_bridge_enabled(void)
+{
+	return cubb_enable && cubb_debugfs_ok;
+}
+
+/*
+ * Called from svc_decode_stream on the service thread when a complete inline
+ * QMD has been accumulated. Only queues the launch: marshalling guest memory
+ * happens on the next service pass so the push pump is never held.
+ */
+void gsp_cuda_bridge_launch(struct nv_vgpu_device *vgpu, u32 h_object,
+			    const u32 qmd[64], u64 qmd_dev_va, u64 push_va,
+			    const u64 *recent_dsts, u32 n_recent_dsts)
+{
+	unsigned long flags;
+	u64 hash = 1469598103934665603ULL;
+	int i;
+
+	if (!gsp_cuda_bridge_enabled())
+		return;
+	(void)vgpu;
+
+	for (i = 0; i < 64; i++) {
+		hash ^= qmd[i];
+		hash *= 1099511628211ULL;
+	}
+	hash ^= (u64)h_object << 32;
+	hash *= 1099511628211ULL;
+	hash ^= qmd_dev_va;
+
+	spin_lock_irqsave(&cubb_req_lock, flags);
+	for (i = 0; i < CUBB2_DEDUP; i++) {
+		if (cubb_dedup[i].hash == hash &&
+		    time_before(jiffies,
+				cubb_dedup[i].when +
+				msecs_to_jiffies(CUBB2_DEDUP_MS))) {
+			spin_unlock_irqrestore(&cubb_req_lock, flags);
+			cubb_dedup_hits++;
+			return;
+		}
+	}
+	cubb_dedup[cubb_dedup_next % CUBB2_DEDUP].hash = hash;
+	cubb_dedup[cubb_dedup_next % CUBB2_DEDUP].when = jiffies;
+	cubb_dedup_next++;
+
+	if (cubb_req_head - cubb_req_tail >= CUBB2_REQ_MAX) {
+		spin_unlock_irqrestore(&cubb_req_lock, flags);
+		cubb_dropped++;
+		pr_info_ratelimited("request ring full, launch dropped (push=0x%llx)\n",
+				    push_va);
+		return;
+	}
+	{
+		struct cubb_req *r =
+			&cubb_req[cubb_req_head % CUBB2_REQ_MAX];
+
+		r->h_object = h_object;
+		r->qmd_dev_va = qmd_dev_va;
+		r->push_va = push_va;
+		r->attempts = 0;
+		memcpy(r->qmd, qmd, sizeof(r->qmd));
+		r->n_recent_dsts = 0;
+		if (recent_dsts && n_recent_dsts) {
+			if (n_recent_dsts > CUBB2_INLINE_MAX)
+				n_recent_dsts = CUBB2_INLINE_MAX;
+			memcpy(r->recent_dsts, recent_dsts,
+			       n_recent_dsts * sizeof(u64));
+			r->n_recent_dsts = n_recent_dsts;
+		}
+	}
+	cubb_req_head++;
+	spin_unlock_irqrestore(&cubb_req_lock, flags);
+}
+
+/* Consume the request the marshaller was working on (service thread only). */
+static void cubb_req_drop_head(void)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&cubb_req_lock, flags);
+	if (cubb_req_tail != cubb_req_head)
+		cubb_req_tail++;
+	spin_unlock_irqrestore(&cubb_req_lock, flags);
+}
+
+/*
+ * A guest VA read can miss because the targeted rm_mem scan for that page is
+ * still in progress (it walks the RAM a slice per call and reports -EAGAIN ->
+ * -ENOENT). Keep the request at the queue head and retry on later passes.
+ */
+#define CUBB2_REQ_TRIES	300
+
+static void cubb_req_note_attempt(const struct cubb_req *r)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&cubb_req_lock, flags);
+	if (cubb_req_tail != cubb_req_head)
+		cubb_req[cubb_req_tail % CUBB2_REQ_MAX].attempts = r->attempts;
+	spin_unlock_irqrestore(&cubb_req_lock, flags);
+}
+
+static bool cubb_req_retry(const struct cubb_req *r, int rc)
+{
+	if ((rc == -ENOENT || rc == -EAGAIN) &&
+	    r->attempts < CUBB2_REQ_TRIES) {
+		cubb_req_note_attempt(r);
+		return true;
+	}
+	return false;
+}
+
+/* ------------------------------------------------------------------ */
+/* request -> pending job (marshalling)                                */
+/* ------------------------------------------------------------------ */
+
+static void cubb_log_raw_qmd(const u32 *qmd)
+{
+	int i;
+
+	pr_info("qmd raw (64 words):\n");
+	for (i = 0; i < 64; i += 4)
+		pr_info("qmd[%02d] %08x %08x %08x %08x\n", i,
+			qmd[i], qmd[i + 1], qmd[i + 2], qmd[i + 3]);
+}
+
+static int cubb_marshal_one(struct nv_vgpu_device *vgpu)
+{
+	struct cubb_req r;
+	struct cubb2_job job;
+	unsigned long flags;
+	u32 gx, gy, gz, bx, by, bz, shared;
+	u64 param_va, sem_va, prog_va;
+	u32 param_size, sem_payload, sem_flags, red_en, one_word, enable;
+	u8 params[CUBB2_PARAM_BLOB];
+	u32 plen, n = 0, opcode = 0, param_count = 0;
+	u32 hint = 0, found_kind = 0;
+	u64 params_off = 0;
+	const char *params_src = "bank+0";
+	u64 dst_va = 0, in_bytes = 0, out_bytes = 0;
+	u64 dst_pa = 0;
+	u32 dst_ap = 0;
+	u8 *in = NULL;
+	int rc, slot;
+
+	spin_lock_irqsave(&cubb_req_lock, flags);
+	if (cubb_req_tail == cubb_req_head) {
+		spin_unlock_irqrestore(&cubb_req_lock, flags);
+		return 0;
+	}
+	r = cubb_req[cubb_req_tail % CUBB2_REQ_MAX];
+	r.attempts++;
+	spin_unlock_irqrestore(&cubb_req_lock, flags);
+
+	gx = r.qmd[12];
+	gy = r.qmd[13] & 0xffff;
+	gz = r.qmd[14] & 0xffff;
+	shared = r.qmd[17] & 0x3ffff;
+	bx = r.qmd[18] >> 16;
+	by = r.qmd[19] & 0xffff;
+	bz = r.qmd[19] >> 16;
+	enable = (r.qmd[4] >> 10) & 1;
+	sem_va = r.qmd[23] | ((u64)(r.qmd[24] & 0xff) << 32);
+	red_en = (r.qmd[24] >> 26) & 1;
+	one_word = (r.qmd[24] >> 31) & 1;
+	sem_payload = r.qmd[25];
+	param_va = r.qmd[32] | ((u64)(r.qmd[33] & 0x1ffff) << 32);
+	param_size = (r.qmd[33] >> 19) << 4;
+	prog_va = r.qmd[48] | ((u64)(r.qmd[49] & 0x1ffff) << 32);
+	sem_flags = (red_en ? 1u : 0u) | (one_word ? 2u : 0u) |
+		    (enable ? 4u : 0u);
+
+	/*
+	 * The parameter bank lives in the client's VA space, and CUDA writes
+	 * the arguments into it with GRCE inline writes at an arbitrary
+	 * offset (observed: mail args at bank+0x160, bank+0 unrelated). The
+	 * frozen grid/block shapes hint at the kernel; locate the actual
+	 * argument block by recent inline-write destination, then by shape
+	 * scan of the bank.
+	 */
+	plen = sizeof(params);
+	if (gx == 1 && gy == 1 && gz == 1 && bx == 64 && by == 1 && bz == 1)
+		hint = 1;
+	else if (gx == 4096 && gy == 1 && gz == 1 && bx == 256 && by == 1 &&
+		 bz == 1)
+		hint = 2;
+
+	rc = cubb_find_params(vgpu, param_va, param_size, r.recent_dsts,
+			      r.n_recent_dsts, hint, &params_off, &found_kind,
+			      params, plen, &params_src);
+	if (rc) {
+		if (cubb_req_retry(&r, rc))
+			return 0;
+		if (rc != -ENODATA) {
+			cubb_req_drop_head();
+			cubb_failed++;
+			cubb_last_err = rc;
+			pr_info_ratelimited("failed seq=0 param locate va=0x%llx size=0x%x rc=%d push=0x%llx h=0x%x\n",
+					    param_va, param_size, rc, r.push_va,
+					    r.h_object);
+			return 0;
+		}
+		/* No argument shape found: legacy bank+0 read. */
+		rc = gsp_sema_service_va_read(vgpu, param_va, params, plen);
+		if (rc) {
+			if (cubb_req_retry(&r, rc))
+				return 0;
+			cubb_req_drop_head();
+			cubb_failed++;
+			cubb_last_err = rc;
+			pr_info_ratelimited("failed seq=0 op=? param bank va=0x%llx size=0x%x rc=%d push=0x%llx h=0x%x\n",
+					    param_va, param_size, rc, r.push_va,
+					    r.h_object);
+			return 0;
+		}
+		params_off = 0;
+		found_kind = 0;
+		params_src = "bank+0";
+	}
+
+	/*
+	 * Classify: the two frozen launch shapes first; then the located
+	 * argument block; otherwise the old content heuristic.
+	 */
+	if (hint)
+		opcode = hint;
+	else if (found_kind)
+		opcode = found_kind;
+	else if (cubb_va_plausible(cubb_rd64(params)) &&
+		 cubb_va_plausible(cubb_rd64(params + 8)) &&
+		 cubb_va_plausible(cubb_rd64(params + 16)) &&
+		 cubb_rd32(params + 24) &&
+		 cubb_rd32(params + 24) <= (64u << 20))
+		opcode = 2;
+	else
+		opcode = 1;
+
+	if (opcode == 1) {
+		/* mail: {ptr, len} */
+		dst_va = cubb_rd64(params);
+		n = cubb_rd32(params + 8);
+		out_bytes = n;
+		in_bytes = 0;
+		param_count = 2;
+		if (n > 4096) {
+			cubb_req_drop_head();
+			cubb_failed++;
+			cubb_last_err = -E2BIG;
+			pr_info_ratelimited("failed seq=0 op=mail len %u too large (dst=0x%llx)\n",
+					    n, dst_va);
+			return 0;
+		}
+	} else {
+		/* vadd: {a, b, c, n} (+ 5th slot) */
+		u64 a = cubb_rd64(params);
+		u64 b = cubb_rd64(params + 8);
+
+		dst_va = cubb_rd64(params + 16);
+		n = cubb_rd32(params + 24);
+		out_bytes = 4ULL * n;
+		in_bytes = 8ULL * n;
+		param_count = 5;
+		if (!n || in_bytes > CUBB2_IN_MAX ||
+		    out_bytes > CUBB2_OUT_MAX) {
+			cubb_req_drop_head();
+			cubb_failed++;
+			cubb_last_err = -E2BIG;
+			pr_info_ratelimited("failed seq=0 op=vadd n=%u in=%llu out=%llu too large\n",
+					    n, in_bytes, out_bytes);
+			return 0;
+		}
+		in = vmalloc(in_bytes);
+		if (!in) {
+			cubb_req_drop_head();
+			cubb_failed++;
+			cubb_last_err = -ENOMEM;
+			pr_info_ratelimited("failed seq=0 op=vadd in alloc %llu bytes\n",
+					    in_bytes);
+			return 0;
+		}
+		rc = gsp_sema_service_va_read(vgpu, a, in, in_bytes / 2);
+		if (!rc)
+			rc = gsp_sema_service_va_read(vgpu, b,
+						      in + in_bytes / 2,
+						      in_bytes / 2);
+		if (rc) {
+			vfree(in);
+			if (cubb_req_retry(&r, rc))
+				return 0;
+			cubb_req_drop_head();
+			cubb_failed++;
+			cubb_last_err = rc;
+			pr_info_ratelimited("failed seq=0 op=vadd src a=0x%llx b=0x%llx rc=%d\n",
+					    a, b, rc);
+			return 0;
+		}
+	}
+
+	if (!n || !out_bytes || out_bytes > CUBB2_OUT_MAX) {
+		vfree(in);
+		cubb_req_drop_head();
+		cubb_failed++;
+		cubb_last_err = -EINVAL;
+		pr_info_ratelimited("failed seq=0 op=%u bad sizes n=%u out=%llu\n",
+				    opcode, n, out_bytes);
+		return 0;
+	}
+
+	if (opcode == 1)
+		pr_info_ratelimited("params off=0x%llx kind=mail ptr=0x%llx len/n=%u\n",
+				    params_off, dst_va, n);
+	else
+		pr_info_ratelimited("params off=0x%llx kind=vadd ptr=0x%llx len/n=%u\n",
+				    params_off, dst_va, n);
+
+	rc = gsp_sema_service_va_resolve(vgpu, dst_va, &dst_pa, &dst_ap);
+
+	memset(&job, 0, sizeof(job));
+	job.magic = CUBB2_JOB_MAGIC;
+	job.version = CUBB2_VERSION;
+	job.opcode = opcode;
+	job.grid[0] = gx;
+	job.grid[1] = gy;
+	job.grid[2] = gz;
+	job.block[0] = bx;
+	job.block[1] = by;
+	job.block[2] = bz;
+	job.shared = shared;
+	job.n = n;
+	job.in_bytes = in_bytes;
+	job.out_bytes = out_bytes;
+	job.sem_va = sem_va;
+	job.sem_payload = sem_payload;
+	job.sem_flags = sem_flags;
+	job.prog_va = prog_va;
+	job.param_count = param_count;
+
+	mutex_lock(&cubb_lock);
+	slot = cubb_pend_slot();
+	if (slot < 0) {
+		mutex_unlock(&cubb_lock);
+		vfree(in);
+		cubb_req_drop_head();
+		cubb_dropped++;
+		pr_info_ratelimited("pending queue full, launch dropped\n");
+		return 0;
+	}
+	job.seq = ++cubb_seq;
+	cubb_pend[slot].valid = true;
+	cubb_pend[slot].seq = job.seq;
+	cubb_pend[slot].h_object = r.h_object;
+	cubb_pend[slot].opcode = opcode;
+	cubb_pend[slot].dst_va = dst_va;
+	cubb_pend[slot].out_bytes = out_bytes;
+	cubb_pend[slot].sem_va = sem_va;
+	cubb_pend[slot].sem_payload = sem_payload;
+	cubb_pend[slot].sem_flags = sem_flags;
+	cubb_pend[slot].job = job;
+	cubb_pend[slot].in = in;
+	cubb_jobs++;
+	cubb_last_seq = job.seq;
+	mutex_unlock(&cubb_lock);
+	cubb_req_drop_head();
+
+	if (cubb_launch_log < 8) {
+		cubb_launch_log++;
+		pr_info("launch seq=%llu op=%s grid=%u,%u,%u block=%u,%u,%u shared=%u n=%u in=%llu out=%llu sem=0x%llx pay=%u flags=0x%x param=0x%llx psize=0x%x poff=0x%llx psrc=%s prog=0x%llx dst=0x%llx dstpa=0x%llx dstap=%u h=0x%x push=0x%llx qmddev=0x%llx\n",
+			job.seq, opcode == 1 ? "mail" : "vadd",
+			gx, gy, gz, bx, by, bz, shared, n, in_bytes,
+			out_bytes, sem_va, sem_payload, sem_flags, param_va,
+			param_size, params_off, params_src, prog_va, dst_va,
+			dst_pa, dst_ap, r.h_object, r.push_va, r.qmd_dev_va);
+		if (gsp_sema_dbg) {
+			cubb_log_raw_qmd(r.qmd);
+			print_hex_dump(KERN_INFO, "cuda-bridge: params ",
+				       DUMP_PREFIX_OFFSET, 16, 1, params,
+				       plen, false);
+		}
+	} else {
+		pr_info_ratelimited("launch seq=%llu op=%s n=%u out=%llu\n",
+				    job.seq, opcode == 1 ? "mail" : "vadd",
+				    n, out_bytes);
+	}
+	return 0;
+}
+
+/* ------------------------------------------------------------------ */
+/* done records -> completion                                          */
+/* ------------------------------------------------------------------ */
+
+static bool cubb_done_pop(struct cubb2_done *out)
+{
+	unsigned long flags;
+	bool have = false;
+
+	spin_lock_irqsave(&cubb_done_lock, flags);
+	if (cubb_done_tail != cubb_done_head) {
+		*out = cubb_done_ring[cubb_done_tail % CUBB2_DONE_RING];
+		cubb_done_tail++;
+		have = true;
+	}
+	spin_unlock_irqrestore(&cubb_done_lock, flags);
+	return have;
+}
+
+static void cubb_finish_one(struct nv_vgpu_device *vgpu,
+			    const struct cubb2_done *d)
+{
+	struct cubb_pend *p;
+	int idx = -1;
+	int rc = 0;
+	u32 sem_val = 0;
+	u32 sem_rc = 0;
+	u64 wrote;
+
+	mutex_lock(&cubb_lock);
+	p = cubb_pend_lookup(d->seq, &idx);
+	if (!p) {
+		mutex_unlock(&cubb_lock);
+		cubb_stale++;
+		pr_info_ratelimited("stale done seq=%llu (no pending job)\n",
+				    d->seq);
+		return;
+	}
+
+	if (d->status != 0) {
+		cubb_failed++;
+		cubb_last_err = (int)d->err_code;
+		pr_info("failed seq=%llu err=%u status=%u op=%u\n",
+			d->seq, d->err_code, d->status, p->opcode);
+		goto out;
+	}
+
+	wrote = min_t(u64, p->out_bytes, cubb_out_len);
+	if (wrote < p->out_bytes) {
+		cubb_failed++;
+		cubb_last_err = -ENODATA;
+		pr_info("failed seq=%llu err=%d (daemon wrote %llu of %llu out bytes)\n",
+			d->seq, -ENODATA, wrote, p->out_bytes);
+		goto out;
+	}
+
+	rc = gsp_sema_service_va_write(vgpu, p->dst_va, cubb_out_buf, wrote);
+	if (rc) {
+		cubb_failed++;
+		cubb_last_err = rc;
+		pr_info("failed seq=%llu err=%d (output write va=0x%llx len=%llu)\n",
+			d->seq, rc, p->dst_va, wrote);
+		goto out;
+	}
+	/*
+	 * Read-back check over the whole output: proves the bytes landed
+	 * through the translation the guest will read (vidmem shadow or
+	 * guest RAM). A missing/mismatching copy must NOT be signalled
+	 * complete, or the guest reads stale data and runs on.
+	 */
+	{
+		u8 check[256];
+		size_t off = 0;
+
+		while (off < wrote) {
+			size_t clen = min_t(size_t, wrote - off,
+					    sizeof(check));
+
+			if (gsp_sema_service_va_read(vgpu, p->dst_va + off,
+						     check, clen) ||
+			    memcmp(check, cubb_out_buf + off, clen)) {
+				cubb_failed++;
+				cubb_last_err = -EIO;
+				pr_info("failed seq=%llu err=%d (output read-back mismatch va=0x%llx off=%zu len=%zu)\n",
+					d->seq, -EIO, p->dst_va, off, clen);
+				goto out;
+			}
+			off += clen;
+		}
+	}
+
+	if ((p->sem_flags & 4u) && p->sem_va) {
+		sem_val = p->sem_payload;
+		if (p->sem_flags & 1u) {
+			u32 cur = 0;
+
+			if (!gsp_sema_service_sem_read(vgpu, p->sem_va, &cur))
+				sem_val = cur + p->sem_payload;
+		}
+		sem_rc = gsp_sema_service_sem_write(vgpu, p->h_object,
+						    p->sem_va, sem_val);
+		if (sem_rc) {
+			cubb_failed++;
+			cubb_last_err = sem_rc;
+			pr_info("failed seq=%llu err=%d sem_va=0x%llx pay=%u\n",
+				d->seq, sem_rc, p->sem_va, sem_val);
+			goto out;
+		}
+	}
+
+	{
+		u64 dpa = 0;
+		u32 dap = 0;
+		u64 sempa = 0;
+		u32 semap = 0;
+
+		(void)gsp_sema_service_va_resolve(vgpu, p->dst_va, &dpa, &dap);
+		if (p->sem_va)
+			(void)gsp_sema_service_va_resolve(vgpu, p->sem_va,
+							  &sempa, &semap);
+		cubb_completed++;
+		pr_info("completed seq=%llu op=%u dst=0x%llx dstpa=0x%llx dstap=%u out_bytes=%llu/%llu sem_va=0x%llx sempa=0x%llx pay=%u bad=%u kernel_ns=%llu\n",
+			d->seq, p->opcode, p->dst_va, dpa, dap, wrote,
+			p->out_bytes, p->sem_va, sempa, sem_val, d->bad,
+			d->kernel_ns);
+	}
+
+out:
+	vfree(p->in);
+	p->in = NULL;
+	p->valid = false;
+	cubb_out_len = 0;
+	if (idx >= 0) {
+		/* compact: shift the younger entries down one slot */
+		int j = idx;
+
+		while (j + 1 < CUBB2_PEND_MAX && cubb_pend[j + 1].valid) {
+			cubb_pend[j] = cubb_pend[j + 1];
+			j++;
+		}
+		cubb_pend[j].in = NULL;
+		cubb_pend[j].valid = false;
+	}
+	mutex_unlock(&cubb_lock);
+}
+
+/*
+ * Service-thread hook: marshal at most one queued launch into a daemon-visible
+ * job, then resolve any completed jobs. Called every service pass; a no-op
+ * when no launch has been seen.
+ */
+void gsp_cuda_bridge_service(struct nv_vgpu_device *vgpu)
+{
+	struct cubb2_done d;
+	int guard = 0;
+
+	if (!gsp_cuda_bridge_enabled())
+		return;
+
+	(void)cubb_marshal_one(vgpu);
+	while (guard++ < CUBB2_DONE_RING && cubb_done_pop(&d)) {
+		if (d.magic != CUBB2_DONE_MAGIC ||
+		    d.version != CUBB2_VERSION) {
+			cubb_malformed++;
+			continue;
+		}
+		cubb_finish_one(vgpu, &d);
+	}
+}
+
+/* ------------------------------------------------------------------ */
+/* debugfs handlers                                                    */
+/* ------------------------------------------------------------------ */
+
+static ssize_t cubb_job_read(struct file *f, char __user *ubuf, size_t count,
+			     loff_t *ppos)
+{
+	u8 rec[CUBB2_RECORD_SIZE];
+	int idx = -1;
+	struct cubb_pend *p = NULL;
+	size_t off = *ppos;
+	size_t n;
+
+	if (!cubb_debugfs_ok)
+		return 0;
+	mutex_lock(&cubb_lock);
+	for (idx = 0; idx < CUBB2_PEND_MAX; idx++) {
+		if (cubb_pend[idx].valid) {
+			p = &cubb_pend[idx];
+			break;
+		}
+	}
+	if (!p) {
+		mutex_unlock(&cubb_lock);
+		return 0;
+	}
+	memcpy(rec, &p->job, sizeof(rec));
+	mutex_unlock(&cubb_lock);
+
+	if (off >= sizeof(rec))
+		return 0;
+	n = min_t(size_t, count, sizeof(rec) - off);
+	if (copy_to_user(ubuf, rec + off, n))
+		return -EFAULT;
+	*ppos = off + n;
+	return n;
+}
+
+static ssize_t cubb_in_read(struct file *f, char __user *ubuf, size_t count,
+			    loff_t *ppos)
+{
+	struct cubb_pend *p = NULL;
+	u64 in_bytes = 0;
+	u8 *in = NULL;
+	size_t off = *ppos;
+	size_t n;
+	int rc = 0;
+	int idx;
+
+	if (!cubb_debugfs_ok)
+		return 0;
+	mutex_lock(&cubb_lock);
+	for (idx = 0; idx < CUBB2_PEND_MAX; idx++) {
+		if (cubb_pend[idx].valid) {
+			p = &cubb_pend[idx];
+			break;
+		}
+	}
+	if (!p) {
+		mutex_unlock(&cubb_lock);
+		return 0;
+	}
+	in = p->in;
+	in_bytes = p->job.in_bytes;
+	if (!in || off >= in_bytes) {
+		mutex_unlock(&cubb_lock);
+		return 0;
+	}
+	n = min_t(size_t, count, in_bytes - off);
+	rc = copy_to_user(ubuf, in + off, n);
+	mutex_unlock(&cubb_lock);
+	if (rc)
+		return -EFAULT;
+	*ppos = off + n;
+	return n;
+}
+
+/*
+ * Accumulate the daemon's output blob. ppos 0 starts a new blob; growth is
+ * amortised (double until the record fits, capped at CUBB2_OUT_MAX).
+ */
+static ssize_t cubb_out_write(struct file *f, const char __user *ubuf,
+			      size_t count, loff_t *ppos)
+{
+	u64 off = *ppos;
+	u64 end;
+	int rc = 0;
+
+	if (!cubb_debugfs_ok)
+		return -ENODEV;
+	if (!count)
+		return 0;
+	if (*ppos < 0)
+		return -EINVAL;
+	end = off + count;
+	if (end > CUBB2_OUT_MAX)
+		return -E2BIG;
+
+	mutex_lock(&cubb_lock);
+	if (off == 0)
+		cubb_out_len = 0;
+	if (end > cubb_out_alloc) {
+		unsigned int want = cubb_out_alloc ? cubb_out_alloc : 4096;
+		u8 *nb;
+
+		while (want < end && want < CUBB2_OUT_MAX)
+			want *= 2;
+		if (want > CUBB2_OUT_MAX)
+			want = CUBB2_OUT_MAX;
+		nb = vmalloc(want);
+		if (!nb) {
+			mutex_unlock(&cubb_lock);
+			return -ENOMEM;
+		}
+		if (cubb_out_len)
+			memcpy(nb, cubb_out_buf, cubb_out_len);
+		vfree(cubb_out_buf);
+		cubb_out_buf = nb;
+		cubb_out_alloc = want;
+	}
+	rc = copy_from_user(cubb_out_buf + off, ubuf, count);
+	if (!rc && end > cubb_out_len)
+		cubb_out_len = end;
+	mutex_unlock(&cubb_lock);
+	if (rc)
+		return -EFAULT;
+	*ppos = end;
+	return count;
+}
+
+/* Accumulate a 256-byte CBD2 record (single write or ppos-chunked). */
+static u8 cubb_done_stage[CUBB2_RECORD_SIZE];
+static size_t cubb_done_stage_len;
+
+static ssize_t cubb_done_write(struct file *f, const char __user *ubuf,
+			       size_t count, loff_t *ppos)
+{
+	/*
+	 * Accept both pwrite(fd, rec, 256, 0) and a long-lived fd writing
+	 * records back to back: a write starting on a record boundary starts
+	 * a fresh record.
+	 */
+	size_t off = (size_t)(*ppos % CUBB2_RECORD_SIZE);
+	unsigned long flags;
+	int rc;
+
+	if (!cubb_debugfs_ok)
+		return -ENODEV;
+	if (!count)
+		return 0;
+	if (count > CUBB2_RECORD_SIZE - off)
+		count = CUBB2_RECORD_SIZE - off;
+
+	mutex_lock(&cubb_lock);
+	if (off == 0)
+		cubb_done_stage_len = 0;
+	rc = copy_from_user(cubb_done_stage + off, ubuf, count);
+	if (rc) {
+		mutex_unlock(&cubb_lock);
+		return -EFAULT;
+	}
+	if (off + count > cubb_done_stage_len)
+		cubb_done_stage_len = off + count;
+
+	if (cubb_done_stage_len >= CUBB2_RECORD_SIZE) {
+		struct cubb2_done *d = (struct cubb2_done *)cubb_done_stage;
+
+		if (d->magic == CUBB2_DONE_MAGIC &&
+		    d->version == CUBB2_VERSION) {
+			spin_lock_irqsave(&cubb_done_lock, flags);
+			if (cubb_done_head - cubb_done_tail <
+			    CUBB2_DONE_RING) {
+				cubb_done_ring[cubb_done_head %
+					       CUBB2_DONE_RING] = *d;
+				cubb_done_head++;
+			} else {
+				cubb_dropped++;
+			}
+			spin_unlock_irqrestore(&cubb_done_lock, flags);
+		} else {
+			cubb_malformed++;
+		}
+		memset(cubb_done_stage, 0, sizeof(cubb_done_stage));
+		cubb_done_stage_len = 0;
+	}
+	mutex_unlock(&cubb_lock);
+	*ppos = off + count;
+	return count;
+}
+
+static ssize_t cubb_status_read(struct file *f, char __user *ubuf,
+				size_t count, loff_t *ppos)
+{
+	char buf[256];
+	int len;
+
+	len = scnprintf(buf, sizeof(buf),
+			"jobs=%llu completed=%llu failed=%llu last_seq=%llu last_err=%d dropped=%llu stale=%llu malformed=%llu dedup=%llu\n",
+			cubb_jobs, cubb_completed, cubb_failed,
+			cubb_last_seq, cubb_last_err, cubb_dropped,
+			cubb_stale, cubb_malformed, cubb_dedup_hits);
+	return simple_read_from_buffer(ubuf, count, ppos, buf, len);
+}
+
+static const struct file_operations cubb_job_fops = {
+	.owner = THIS_MODULE,
+	.open = simple_open,
+	.read = cubb_job_read,
+	.llseek = default_llseek,
+};
+
+static const struct file_operations cubb_in_fops = {
+	.owner = THIS_MODULE,
+	.open = simple_open,
+	.read = cubb_in_read,
+	.llseek = default_llseek,
+};
+
+static const struct file_operations cubb_out_fops = {
+	.owner = THIS_MODULE,
+	.open = simple_open,
+	.write = cubb_out_write,
+	.llseek = default_llseek,
+};
+
+static const struct file_operations cubb_done_fops = {
+	.owner = THIS_MODULE,
+	.open = simple_open,
+	.write = cubb_done_write,
+	.llseek = default_llseek,
+};
+
+static const struct file_operations cubb_status_fops = {
+	.owner = THIS_MODULE,
+	.open = simple_open,
+	.read = cubb_status_read,
+	.llseek = default_llseek,
+};
+
+/* ------------------------------------------------------------------ */
+/* lifecycle                                                           */
+/* ------------------------------------------------------------------ */
+
+int gsp_cuda_bridge_init(void)
+{
+	cubb_debugfs_init();
+	return cubb_debugfs_ok ? 0 : -ENODEV;
+}
+
+void gsp_cuda_bridge_fini(void)
+{
+	int i;
+
+	if (cubb_out_buf) {
+		vfree(cubb_out_buf);
+		cubb_out_buf = NULL;
+		cubb_out_alloc = 0;
+		cubb_out_len = 0;
+	}
+	mutex_lock(&cubb_lock);
+	for (i = 0; i < CUBB2_PEND_MAX; i++) {
+		vfree(cubb_pend[i].in);
+		cubb_pend[i].in = NULL;
+		cubb_pend[i].valid = false;
+	}
+	mutex_unlock(&cubb_lock);
+	cubb_req_head = cubb_req_tail = 0;
+	cubb_done_head = cubb_done_tail = 0;
+	cubb_debugfs_fini();
+}
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 3d1ccea..bead218 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -440,8 +440,34 @@ bool gsp_phys_boot_requested(void)
  * carries both PA and VA in one entry, on the non-KMD V5 channel.
  */
 #define GSP_PHYS_RMCALL_GR_PROMOTE_GUEST_SET		19u
+/*
+ * Mode 20 (2026-10-02, untested): the mode-6 no-entry hVirtMemory promote on
+ * the V6 KMD-managed channel (SKIP_CTXBUFFER_ALLOC). Mode 6 was refused with
+ * NV_ERR_INVALID_ARGUMENT (0x1f) on the V5 channel because the server's
+ * client-promote path rejects the hVirtMemory (pVirtMemDesc) branch unless the
+ * channel is KMD-managed; V6 sets exactly that flag. The object alloc stays on
+ * the default external VAS, so the shared-management ctx-buffer map path that
+ * wedged grsvas2/grsvas3 never runs. The promote asks the server to create the
+ * virtual-context memdesc/vaList entry for the context window itself, after
+ * which the KMD commit path can validate ENG_CTX_VALID.
+ */
+#define GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT		20u
+/*
+ * Mode 21 (2026-10-02, untested): the mode-7 no-entry bare (virtAddress, size)
+ * promote on the V6 KMD-managed channel - the counterpart of mode 20 without
+ * the hVirtMemory object.
+ */
+#define GSP_PHYS_RMCALL_GR_PROMOTE_KMD_BARE_VA		21u
+/*
+ * Mode 22 (2026-10-02, untested): install the physical context memdesc with
+ * GPU_INITIALIZE_CTX (accepted alone in modes 10/11), then send mode 20's
+ * no-entry hVirtMemory promote. If mode 20's plain 0x40 was the promote's
+ * physical-memdesc precondition, this combination satisfies it before the
+ * virtual-memdesc promote.
+ */
+#define GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT	22u
 #define GSP_PHYS_RMCALL_GR_PROMOTE_MAX \
-	(GSP_PHYS_RMCALL_GR_PROMOTE_GUEST_SET)
+	(GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT)
 
 /*
  * Work-submit token notifier slot. The default index is
@@ -483,9 +509,17 @@ bool gsp_phys_boot_requested(void)
  */
 #define GSP_PHYS_RMCALL_VAS_INDEX_GPU_NEW		0u
 #define GSP_PHYS_RMCALL_VAS_FLAG_IS_EXTERNALLY_OWNED	BIT(3)
+#define GSP_PHYS_RMCALL_VAS_FLAG_SHARED_MANAGEMENT	BIT(2)
 #define GSP_PHYS_RMCALL_VAS_FLAG_ALLOW_ZERO_ADDRESS	BIT(8)
 #define GSP_PHYS_RMCALL_VAS_SIZE			0x10000000ULL
 /*
+ * Shared-management VAS size: the normal KMD/UVM client passes vaBase=0 and
+ * rm_va_size = 128 TB on Volta (uvm_volta.c:48-49); the split-VAS server-RM
+ * carveout math (gpu_vaspace.c:440-470) underflows with a small (256 MB)
+ * size and asserts the alloc with NV_ERR_INVALID_ARGUMENT.
+ */
+#define GSP_PHYS_RMCALL_VAS_SIZE_SHARED			0x200000000000ULL
+/*
  * Turing/TU104 uses the GP10X GMMU root format: the root level has 2 index
  * bits, i.e. exactly 4 entries covering 2^47 bytes each, and the server caps
  * the external root at vaLimitMax = 2^49-1 with vaLimitNew =
@@ -2969,6 +3003,16 @@ static void gsp_phys_fault_corroborate(struct gsp_phys_fault_info *out);
 #define GSP_PHYS_NOCAT_OFF_DIAGBUF	180
 #define GSP_PHYS_NOCAT_MIN_PAYLOAD	GSP_PHYS_NOCAT_OFF_DIAGBUF
 #define GSP_PHYS_NOCAT_FULL_PAYLOAD	1208
+/*
+ * Assert diagnostic buffer (RM_NOCAT_ASSERT_DIAG_BUFFER, g_journal_nvoc.h):
+ * ten u32 call-stack entries (each already relative to the NOCAT load
+ * address) followed by a u32 repeat count. The stack entries are the exact
+ * call chain of the assert, so a dump makes the failing path nameable
+ * offline without another run.
+ */
+#define GSP_PHYS_NOCAT_DIAG_STACK	10
+#define GSP_PHYS_NOCAT_OFF_DIAGCOUNT	(GSP_PHYS_NOCAT_OFF_DIAGBUF + 4 * GSP_PHYS_NOCAT_DIAG_STACK)
+#define GSP_PHYS_NOCAT_DIAG_NEEDED	(GSP_PHYS_NOCAT_OFF_DIAGCOUNT + 4)
 
 /* Local re-declaration of the reference record, only for the offset asserts. */
 struct gsp_phys_nocat_journal_record {
@@ -3116,10 +3160,33 @@ static void gsp_phys_nocat_record(const struct gsp_phys_rx_record *rec)
 	gsp_phys_nocat_accumulate(rec_type, error_code, subsystem, bugcheck,
 				  source, engine, rec->seq_num);
 
-	if (gsp_phys_nocat_param)
+	if (gsp_phys_nocat_param) {
+		u32 diag_len = 0;
+		u32 count = 0;
+		u32 i;
+		char stack[GSP_PHYS_NOCAT_DIAG_STACK * 12];
+		size_t pos = 0;
+
 		pr_info("vgsp-nocat: rec seq=%u recType=%u errorCode=0x%llx subsystem=0x%x bugcheck=0x%x payload=%u source=\"%s\" faultingEngine=\"%s\"\n",
 			rec->seq_num, rec_type, error_code, subsystem,
 			bugcheck, len, source, engine);
+		if (len >= GSP_PHYS_NOCAT_OFF_DIAGBUFLEN + 4)
+			diag_len = get_unaligned_le32(p +
+					GSP_PHYS_NOCAT_OFF_DIAGBUFLEN);
+		if (rec_type == 5 && diag_len >= 4 * GSP_PHYS_NOCAT_DIAG_STACK + 4 &&
+		    len >= GSP_PHYS_NOCAT_DIAG_NEEDED) {
+			for (i = 0; i < GSP_PHYS_NOCAT_DIAG_STACK; i++) {
+				u32 v = get_unaligned_le32(p +
+					GSP_PHYS_NOCAT_OFF_DIAGBUF + 4 * i);
+				pos += scnprintf(stack + pos, sizeof(stack) - pos,
+						 "%s0x%x", i ? "," : "", v);
+			}
+			count = get_unaligned_le32(p +
+					GSP_PHYS_NOCAT_OFF_DIAGCOUNT);
+			pr_info("vgsp-nocat: rec seq=%u assertCallStack count=%u [%s] (offsets from load base 0x4000000)\n",
+				rec->seq_num, count, stack);
+		}
+	}
 }
 
 static void gsp_phys_nocat_summary_dump(void)
@@ -6732,6 +6799,12 @@ static const char *gsp_phys_rmcall_gr_promote_name(void)
 		return "mode 8 shape plus a client FERMI_CONTEXT_SHARE_A (hContextShare) that allocates the GraphicsSubcontext";
 	case GSP_PHYS_RMCALL_GR_PROMOTE_GUEST_SET:
 		return "stock guest's 9-entry promote set (MAIN 2 MB PA+VA, PATCH/global/FECS/priv maps) on V5";
+	case GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT:
+		return "KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC) + no-entry hVirtMemory promote";
+	case GSP_PHYS_RMCALL_GR_PROMOTE_KMD_BARE_VA:
+		return "KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC) + no-entry bare VA+size promote";
+	case GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT:
+		return "KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC) + GPU_INITIALIZE_CTX then no-entry hVirtMemory promote";
 	default:
 		return "off (PROMOTE_CTX never sent)";
 	}
@@ -6907,7 +6980,13 @@ static int gsp_phys_rmcall_alloc_channel(void __iomem *bar0)
 	     boot_stage.rmcall_gr_promote ==
 			GSP_PHYS_RMCALL_GR_PROMOTE_KMD_PM ||
 	     boot_stage.rmcall_gr_promote ==
-			GSP_PHYS_RMCALL_GR_PROMOTE_CTXSHARE)) {
+			GSP_PHYS_RMCALL_GR_PROMOTE_CTXSHARE ||
+	     boot_stage.rmcall_gr_promote ==
+			GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT ||
+	     boot_stage.rmcall_gr_promote ==
+			GSP_PHYS_RMCALL_GR_PROMOTE_KMD_BARE_VA ||
+	     boot_stage.rmcall_gr_promote ==
+			GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT)) {
 		vars = &gsp_phys_rmcall_channel_variant_skipctx;
 		attempts = 1;
 	}
@@ -7843,7 +7922,10 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0)
 	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD ||
 	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_FB ||
 	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_PM ||
-	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_CTXSHARE) &&
+	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_CTXSHARE ||
+	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT ||
+	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_BARE_VA ||
+	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT) &&
 	    boot_stage.rmcall_channel_variant !=
 			gsp_phys_rmcall_channel_variant_skipctx.id) {
 		pr_err("vgsp-boot: rm-call: mode %u requires the SKIP_CTXBUFFER_ALLOC channel (V%u %s); winning variant is V%u; no PROMOTE_CTX is sent and the ladder aborts with no retry\n",
@@ -7981,17 +8063,21 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0)
 		}
 		boot_stage.rmcall_gr_promote_step = 2;
 	} else if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT ||
-		   mode == GSP_PHYS_RMCALL_GR_PROMOTE_BARE_VA) {
+		   mode == GSP_PHYS_RMCALL_GR_PROMOTE_BARE_VA ||
+		   mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT ||
+		   mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_BARE_VA) {
 		/*
-		 * Modes 6/7: the documented no-entry branches. Mode 6 first
-		 * reserves the context window with an NV01_MEMORY_VIRTUAL
-		 * object on the ladder VAS and passes it as hVirtMemory
+		 * Modes 6/7/20: the documented no-entry branches. Modes 6/20
+		 * first reserve the context window with an NV01_MEMORY_VIRTUAL
+		 * object on the ladder VAS and pass it as hVirtMemory
 		 * (hClient is its owner); mode 7 sends the bare
 		 * (virtAddress, size) pair with no hVirtMemory. entryCount=0
-		 * in both, so the server creates the memdesc/vaList entry
-		 * instead of resolving one of our raw-PTE entries.
+		 * in all, so the server creates the memdesc/vaList entry
+		 * instead of resolving one of our raw-PTE entries. Mode 20 is
+		 * the same shape as mode 6 but on the V6 KMD-managed channel.
 		 */
-		if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT) {
+		if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT ||
+		    mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT) {
 			ret = gsp_phys_rmcall_alloc_gr_hvirt(bar0);
 			if (ret) {
 				kvfree(p);
@@ -8009,11 +8095,13 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0)
 				GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX,
 				p, sizeof(*p), 24, NULL, 0,
 				&rpc_result, &status,
-				mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT ?
+				(mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT ||
+				 mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT) ?
 				"GPU_PROMOTE_CTX (hVirtMemory)" :
 				"GPU_PROMOTE_CTX (bare VA+size)");
 		pr_info("vgsp-boot: rm-call: PROMOTE_CTX %s: entryCount=0 hVirtMemory=0x%x virtAddress=0x%llx size=0x%x status=0x%x rpc_result=0x%x (hClient=0x%x hObject=0x%x hChanClient=0x%x engineType=0x%x gr_promote_param=%u)\n",
-			mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT ?
+			(mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT ||
+			 mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT) ?
 				"hVirtMemory" : "bare VA+size",
 			le32_to_cpu(p->h_virt_memory),
 			boot_stage.rmcall_ctx_va, boot_stage.rmcall_ctx_size,
@@ -8022,7 +8110,8 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0)
 			GSP_PHYS_RMCALL_ENGINE_GR0, mode);
 		if (ret) {
 			pr_err("vgsp-boot: rm-call: PROMOTE_CTX %s failed (status=0x%x rpc_result=0x%x, %s, ret=%d); the GR context is not committed and the ladder aborts before BIND with no retry\n",
-			       mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT ?
+			       (mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT ||
+				mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT) ?
 				"hVirtMemory" : "bare VA+size",
 			       status, rpc_result,
 			       gsp_phys_rmcall_status_str(status), ret);
@@ -8063,9 +8152,10 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0)
 		}
 		boot_stage.rmcall_gr_promote_step = 1;
 	} else if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_OLDIFACE_VA ||
-		   mode == GSP_PHYS_RMCALL_GR_PROMOTE_OLDIFACE_FB) {
+		   mode == GSP_PHYS_RMCALL_GR_PROMOTE_OLDIFACE_FB ||
+		   mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT) {
 		/*
-		 * Modes 10/11: the old-interface client-managed flow. Step 1
+		 * Modes 10/11/22: the old-interface client-managed flow. Step 1
 		 * sends GPU_INITIALIZE_CTX (0x2080012d) with the physical
 		 * context buffer, which installs the PA/size/physAttr
 		 * memdesc on the GraphicsContext (grctxInitializeAPI) that
@@ -8076,7 +8166,8 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0)
 		 * entryCount=0 and the bare (virtAddress, size) pair; the
 		 * server builds the virtual memdesc itself and, on the V6
 		 * KMD-managed channel, binds it without the VA/bookkeeping
-		 * resolve.
+		 * resolve. Mode 22 is mode 10's step 1 followed by mode 20's
+		 * hVirtMemory no-entry promote instead of the bare pair.
 		 */
 		struct gsp_phys_rmcall_init_ctx_params *ic;
 		u32 ic_phys_attr = (mode ==
@@ -8126,6 +8217,16 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0)
 		p->virt_address = cpu_to_le64(boot_stage.rmcall_ctx_va);
 		p->size = cpu_to_le64(boot_stage.rmcall_ctx_size);
 		p->entry_count = cpu_to_le32(0);
+		if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT) {
+			ret = gsp_phys_rmcall_alloc_gr_hvirt(bar0);
+			if (ret) {
+				kvfree(p);
+				return ret;
+			}
+			p->h_client = cpu_to_le32(boot_stage.rmcall_client);
+			p->h_virt_memory =
+				cpu_to_le32(boot_stage.rmcall_gr_hvirt);
+		}
 		rpc_result = 0;
 		status = 0;
 		ret = gsp_phys_rmcall_control(bar0,
@@ -8133,14 +8234,21 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0)
 				GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX,
 				p, sizeof(*p), 24, NULL, 0,
 				&rpc_result, &status,
+				(mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT) ?
+				"GPU_PROMOTE_CTX (hVirtMemory after init)" :
 				"GPU_PROMOTE_CTX (KMD bare VA+size after init)");
-		pr_info("vgsp-boot: rm-call: PROMOTE_CTX bare VA+size after init: entryCount=0 virtAddress=0x%llx size=0x%x status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x gr_promote_param=%u)\n",
+		pr_info("vgsp-boot: rm-call: PROMOTE_CTX %s: entryCount=0 hVirtMemory=0x%x virtAddress=0x%llx size=0x%x status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x gr_promote_param=%u)\n",
+			(mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT) ?
+				"hVirtMemory after init" : "bare VA+size after init",
+			le32_to_cpu(p->h_virt_memory),
 			boot_stage.rmcall_ctx_va, boot_stage.rmcall_ctx_size,
 			status, rpc_result, boot_stage.rmcall_channel,
 			boot_stage.rmcall_client,
 			GSP_PHYS_RMCALL_ENGINE_GR0, mode);
 		if (ret) {
-			pr_err("vgsp-boot: rm-call: PROMOTE_CTX bare VA+size failed (status=0x%x rpc_result=0x%x, %s, ret=%d); last completed promote step: INITIALIZE_CTX accepted (1/2); the GR context is not committed and the ladder aborts before BIND with no retry\n",
+			pr_err("vgsp-boot: rm-call: PROMOTE_CTX %s failed (status=0x%x rpc_result=0x%x, %s, ret=%d); last completed promote step: INITIALIZE_CTX accepted (1/2); the GR context is not committed and the ladder aborts before BIND with no retry\n",
+			       (mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT) ?
+				"hVirtMemory after init" : "bare VA+size",
 			       status, rpc_result,
 			       gsp_phys_rmcall_status_str(status), ret);
 			kvfree(p);
@@ -8632,17 +8740,41 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0)
 	gsp_phys_rmcall_gr_promote_snapshot();
 
 	/*
-	 * The TSG below carries this externally-owned FERMI_VASPACE_A, whose
-	 * root page directory is committed from the zeroed VASPDB chunk before
-	 * any channel alloc (the instance-block setup programs that base and
-	 * fails 0x40 when it is NULL, kern_gmmu_gp100.c:150-152). Either call
-	 * aborts the ladder: the variant walk cannot succeed without them.
+	 * The TSG below carries this explicitly allocated FERMI_VASPACE_A,
+	 * whose root page directory is committed from the zeroed VASPDB chunk
+	 * before any channel alloc (the instance-block setup programs that
+	 * base and fails 0x40 when it is NULL, kern_gmmu_gp100.c:150-152).
+	 * Either call aborts the ladder: the variant walk cannot succeed
+	 * without them.
+	 *
+	 * History (2026-10-02): the VAS was allocated IS_EXTERNALLY_OWNED to
+	 * mirror the guest's UVM-owned address space. That flag tells the RM
+	 * that the client owns all page tables and VA/context commitment, so
+	 * the server never books the GR engine-context VA itself and every
+	 * promote either asserts ("Externally owned object not found",
+	 * kchannelMapEngineCtxBuf) or silently leaves the channel
+	 * CTXNOTVALID. The normal KMD client instead allocates its VAS with
+	 * SHARED_MANAGEMENT (nv_gpu_ops.c:2495): the client provides the root
+	 * PDB (SET_PAGE_DIRECTORY still legal, gpu_vaspace.c:3108) while the
+	 * RM manages the lower page tables and maps/commits engine contexts.
+	 * gsp_phys_rmcall_vas_shared=1 selects that variant; the default 0
+	 * keeps the byte-identical external path.
 	 */
 	memset(&vas, 0, sizeof(vas));
 	vas.index = cpu_to_le32(GSP_PHYS_RMCALL_VAS_INDEX_GPU_NEW);
-	vas.flags = cpu_to_le32(GSP_PHYS_RMCALL_VAS_FLAG_IS_EXTERNALLY_OWNED |
-				GSP_PHYS_RMCALL_VAS_FLAG_ALLOW_ZERO_ADDRESS);
-	vas.va_size = cpu_to_le64(GSP_PHYS_RMCALL_VAS_SIZE);
+	vas.flags = cpu_to_le32(gsp_phys_rmcall_vas_shared_value() ?
+				(GSP_PHYS_RMCALL_VAS_FLAG_SHARED_MANAGEMENT |
+				 GSP_PHYS_RMCALL_VAS_FLAG_ALLOW_ZERO_ADDRESS) :
+				(GSP_PHYS_RMCALL_VAS_FLAG_IS_EXTERNALLY_OWNED |
+				 GSP_PHYS_RMCALL_VAS_FLAG_ALLOW_ZERO_ADDRESS));
+	vas.va_size = cpu_to_le64(gsp_phys_rmcall_vas_shared_value() ?
+				  GSP_PHYS_RMCALL_VAS_SIZE_SHARED :
+				  GSP_PHYS_RMCALL_VAS_SIZE);
+	pr_info("vgsp-boot: rm-call: FERMI_VASPACE_A flags=0x%x va_size=0x%llx (%s)\n",
+		le32_to_cpu(vas.flags), le64_to_cpu(vas.va_size),
+		gsp_phys_rmcall_vas_shared_value() ?
+			"shared management (RM owns page tables/commit)" :
+			"externally owned (client owns page tables/commit)");
 	ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client,
 				    boot_stage.rmcall_device,
 				    GSP_PHYS_RMCALL_HANDLE_VAS,
diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c
index 1fa1670..91f7a77 100644
--- a/kernel/gsp_phys_gpu.c
+++ b/kernel/gsp_phys_gpu.c
@@ -424,8 +424,8 @@ static int gsp_phys_rmcall_gr_promote_set(const char *val,
 		return ret;
 
 	if (gsp_phys_rmcall_gr_promote_param < 0 ||
-	    gsp_phys_rmcall_gr_promote_param > 19) {
-		pr_warn("vgsp: gsp_phys_rmcall_gr_promote=%d is outside 0..19; clamping to 0 (no PROMOTE_CTX)\n",
+	    gsp_phys_rmcall_gr_promote_param > 22) {
+		pr_warn("vgsp: gsp_phys_rmcall_gr_promote=%d is outside 0..22; clamping to 0 (no PROMOTE_CTX)\n",
 			gsp_phys_rmcall_gr_promote_param);
 		gsp_phys_rmcall_gr_promote_param = 0;
 	}
@@ -444,7 +444,7 @@ module_param_cb(gsp_phys_rmcall_gr_promote,
 		&gsp_phys_rmcall_gr_promote_ops,
 		&gsp_phys_rmcall_gr_promote_param, 0644);
 MODULE_PARM_DESC(gsp_phys_rmcall_gr_promote,
-		 "GR context-promote bisection (default 0: never send PROMOTE_CTX; 1: single entry init+nonmapped=0 size=raw+subctx, the p6h1b shape; 2: two-phase PA-only nonmapped then VA-only with the same fields; 3: single entry size=raw; 4: two-phase with a zero-field VA-only step; 5: batched entryCount=2 PA-only init + zero-field VA-only; 6: no-entry hVirtMemory, NV01_MEMORY_VIRTUAL (class 0x70) on the VAS at the context window + PROMOTE_CTX entryCount=0; 7: no-entry bare virtAddress+size, entryCount=0; 8: single MAIN entry like 1 on the KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC), server creates the virtual context memdesc itself; 9: same V6 channel with the stock client's zero-field VA-only MAIN entry, bInitialize=0; 10: same V6 channel, GPU_INITIALIZE_CTX (0x2080012d) installs the physical context memdesc then PROMOTE_CTX entryCount=0 bare virtAddress+size; 11: mode 10 with a server FBMEM context buffer, VIDMEM physAttr 0x4; 12: V5 UVM/vGPU-faithful PA-only MAIN entry (bInitialize=1 bNonmapped=1, no VA) and continue to BIND/SCHEDULE/submit; 13: mode 12 plus a PATCH PA-only promote first, sized from GET_ENGINE_CONTEXT_PROPERTIES engineId 0x10; 14: KMD channel (V6) + PATCH PA-only then the single-entry MAIN init+VA (the flow that books the context VA and commits ENG_CTX_VALID); 15: mode 14 with a server FBMEM MAIN context buffer and VIDMEM physAttr 0x4; 16: mode 14 plus a PM ctxsw PA-only promote first; 17: no client PROMOTE_CTX, paired with gsp_phys_srv_ctxbuf=1 so the GSP-RM server-manages and commits the context; 18: mode 8 shape plus a client FERMI_CONTEXT_SHARE_A on the TSG (hContextShare) whose construct allocates the GraphicsSubcontext; 19: the stock guest's exact 9-entry promote set (MAIN 2 MB PA+VA, PATCH, global buffers, FECS event, private access maps) on V5); values outside 0..19 clamp to 0 with a warning");
+		 "GR context-promote bisection (default 0: never send PROMOTE_CTX; 1: single entry init+nonmapped=0 size=raw+subctx, the p6h1b shape; 2: two-phase PA-only nonmapped then VA-only with the same fields; 3: single entry size=raw; 4: two-phase with a zero-field VA-only step; 5: batched entryCount=2 PA-only init + zero-field VA-only; 6: no-entry hVirtMemory, NV01_MEMORY_VIRTUAL (class 0x70) on the VAS at the context window + PROMOTE_CTX entryCount=0; 7: no-entry bare virtAddress+size, entryCount=0; 8: single MAIN entry like 1 on the KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC), server creates the virtual context memdesc itself; 9: same V6 channel with the stock client's zero-field VA-only MAIN entry, bInitialize=0; 10: same V6 channel, GPU_INITIALIZE_CTX (0x2080012d) installs the physical context memdesc then PROMOTE_CTX entryCount=0 bare virtAddress+size; 11: mode 10 with a server FBMEM context buffer, VIDMEM physAttr 0x4; 12: V5 UVM/vGPU-faithful PA-only MAIN entry (bInitialize=1 bNonmapped=1, no VA) and continue to BIND/SCHEDULE/submit; 13: mode 12 plus a PATCH PA-only promote first, sized from GET_ENGINE_CONTEXT_PROPERTIES engineId 0x10; 14: KMD channel (V6) + PATCH PA-only then the single-entry MAIN init+VA (the flow that books the context VA and commits ENG_CTX_VALID); 15: mode 14 with a server FBMEM MAIN context buffer and VIDMEM physAttr 0x4; 16: mode 14 plus a PM ctxsw PA-only promote first; 17: no client PROMOTE_CTX, paired with gsp_phys_srv_ctxbuf=1 so the GSP-RM server-manages and commits the context; 18: mode 8 shape plus a client FERMI_CONTEXT_SHARE_A on the TSG (hContextShare) whose construct allocates the GraphicsSubcontext; 19: the stock guest's exact 9-entry promote set (MAIN 2 MB PA+VA, PATCH, global buffers, FECS event, private access maps) on V5; 20: mode 6's no-entry hVirtMemory promote on the KMD-managed V6 channel (SKIP_CTXBUFFER_ALLOC), untested - the combination the V5 refusal (0x1f) points at); values outside 0..20 clamp to 0 with a warning");
 
 int gsp_phys_rmcall_gr_promote_value(void)
 {
@@ -453,6 +453,34 @@ int gsp_phys_rmcall_gr_promote_value(void)
 EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_promote_value);
 
 /*
+ * Ladder VAS ownership selector (gsp_phys_rmcall_vas_shared; default 0).
+ * The H1 GR promote wall is a direct consequence of the ladder's
+ * externally-owned FERMI_VASPACE_A: with IS_EXTERNALLY_OWNED the RM treats
+ * the client (UVM) as the owner of all page tables, VA bookings and engine
+ * context commitment, so the server never creates the context VA booking
+ * itself and the kickoff either asserts in kchannelMapEngineCtxBuf
+ * ("Externally owned object not found") or leaves the channel CTXNOTVALID.
+ * The normal KMD client instead allocates the VAS with SHARED_MANAGEMENT
+ * (nv_gpu_ops.c:2495): SET_PAGE_DIRECTORY is still legal (gpu_vaspace.c:
+ * 3108) and migrates the ladder's committed root PDB into the RM page
+ * walker, while the RM allocates the lower tables and maps/commits engine
+ * contexts. 1 selects the shared-management VAS; 0 keeps the byte-identical
+ * external path used by every previous cycle. Read once, at the stage-3
+ * VAS alloc; a mid-session change is ignored (the VAS already exists).
+ */
+static bool gsp_phys_rmcall_vas_shared;
+module_param_named(gsp_phys_rmcall_vas_shared, gsp_phys_rmcall_vas_shared,
+		   bool, 0644);
+MODULE_PARM_DESC(gsp_phys_rmcall_vas_shared,
+		 "Ladder FERMI_VASPACE_A ownership (default 0: IS_EXTERNALLY_OWNED, the client owns page tables and VA/context commitment - the state under which every GR promote failed; 1: SHARED_MANAGEMENT, the client supplies the root PDB but the RM owns the lower page tables and maps/commits engine contexts, like the normal KMD client)");
+
+bool gsp_phys_rmcall_vas_shared_value(void)
+{
+	return gsp_phys_rmcall_vas_shared;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_rmcall_vas_shared_value);
+
+/*
  * One-way power latch (default). Re-enabling runtime PM and D3cold after a
  * claim lets the board power-gate and the host hard-freeze on resume
  * (observed 2026-10-01); keep the pre-claim D0 policy instead and require
diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h
index 7d9b695..74e3103 100644
--- a/kernel/gsp_phys_gpu.h
+++ b/kernel/gsp_phys_gpu.h
@@ -210,6 +210,19 @@ int gsp_phys_rmcall_engine_value(void);
 int gsp_phys_rmcall_gr_promote_value(void);
 
 /*
+ * Ladder VAS ownership selector (gsp_phys_rmcall_vas_shared; default 0).
+ * 0 = FERMI_VASPACE_A is allocated IS_EXTERNALLY_OWNED (historical ladder
+ * behaviour: the client owns page tables, VA bookings and engine-context
+ * commitment, which is why every host GR promote failed or wedged).
+ * 1 = SHARED_MANAGEMENT: the ladder still supplies and commits its root PDB
+ * via SET_PAGE_DIRECTORY (legal for shared management, gpu_vaspace.c:3108)
+ * while the RM owns the lower page tables and maps/commits engine contexts
+ * itself, like the normal KMD client (nv_gpu_ops.c:2495). Read at the
+ * stage-3 VAS alloc; a mid-session change is ignored.
+ */
+bool gsp_phys_rmcall_vas_shared_value(void);
+
+/*
  * Translate a guest GPA to the pinned host PA (gsp_phys_guest_mem ladder
  * support). The mdev is registered with vfio_register_emulated_iommu_dev():
  * its VFIO type-1 container keeps the guest DMA mappings privately, so
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index ae74881..8b8708d 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -198,6 +198,9 @@ struct svc_channel {
 				 * (registered via the sema_counter debug param);
 				 * 0 = not registered */
 	u32	counter_last;	/* last value published to counter_pa */
+	u64	pushpg_va;	/* last push page logged as pin-vs-window
+				 * divergent (rate limit: one line per page) */
+	u8	pushpg_n;	/* pushpg lines emitted for this channel (cap) */
 	u32	pending_rel_pay;	/* release payload decoded while counter_pa
 				 * was still unknown (the injected pin for the
 				 * counter page can land a moment after the
@@ -246,6 +249,10 @@ struct svc_pin {
 	u64 va_page;
 	u64 pa_page;
 	int ap;		/* SVC_APERTURE_* of the mapped page */
+	u8 win;		/* linear push-window guess (svc_pb_pair): the page
+			 * was NOT read back, only its window base was. Push
+			 * reads skip these and resolve each page themselves;
+			 * every other path still uses them as before. */
 };
 
 struct svc_vaspace {
@@ -405,6 +412,8 @@ struct sema_svc {
 	u64			entry_log;		/* per-push summary budget */
 	u64			ops_denied;		/* work refused: implausible stream */
 	u64			false_trans;		/* resolvable VA mapped to non-push */
+	u64			sem_write_reresolve;	/* writes re-resolved after read-back */
+	u64			pushpg_fallback;	/* injected pin rejected by content */
 	u64			gpfifo_skipped;		/* stale entries consumed */
 	u64			releases_seen;
 	u64			releases_applied;
@@ -415,6 +424,12 @@ struct sema_svc {
 	u64			releases_stale;		/* payload <= current */
 	u64			releases_jumped;	/* applied with gap > 1 */
 	u64			releases_faked;		/* skipped-entry fallback */
+	u64			releases_forwarded;	/* exact slot known, not translatable:
+						 * expectation advanced, write queued */
+	u64			gpfifo_held;		/* passes an entry was left pending
+						 * for a translation/grace window */
+	u64			gpfifo_starved;		/* entries consumed without a
+						 * deliverable exact slot */
 	u64			counter_rel_applied;	/* releases to registered counters */
 	u64			counter_rel_dbg;	/* rel-apply diagnostic budget */
 	u64			counter_pub_dbg;	/* counter-publish diagnostic budget */
@@ -480,6 +495,7 @@ struct sema_svc {
 	u64			fwd_hold_engaged;
 	u64			fwd_hold_timeouts;
 	u64			fwd_hold_canceled;
+	u64			rel_hold_direct;	/* decoded releases applied without a forwarded copy */
 	/*
 	 * P6c completion relay (gsp_phys_relay=1 only): relay selections armed
 	 * on a deferred guest request (fwd_relay_requested), relays whose
@@ -568,6 +584,19 @@ static int svc_write(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len
 #define SVC_APERTURE_FB		2
 
 /*
+ * Aperture selectors are codes (1=SYS, 2=FB), never booleans. The idiom
+ * `ap ? SVC_APERTURE_FB : SVC_APERTURE_SYS` turns a valid SYS result (1)
+ * into an FB access into the 256 MiB BAR1 vidmem aperture: for a guest PA
+ * above the aperture the access fails, and the pool-semaphore read path then
+ * erased the verified cache entry it had just stored. Normalise every
+ * resolver output through this (0 also normalises to SYS).
+ */
+static inline u32 svc_ap_of(int ap)
+{
+	return ap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS;
+}
+
+/*
  * Aperture-aware access: channels describe their USERD/instance blocks with an
  * address-space tag, and "frame buffer" addresses are offsets into the
  * emulated vidmem (BAR1 backing), not guest RAM.
@@ -1261,7 +1290,7 @@ static void svc_lat_store(u64 va, u64 addr, int ap)
 }
 
 static int pin_lookup_tab(const struct svc_pin *tab, u32 slots, u32 probe_max,
-			  u64 va, u64 *pa, int *ap)
+			  u64 va, u64 *pa, int *ap, bool skip_win)
 {
 	u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
 	u32 mask = slots - 1;
@@ -1274,6 +1303,13 @@ static int pin_lookup_tab(const struct svc_pin *tab, u32 slots, u32 probe_max,
 		if (!p->va_page)
 			return -ENOENT;
 		if (p->va_page == page) {
+			/*
+			 * The push read path asks for validated pins only:
+			 * a window guess must not shadow the per-page
+			 * sources it would otherwise resolve from.
+			 */
+			if (skip_win && p->win)
+				return -ENOENT;
 			*pa = p->pa_page | (va & (SEMA_PAGE_SIZE - 1));
 			*ap = p->ap;
 			return 0;
@@ -1283,7 +1319,7 @@ static int pin_lookup_tab(const struct svc_pin *tab, u32 slots, u32 probe_max,
 }
 
 static void pin_store_tab(struct svc_pin *tab, u32 slots, u32 probe_max,
-			  u32 *n, u64 *drops, u64 va, u64 pa, int ap)
+			  u32 *n, u64 *drops, u64 va, u64 pa, int ap, u8 win)
 {
 	u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
 	u32 mask = slots - 1;
@@ -1299,6 +1335,7 @@ static void pin_store_tab(struct svc_pin *tab, u32 slots, u32 probe_max,
 			p->va_page = page;
 			p->pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1);
 			p->ap = ap;
+			p->win = win;
 			return;
 		}
 	}
@@ -1307,13 +1344,32 @@ static void pin_store_tab(struct svc_pin *tab, u32 slots, u32 probe_max,
 
 static int svc_pin_lookup(u64 va, u64 *pa, int *ap)
 {
-	return pin_lookup_tab(svc.pin, SVC_PIN_SLOTS, SVC_PIN_PROBE, va, pa, ap);
+	return pin_lookup_tab(svc.pin, SVC_PIN_SLOTS, SVC_PIN_PROBE, va, pa, ap,
+			      false);
+}
+
+/*
+ * Validated self-pins only (skip_win): what the push read path may use
+ * without shadowing a better per-page source. Window guesses are served by
+ * svc_pb_win_lookup() as the explicit last-resort fallback instead.
+ */
+static int svc_pin_lookup_valid(u64 va, u64 *pa, int *ap)
+{
+	return pin_lookup_tab(svc.pin, SVC_PIN_SLOTS, SVC_PIN_PROBE, va, pa, ap,
+			      true);
 }
 
 static void svc_pin_store(u64 va, u64 pa, int ap)
 {
 	pin_store_tab(svc.pin, SVC_PIN_SLOTS, SVC_PIN_PROBE, &svc.pin_n,
-		      &svc.pin_drop, va, pa, ap);
+		      &svc.pin_drop, va, pa, ap, 0);
+}
+
+/* The linear window convention of svc_pb_pair: a guess, never validated. */
+static void svc_pin_store_win(u64 va, u64 pa, int ap)
+{
+	pin_store_tab(svc.pin, SVC_PIN_SLOTS, SVC_PIN_PROBE, &svc.pin_n,
+		      &svc.pin_drop, va, pa, ap, 1);
 }
 
 /*
@@ -1327,13 +1383,14 @@ static void svc_pin_store(u64 va, u64 pa, int ap)
  */
 static int svc_ipin_lookup(u64 va, u64 *pa, int *ap)
 {
-	return pin_lookup_tab(svc.ipin, SVC_IPIN_SLOTS, SVC_IPIN_PROBE, va, pa, ap);
+	return pin_lookup_tab(svc.ipin, SVC_IPIN_SLOTS, SVC_IPIN_PROBE, va, pa,
+			      ap, false);
 }
 
 static void svc_ipin_store(u64 va, u64 pa, int ap)
 {
 	pin_store_tab(svc.ipin, SVC_IPIN_SLOTS, SVC_IPIN_PROBE, &svc.ipin_n,
-		      &svc.ipin_drop, va, pa, ap);
+		      &svc.ipin_drop, va, pa, ap, 0);
 }
 
 /* Any pinned translation: injected truth first, then self-discovered pins. */
@@ -1393,6 +1450,22 @@ static void svc_tr_note_failure(u64 page_va)
 	tr_fail_next = (tr_fail_next + 1) % SVC_TR_FAIL_MAX;
 }
 
+/*
+ * A successful resolve invalidates any earlier failure hold for the page:
+ * the hold exists to stop rescanning a VA that was not resolvable, and a
+ * fresh verified translation proves the guest has since mapped it. Without
+ * this, a page resolved through a pin/cache after a failed scan stayed
+ * blocked from the targeted scan for the rest of the 60 s window.
+ */
+static void svc_tr_clear_failure(u64 page_va)
+{
+	u32 i;
+
+	for (i = 0; i < SVC_TR_FAIL_MAX; i++)
+		if (tr_failed[i].va == page_va)
+			tr_failed[i].va = 0;
+}
+
 static int svc_targeted_resolve(struct nv_vgpu_device *vgpu, u64 gpu_va,
 				u64 *pa_out)
 {
@@ -1475,6 +1548,7 @@ static int svc_targeted_resolve(struct nv_vgpu_device *vgpu, u64 gpu_va,
 			*pa_out = pa | (gpu_va & (SEMA_PAGE_SIZE - 1));
 			svc_pin_store(page_va, pa, SVC_APERTURE_SYS);
 			svc_lat_store(page_va, pa, SVC_APERTURE_SYS);
+			svc_tr_clear_failure(page_va);
 			tr_va = 0;
 			tr_scans++;
 			pr_info("sema-svc: targeted resolve va=0x%llx -> pa=0x%llx (gpa=0x%llx off=%u, scan #%llu)\n",
@@ -1608,34 +1682,369 @@ static int svc_resolve_va(struct nv_vgpu_device *vgpu, u64 va, u64 *addr,
 	 * resolvable stays resolvable for the boot.
 	 */
 	svc_pin_store(va, *addr, ap);
-	*aperture = (ap == SVC_APERTURE_FB) ? 1 : 0;
+	/*
+	 * Raw aperture code, matching svc_lookup_trusted/svc_pool_pa_resolve:
+	 * a boolean 0/1 encoding here invited truthiness conversions
+	 * downstream that mapped SYS (1) to FB.
+	 */
+	*aperture = svc_ap_of(ap);
 	return 0;
 }
 
+/*
+ * CE channel-pool semaphores: the completion words UVM allocates for the CE
+ * channels (observed at 0x121000000 + 4*ce_index inside the CE window
+ * 0x120000000..0x121400000). They are sysmem objects the client polls
+ * directly, and the learned cache / content-scanned VA map can hold a
+ * stale/wrong PA for them: a release written through such a translation
+ * lands in a page the client never reads (observed: the pool semaphore
+ * stayed at 1 while the guest spun in
+ * uvm_page_table_range_vec_init -> uvm_tracker_wait). Pool releases resolve
+ * pin -> verified cache -> targeted rm_mem record, never the learned
+ * sources first; the GR/CE completion counters (0x2040xxxxx) are outside
+ * this range and keep their exact-slot eager logic untouched.
+ */
+#define SVC_POOL_SEM_VA_LO	0x121000000ULL
+#define SVC_POOL_SEM_VA_HI	0x122000000ULL
+
+static inline bool svc_pool_sem_va(u64 va)
+{
+	return va >= SVC_POOL_SEM_VA_LO && va < SVC_POOL_SEM_VA_HI;
+}
+
+/*
+ * Verified pool-semaphore translations, keyed by page VA. An entry is only
+ * created from an injected pin or from a targeted rm_mem resolve whose page
+ * was read back successfully; the learned cache and the discovered map are
+ * never cached here. Plain per-page state on the service thread, mirroring
+ * counter_pa / svc_cnt_der lifetime; a failed write or an unreadable cached
+ * page forgets the entry so the next pass re-resolves from the live record.
+ */
+#define SVC_POOL_PA_MAX		64
+#define SVC_POOL_PA_LOG_MAX	32
+#define SVC_POOL_PA_HIT_LOG_MAX	32
+#define SVC_POOL_WR_LOG_MAX	64
+
+struct svc_pool_pa {
+	u64	va_page;
+	u64	pa_page;
+	int	ap;
+};
+
+static struct svc_pool_pa svc_pool_pa[SVC_POOL_PA_MAX];
+static u32 svc_pool_pa_n;
+static u32 svc_pool_pa_next;
+static u64 svc_pool_pa_logged[SVC_POOL_PA_LOG_MAX];
+static u32 svc_pool_pa_logged_n;
+static u64 svc_pool_pa_miss_log;
+static u32 svc_pool_pa_hit_log;
+static u32 svc_pool_wr_log;
+
+/* First-use-per-page proof line; bounded for the boot. */
+static void svc_pool_pa_note(u32 h, u64 va, u64 pa, const char *src)
+{
+	u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
+	u32 i;
+
+	for (i = 0; i < svc_pool_pa_logged_n; i++)
+		if (svc_pool_pa_logged[i] == page)
+			return;
+	if (svc_pool_pa_logged_n >= SVC_POOL_PA_LOG_MAX)
+		return;
+	svc_pool_pa_logged[svc_pool_pa_logged_n++] = page;
+	pr_info("vgsp-sema: pool-pa ch=0x%x va=0x%llx pa=0x%llx src=%s\n",
+		h, va, pa, src);
+}
+
+static void svc_pool_pa_miss_note(u64 va)
+{
+	if (svc_pool_pa_miss_log >= 16)
+		return;
+	svc_pool_pa_miss_log++;
+	pr_info("vgsp-sema: sem-pa-miss va=0x%llx (using fallback)\n", va);
+}
+
+/*
+ * Bounded per-write proof line: the first 64 pool writes always print, later
+ * ones are rate-limited. Every release apply funnels through svc_sem_write
+ * (direct release, deferred pending apply with h=0, and hold-expired apply),
+ * so this covers all three with the handle of the originating call site.
+ */
+static void svc_pool_wr_note(u32 h, u64 va, u64 pa, const char *src,
+			     u32 cur, u32 val, const char *res)
+{
+	if (svc_pool_wr_log < SVC_POOL_WR_LOG_MAX) {
+		svc_pool_wr_log++;
+		pr_info("vgsp-sema: pool-wr ch=0x%x va=0x%llx pa=0x%llx src=%s cur=%u -> val=%u %s\n",
+			h, va, pa, src, cur, val, res);
+	} else {
+		pr_info_ratelimited("vgsp-sema: pool-wr ch=0x%x va=0x%llx pa=0x%llx src=%s cur=%u -> val=%u %s\n",
+			h, va, pa, src, cur, val, res);
+	}
+}
+
+static int svc_pool_pa_lookup(u64 va, u64 *pa, int *ap)
+{
+	u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
+	u32 i;
+
+	for (i = 0; i < svc_pool_pa_n; i++) {
+		if (svc_pool_pa[i].va_page == page) {
+			*pa = svc_pool_pa[i].pa_page |
+			      (va & (SEMA_PAGE_SIZE - 1));
+			*ap = svc_pool_pa[i].ap;
+			return 0;
+		}
+	}
+	return -ENOENT;
+}
+
+static void svc_pool_pa_store(u64 va, u64 pa, int ap)
+{
+	u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
+	u32 i;
+
+	for (i = 0; i < svc_pool_pa_n; i++) {
+		if (svc_pool_pa[i].va_page == page) {
+			svc_pool_pa[i].pa_page =
+				pa & ~(u64)(SEMA_PAGE_SIZE - 1);
+			svc_pool_pa[i].ap = ap;
+			return;
+		}
+	}
+	if (svc_pool_pa_n < SVC_POOL_PA_MAX) {
+		i = svc_pool_pa_n++;
+	} else {
+		i = svc_pool_pa_next++ % SVC_POOL_PA_MAX;
+	}
+	svc_pool_pa[i].va_page = page;
+	svc_pool_pa[i].pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1);
+	svc_pool_pa[i].ap = ap;
+}
+
+static void svc_pool_pa_forget(u64 va)
+{
+	u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
+	u32 i;
+
+	for (i = 0; i < svc_pool_pa_n; i++)
+		if (svc_pool_pa[i].va_page == page)
+			svc_pool_pa[i].va_page = 0;
+}
+
+/*
+ * Resolve one pool-semaphore target the way the successful deferred apply did:
+ * injected pin (host-verified ground truth), the verified cache, then the
+ * targeted rm_mem record scan, whose result is read back before it is cached.
+ * The learned cache, the discovered map and the page-directory walks are
+ * deliberately skipped: each can answer with a stale page for a pool
+ * semaphore, and a write there is silently lost. Returns -ENOENT when no
+ * source applies; the caller then uses the historical order. The cache is
+ * keyed by VA page only and shared by every caller: the handle is logging
+ * only, never part of the key. `src`, when non-NULL, names the source that
+ * answered (for the write proof line).
+ */
+static int svc_pool_pa_resolve(struct nv_vgpu_device *vgpu, u32 h, u64 va,
+			       u64 *addr, int *ap, const char **src)
+{
+	u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
+	u64 pa = 0;
+	int a = SVC_APERTURE_SYS;
+	u32 probe = 0;
+	u64 wpa = 0;
+	int wap = SVC_APERTURE_SYS;
+
+	if (!svc_ipin_lookup(va, &pa, &a)) {
+		svc_tr_clear_failure(page);
+		if (src)
+			*src = "pin";
+		svc_pool_pa_note(h, va, pa, "pin");
+		*addr = pa;
+		*ap = a;
+		return 0;
+	}
+	if (!svc_pool_pa_lookup(va, &pa, &a)) {
+		svc_tr_clear_failure(page);
+		if (svc_pool_pa_hit_log < SVC_POOL_PA_HIT_LOG_MAX) {
+			svc_pool_pa_hit_log++;
+			pr_info("vgsp-sema: pool-pa hit ch=0x%x va=0x%llx pa=0x%llx\n",
+				h, va, pa);
+		}
+		if (src)
+			*src = "cache";
+		*addr = pa;
+		*ap = a;
+		return 0;
+	}
+	/*
+	 * Structural ground truth before the content scan: the client's own
+	 * page tables (the self-discovered directory; the driver-published
+	 * directories are tried by svc_lookup_trusted). A walk result is
+	 * deterministic and cannot pick a stale rm_mem duplicate. It is only
+	 * accepted when the exact release offset is readable through it, the
+	 * same read-back proof every other source must pass. On every boot
+	 * where the directory does not contain this VA the walk misses and
+	 * the sequence falls through unchanged.
+	 */
+	if (!svc_walk_discovered(vgpu, page, &wpa, &wap) &&
+	    wap == SVC_APERTURE_SYS) {
+		u64 exact = (wpa & ~(u64)(SEMA_PAGE_SIZE - 1)) |
+			    (va & (SEMA_PAGE_SIZE - 1));
+
+		if (!svc_read_ap(vgpu, exact, SVC_APERTURE_SYS, &probe,
+				 sizeof(probe))) {
+			svc_pool_pa_store(va, exact, SVC_APERTURE_SYS);
+			svc_tr_clear_failure(page);
+			if (src)
+				*src = "walk";
+			svc_pool_pa_note(h, va, exact, "walk");
+			*addr = exact;
+			*ap = SVC_APERTURE_SYS;
+			return 0;
+		}
+	}
+	if (!svc_targeted_resolve(vgpu, va, &pa)) {
+		/* Read back before trusting: an unreadable page is not kept. */
+		if (!svc_read_ap(vgpu, pa, SVC_APERTURE_SYS, &probe,
+				 sizeof(probe))) {
+			svc_pool_pa_store(va, pa, SVC_APERTURE_SYS);
+			svc_tr_clear_failure(page);
+			if (src)
+				*src = "targeted";
+			svc_pool_pa_note(h, va, pa, "targeted");
+			*addr = pa;
+			*ap = SVC_APERTURE_SYS;
+			return 0;
+		}
+	}
+	return -ENOENT;
+}
+
 static int svc_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val)
 {
 	u64 addr = 0;
 	u32 ap = 0;
 
+	if (svc_pool_sem_va(va)) {
+		if (!svc_pool_pa_resolve(vgpu, 0, va, &addr, &ap, NULL)) {
+			if (!svc_read_ap(vgpu, addr, svc_ap_of(ap), val,
+					 sizeof(*val)))
+				return 0;
+			/*
+			 * The verified translation became unreadable (the
+			 * guest reallocated the pool). Re-resolve the live
+			 * record once and retry: only a targeted resolve that
+			 * also fails drops the entry, so one transient read
+			 * failure cannot evict a translation the following
+			 * write depends on (this was the v17 bypass).
+			 */
+			if (!svc_targeted_resolve(vgpu, va, &addr)) {
+				svc_pool_pa_store(va, addr, SVC_APERTURE_SYS);
+				if (!svc_read_ap(vgpu, addr,
+						 SVC_APERTURE_SYS, val,
+						 sizeof(*val)))
+					return 0;
+			}
+			svc_pool_pa_forget(va);
+		}
+	}
 	if (svc_resolve_va(vgpu, va, &addr, &ap))
 		return -ENOENT;
-	return svc_read_ap(vgpu, addr,
-			   ap ? SVC_APERTURE_FB : SVC_APERTURE_SYS,
-			   val, sizeof(*val));
+	return svc_read_ap(vgpu, addr, svc_ap_of(ap), val, sizeof(*val));
 }
 
-static int svc_sem_write(struct nv_vgpu_device *vgpu, u64 va, u32 val)
+static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val)
 {
-	u64 addr = 0;
+	u64 addr = 0, old_addr;
 	u32 ap = 0;
+	u32 back = 0;
+	u32 cur = 0;
+	const char *src = "fallback";
+	bool is_pool = svc_pool_sem_va(va);
+	int wap, rc;
+	bool verified = false;
 
-	if (svc_resolve_va(vgpu, va, &addr, &ap))
-		return -ENOENT;
-	if (ap)
+	/*
+	 * Pool semaphores resolve pin -> verified cache -> targeted rm_mem
+	 * record before any learned translation; the learned cache and the
+	 * discovered map may hold a stale PA for them and a write through it
+	 * is lost (see svc_pool_pa_resolve). Every other VA keeps the
+	 * historical order below.
+	 */
+	if (is_pool) {
+		if (!svc_pool_pa_resolve(vgpu, h, va, &addr, &ap, &src))
+			verified = true;
+		else
+			svc_pool_pa_miss_note(va);
+	}
+	if (!verified && svc_resolve_va(vgpu, va, &addr, &ap)) {
+		/*
+		 * No trusted translation at all. One last exact resolve: the
+		 * targeted rm_mem scan validates the live record header for
+		 * this VA and pins the page. Bounded: one slice per call and a
+		 * full-scan miss is cached for SVC_TR_FAIL_HOLD, so a genuinely
+		 * unmapped semaphore is not rescanned on every pass.
+		 */
+		if (svc_targeted_resolve(vgpu, va, &addr))
+			return -ENOENT;
+		ap = SVC_APERTURE_SYS;
+		src = "targeted-direct";
+	}
+	/*
+	 * Raw aperture code, never a truthiness test: a SYS resolve (1) used
+	 * to turn into FB here and the release landed in the vidmem shadow.
+	 */
+	wap = svc_ap_of(ap);
+	if (wap == SVC_APERTURE_FB)
 		svc.vidmem_writes++;
-	return svc_write_ap(vgpu, addr,
-			    ap ? SVC_APERTURE_FB : SVC_APERTURE_SYS,
-			    &val, sizeof(val));
+	old_addr = addr;
+	if (is_pool)
+		(void)svc_read_ap(vgpu, addr, wap, &cur, sizeof(cur));
+	rc = svc_write_ap(vgpu, addr, wap, &val, sizeof(val));
+	if (!rc && !svc_read_ap(vgpu, addr, wap, &back, sizeof(back)) &&
+	    back == val) {
+		if (is_pool)
+			svc_pool_wr_note(h, va, addr, src, cur, val, "ok");
+		return 0;
+	}
+
+	/*
+	 * The value is not visible through the translation we just wrote:
+	 * the entry can be a stale content-scanned cache line or an
+	 * unvalidated self-pin whose page is not the one the client polls.
+	 * Re-resolve the exact VA through the targeted rm_mem record scan
+	 * (the live record names the client's current PA) and rewrite once.
+	 * Bounded: exactly one re-resolve; its result is read back but never
+	 * re-resolved again, so a hopeless VA costs one extra attempt.
+	 */
+	if (is_pool)
+		svc_pool_pa_forget(va);	/* stale: rebuild from the live record */
+	if (!svc_targeted_resolve(vgpu, va, &addr)) {
+		back = 0;
+		if (!svc_write_ap(vgpu, addr, SVC_APERTURE_SYS, &val,
+				  sizeof(val)) &&
+		    !svc_read_ap(vgpu, addr, SVC_APERTURE_SYS, &back,
+				 sizeof(back)) && back == val) {
+			if (is_pool) {
+				svc_pool_pa_store(va, addr, SVC_APERTURE_SYS);
+				svc_pool_wr_note(h, va, addr, "targeted", cur,
+						 val, "reresolve");
+			}
+			svc.sem_write_reresolve++;
+			pr_info_ratelimited("vgsp-sema: sem-write re-resolve va=0x%llx pa=0x%llx old=0x%llx new=0x%x\n",
+				va, addr, old_addr, val);
+			return 0;
+		}
+	}
+	if (!rc) {
+		if (is_pool)
+			svc_pool_wr_note(h, va, old_addr, src, cur, val,
+					 "advisory");
+		return 0;	/* the original write landed; read-back is advisory */
+	}
+	if (is_pool)
+		svc_pool_wr_note(h, va, old_addr, src, cur, val, "fail");
+	return rc;
 }
 
 /* ------------------------------------------------------------------ */
@@ -1981,8 +2390,12 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu)
 		svc.memsets_fail, svc.ring_ops, svc.ops_denied,
 		svc.false_trans, svc.gpfifo_skipped, svc.releases_faked,
 		svc.releases_recovered, svc.counter_rel_applied);
-	pr_info("sema-svc:   counter pub_calls=%llu gr_rel=%llu\n",
-		svc.counter_pub_calls, svc.gr_rel_applied);
+	pr_info("sema-svc:   drain held=%llu starved=%llu forwarded=%llu faked=%llu rec=%llu\n",
+		svc.gpfifo_held, svc.gpfifo_starved, svc.releases_forwarded,
+		svc.releases_faked, svc.releases_recovered);
+	pr_info("sema-svc:   counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu\n",
+		svc.counter_pub_calls, svc.gr_rel_applied,
+		svc.sem_write_reresolve, svc.pushpg_fallback);
 	pr_info("sema-svc:   gpfifo done=%llu nomap=%llu vidmem_writes=%llu ring_auto=%u pb=%u/%u pins=%u/%llu ipins=%u/%llu\n",
 		svc.gpfifo_entries_done, svc.gpfifo_nomap,
 		svc.vidmem_writes, svc_ring_n, svc_pb_n, svc_pb_passes,
@@ -2010,13 +2423,13 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu)
 	    svc.fwd_rel_fallback || svc.fwd_rel_pending ||
 	    svc.fwd_rel_overflow || svc.fwd_hold_engaged ||
 	    svc.fwd_hold_timeouts || svc.fwd_hold_canceled)
-		pr_info("sema-svc:   work_guard deferred=%llu applied=%llu fallback=%llu pending=%llu overflow=%llu engaged=%llu timeouts=%llu canceled=%llu copy_skipped=%llu copy_soft=%llu copy_fail=%llu\n",
+		pr_info("sema-svc:   work_guard deferred=%llu applied=%llu fallback=%llu pending=%llu overflow=%llu engaged=%llu timeouts=%llu canceled=%llu copy_skipped=%llu copy_soft=%llu copy_fail=%llu direct=%llu\n",
 			svc.fwd_rel_deferred, svc.fwd_rel_applied,
 			svc.fwd_rel_fallback, svc.fwd_rel_pending,
 			svc.fwd_rel_overflow, svc.fwd_hold_engaged,
 			svc.fwd_hold_timeouts, svc.fwd_hold_canceled,
 			svc.fwd_copy_skipped, svc.fwd_copy_soft,
-			svc.fwd_copy_fail);
+			svc.fwd_copy_fail, svc.rel_hold_direct);
 	/* show a few entries so mappings can be verified from dmesg */
 	if (gsp_sema_dbg) {
 		u32 shown;
@@ -2063,6 +2476,28 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu)
 #define GRCE_M_LAUNCH_DMA	0x1b0
 #define GRCE_M_LOAD_INLINE_DATA	0x1b4
 
+/*
+ * C5C0 (compute class) inline-QMD launch methods: SET_INLINE_QMD_ADDRESS_A/B
+ * carry the shifted QMD device address, LOAD_INLINE_QMD_DATA the 64 QMD words.
+ * The driver emits them as one INC run covering 0x318..0x41c (observed:
+ * op=1 count=66 at 0x318) or as several runs; the host CUDA bridge
+ * (gsp_cuda_bridge.c) accumulates the words and turns a completed QMD into a
+ * daemon job.
+ */
+#define SVC_QMD_ADDR_A		0x318
+#define SVC_QMD_ADDR_B		0x31c
+#define SVC_QMD_DATA		0x320
+#define SVC_QMD_WORDS		64
+
+/*
+ * Recent GRCE LOAD_INLINE_DATA destinations, newest at (next - 1). CUDA
+ * writes the launch arguments into the QMD's parameter bank at an arbitrary
+ * offset with these writes (observed: mail args at bank+0x160), so the host
+ * CUDA bridge searches this ring for the argument block instead of reading
+ * bank+0. Service thread only, so no lock.
+ */
+#define SVC_GRCE_DST_MAX	32
+
 /* LAUNCH_DMA fields. */
 #define DMA_DATA_TRANSFER_TYPE(v)	((v) & 0x3)
 #define DMA_SEMAPHORE_TYPE(v)		(((v) >> 3) & 0x3)
@@ -2116,8 +2551,37 @@ struct svc_push_state {
 	u64	rel_va;		/* last release decoded in this push */
 	u32	rel_payload;
 	u32	have_rel;
+	/* inline-QMD launch accumulation (GR compute channel) */
+	u64	qmd_dev_va;	/* devVA decoded from ADDRESS_A/B */
+	u32	qmd_hi, qmd_lo;	/* ADDRESS_A / ADDRESS_B payloads */
+	u32	qmd_have;	/* A/B seen bitmap */
+	u32	qmd_n;		/* QMD words collected so far */
+	u64	qmd_bits;	/* received-word bitmap */
+	u64	push_va;	/* push VA, for launch logging */
+	u32	qmd[SVC_QMD_WORDS];
 };
 
+/* Host CUDA bridge (gsp_cuda_bridge.c): launch transport + service hook. */
+void gsp_cuda_bridge_launch(struct nv_vgpu_device *vgpu, u32 h_object,
+			    const u32 qmd[SVC_QMD_WORDS], u64 qmd_dev_va,
+			    u64 push_va, const u64 *recent_dsts,
+			    u32 n_recent_dsts);
+bool gsp_cuda_bridge_enabled(void);
+void gsp_cuda_bridge_service(struct nv_vgpu_device *vgpu);
+int gsp_cuda_bridge_init(void);
+void gsp_cuda_bridge_fini(void);
+
+/* VA/semaphore helpers exported to gsp_cuda_bridge.c. */
+int gsp_sema_service_va_read(struct nv_vgpu_device *vgpu, u64 va, void *buf,
+			     size_t len);
+int gsp_sema_service_va_write(struct nv_vgpu_device *vgpu, u64 va,
+			      const void *buf, size_t len);
+int gsp_sema_service_va_resolve(struct nv_vgpu_device *vgpu, u64 va, u64 *pa,
+				u32 *ap);
+int gsp_sema_service_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val);
+int gsp_sema_service_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va,
+			       u32 val);
+
 /* ------------------------------------------------------------------ */
 /* CE execution                                                       */
 /* ------------------------------------------------------------------ */
@@ -2782,15 +3246,35 @@ static bool svc_stream_parse(const u8 *buf, u32 len)
 		if (off + 4 > len)
 			break;
 		memcpy(&hdr, buf + off, 4);
-		if (hdr == 0)
+		if (hdr == 0) {
+			/*
+			 * UVM's large page-table pushes begin with a
+			 * zero-filled header region (observed 0x1004
+			 * bytes) before the first method, so zeros before
+			 * the first method are leading padding; zeros
+			 * after one are trailing padding and end the
+			 * stream.
+			 */
+			if (!methods) {
+				off += 4;
+				continue;
+			}
 			break;	/* trailing zero padding */
+		}
 		op = (hdr >> 29) & 0x7;
 		count = (hdr >> 16) & 0xfff;
 		switch (op) {
 		case 1:		/* INC_METHOD */
 		case 3:		/* NON_INC_METHOD */
 		case 5:		/* ONE_INC */
-			if (!count || count > 64 ||
+			/*
+			 * The count field is 12 bits, and UVM emits
+			 * NON_INC runs of 1024 words (the page-table
+			 * pushes); the old 64 cap rejected exactly those,
+			 * which made the content rescue unable to ever
+			 * validate a large push.
+			 */
+			if (!count || count > 0xfff ||
 			    off + 4 + count * 4 > len)
 				return false;
 			off += 4 + count * 4;
@@ -2851,7 +3335,15 @@ static u64 svc_rel_va_seen;
 static u32 svc_rel_count;
 static u32 svc_dbg_streams;
 
-static void svc_counter_learn(u32 h, u64 sem_va);
+/*
+ * Ring of recent GRCE inline-write destinations (newest at next-1), consumed
+ * by the CUDA bridge to locate a launch's argument block inside the QMD
+ * parameter bank. Written and read on the service thread only.
+ */
+static u64 svc_grce_dst_ring[SVC_GRCE_DST_MAX];
+static u32 svc_grce_dst_ring_next;
+
+static void svc_counter_learn(struct nv_vgpu_device *vgpu, u32 h, u64 sem_va);
 static void svc_pending_add(u64 va, u32 payload, u32 op);
 static int svc_rescue_push_page(struct nv_vgpu_device *vgpu,
 				struct svc_channel *ch, u64 entry_va,
@@ -2902,7 +3394,7 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h,
 	 * it from the pin table first.
 	 */
 	if (sem_va >= 0x204000000ULL && sem_va < 0x204100000ULL)
-		svc_counter_learn(h, sem_va);
+		svc_counter_learn(vgpu, h, sem_va);
 
 	spin_lock_irqsave(&svc.chan_lock, flags);
 	for (i = 0; i < SVC_MAX_CHANNELS; i++) {
@@ -3019,12 +3511,88 @@ static void svc_counter_flush_pending(struct nv_vgpu_device *vgpu,
 }
 
 /*
+ * In-kernel counter-slot derivation.
+ *
+ * The authoritative counter translation is the external injected pin, but the
+ * pin watcher can land tens of seconds after the first release of a channel
+ * (observed: a GR release at 7586.5 s whose counter was only learned at
+ * 7609.9 s), and every release queued in that window leaves the client's
+ * tracker polling a value that never arrives. On an injected-pin miss,
+ * resolve the slot from the module's own translations and register it - but
+ * only translations this service already treats as structural truth:
+ *
+ *  - the page directories the driver itself published (svc_walk_vaspaces),
+ *  - the self-discovered directory (svc_walk_discovered; accepted only after
+ *    it reproduced several already-known VA->PA pairs), and
+ *  - the targeted rm_mem resolve (svc_targeted_resolve; validates the rm_mem
+ *    record header for this exact VA).
+ *
+ * The content-scanned VA map and the learned cache are deliberately NOT
+ * consulted: both can hold a stale translation for a counter page, and a
+ * write through one lands in a page the client never polls (see
+ * svc_counter_autoreg). A successful derivation is proven readable before it
+ * is returned, cached per page so the next release does not rescan, and never
+ * overrides an injected pin: svc_counter_learn re-checks the pin table first
+ * and re-registers when the injected PA differs.
+ *
+ * Service-thread only (like every other release/decode path); the cache is
+ * plain per-page state, mirroring counter_pa's lifetime.
+ */
+#define SVC_CNT_DER_MAX	16
+static struct {
+	u64 va_page;
+	u64 pa_page;
+} svc_cnt_der[SVC_CNT_DER_MAX];
+static u32 svc_cnt_der_n;
+
+static int svc_counter_derive(struct nv_vgpu_device *vgpu, u64 va, u64 *pa_out)
+{
+	u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
+	u64 off = va & (SEMA_PAGE_SIZE - 1);
+	u64 pa = 0;
+	int ap = SVC_APERTURE_SYS;
+	u32 probe = 0;
+	u32 i;
+	bool fresh = true;
+
+	for (i = 0; i < svc_cnt_der_n; i++) {
+		if (svc_cnt_der[i].va_page == page) {
+			pa = svc_cnt_der[i].pa_page | off;
+			fresh = false;
+			break;
+		}
+	}
+	if (fresh) {
+		if (svc_walk_vaspaces(vgpu, page, &pa, &ap) &&
+		    svc_walk_discovered(vgpu, page, &pa, &ap) &&
+		    svc_targeted_resolve(vgpu, page, &pa))
+			return -ENOENT;
+		if (ap != SVC_APERTURE_SYS)
+			return -ENOENT;
+		pa = (pa & ~(u64)(SEMA_PAGE_SIZE - 1)) | off;
+	}
+	/* Read the slot before trusting the mapping: an unreadable page is
+	 * not registered, so the caller keeps the "slot pending" fallback. */
+	if (svc_read_ap(vgpu, pa, SVC_APERTURE_SYS, &probe, sizeof(probe)))
+		return -EIO;
+	if (fresh && svc_cnt_der_n < SVC_CNT_DER_MAX) {
+		svc_cnt_der[svc_cnt_der_n].va_page = page;
+		svc_cnt_der[svc_cnt_der_n].pa_page =
+			pa & ~(u64)(SEMA_PAGE_SIZE - 1);
+		svc_cnt_der_n++;
+	}
+	*pa_out = pa;
+	return 0;
+}
+
+/*
  * Auto-register a GR channel's completion-counter slot from the observed
  * layout, without waiting for a readable push: the client zips GR channels
  * in ascending handle order with descending counter slots, so with n_gr
- * channels the rank-r channel owns slot 0x20400fec0 + (n_gr-1-r)*0x10. Only
- * a host-verified (pinned) translation is used, so the write target is the
- * page the client actually polls. This lets the publish path advance every
+ * channels the rank-r channel owns slot 0x20400fec0 + (n_gr-1-r)*0x10. The
+ * slot PA comes from a host-verified pin or from the in-kernel structural
+ * derivation (svc_counter_derive), so the write target is a page the client
+ * actually polls. This lets the publish path advance every
  * channel's counter even when some GR pushes are unreadable (observed: the
  * lower ranks' 676-byte context-init pushes stayed unresolved while the
  * other 13 channels' counters advanced, and the client timed out waiting).
@@ -3034,6 +3602,7 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu,
 {
 	u64 va, pa = 0;
 	int ap = SVC_APERTURE_SYS;
+	bool derived = false;
 
 	if (ch->engine_type || n_gr <= 0)
 		return;
@@ -3045,52 +3614,76 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu,
 		return;
 	va = 0x20400fec0ULL + (u64)(n_gr - 1 - ch->gr_index) * 0x10ULL;
 	/*
-	 * Only the authoritative injected-pin table may register a counter
-	 * slot: the self-discovered map can hold a stale translation for this
-	 * VA (content-scanned page), and a write to that page never reaches
-	 * the client. If no injected pin exists yet, leave counter_pa zero so
-	 * a later pass (once the pin lands) registers it.
+	 * The injected-pin table is the authoritative source; only it may
+	 * shadow a self-discovered map entry (which can hold a stale
+	 * translation for this VA - a content-scanned page - and a write
+	 * there never reaches the client). On a miss, derive the slot
+	 * in-kernel from the structural translations and the read-back proof
+	 * instead of waiting for the pin watcher; the layout formula only
+	 * names which slot belongs to this rank. If no translation exists,
+	 * leave counter_pa zero so a later pass registers it.
 	 */
-	if (svc_ipin_lookup(va, &pa, &ap) || ap != SVC_APERTURE_SYS)
-		return;
+	if (svc_ipin_lookup(va, &pa, &ap) || ap != SVC_APERTURE_SYS) {
+		if (!gsp_sema_counter_derive ||
+		    svc_counter_derive(vgpu, va, &pa))
+			return;
+		derived = true;
+	}
 	ch->counter_pa = pa & ~0x3ULL;
-	pr_info("sema-chan: counter auto h=0x%x rank=%u pa=0x%llx (slot va=0x%llx)\n",
-		ch->h_object, ch->gr_index, ch->counter_pa, va);
+	if (derived)
+		pr_info("sema-chan: counter derived h=0x%x rank=%u pa=0x%llx (slot va=0x%llx)\n",
+			ch->h_object, ch->gr_index, ch->counter_pa, va);
+	else
+		pr_info("sema-chan: counter auto h=0x%x rank=%u pa=0x%llx (slot va=0x%llx)\n",
+			ch->h_object, ch->gr_index, ch->counter_pa, va);
 	svc_counter_flush_pending(vgpu, ch);
 }
 
 /*
  * Learn a GR channel's completion-counter guest PA from a release VA the
  * push carried, so the publish path can advance the exact slot the client
- * polls without any guest-side enumeration. Only a host-verified translation
- * (pin/ipin) is trusted; the release VA of a GR push is the counter slot
- * itself (0x20400fXX observed). No-op once registered.
+ * polls without any guest-side enumeration. The release VA of a GR push is
+ * the counter slot itself (0x20400fXX observed). An injected pin is the
+ * authoritative source; on a miss the slot is derived in-kernel
+ * (svc_counter_derive) so the release is authored immediately rather than
+ * waiting for the external pin watcher. No-op once registered at the same PA.
  */
-static void svc_counter_learn(u32 h, u64 sem_va)
+static void svc_counter_learn(struct nv_vgpu_device *vgpu, u32 h, u64 sem_va)
 {
 	unsigned long flags;
 	u64 pa = 0;
 	int ap = SVC_APERTURE_SYS;
+	bool derived = false;
 	int i;
 
 	if (!h || sem_va < 0x204000000ULL || sem_va >= 0x204100000ULL)
 		return;
 	/*
-	 * Authoritative injected pins only (see svc_counter_autoreg): a stale
-	 * self-discovered entry would make every release land on a page the
-	 * client never polls. Re-learn when the injected PA differs, so a pin
-	 * that lands after an earlier stale registration still wins.
+	 * Injected pins first (see svc_counter_autoreg): they stay
+	 * authoritative and re-learn when their PA differs, so a pin that
+	 * lands after a derived/stale registration still wins. On a miss,
+	 * derive from the structural translations only - never from the
+	 * content-scanned map or the learned cache, which can hold a stale
+	 * page for this VA.
 	 */
-	if (svc_ipin_lookup(sem_va, &pa, &ap) || ap != SVC_APERTURE_SYS)
-		return;
+	if (svc_ipin_lookup(sem_va, &pa, &ap) || ap != SVC_APERTURE_SYS) {
+		if (!gsp_sema_counter_derive ||
+		    svc_counter_derive(vgpu, sem_va, &pa))
+			return;
+		derived = true;
+	}
 
 	spin_lock_irqsave(&svc.chan_lock, flags);
 	for (i = 0; i < SVC_MAX_CHANNELS; i++) {
 		if (svc.chan[i].valid && svc.chan[i].h_object == h) {
 			if (svc.chan[i].counter_pa != (pa & ~0x3ULL)) {
 				svc.chan[i].counter_pa = pa & ~0x3ULL;
-				pr_info("sema-chan: counter learned h=0x%x pa=0x%llx (from release va=0x%llx)\n",
-					h, pa & ~0x3ULL, sem_va);
+				if (derived)
+					pr_info("sema-chan: counter derived h=0x%x pa=0x%llx (from release va=0x%llx)\n",
+						h, pa & ~0x3ULL, sem_va);
+				else
+					pr_info("sema-chan: counter learned h=0x%x pa=0x%llx (from release va=0x%llx)\n",
+						h, pa & ~0x3ULL, sem_va);
 			}
 			break;
 		}
@@ -3114,14 +3707,14 @@ static bool svc_rel_apply_906f(struct nv_vgpu_device *vgpu,
 	u32 cur = 0;
 
 	if (ch->h_object && svc_counter_va(va)) {
-		svc_counter_learn(ch->h_object, va);
+		svc_counter_learn(vgpu, ch->h_object, va);
 		svc_counter_release_apply(vgpu, ch->h_object, va, payload);
 		return false;
 	}
 	if (!svc_sem_read(vgpu, va, &cur) && payload > cur) {
 		svc.pushes_pending++;
 		cur = (op == HOST_SEM_OP_REDUCTION) ? cur + payload : payload;
-		if (!svc_sem_write(vgpu, va, cur)) {
+		if (!svc_sem_write(vgpu, ch->h_object, va, cur)) {
 			svc.releases_applied++;
 			svc.post_hint_until = jiffies + 10 * HZ;
 			return true;
@@ -3137,7 +3730,7 @@ static bool svc_rel_apply_reportsem(struct nv_vgpu_device *vgpu,
 	u32 cur = 0;
 
 	if (ch->h_object && svc_counter_va(va)) {
-		svc_counter_learn(ch->h_object, va);
+		svc_counter_learn(vgpu, ch->h_object, va);
 		svc_counter_release_apply(vgpu, ch->h_object, va, payload);
 		return false;
 	}
@@ -3148,7 +3741,7 @@ static bool svc_rel_apply_reportsem(struct nv_vgpu_device *vgpu,
 	if (payload <= cur)
 		return false;
 	svc.pushes_pending++;
-	if (!svc_sem_write(vgpu, va, payload)) {
+	if (!svc_sem_write(vgpu, ch->h_object, va, payload)) {
 		svc.releases_applied++;
 		svc.post_hint_until = jiffies + 10 * HZ;
 		return true;
@@ -3175,14 +3768,14 @@ static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu,
 		 */
 		if (ch->engine_type == 0 && svc_counter_va(sem_va)) {
 			/* handled by the exact-slot path */
-		} else if (!svc_sem_write(vgpu, sem_va, cur)) {
+		} else if (!svc_sem_write(vgpu, ch->h_object, sem_va, cur)) {
 			svc.releases_applied++;
 			svc.post_hint_until = jiffies + 10 * HZ;
 			return true;
 		}
 	}
 	if (ch->engine_type == 0)
-		svc_counter_learn(ch->h_object, sem_va);
+		svc_counter_learn(vgpu, ch->h_object, sem_va);
 	svc_counter_release_apply(vgpu, ch->h_object, sem_va, payload);
 	return false;
 }
@@ -3235,19 +3828,109 @@ static void svc_fwd_relay_note(struct nv_vgpu_device *vgpu,
 		    DMA_SEM_REDUCTION(launch_dma) == SEM_REDUCTION_INC)
 			return;
 	}
-	if (svc_resolve_va(vgpu, r->va, &addr, &ap))
+	if (svc_pool_sem_va(r->va)) {
+		if (svc_pool_pa_resolve(vgpu, ch->h_object, r->va, &addr,
+					&ap, NULL))
+			return;
+	} else if (svc_resolve_va(vgpu, r->va, &addr, &ap)) {
 		return;
+	}
 	if (svc_sem_read(vgpu, r->va, &cur))
 		return;
 	/* Forward-only pre-check: only a word behind the payload is relayed. */
 	if (cur >= r->payload)
 		return;
 	r->tgt_ok = true;
-	r->tgt_kind = ap ? SVC_APERTURE_FB : SVC_APERTURE_SYS;
+	r->tgt_kind = svc_ap_of(ap);
 	r->tgt_page = addr & ~(u64)(SEMA_PAGE_SIZE - 1);
 	r->tgt_off = (u32)(addr & (SEMA_PAGE_SIZE - 1));
 }
 
+/* Whole-boot budget for the "no forwarded copy" proof line. */
+static u32 svc_fwd_rel_direct_log;
+
+/*
+ * One rate-limited line per decoded release that was applied directly because
+ * no forwarded hardware copy ever engaged (or the bounded hold window expired
+ * without one). A decoded release is never left to a copy that will not run;
+ * this line is the next-run proof of that path.
+ */
+static void svc_fwd_rel_direct_note(u32 h, u64 va, u32 payload)
+{
+	svc.rel_hold_direct++;
+	if (svc_fwd_rel_direct_log >= 32)
+		return;
+	svc_fwd_rel_direct_log++;
+	pr_info("vgsp-work: rel-hold direct h=0x%x va=0x%llx pay=%u (no work)\n",
+		h, va, payload);
+}
+
+/*
+ * Accumulate one method run's inline-QMD words; hand a complete 64-word QMD
+ * to the host CUDA bridge. The run may cover the address registers and the
+ * data registers as a single INC/NON_INC/ONE_INC method (observed: op=1
+ * count=66 starting at 0x318), so every data word is mapped to its register
+ * offset. Each completed QMD resets the accumulator, so a push may carry
+ * more than one launch.
+ */
+static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
+			  struct svc_push_state *st, u32 addr, u32 count,
+			  const u8 *data)
+{
+	u32 i;
+
+	for (i = 0; i < count; i++) {
+		u32 a = addr + 4 * i;
+		u32 v;
+
+		memcpy(&v, data + 4 * i, 4);
+		if (a == SVC_QMD_ADDR_A) {
+			st->qmd_hi = v;
+			st->qmd_have |= 1;
+		} else if (a == SVC_QMD_ADDR_B) {
+			st->qmd_lo = v;
+			st->qmd_have |= 2;
+		} else if (a >= SVC_QMD_DATA &&
+			   a < SVC_QMD_DATA + SVC_QMD_WORDS * 4) {
+			u32 idx = (a - SVC_QMD_DATA) / 4;
+
+			if (!(st->qmd_bits & (1ULL << idx))) {
+				st->qmd_bits |= 1ULL << idx;
+				st->qmd[idx] = v;
+				st->qmd_n++;
+			}
+		}
+	}
+	if (st->qmd_have == 3)
+		st->qmd_dev_va = (((u64)st->qmd_hi << 32) | st->qmd_lo) << 8;
+	if (st->qmd_n == SVC_QMD_WORDS) {
+		u64 dsts[SVC_GRCE_DST_MAX];
+		u32 have, i, n = 0;
+
+		/*
+		 * Hand the bridge the recent inline-write destinations,
+		 * newest first, so it can find the argument block CUDA
+		 * placed inside the QMD parameter bank (never assume
+		 * bank+0). The current push's writes are already in the
+		 * ring because the QMD run follows them.
+		 */
+		have = svc_grce_dst_ring_next < SVC_GRCE_DST_MAX ?
+		       svc_grce_dst_ring_next : SVC_GRCE_DST_MAX;
+		for (i = 0; i < have; i++) {
+			u32 idx = (svc_grce_dst_ring_next - 1 - i) %
+				  SVC_GRCE_DST_MAX;
+
+			dsts[n++] = svc_grce_dst_ring[idx];
+		}
+		gsp_cuda_bridge_launch(vgpu, ch->h_object, st->qmd,
+				       st->qmd_dev_va, st->push_va,
+				       dsts, n);
+		st->qmd_n = 0;
+		st->qmd_bits = 0;
+		st->qmd_have = 0;
+	}
+}
+
 static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 			      u32 len, u64 page_pa, bool allow_ops,
 			      struct svc_channel *ch, u32 entry_idx,
@@ -3279,6 +3962,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 	}
 
 	memset(&st, 0, sizeof(st));
+	st.push_va = push_va;
 	/*
 	 * P6b wave 1: clear the guest-forwarding candidate at the start of
 	 * every decode pass; the first eligible copy captured below is the one
@@ -3364,6 +4048,19 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 					trusted, allow_ops, st.have_out,
 					st.line_length, ch->h_object);
 			}
+			/*
+			 * Record the destination for the CUDA bridge's
+			 * argument-block search, whether or not the write was
+			 * executed: the launch QMD later in this push needs it
+			 * to find where CUDA placed the kernel arguments in
+			 * the QMD parameter bank.
+			 */
+			if (st.have_out && st.offset_out) {
+				svc_grce_dst_ring[svc_grce_dst_ring_next %
+						  SVC_GRCE_DST_MAX] =
+					st.offset_out;
+				svc_grce_dst_ring_next++;
+			}
 			off += 4 + count * 4;
 			continue;
 		}
@@ -3375,12 +4072,28 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 			continue;
 		}
 
-		if ((op != 1 && op != 3 && op != 5) || !count || count > 64 ||
+		if ((op != 1 && op != 3 && op != 5) || !count || count > 0xfff ||
 		    off + 4 + count * 4 > len) {
 			off += 4;	/* resync */
 			continue;
 		}
 
+		/*
+		 * CUDA launch (C5C0 compute class, subchannel >= 1): capture
+		 * the inline-QMD method run for the host CUDA bridge. The run
+		 * is skipped whole exactly as before (release/decode behaviour
+		 * is untouched); a completed 64-word QMD queues a daemon job.
+		 */
+		if (ch->engine_type == 0 && subch >= 1 &&
+		    gsp_cuda_bridge_enabled() &&
+		    addr >= SVC_QMD_ADDR_A &&
+		    addr <= SVC_QMD_DATA + (SVC_QMD_WORDS - 1) * 4) {
+			svc_qmd_accum(vgpu, ch, &st, addr, count,
+				      buf + off + 4);
+			off += 4 + count * 4;
+			continue;
+		}
+
 		if (op == 3 || op == 5) {
 			/*
 			 * NON_INC / ONE_INC methods carry their own data
@@ -3502,7 +4215,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 				 * only the apply moves. Overflow applies inline.
 				 */
 				if (gsp_phys_work_defer_release_enabled() &&
-				    svc_fwd_cand.valid) {
+				    svc_fwd_cand.valid &&
+				    gsp_phys_guest_mem_enabled()) {
 					if (svc_fwd_rel_n < SVC_FWD_REL_MAX) {
 						struct svc_fwd_rel *r =
 							&svc_fwd_rel[svc_fwd_rel_n++];
@@ -3617,7 +4331,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 					 * Overflow applies inline.
 					 */
 					if (gsp_phys_work_defer_release_enabled() &&
-					    svc_fwd_cand.valid) {
+					    svc_fwd_cand.valid &&
+					    gsp_phys_guest_mem_enabled()) {
 						if (svc_fwd_rel_n < SVC_FWD_REL_MAX) {
 							struct svc_fwd_rel *r =
 								&svc_fwd_rel[svc_fwd_rel_n++];
@@ -3952,7 +4667,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 				 * release as decoded. Overflow applies inline.
 				 */
 				if (gsp_phys_work_defer_release_enabled() &&
-				    svc_fwd_cand.valid) {
+				    svc_fwd_cand.valid &&
+				    gsp_phys_guest_mem_enabled()) {
 					if (svc_fwd_rel_n < SVC_FWD_REL_MAX) {
 						struct svc_fwd_rel *r =
 							&svc_fwd_rel[svc_fwd_rel_n++];
@@ -3996,12 +4712,29 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 							svc_rel_count++;
 						}
 					}
-				} else if (svc_rel_apply_launch(vgpu, ch, sem_va,
-								st.sem_payload,
-								cur, pending, inc,
-								inc_apply)) {
-					svc_rel_va_seen = sem_va;
-					svc_rel_count++;
+				} else {
+					/*
+					 * Deferral is enabled but no forwarded
+					 * hardware copy can run for this push
+					 * (guest-memory copy path off, or no
+					 * candidate): apply the decoded release
+					 * directly, exactly once. The note is
+					 * the proof line that a decoded release
+					 * is never left to a copy that will
+					 * not engage.
+					 */
+					if (gsp_phys_work_defer_release_enabled() &&
+					    svc_fwd_cand.valid)
+						svc_fwd_rel_direct_note(ch->h_object,
+									sem_va,
+									st.sem_payload);
+					if (svc_rel_apply_launch(vgpu, ch, sem_va,
+								 st.sem_payload,
+								 cur, pending, inc,
+								 inc_apply)) {
+						svc_rel_va_seen = sem_va;
+						svc_rel_count++;
+					}
 				}
 				/* release ends the push */
 				n_ops = 0;
@@ -4096,7 +4829,7 @@ static void svc_apply_pending(struct nv_vgpu_device *vgpu)
 			svc_pending[i] = svc_pending[--svc_pending_n];
 			continue;
 		}
-		if (!svc_sem_write(vgpu, p->va,
+		if (!svc_sem_write(vgpu, 0, p->va,
 				   p->op ? cur + p->payload : p->payload)) {
 			svc.releases_applied++;
 			svc.post_hint_until = jiffies + 10 * HZ;
@@ -4239,7 +4972,7 @@ static u32 __maybe_unused __svc_apply_page_data_disabled(struct nv_vgpu_device *
 		    (lo & 0xfff) < 0x20 && svc_pb_first_push(buf))
 			svc_pb_add(pa_page, lo & 0xfff);
 
-		if (svc_sem_write(vgpu, va, payload) == 0) {
+		if (svc_sem_write(vgpu, 0, va, payload) == 0) {
 			svc.releases_applied++;
 			svc.post_hint_until = jiffies + 10 * HZ;
 			applied++;
@@ -4595,6 +5328,223 @@ static int svc_read_va_trusted(struct nv_vgpu_device *vgpu, u64 va, void *buf,
 }
 
 /*
+ * Linear fallback for a push-buffer window page.
+ *
+ * svc_pb_pair() pairs the window base VA with the base PA it found by content
+ * and pins page N at base_pa + N*0x1000. That holds only while the guest's
+ * push-buffer allocation is physically contiguous, which is not guaranteed
+ * and was observed to change run-to-run. This returns the guess for a page in
+ * the channel's window; callers consult it only after every validated
+ * per-page source missed. Returns 0 with the page-aligned PA, -ENOENT when
+ * the page is outside every window known for the channel's rank.
+ */
+static int svc_pb_win_lookup(const struct svc_channel *ch, u64 va_page,
+			     u64 *pa_page)
+{
+	u32 slot, i;
+
+	if (!ch || !ch->engine_type || ch->ce_index >= 32)
+		return -ENOENT;
+	slot = 4 * ch->ce_index;
+	for (i = 0; i < svc_pb_n; i++) {
+		u64 pa;
+
+		if (svc_pb[i].slot != slot || !svc_pb[i].va ||
+		    !svc_pb[i].pa_page)
+			continue;
+		if (va_page < svc_pb[i].va ||
+		    va_page >= svc_pb[i].va + 0x100000ULL)
+			continue;
+		pa = svc_pb[i].pa_page + (va_page - svc_pb[i].va);
+		if (pa >= SEMA_PA_MAX)
+			return -ENOENT;
+		*pa_page = pa;
+		return 0;
+	}
+	return -ENOENT;
+}
+
+#define SVC_PUSH_PG_LOG_MAX	4
+
+/*
+ * One line when a push page resolved from a validated pin while the channel's
+ * linear window guess would have answered differently - the non-contiguous
+ * case the window convention gets wrong. Capped per channel and one line per
+ * page so a boot log shows whether pins carried the read without per-push
+ * spam. Behind sema_dbg like the other decode diagnostics.
+ */
+static void svc_pushpg_note(struct svc_channel *ch, u64 page_va, u64 pin_pa,
+			    u64 win_pa)
+{
+	if (!gsp_sema_dbg || ch->pushpg_n >= SVC_PUSH_PG_LOG_MAX)
+		return;
+	if ((pin_pa & ~0xfffULL) == (win_pa & ~0xfffULL))
+		return;			/* the window guess already agrees */
+	if (ch->pushpg_va == page_va)
+		return;
+	ch->pushpg_va = page_va;
+	ch->pushpg_n++;
+	pr_info("sema-pushpg: h=0x%x va=0x%llx pin=1 pa=0x%llx win=0x%llx\n",
+		ch->h_object, page_va, pin_pa & ~0xfffULL,
+		win_pa & ~0xfffULL);
+}
+
+/*
+ * Cheap candidate-arbitration prefix: a real push starts with a method header
+ * (op 1/3/5 with a non-zero count, or a header-only 4/7), optionally after
+ * the zero-filled header region UVM emits before large page-table pushes
+ * (observed up to 0x1004 bytes). A zero page or unrelated data practically
+ * never starts with one, so a wrong injected pin can be rejected before the
+ * pump's full-stream parse. An all-zero chunk cannot refute the pin and is
+ * accepted; the full parse is the final arbiter. The run length is
+ * deliberately not bounded by the first chunk: a valid method run may span
+ * into the next page and the full parse settles that.
+ */
+static bool svc_push_prefix_ok(const u8 *buf, size_t len)
+{
+	u32 hdr, op, count;
+	size_t off = 0;
+
+	while (off + 4 <= len) {
+		memcpy(&hdr, buf + off, 4);
+		if (hdr != 0)
+			break;
+		off += 4;
+	}
+	if (off + 4 > len)
+		return true;		/* all zeros: not refutable here */
+	op = (hdr >> 29) & 0x7;
+	if (op == 4 || op == 7)		/* header-only methods */
+		return true;
+	if (op != 1 && op != 3 && op != 5)
+		return false;
+	count = (hdr >> 16) & 0xfff;
+	return count != 0;
+}
+
+/* Fallback-line budget for the whole boot. */
+static u32 svc_pushpg_fb_log;
+
+/*
+ * Read a push the way the engine does: every 4 KB page of the range is
+ * translated independently.
+ *
+ * The old push read trusted the window's linear mapping for the whole range
+ * (through the pins svc_pb_pair installed). When the guest's push-buffer
+ * allocation is not physically contiguous, the continuation page answered
+ * with the wrong PA, the method-stream plausibility check failed, and the
+ * release the client waited on was never applied (observed: push #25 on
+ * h=0xcaf00012 at va=0x120000f58 crossing into a page whose PA was not
+ * base+0x1000; the pool semaphore stalled at 24 and the guest UVM hung).
+ *
+ * Order per page:
+ *   1. injected pin (svc_ipin_lookup): host-verified ground truth, but the
+ *      content arbitrates - a pin whose first chunk does not start like a
+ *      method stream is rejected and the remaining candidates are tried
+ *      (observed: a wrong injected pin for the push page made every push
+ *      decode no release, starving the client's tracker);
+ *   2. validated self-pin (svc_pin_lookup_valid): read-back-validated pages
+ *      (content rescue, first-push pairing), window guesses excluded;
+ *   3. the channel's linear window mapping (svc_pb_win_lookup): last resort
+ *      for a page no validated source knows;
+ *   4. outside any window: the remaining trusted sources
+ *      (svc_lookup_trusted), exactly the old behaviour for GR/other pushes.
+ *
+ * For a contiguous allocation every path resolves to the same PA the old
+ * read returned, so the normal case is byte-for-byte unchanged; only a
+ * non-contiguous or wrongly-pinned page changes behaviour (to its next
+ * validated candidate). Reads stay defensively copied through
+ * svc_read_ap()/vfio_dma_rw(), which enforce the aperture bounds.
+ */
+static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu,
+			       struct svc_channel *ch, u64 va, void *buf,
+			       size_t len, bool skip_ipin)
+{
+	u8 *p = buf;
+
+	while (len) {
+		u64 page_va = va & ~(u64)(SEMA_PAGE_SIZE - 1);
+		size_t chunk = min_t(size_t, len,
+				     SEMA_PAGE_SIZE - (va & (SEMA_PAGE_SIZE - 1)));
+		u64 pa = 0, win_pa = 0;
+		int ap = SVC_APERTURE_SYS;
+		bool have_win, done = false;
+		bool first = (p == buf);
+		int last_rc = -ENOENT;
+		u32 src;
+
+		have_win = !svc_pb_win_lookup(ch, page_va, &win_pa);
+
+		for (src = 0; src < 4 && !done; src++) {
+			bool via_ipin = false;
+
+			pa = 0;
+			ap = SVC_APERTURE_SYS;
+			switch (src) {
+			case 0:
+				if (skip_ipin ||
+				    svc_ipin_lookup(va, &pa, &ap))
+					continue;
+				via_ipin = true;
+				if (have_win)
+					svc_pushpg_note(ch, page_va, pa,
+							win_pa);
+				break;
+			case 1:
+				if (svc_pin_lookup_valid(va, &pa, &ap))
+					continue;
+				if (have_win)
+					svc_pushpg_note(ch, page_va, pa,
+							win_pa);
+				break;
+			case 2:
+				if (!have_win)
+					continue;
+				pa = win_pa + (va & (SEMA_PAGE_SIZE - 1));
+				ap = SVC_APERTURE_SYS;
+				break;
+			default:
+				/* learned cache / discovered map / page
+				 * directories / targeted rm_mem resolve */
+				if (svc_lookup_trusted(vgpu, va, &pa, &ap))
+					continue;
+				break;
+			}
+
+			if (ap == SVC_APERTURE_SYS && pa >= SEMA_PA_MAX) {
+				last_rc = -EIO;
+				continue;
+			}
+			if (svc_read_ap(vgpu, pa,
+					ap == SVC_APERTURE_FB ?
+						SVC_APERTURE_FB :
+						SVC_APERTURE_SYS,
+					p, chunk)) {
+				last_rc = -EIO;
+				continue;
+			}
+			if (via_ipin && first &&
+			    !svc_push_prefix_ok(p, chunk))
+				continue;	/* content says: wrong pin */
+			done = true;
+		}
+		if (!done)
+			return last_rc;
+		p += chunk;
+		va += chunk;
+		len -= chunk;
+	}
+	return 0;
+}
+
+static int svc_read_va_push(struct nv_vgpu_device *vgpu,
+			    struct svc_channel *ch, u64 va, void *buf,
+			    size_t len)
+{
+	return svc_read_va_push_ex(vgpu, ch, va, buf, len, false);
+}
+
+/*
  * Read the GPFIFO get/put pair from a candidate USERD. Returns 0 when the
  * values are plausible for a channel with `entries` GPFIFO slots.
  */
@@ -4815,8 +5765,18 @@ static int svc_chan_entry0(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
  */
 #define SVC_GR_RING_STRIDE	0x3000ULL
 #define SVC_GR_USERD_OFF	0x2000ULL
-
 bool gsp_sema_gr_rings = true;
+
+/*
+ * In-kernel GR completion-counter slot derivation (svc_counter_derive): on an
+ * injected-pin miss, resolve a release's counter slot from the structural
+ * translations (published page directories, self-discovered directory,
+ * targeted rm_mem record) instead of holding the release until the external
+ * pin watcher lands. Default on; set 0 to restore the injected-pin-only
+ * behaviour for an A/B without a rebuild.
+ */
+bool gsp_sema_counter_derive = true;
+
 /*
  * GR (compute) entries whose push page is not translatable are left pending
  * for this long before the pump consumes them as stale. The push pages are
@@ -4968,12 +5928,54 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu,
 
 	/*
 	 * The candidate chunk did not validate. If another source already
-	 * translates this ring VA (bridge or manual pin), trust it.
+	 * translates this ring VA (bridge or manual pin), do not just trust
+	 * it: read the first GPFIFO entry through that translation exactly
+	 * like the FB candidate above and validate it with the same checks.
+	 * Previously this branch latched the rank without storing the
+	 * translation or the USERD, so for a sysmem pin svc_chan_userd() had
+	 * no candidate (its FB-only ring offset path is skipped) and the
+	 * pump stalled before ever draining the ring: the GR push decode,
+	 * its release and the client's completion counter all never
+	 * happened. Only a validated read may latch the rank, so a stale or
+	 * wrong pin still falls through to the "not at fb" retry.
 	 */
 	if (!svc_pin_lookup_any(ch->gpfifo_va, &pin_pa, &pin_ap)) {
-		set_bit(ch->gr_index, &svc_gr_ring_done);
-		svc_ring_n++;
-		return;
+		if (!svc_read_ap(vgpu, pin_pa, svc_ap_of(pin_ap), e,
+				 sizeof(e))) {
+			memcpy(&e0, e, 4);
+			memcpy(&e1, e + 4, 4);
+			va = ((u64)(e1 & 0xff) << 32) |
+			     (u64)(e0 & 0xfffffffc);
+			len = ((e1 >> 10) & 0x1fffff) << 2;
+			if (!(va & 0xfffff) && va >= SEMA_VA_LO &&
+			    va < SEMA_VA_HI && len >= 4 && len <= 0x10000) {
+				/*
+				 * Best effort only: a wrapped GP_PUT
+				 * legitimately exceeds the entry count, so a
+				 * failed probe must not reject the ring; the
+				 * pump reads the raw put/get itself and
+				 * reduces put modulo the entry count.
+				 */
+				(void)svc_userd_probe(vgpu,
+						      pin_pa + SVC_GR_USERD_OFF,
+						      svc_ap_of(pin_ap),
+						      ch->gpfifo_entries, &put,
+						      &get);
+				svc_pin_store(ch->gpfifo_va, pin_pa, pin_ap);
+				svc_lat_store(ch->gpfifo_va, pin_pa, pin_ap);
+				ch->userd_pa = pin_pa + SVC_GR_USERD_OFF;
+				ch->userd_as = svc_ap_of(pin_ap);
+				ch->userd_ok = true;
+				svc_gr_ring_base = base;
+				set_bit(ch->gr_index, &svc_gr_ring_done);
+				svc_ring_n++;
+				pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx (entry0 va=0x%llx len=%u put=%u get=%u)\n",
+					ch->h_object, ch->gr_index,
+					ch->gpfifo_va, pin_pa, va, len, put,
+					get);
+				return;
+			}
+		}
 	}
 
 	if (!test_and_set_bit(ch->gr_index, &svc_gr_ring_tried))
@@ -5053,8 +6055,19 @@ static void svc_pb_pair(struct nv_vgpu_device *vgpu, struct svc_channel *ch)
 	    len < 4 || len > 0x10000)
 		return;
 
+	/*
+	 * Install the window mapping as tagged *guesses*: only the base page
+	 * was paired against real content (svc_chan_entry0 read the first
+	 * push there). The rest of the 1 MB window is NOT assumed physically
+	 * contiguous - a push may cross into a page whose PA is not
+	 * base+N*0x1000 (observed: a push at window offset 0xf58 whose
+	 * continuation page lived one page *below* the base). Push reads
+	 * skip these guesses and resolve each page on its own
+	 * (svc_read_va_push), using this window mapping only as a last-resort
+	 * fallback; every other consumer still sees the pins as before.
+	 */
 	for (off = 0; off < 0x100000; off += SEMA_PAGE_SIZE) {
-		svc_pin_store(va + off, hit_pa + off, SVC_APERTURE_SYS);
+		svc_pin_store_win(va + off, hit_pa + off, SVC_APERTURE_SYS);
 		svc_lat_store(va + off, hit_pa + off, SVC_APERTURE_SYS);
 		svc_mark_hot(hit_pa + off);
 	}
@@ -5433,14 +6446,47 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu,
 static void svc_init_expected_payload(struct nv_vgpu_device *vgpu,
 				      struct svc_channel *ch)
 {
-	u64 pool_va = 0x121000000ULL + 4ULL * ch->ce_index;
+	/*
+	 * The exact slot is the release VA this channel's last decoded push
+	 * carried. Until one is decoded, the rank-derived VA is a guess (the
+	 * UVM pool index is allocated in channel creation order, not by
+	 * rank); it is only used to seed the expectation early. Once a push
+	 * decodes, last_rel_va is the authoritative slot and the expectation
+	 * (and the automatic recovery write) follows it.
+	 */
+	u64 pool_va;
 	u64 pa = 0;
 	int ap = SVC_APERTURE_SYS;
 	u32 cur = 0;
 	u32 want;
 
-	if (svc_lookup_trusted(vgpu, pool_va, &pa, &ap))
+	/*
+	 * Only CE channels own a UVM tracking-semaphore pool slot; a GR
+	 * channel's completion signal is its per-channel counter. Deriving a
+	 * pool VA for a GR channel and resolving it here used to start a
+	 * targeted RAM scan per GR channel per pass, starving the pump.
+	 */
+	if (!ch->engine_type)
+		return;
+
+	pool_va = ch->last_rel_va ? ch->last_rel_va :
+		  0x121000000ULL + 4ULL * ch->ce_index;
+
+	/*
+	 * The expectation is a hint, never a gate. A known exact slot may go
+	 * through the verified pool resolver (injected pin -> validated cache
+	 * -> walk -> targeted scan); a rank-derived guess uses the learned
+	 * map only, so seeding an unknown slot can never launch a multi-
+	 * hundred-millisecond RAM scan from the per-pass hot path.
+	 */
+	if (ch->last_rel_va) {
+		if (svc_pool_pa_resolve(vgpu, ch->h_object, pool_va, &pa, &ap,
+					NULL) &&
+		    svc_lookup_trusted(vgpu, pool_va, &pa, &ap))
+			return; /* pool page not mapped yet */
+	} else if (svc_lookup_trusted(vgpu, pool_va, &pa, &ap)) {
 		return; /* pool page not mapped yet */
+	}
 
 	if (svc_read_ap(vgpu, pa, ap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS,
 			&cur, sizeof(cur)))
@@ -5518,10 +6564,23 @@ static void svc_counter_publish(struct nv_vgpu_device *vgpu,
 
 /*
  * Acknowledge a CE tracking-semaphore release without reading the push that
- * carried it: write the expected payload into the semaphore the channel last
- * released to (learned from a readable push; the pool index is allocated by
- * UVM and does not match the channel rank). Returns false when there is
- * nothing to fall back on. Caller owns expected_sem_payload bookkeeping.
+ * carried it: write the expected payload into the exact semaphore slot the
+ * channel last released to. That slot is push/CE-side evidence: the decoded
+ * release VA itself (uvm_channel_tracking_semaphore_get_gpu_va() =
+ * pool-page GPU VA + semaphore->index * 4; gsp_sema_service.c pool-wr lines
+ * show one slot per CE channel). The pool index is allocated by UVM in
+ * channel creation order and does NOT generally match the channel rank, so
+ * the rank-derived VA (`0x121000000 + 4*ce_index`) is a guess: it is only
+ * used when the operator explicitly asked for it via gsp_sema_fake_rel /
+ * gsp_sema_rel_recover. The automatic recovery path (no module parameter)
+ * requires the decoded VA and never writes into an unproven slot.
+ *
+ * Returns false only when there is no exact slot to fall back on at all
+ * (no decoded release VA and no operator-selected rank guess). A known
+ * slot is always acknowledged: when its page is not translatable the
+ * expected write is queued for the pending-release retry pass, the
+ * expectation still advances and the caller consumes the GPFIFO entry.
+ * A missing translation must never hold/starve the channel.
  */
 static bool svc_fake_release(struct nv_vgpu_device *vgpu, struct svc_channel *ch)
 {
@@ -5529,31 +6588,82 @@ static bool svc_fake_release(struct nv_vgpu_device *vgpu, struct svc_channel *ch
 	u64 spa = 0;
 	int sap = SVC_APERTURE_SYS;
 	u32 scur = 0;
+	u32 deliver;
+	bool have_pa = true;
+	bool delivered = false;
 
 	if (!ch->engine_type || !ch->expected_sem_payload)
 		return false;
 
-	pool_va = ch->last_rel_va ? ch->last_rel_va :
-		  0x121000000ULL + 4ULL * ch->ce_index;
+	pool_va = ch->last_rel_va;
+	if (!pool_va) {
+		if (!gsp_sema_fake_rel && !gsp_sema_rel_recover)
+			return false;
+		pool_va = 0x121000000ULL + 4ULL * ch->ce_index;
+	}
+	deliver = ch->expected_sem_payload;
 
 	/*
 	 * The release can only be delivered when the pool page translates.
-	 * A missing translation is transient (the client's pool page is pinned
-	 * a moment after the channel appears), so report failure and let the
-	 * caller keep the GPFIFO entry pending: consuming it without writing
-	 * the semaphore leaves the client waiting forever.
+	 * svc_pool_pa_resolve() is the verified resolver (injected pin ->
+	 * read-back-validated cache -> walk -> targeted rm_mem record with
+	 * read-back); a translated page is still read at the exact release
+	 * offset before the write. Failure here is not fatal: the exact slot
+	 * is known, so the write is queued and the entry is released.
 	 */
-	if (svc_lookup_trusted(vgpu, pool_va, &spa, &sap))
-		return false;
-	if (svc_read_ap(vgpu, spa, sap, &scur, sizeof(scur)))
-		return false;
-
-	if (scur < ch->expected_sem_payload) {
-		u32 nv = ch->expected_sem_payload;
+	if (svc_pool_sem_va(pool_va)) {
+		if (svc_pool_pa_resolve(vgpu, ch->h_object, pool_va, &spa,
+					&sap, NULL) &&
+		    svc_lookup_trusted(vgpu, pool_va, &spa, &sap))
+			have_pa = false;
+	} else if (svc_lookup_trusted(vgpu, pool_va, &spa, &sap)) {
+		have_pa = false;
+	}
+	if (have_pa &&
+	    !svc_read_ap(vgpu, spa, svc_ap_of(sap), &scur, sizeof(scur))) {
+		if (scur >= ch->expected_sem_payload) {
+			/* Another path already delivered this payload. */
+			delivered = true;
+		} else {
+			u32 nv = ch->expected_sem_payload;
+			u32 back = 0;
 
-		if (svc_write_ap(vgpu, spa, sap, &nv, sizeof(nv)))
-			return false;
-		svc.releases_applied++;
+			/*
+			 * UVM's update_completed_value() asserts on a jump
+			 * beyond UVM_GPU_SEMAPHORE_MAX_JUMP (a stale
+			 * expectation must never slam the whole page): if the
+			 * expectation is far ahead, deliver exactly one step
+			 * and re-seed it.
+			 */
+			if (nv - scur > 0x1000) {
+				nv = scur + 1;
+				ch->expected_sem_payload = nv + 1;
+			}
+			deliver = nv;
+			if (!svc_write_ap(vgpu, spa, svc_ap_of(sap), &nv,
+					  sizeof(nv)) &&
+			    !svc_read_ap(vgpu, spa, svc_ap_of(sap), &back,
+					 sizeof(back)) &&
+			    back == nv) {
+				delivered = true;
+				if (svc_pool_sem_va(pool_va))
+					svc_pool_pa_store(pool_va, spa,
+							  svc_ap_of(sap));
+				svc.releases_applied++;
+				if (!gsp_sema_fake_rel)
+					svc.releases_recovered++;
+				svc.post_hint_until = jiffies + 10 * HZ;
+			}
+		}
+	}
+	if (!delivered) {
+		/*
+		 * The exact target is known but not translatable right now.
+		 * Never hold the channel: queue the exact-slot write for the
+		 * retry pass so it still lands when the mapping appears.
+		 */
+		svc_pending_add(pool_va, deliver, 0);
+		svc.releases_forwarded++;
 	}
 	svc.releases_faked++;
 	svc_rel_count++;
@@ -5969,6 +7079,22 @@ static void svc_fwd_rel_apply_all(struct nv_vgpu_device *vgpu,
 	svc_fwd_rel_apply_all_ex(vgpu, ch, rel, n, SVC_FWD_REL_SKIP_NONE);
 }
 
+/*
+ * Proof line for a held record set resolved without a successful forwarded
+ * copy (failure/timeout/abort): the releases below are applied directly, so
+ * none is lost to a hardware copy that never ran. Called with svc_fwd_hold
+ * populated, before svc_fwd_hold_clear().
+ */
+static void svc_fwd_hold_direct_note(void)
+{
+	u8 i;
+
+	for (i = 0; i < svc_fwd_hold.nrel; i++)
+		svc_fwd_rel_direct_note(svc_fwd_hold.ch_handle,
+					svc_fwd_hold.rel[i].va,
+					svc_fwd_hold.rel[i].payload);
+}
+
 static bool svc_fwd_hold_expired(void)
 {
 	return time_after(jiffies, svc_fwd_hold.since +
@@ -6194,6 +7320,8 @@ static void svc_fwd_hold_expire_check(struct nv_vgpu_device *vgpu,
 		svc_fwd_rel_apply_all(hv, &shim, svc_fwd_hold.rel,
 				      svc_fwd_hold.nrel);
 	}
+	if (!(done && svc_fwd_req.hw_rc == 0))
+		svc_fwd_hold_direct_note();
 	if (done && svc_fwd_req.hw_rc == 0) {
 		svc.fwd_rel_applied++;
 	} else {
@@ -6295,6 +7423,7 @@ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu,
 			svc.fwd_relay_fallback++;
 		svc_fwd_rel_apply_all(hv, ch, svc_fwd_hold.rel,
 				      svc_fwd_hold.nrel);
+		svc_fwd_hold_direct_note();
 		svc.fwd_rel_fallback++;
 		if (!done)
 			svc.fwd_hold_timeouts++;
@@ -6320,58 +7449,52 @@ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu,
 }
 
 /*
- * Channel vanished while a hold was active. A successfully completed work
- * item or an abort (quiesce / channel delete seen before the snapshot lost
- * the channel) applies the recorded releases through a shim channel - holds
- * exist only for CE channels and the vanished channel object is not required;
- * only a copy that did not complete successfully (timeout/failure/cancel)
- * drops without applying. The stored engage-time device is preferred over the
- * caller's so the releases never land in a different device's address space.
+ * Channel vanished while a hold was active. Any resolved hold (completed,
+ * aborted, failed or expired) applies the recorded releases through a shim
+ * channel - holds exist only for CE channels and the vanished channel object
+ * is not required. A copy that did not complete successfully is no longer
+ * dropped: the releases land through the same bounded, forward-only apply, so
+ * a decoded release is never lost to a hardware copy that did not run. The
+ * stored engage-time device is preferred over the caller's so the releases
+ * never land in a different device's address space.
  */
 static void svc_fwd_hold_gone_finish(struct nv_vgpu_device *vgpu)
 {
 	struct nv_vgpu_device *hv = svc_fwd_hold.vgpu ?
 				    svc_fwd_hold.vgpu : vgpu;
+	struct svc_channel shim;
 	bool done = smp_load_acquire(&svc_fwd_req.done);
 	bool aborted = READ_ONCE(svc_fwd_hold.aborted);
 
 	if (!done && !aborted && !svc_fwd_hold_expired())
 		return;
-	if ((done && svc_fwd_req.hw_rc == 0) || aborted) {
-		struct svc_channel shim;
-
-		/*
-		 * P6b wave 2: an abort (or a vanished channel after a
-		 * failure/timeout) applies its releases through the shim;
-		 * author the skipped data once first unless the hardware copy
-		 * itself completed successfully.
-		 */
-		if (!(done && svc_fwd_req.hw_rc == 0))
-			svc_fwd_hold_soft_copy(hv);
-		if (hv) {
-			memset(&shim, 0, sizeof(shim));
-			shim.valid = true;
-			shim.h_object = svc_fwd_hold.ch_handle;
-			shim.engine_type = 1;	/* holds are CE-only */
-			svc_fwd_rel_apply_all(hv, &shim, svc_fwd_hold.rel,
-					      svc_fwd_hold.nrel);
-		}
-		if (done && svc_fwd_req.hw_rc == 0)
-			svc.fwd_rel_applied++;
-		else {
-			svc.fwd_rel_fallback++;
-			if (!done)
-				svc.fwd_hold_timeouts++;
-		}
+	/*
+	 * P6b wave 2: author the skipped data once first unless the hardware
+	 * copy itself completed successfully.
+	 */
+	if (!(done && svc_fwd_req.hw_rc == 0))
+		svc_fwd_hold_soft_copy(hv);
+	if (hv) {
+		memset(&shim, 0, sizeof(shim));
+		shim.valid = true;
+		shim.h_object = svc_fwd_hold.ch_handle;
+		shim.engine_type = 1;	/* holds are CE-only */
+		svc_fwd_rel_apply_all(hv, &shim, svc_fwd_hold.rel,
+				      svc_fwd_hold.nrel);
+	}
+	if (done && svc_fwd_req.hw_rc == 0) {
+		svc.fwd_rel_applied++;
 	} else {
 		/*
-		 * Canceled (expired without a successful copy): wave 1.5
-		 * drops the releases, but the skipped data is still authored
-		 * once before the hold is released so the destination is
-		 * correct if anything else observes it.
+		 * No successful hardware copy ever engaged (failed,
+		 * aborted or the bounded window expired): the recorded
+		 * releases were applied directly above, exactly once.
 		 */
-		svc_fwd_hold_soft_copy(hv);
+		svc_fwd_hold_direct_note();
+		svc.fwd_rel_fallback++;
 		svc.fwd_hold_canceled++;
+		if (!done)
+			svc.fwd_hold_timeouts++;
 	}
 	/*
 	 * P6c: a vanished channel cannot run the relay verification edge; a
@@ -6409,10 +7532,12 @@ static void svc_fwd_hold_stop_flush(struct nv_vgpu_device *vgpu)
 		svc_fwd_rel_apply_all(hv, &shim, svc_fwd_hold.rel,
 				      svc_fwd_hold.nrel);
 	}
-	if (done && svc_fwd_req.hw_rc == 0)
+	if (done && svc_fwd_req.hw_rc == 0) {
 		svc.fwd_rel_applied++;
-	else
+	} else {
+		svc_fwd_hold_direct_note();
 		svc.fwd_rel_fallback++;
+	}
 	if (svc_fwd_hold.relay_armed)
 		svc.fwd_relay_fallback++;
 	svc_fwd_hold_clear();
@@ -6976,6 +8101,22 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 		bool held = false;
 
 		/*
+		 * Safety net: a GR channel's completion-counter slot must be
+		 * registered - and any release decoded before the slot was
+		 * known flushed - even when this pass cannot read the ring or
+		 * USERD and one of the early-outs below skips the publish at
+		 * the end of the pass. Without it a release recorded while
+		 * push decode stayed broken never lands and the client's
+		 * userspace tracker polls forever. Idempotent and cheap: once
+		 * the slot is registered only the pending-payload check
+		 * remains.
+		 */
+		if (!ch->engine_type) {
+			svc_counter_autoreg(vgpu, ch, n_gr);
+			svc_counter_flush_pending(vgpu, ch);
+		}
+
+		/*
 		 * P6c: while a relay hold owns this channel, the hold's
 		 * verified success edge is the single completion post; skip
 		 * the per-pass notifier so it cannot fire before the guest
@@ -7146,8 +8287,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 				if (pb) {
 					int rescue_rc;
 					bool unreadable =
-						svc_read_va_trusted(vgpu, va,
-								    pb, len) != 0;
+						svc_read_va_push(vgpu, ch, va,
+								 pb, len) != 0;
 					bool injected = false;
 					bool allow_ops = (ch->engine_type != 0);
 
@@ -7157,14 +8298,16 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 					 * (content-scanned map; observed firmware
 					 * and kernel data pages). Handle it like
 					 * an unreadable push so the content rescue
-					 * can find the real page. Exception: a
-					 * host-injected translation is verified
-					 * truth, and real CE release pushes start
-					 * with an INC_METHOD whose address is not
-					 * zero (observed 0x200381c0, addr 0x700):
-					 * decode those release-only instead of
-					 * discarding them, so the client's
-					 * tracking-semaphore releases still apply.
+					 * can find the real page. An injected pin
+					 * is treated as verified truth by default,
+					 * but content is the arbiter: a pin whose
+					 * page does not parse is re-read from the
+					 * remaining candidates (validated pin /
+					 * window / trusted) and only a plausible
+					 * read replaces it; if that also fails the
+					 * old release-only decode of the injected
+					 * page stands, so the client's tracking-
+					 * semaphore releases still apply.
 					 */
 					if (!unreadable) {
 						u64 ipa;
@@ -7174,8 +8317,34 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 									    &iap) == 0);
 						if (!svc_method_stream_plausible(pb, len)) {
 							svc.false_trans++;
-							if (!injected)
+							if (injected) {
+								u8 *fb = kmalloc(len,
+										 GFP_KERNEL);
+
+								if (fb) {
+									if (!svc_read_va_push_ex(vgpu, ch, va,
+												 fb, len, true) &&
+									    svc_method_stream_plausible(fb, len)) {
+										memcpy(pb, fb, len);
+										injected = false;
+										svc.pushpg_fallback++;
+										if (gsp_sema_dbg &&
+										    svc_pushpg_fb_log < 16) {
+											u64 wpa = 0;
+											bool win = !svc_pb_win_lookup(ch,
+												va & ~(u64)(SEMA_PAGE_SIZE - 1), &wpa);
+
+											svc_pushpg_fb_log++;
+											pr_info("sema-pushpg: h=0x%x va=0x%llx fallback pin=0 win=%d\n",
+												ch->h_object, va,
+												win ? 1 : 0);
+										}
+									}
+									kfree(fb);
+								}
+							} else {
 								unreadable = true;
+							}
 						}
 					}
 
@@ -7306,6 +8475,16 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 									svc_fwd_rel_apply_all(vgpu, ch,
 											      svc_fwd_rel,
 											      svc_fwd_rel_n);
+									{
+										u8 ri;
+
+										for (ri = 0;
+										     ri < svc_fwd_rel_n;
+										     ri++)
+											svc_fwd_rel_direct_note(ch->h_object,
+														svc_fwd_rel[ri].va,
+														svc_fwd_rel[ri].payload);
+									}
 									/*
 									 * P6c: a
 									 * selected
@@ -7339,7 +8518,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 								u32 rel_n2 = svc_rel_count;
 
 								if (!svc_rescue_push_page(vgpu, ch, va, len) &&
-								    !svc_read_va_trusted(vgpu, va, pb, len))
+								    !svc_read_va_push(vgpu, ch, va, pb, len))
 									svc_decode_stream(vgpu, pb,
 											  len, 0, false,
 											  ch, idx, va, true);
@@ -7383,6 +8562,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 									if (time_before(jiffies,
 											ch->defer_since +
 											msecs_to_jiffies(gsp_sema_ce_grace_ms))) {
+										svc.gpfifo_held++;
 										svc.gpfifo_nomap++;
 										kfree(pb);
 										break;
@@ -7390,16 +8570,15 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 									pr_info("sema-chan: skipping stale entry h=0x%x va=0x%llx len=%u after %u ms grace\n",
 										ch->h_object, va, len,
 										gsp_sema_ce_grace_ms);
-									if ((gsp_sema_fake_rel ||
-									     gsp_sema_rel_recover) &&
-									    !svc_fake_release(vgpu, ch)) {
-										if (!ch->defer_since)
-											ch->defer_since =
-												jiffies;
-										svc.gpfifo_nomap++;
-										kfree(pb);
-										break;
-									}
+									/*
+									 * Grace expired: deliver through the
+									 * exact-slot fallback and consume the
+									 * entry regardless. A missing
+									 * translation is queued for the retry
+									 * pass; it must never hold the channel.
+									 */
+									if (!svc_fake_release(vgpu, ch))
+										svc.gpfifo_starved++;
 								} else {
 									ch->expected_sem_payload++;
 								}
@@ -7423,28 +8602,35 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 						 * lets the client read unstaged
 						 * memory (observed: cuMemcpyDtoH
 						 * returning zeros after its HtoD push
-						 * was skipped unreadable). The push
-						 * page's translation usually lands
-						 * within one service pass (content
-						 * discovery or the host watcher), so
-						 * keep the entry pending for the grace
-						 * window first and only then fall back
-						 * to acknowledging the release.
+						 * was skipped unreadable). With a
+						 * decoded release VA (last_rel_va)
+						 * the exact slot is already known:
+						 * deliver or queue it and consume the
+						 * entry at once - never hold the
+						 * channel for a translation that may
+						 * never arrive. Only a channel with no
+						 * exact slot gets one bounded grace so
+						 * a late translation can teach it the
+						 * slot; after that it is consumed
+						 * either way (the release content
+						 * sweep is the safety net).
 						 */
-						if (gsp_sema_fake_rel && ch->engine_type) {
+						if (ch->engine_type &&
+						    !ch->last_rel_va &&
+						    !gsp_sema_fake_rel) {
 							if (!ch->defer_since)
 								ch->defer_since =
 									jiffies;
 							if (time_before(jiffies,
 									ch->defer_since +
 									msecs_to_jiffies(gsp_sema_ce_grace_ms))) {
+								svc.gpfifo_held++;
 								svc.gpfifo_nomap++;
 								kfree(pb);
 								break;
 							}
 						}
-if (gsp_sema_fake_rel &&
-					    svc_fake_release(vgpu, ch)) {
+						if (svc_fake_release(vgpu, ch)) {
 							ch->defer_since = 0;
 							svc_replay_note(ch, idx, put);
 							ch->last_get = (idx + 1) %
@@ -7454,29 +8640,6 @@ if (gsp_sema_fake_rel &&
 							kfree(pb);
 							continue;
 						}
-						if (gsp_sema_fake_rel &&
-						    ch->engine_type) {
-							/*
-							 * CE channel and the
-							 * pool page is not
-							 * translated yet: the
-							 * pin lands a moment
-							 * later, so keep the
-							 * entry pending
-							 * instead of burning a
-							 * multi-second RAM
-							 * rescue. Consuming it
-							 * now would drop the
-							 * release the client
-							 * waits on.
-							 */
-							if (!ch->defer_since)
-								ch->defer_since =
-									jiffies;
-							svc.gpfifo_nomap++;
-							kfree(pb);
-							break;
-						}
 						if (svc.false_trans && svc.false_trans < 8)
 							pr_info("sema-svc: false translation for push va=0x%llx (h=0x%x len=%u) - rescuing\n",
 								va, ch->h_object, len);
@@ -7485,8 +8648,8 @@ if (gsp_sema_fake_rel &&
 						if (!rescue_rc) {
 							ch->rescue_fail = 0;
 							/* Rescue succeeded, retry the read. */
-							if (!svc_read_va_trusted(vgpu, va, pb,
-										 len)) {
+							if (!svc_read_va_push(vgpu, ch, va, pb,
+									      len)) {
 								/*
 								 * Never run work decoded from a
 								 * rescued page: the rescue matches
@@ -7574,6 +8737,7 @@ if (gsp_sema_fake_rel &&
 							if (time_before(jiffies,
 									ch->defer_since +
 									msecs_to_jiffies(gsp_sema_ce_grace_ms))) {
+								svc.gpfifo_held++;
 								svc.gpfifo_nomap++;
 								kfree(pb);
 								break;
@@ -7581,37 +8745,26 @@ if (gsp_sema_fake_rel &&
 							pr_info("sema-chan: skipping stale entry h=0x%x va=0x%llx len=%u after %u ms grace\n",
 								ch->h_object, va, len,
 								gsp_sema_ce_grace_ms);
-							if (gsp_sema_fake_rel ||
-							    gsp_sema_rel_recover) {
+							{
 								u64 faked_before =
 									svc.releases_faked;
 
-								if (!svc_fake_release(vgpu, ch)) {
-									/*
-									 * The release could not be
-									 * delivered (pool page not
-									 * translated yet): keep the
-									 * entry pending so a pin that
-									 * lands shortly still applies
-									 * it instead of silently
-									 * dropping the client's wait.
-									 */
-									if (!ch->defer_since)
-										ch->defer_since =
-											jiffies;
-									svc.gpfifo_nomap++;
-									kfree(pb);
-									break;
-								}
+								/*
+								 * Deliver through the
+								 * exact-slot fallback and
+								 * consume regardless; a
+								 * missing translation is
+								 * queued for the retry pass.
+								 */
+								if (!svc_fake_release(vgpu, ch))
+									svc.gpfifo_starved++;
 								if (!gsp_sema_fake_rel &&
 								    svc.releases_faked !=
-								    faked_before) {
-									svc.releases_recovered++;
+								    faked_before)
 									pr_info("sema-chan: recovery release h=0x%x rank=%u expected=%u (push page unreadable after rescue)\n",
 										ch->h_object,
 										ch->ce_index,
 										ch->expected_sem_payload);
-								}
 							}
 							ch->rescue_fail = 0;
 							ch->defer_since = 0;
@@ -7657,24 +8810,19 @@ if (gsp_sema_fake_rel &&
 							pr_info("sema-chan: skipping stale entry h=0x%x va=0x%llx len=%u after %u failed rescues\n",
 								ch->h_object, va, len,
 								ch->rescue_fail);
-							if (gsp_sema_fake_rel &&
-							    !svc_fake_release(vgpu, ch)) {
-								/*
-								 * The release could not be
-								 * delivered (pool page not
-								 * translated yet): keep the
-								 * entry pending so a pin that
-								 * lands shortly still applies
-								 * it instead of silently
-								 * dropping the client's wait.
-								 */
-								if (!ch->defer_since)
-									ch->defer_since =
-										jiffies;
-								svc.gpfifo_nomap++;
-								kfree(pb);
-								break;
-							}
+							/*
+							 * Never hold on a missing
+							 * exact slot: consume the
+							 * entry and let the content
+							 * sweep / counter publish carry
+							 * the client's wait. GR
+							 * channels have no exact slot
+							 * (svc_fake_release returns
+							 * false) and are counted as
+							 * consumed-without-delivery.
+							 */
+							if (!svc_fake_release(vgpu, ch))
+								svc.gpfifo_starved++;
 							ch->rescue_fail = 0;
 							ch->defer_since = 0;
 							if (gsp_sema_fake_rel ||
@@ -7738,11 +8886,15 @@ if (gsp_sema_fake_rel &&
 		held = svc_fwd_hold.active &&
 		       svc_fwd_hold.vgpu == vgpu &&
 		       svc_fwd_hold.ch_handle == ch->h_object;
-		/* Advance the client-visible completion counter. */
-		if (!held) {
-			svc_counter_autoreg(vgpu, ch, n_gr);
+		/*
+		 * Advance the client-visible completion counter. The slot was
+		 * already registered (and any pending release flushed) by the
+		 * safety net at the top of the pass, so this must not
+		 * re-register: once per pass is enough and the derivation is
+		 * not free.
+		 */
+		if (!held)
 			svc_counter_publish(vgpu, ch, put);
-		}
 
 		spin_lock_irqsave(&svc.chan_lock, flags);
 		for (k = 0; k < SVC_MAX_CHANNELS; k++) {
@@ -8365,6 +9517,12 @@ static int svc_thread(void *arg)
 		}
 
 		/*
+		 * Host CUDA bridge: marshal one queued launch into a daemon
+		 * job and resolve completions. Inert until a launch is seen.
+		 */
+		gsp_cuda_bridge_service(vgpu);
+
+		/*
 		 * Releases whose target VA was not translatable when the scan
 		 * saw them: retry every pass, so a pin or a learned pair applies
 		 * them immediately instead of on the next full sweep.
@@ -8511,6 +9669,8 @@ int gsp_sema_service_start(void)
 		svc.task = NULL;
 		return ret;
 	}
+	if (gsp_cuda_bridge_init())
+		pr_warn("sema-svc: cuda bridge debugfs unavailable\n");
 	return 0;
 }
 
@@ -8544,4 +9704,97 @@ void gsp_sema_service_stop(void)
 		destroy_workqueue(svc_fwd_wq);
 		svc_fwd_wq = NULL;
 	}
+	/* Thread is gone: release the bridge's pending jobs and debugfs files. */
+	gsp_cuda_bridge_fini();
+}
+
+/* ------------------------------------------------------------------ */
+/* host CUDA bridge exports (gsp_cuda_bridge.c)                        */
+/* ------------------------------------------------------------------ */
+
+/*
+ * Page-wise GPU-VA read for the bridge: resolve each page through the same
+ * trusted sources every operand read uses (pins, learned cache, discovered
+ * map, published page directories, targeted rm_mem resolve) and read through
+ * the aperture the VA maps to (vidmem shadow vs guest RAM via vfio_dma_rw).
+ */
+int gsp_sema_service_va_read(struct nv_vgpu_device *vgpu, u64 va, void *buf,
+			     size_t len)
+{
+	u8 *p = buf;
+
+	while (len) {
+		size_t chunk = min_t(size_t, len,
+				     SEMA_PAGE_SIZE -
+				     (va & (SEMA_PAGE_SIZE - 1)));
+		u64 addr = 0;
+		int ap = SVC_APERTURE_SYS;
+
+		if (svc_lookup(vgpu, va, &addr, &ap))
+			return -ENOENT;
+		if (svc_read_ap(vgpu, addr, svc_ap_of(ap), p, chunk))
+			return -EIO;
+		p += chunk;
+		va += chunk;
+		len -= chunk;
+	}
+	return 0;
+}
+
+int gsp_sema_service_va_write(struct nv_vgpu_device *vgpu, u64 va,
+			      const void *buf, size_t len)
+{
+	const u8 *p = buf;
+
+	while (len) {
+		size_t chunk = min_t(size_t, len,
+				     SEMA_PAGE_SIZE -
+				     (va & (SEMA_PAGE_SIZE - 1)));
+		u64 addr = 0;
+		int ap = SVC_APERTURE_SYS;
+
+		if (svc_lookup(vgpu, va, &addr, &ap))
+			return -ENOENT;
+		if (svc_write_ap(vgpu, addr, svc_ap_of(ap), (void *)p, chunk))
+			return -EIO;
+		p += chunk;
+		va += chunk;
+		len -= chunk;
+	}
+	return 0;
+}
+
+int gsp_sema_service_va_resolve(struct nv_vgpu_device *vgpu, u64 va, u64 *pa,
+				u32 *ap)
+{
+	u64 addr = 0;
+	int iap = SVC_APERTURE_SYS;
+
+	if (svc_lookup(vgpu, va, &addr, &iap))
+		return -ENOENT;
+	if (pa)
+		*pa = addr;
+	if (ap)
+		*ap = svc_ap_of(iap);
+	return 0;
+}
+
+int gsp_sema_service_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val)
+{
+	return svc_sem_read(vgpu, va, val);
+}
+
+int gsp_sema_service_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va,
+			       u32 val)
+{
+	int rc = svc_sem_write(vgpu, h, va, val);
+
+	/*
+	 * A bridge-completed job is a work completion like any pump release:
+	 * keep the completion hint window open so a guest that starts waiting
+	 * after the write still receives the completion event.
+	 */
+	if (!rc)
+		svc.post_hint_until = jiffies + 10 * HZ;
+	return rc;
 }
diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h
index d5a62e9..530663e 100644
--- a/kernel/gsp_sema_service.h
+++ b/kernel/gsp_sema_service.h
@@ -155,6 +155,16 @@ void gsp_sema_service_fwd_fault_stats(u64 *matched, u64 *unmatched,
 extern bool gsp_sema_gr_rings;
 
 /*
+ * In-kernel GR completion-counter slot derivation (svc_counter_derive): on an
+ * injected-pin miss, resolve a counter slot from the structural translations
+ * (the driver-published page directories, the self-discovered directory and
+ * the targeted rm_mem record) and author the release there instead of waiting
+ * for the external pin watcher. Default on; set 0 to restore the
+ * injected-pin-only behaviour for an A/B without a rebuild.
+ */
+extern bool gsp_sema_counter_derive;
+
+/*
  * Acknowledge tracking-semaphore releases from CE pushes the pump could not
  * read: write the expected payload into the channel's pool semaphore so the
  * driver's wait clears immediately instead of after the content sweep.
diff --git a/kernel/main.c b/kernel/main.c
index 39c71c3..7001b3d 100644
--- a/kernel/main.c
+++ b/kernel/main.c
@@ -2551,6 +2551,16 @@ module_param_named(sema_gr_rings, gsp_sema_gr_rings, bool, 0644);
 MODULE_PARM_DESC(sema_gr_rings, "Auto-discover+pin GR channel rings (default 1)");
 
 /*
+ * In-kernel GR completion-counter slot derivation from the structural
+ * translations (published page directories, self-discovered directory,
+ * targeted rm_mem record), used when a release's counter page has no injected
+ * pin yet. Default on; set 0 to restore the injected-pin-only behaviour for
+ * an A/B without a rebuild.
+ */
+module_param_named(sema_counter_derive, gsp_sema_counter_derive, bool, 0644);
+MODULE_PARM_DESC(sema_counter_derive, "Derive GR completion-counter slots in-kernel from structural translations (default 1)");
+
+/*
  * Acknowledge releases from unreadable CE pushes directly (tracking-semaphore
  * fallback). The push's data writes are lost either way when the content
  * rescue cannot find the page; writing the expected payload keeps the