vgsp: host CUDA bridge + sema release recovery; greeneye simulator wave

- kernel: gsp_cuda_bridge (QMD launch capture, debugfs job/out/done, param+output resolution cached from inline writes), sema drain fixes (never hold/starve a channel), GR ring autopin + counter autoreg, per-page push reads, counter-derive gate; module builds clean
- tools/cuda-bridge: CUDA driver-API daemon executing mail/vadd kernels on the physical GPU, CUBB2/CBD2 v2 protocol, layout check + self-tests
- tools/guest-diag + tools/cuda-ab: scanner/pin fixes, victory push-only mode
- debug/risc-v-gsp-rm-vm: Phase S simulator wave (GSP device model, boot ladder, triage/re-demo harnesses, docs 30-41), GPGPU-sim scaffolding
- STATE-CURRENT/docs updates

1 files changed, 1328 insertions, 0 deletions

diff --git a/kernel/gsp_cuda_bridge.c b/kernel/gsp_cuda_bridge.c
new file mode 100644
index 0000000..aaad5ee
--- /dev/null
+++ b/kernel/gsp_cuda_bridge.c
@@ -0,0 +1,1328 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Host CUDA bridge transport (v2): module side of the debugfs protocol under
+ * /sys/kernel/debug/vgsp/. The guest's queued kernel launches (inline-QMD
+ * method runs on the GR compute channel, see gsp_sema_service.c) are turned
+ * into CUBB2 job records; a userspace daemon executes them on the physical
+ * GPU and returns a CBD2 done record plus the output payload. Because the
+ * guest's CUDA buffers live in this module's emulated vidmem shadow, the
+ * daemon cannot reach them: the module marshals the inputs out and copies
+ * the daemon's output back into the resolved destination, then writes the
+ * QMD release semaphore so the guest's cudaDeviceSynchronize completes.
+ *
+ * Files (all under the existing vgsp debugfs directory):
+ *   cuda_bridge_job    0444  module serves the oldest pending 256-byte job
+ *   cuda_bridge_in     0444  module serves that job's input blob (ppos)
+ *   cuda_bridge_out    0200  daemon writes its output blob (ppos)
+ *   cuda_bridge_done   0200  daemon writes a 256-byte CBD2 record
+ *   cuda_bridge_status 0444  text counters
+ *
+ * Everything is inert until a launch is detected; the whole path can be
+ * switched off with the cuda_bridge module parameter.
+ */
+
+#define pr_fmt(fmt) "cuda-bridge: " fmt
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/debugfs.h>
+#include <linux/fs.h>
+#include <linux/mutex.h>
+#include <linux/spinlock.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <linux/uaccess.h>
+#include <linux/jiffies.h>
+#include <linux/string.h>
+
+#include "nvidia-vgpu-vfio.h"
+
+/* Records and limits (see tools/cuda-bridge/PROTOCOL.md, v2). */
+#define CUBB2_JOB_MAGIC		0x32424243u	/* 'CBB2' */
+#define CUBB2_DONE_MAGIC	0x32444243u	/* 'CBD2' */
+#define CUBB2_VERSION		2
+#define CUBB2_RECORD_SIZE	256
+#define CUBB2_IN_MAX		(32u * 1024u * 1024u)
+#define CUBB2_OUT_MAX		(32u * 1024u * 1024u)
+#define CUBB2_REQ_MAX		16
+#define CUBB2_PEND_MAX		4
+#define CUBB2_DONE_RING		16
+#define CUBB2_NAME_MAX		64
+
+/*
+ * Parameter-bank location. CUDA writes the launch arguments into the QMD's
+ * parameter bank with GRCE LOAD_INLINE_DATA at an arbitrary offset (observed:
+ * mail args at bank+0x160 while bank+0 held unrelated constants), so the
+ * bridge searches the recorded recent inline-write destinations inside the
+ * bank, then scans the bank itself for a mail/vadd argument shape.
+ */
+#define CUBB2_INLINE_MAX	16
+#define CUBB2_PARAM_BLOB	40
+#define CUBB2_PARAM_SCAN_MAX	(128u * 1024u)
+#define CUBB2_PAGE		4096
+
+/*
+ * Natural-alignment layout (x86_64 kernel and daemon agree):
+ * in_bytes at 0x38, sem_flags 0x58, prog_va 0x60, rsv at 0x70.
+ */
+struct cubb2_job {
+	u32 magic;
+	u32 version;
+	u64 seq;
+	u32 opcode;
+	u32 grid[3];
+	u32 block[3];
+	u32 shared;
+	u32 n;
+	u64 in_bytes;
+	u64 out_bytes;
+	u64 sem_va;
+	u64 sem_payload;
+	u32 sem_flags;
+	u32 pad;
+	u64 prog_va;
+	u32 param_count;
+	u32 pad2;
+	u8  rsv[0x100 - 0x70];
+};
+
+/* status 0 = ok, 1 = bad job, 2 = CUDA error, 3 = RAM/mapping error. */
+struct cubb2_done {
+	u32 magic;
+	u32 version;
+	u64 seq;
+	u32 status;
+	u32 bad;
+	u64 checksum;
+	u64 kernel_ns;
+	u32 out_bytes_written;
+	u32 err_code;
+	char device_name[CUBB2_NAME_MAX];
+	u64 host_ns;
+	u8  rsv[0x100 - 0x78];
+};
+
+static_assert(sizeof(struct cubb2_job) == CUBB2_RECORD_SIZE,
+	      "cubb2_job must be 256 bytes");
+static_assert(sizeof(struct cubb2_done) == CUBB2_RECORD_SIZE,
+	      "cubb2_done must be 256 bytes");
+static_assert(offsetof(struct cubb2_job, prog_va) == 0x60,
+	      "cubb2_job natural layout changed");
+static_assert(offsetof(struct cubb2_done, host_ns) == 0x70,
+	      "cubb2_done natural layout changed");
+
+/* Entry points (also declared in gsp_sema_service.c). */
+bool gsp_cuda_bridge_enabled(void);
+void gsp_cuda_bridge_launch(struct nv_vgpu_device *vgpu, u32 h_object,
+			    const u32 qmd[64], u64 qmd_dev_va, u64 push_va,
+			    const u64 *recent_dsts, u32 n_recent_dsts);
+void gsp_cuda_bridge_service(struct nv_vgpu_device *vgpu);
+int gsp_cuda_bridge_init(void);
+void gsp_cuda_bridge_fini(void);
+
+/* Helpers exported by gsp_sema_service.c (declared here, not in the header,
+ * so the bridge stays a leaf of the sema service). */
+int gsp_sema_service_va_read(struct nv_vgpu_device *vgpu, u64 va, void *buf,
+			     size_t len);
+int gsp_sema_service_va_write(struct nv_vgpu_device *vgpu, u64 va,
+			      const void *buf, size_t len);
+int gsp_sema_service_va_resolve(struct nv_vgpu_device *vgpu, u64 va, u64 *pa,
+				u32 *ap);
+int gsp_sema_service_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val);
+int gsp_sema_service_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va,
+			       u32 val);
+extern bool gsp_sema_dbg;
+
+static bool cubb_enable = true;
+module_param_named(cuda_bridge, cubb_enable, bool, 0644);
+MODULE_PARM_DESC(cuda_bridge,
+		 "Enable the host CUDA bridge launch transport (default 1)");
+
+/* ------------------------------------------------------------------ */
+/* launch requests: decode-thread producer, service-thread consumer     */
+/* ------------------------------------------------------------------ */
+
+struct cubb_req {
+	u32 h_object;
+	u64 qmd_dev_va;
+	u64 push_va;
+	u32 qmd[64];
+	u32 attempts;
+	u32 n_recent_dsts;
+	u64 recent_dsts[CUBB2_INLINE_MAX];
+};
+
+static struct cubb_req cubb_req[CUBB2_REQ_MAX];
+static u32 cubb_req_head;	/* producer */
+static u32 cubb_req_tail;	/* consumer */
+static DEFINE_SPINLOCK(cubb_req_lock);
+
+/*
+ * A pin-batch rewind replays recently consumed GPFIFO entries, so the same
+ * launch push is decoded again and would queue a duplicate job (victory24
+ * showed the same push three times). Dedupe on a content hash of the QMD:
+ * replaying one push hashes identically, a genuine relaunch has a different
+ * completion semaphore/payload inside the QMD.
+ */
+#define CUBB2_DEDUP		8
+#define CUBB2_DEDUP_MS		5000
+struct cubb_dedup {
+	u64 hash;
+	unsigned long when;
+};
+static struct cubb_dedup cubb_dedup[CUBB2_DEDUP];
+static u32 cubb_dedup_next;
+static u64 cubb_dedup_hits;
+
+/* ------------------------------------------------------------------ */
+/* pending jobs (daemon-visible) and daemon write-back state            */
+/* ------------------------------------------------------------------ */
+
+struct cubb_pend {
+	bool valid;
+	u64 seq;
+	u32 h_object;
+	u32 opcode;
+	u64 dst_va;
+	u64 out_bytes;
+	u64 sem_va;
+	u32 sem_payload;
+	u32 sem_flags;
+	struct cubb2_job job;
+	u8 *in;			/* vmalloc'd input blob */
+};
+
+static struct cubb_pend cubb_pend[CUBB2_PEND_MAX];
+static u64 cubb_seq;
+static DEFINE_MUTEX(cubb_lock);
+
+/* daemon out blob, accumulated by cuda_bridge_out writes */
+static u8 *cubb_out_buf;
+static unsigned int cubb_out_alloc;
+static size_t cubb_out_len;
+
+/* done ring, producer = daemon write handler, consumer = service thread */
+static struct cubb2_done cubb_done_ring[CUBB2_DONE_RING];
+static u32 cubb_done_head;
+static u32 cubb_done_tail;
+static DEFINE_SPINLOCK(cubb_done_lock);
+
+/* counters (plain reads from the status file) */
+static u64 cubb_jobs;
+static u64 cubb_completed;
+static u64 cubb_failed;
+static u64 cubb_dropped;	/* request ring/marshal/allocation drops */
+static u64 cubb_stale;		/* done records for unknown seq */
+static u64 cubb_malformed;	/* done records with bad magic/version */
+static u64 cubb_last_seq;
+static int cubb_last_err;
+static u64 cubb_launch_log;	/* verbose header budget */
+
+/* ------------------------------------------------------------------ */
+/* debugfs                                                             */
+/* ------------------------------------------------------------------ */
+
+static bool cubb_root_owned;
+static bool cubb_debugfs_ok;
+
+static void cubb_debugfs_fini(void);
+static const struct file_operations cubb_job_fops;
+static const struct file_operations cubb_in_fops;
+static const struct file_operations cubb_out_fops;
+static const struct file_operations cubb_done_fops;
+static const struct file_operations cubb_status_fops;
+
+static struct dentry *cubb_find_root(void)
+{
+	struct dentry *root = debugfs_lookup("vgsp", NULL);
+
+	if (IS_ERR(root))
+		return NULL;
+	return root;
+}
+
+static void cubb_debugfs_init(void)
+{
+	struct dentry *root, *f;
+
+	cubb_root_owned = false;
+	root = cubb_find_root();
+	if (!root) {
+		root = debugfs_create_dir("vgsp", NULL);
+		if (IS_ERR_OR_NULL(root)) {
+			pr_warn("debugfs dir vgsp unavailable\n");
+			return;
+		}
+		cubb_root_owned = true;
+	}
+
+	f = debugfs_create_file("cuda_bridge_job", 0444, root, NULL,
+				&cubb_job_fops);
+	if (IS_ERR_OR_NULL(f))
+		goto fail;
+	f = debugfs_create_file("cuda_bridge_in", 0444, root, NULL,
+				&cubb_in_fops);
+	if (IS_ERR_OR_NULL(f))
+		goto fail;
+	f = debugfs_create_file("cuda_bridge_out", 0200, root, NULL,
+				&cubb_out_fops);
+	if (IS_ERR_OR_NULL(f))
+		goto fail;
+	f = debugfs_create_file("cuda_bridge_done", 0200, root, NULL,
+				&cubb_done_fops);
+	if (IS_ERR_OR_NULL(f))
+		goto fail;
+	f = debugfs_create_file("cuda_bridge_status", 0444, root, NULL,
+				&cubb_status_fops);
+	if (IS_ERR_OR_NULL(f))
+		goto fail;
+
+	cubb_debugfs_ok = true;
+	dput(root);
+	pr_info("debugfs ready (/sys/kernel/debug/vgsp/cuda_bridge_*)\n");
+	return;
+
+fail:
+	dput(root);
+	cubb_debugfs_fini();
+	pr_warn("debugfs file creation failed\n");
+}
+
+static void cubb_debugfs_fini(void)
+{
+	struct dentry *root;
+
+	cubb_debugfs_ok = false;
+	root = cubb_find_root();
+	if (!root)
+		return;
+	debugfs_lookup_and_remove("cuda_bridge_job", root);
+	debugfs_lookup_and_remove("cuda_bridge_in", root);
+	debugfs_lookup_and_remove("cuda_bridge_out", root);
+	debugfs_lookup_and_remove("cuda_bridge_done", root);
+	debugfs_lookup_and_remove("cuda_bridge_status", root);
+	dput(root);
+	if (cubb_root_owned) {
+		debugfs_lookup_and_remove("vgsp", NULL);
+		cubb_root_owned = false;
+	}
+}
+
+/* ------------------------------------------------------------------ */
+/* staging helpers                                                     */
+/* ------------------------------------------------------------------ */
+
+static u32 cubb_rd32(const u8 *p)
+{
+	u32 v;
+
+	memcpy(&v, p, 4);
+	return v;
+}
+
+static u64 cubb_rd64(const u8 *p)
+{
+	u64 v;
+
+	memcpy(&v, p, 8);
+	return v;
+}
+
+/* A pointer the guest could have given the GPU (user/device VA range). */
+static bool cubb_va_plausible(u64 v)
+{
+	return v >= 0x10000 && v < (1ULL << 49);
+}
+
+/* mail arguments: {u64 output ptr, u32 len}, len 1..4096. */
+static bool cubb_mail_shape(const u8 *p, size_t len, u64 *ptr, u32 *n)
+{
+	u64 v;
+	u32 l;
+
+	if (len < 12)
+		return false;
+	v = cubb_rd64(p);
+	l = cubb_rd32(p + 8);
+	if (!cubb_va_plausible(v) || !l || l > 4096)
+		return false;
+	if (ptr)
+		*ptr = v;
+	if (n)
+		*n = l;
+	return true;
+}
+
+/* vadd arguments: {u64 a, b, c, u32 n}, n == 1 MiB (frozen launch shape). */
+static bool cubb_vadd_shape(const u8 *p, size_t len, u64 *out, u32 *n)
+{
+	u64 a, b, c;
+	u32 k;
+
+	if (len < 32)
+		return false;
+	a = cubb_rd64(p);
+	b = cubb_rd64(p + 8);
+	c = cubb_rd64(p + 16);
+	k = cubb_rd32(p + 24);
+	if (!cubb_va_plausible(a) || !cubb_va_plausible(b) ||
+	    !cubb_va_plausible(c) || k != (1u << 20))
+		return false;
+	if (out)
+		*out = c;
+	if (n)
+		*n = k;
+	return true;
+}
+
+static int cubb_shape_kind(const u8 *p, size_t len)
+{
+	if (cubb_mail_shape(p, len, NULL, NULL))
+		return 1;
+	if (cubb_vadd_shape(p, len, NULL, NULL))
+		return 2;
+	return 0;
+}
+
+/*
+ * Locate the launch's argument block inside the QMD parameter bank. CUDA
+ * writes the arguments with GRCE LOAD_INLINE_DATA at an arbitrary bank
+ * offset, so never assume +0: first try the recorded recent inline-write
+ * destinations that fall inside [bank_va, bank_va + bank_size), newest
+ * first, then scan the bank for a mail/vadd argument shape. `want` names the
+ * shape implied by grid/block (0 = unknown). On a match returns 0 and fills
+ * off/kind/params/src; -ENODATA means the bank was readable but no shape was
+ * found (caller may fall back to bank+0); -ENOENT means a read failed (the
+ * page resolve may still be in progress) and nothing matched, so the caller
+ * should retry.
+ */
+static int cubb_find_params(struct nv_vgpu_device *vgpu, u64 bank_va,
+			    u32 bank_size, const u64 *dsts, u32 n_dsts,
+			    u32 want, u64 *off, u32 *kind, u8 *params,
+			    size_t plen, const char **src)
+{
+	u8 cand[CUBB2_PARAM_BLOB];
+	u8 *page;
+	u64 hi = bank_va + bank_size;
+	u32 i, pgoff, o, scan;
+	bool read_fail = false;
+
+	*off = 0;
+	*kind = 0;
+	if (bank_size < 16)
+		return -ENODATA;
+
+	/* (a) most recently written inline destinations inside the bank */
+	for (i = 0; i < n_dsts; i++) {
+		u64 va = dsts[i];
+		size_t need = want == 2 ? 32 : 12;
+		size_t got = min_t(size_t, plen, sizeof(cand));
+		int k;
+
+		if (!va || va < bank_va || va + need > hi)
+			continue;
+		if (va + got > hi)
+			got = hi - va;
+		if (gsp_sema_service_va_read(vgpu, va, cand, got)) {
+			read_fail = true;
+			continue;
+		}
+		k = cubb_shape_kind(cand, got);
+		if (!k || (want && k != want))
+			continue;
+		*off = va - bank_va;
+		*kind = k;
+		memcpy(params, cand, min_t(size_t, got, plen));
+		*src = "inline";
+		return 0;
+	}
+
+	/* (b) scan the bank for an argument shape (straddle-safe per page) */
+	scan = bank_size > CUBB2_PARAM_SCAN_MAX ? CUBB2_PARAM_SCAN_MAX :
+						 bank_size;
+	page = vmalloc(CUBB2_PAGE + CUBB2_PARAM_BLOB);
+	if (!page)
+		return read_fail ? -ENOENT : -ENODATA;
+	for (pgoff = 0; pgoff + 32 <= scan; pgoff += CUBB2_PAGE) {
+		u32 available = scan - pgoff;
+		u32 read_len = available > CUBB2_PAGE + CUBB2_PARAM_BLOB ?
+			       CUBB2_PAGE + CUBB2_PARAM_BLOB : available;
+
+		if (gsp_sema_service_va_read(vgpu, bank_va + pgoff, page,
+					     read_len)) {
+			read_fail = true;
+			continue;
+		}
+		for (o = 0; o < available && o < CUBB2_PAGE &&
+			    o + 32 <= read_len; o += 8) {
+			int k = cubb_shape_kind(page + o, read_len - o);
+
+			if (!k || (want && k != want))
+				continue;
+			*off = pgoff + o;
+			*kind = k;
+			memcpy(params, page + o,
+			       min_t(size_t, read_len - o, plen));
+			*src = "scan";
+			vfree(page);
+			return 0;
+		}
+	}
+	vfree(page);
+	return read_fail ? -ENOENT : -ENODATA;
+}
+
+static struct cubb_pend *cubb_pend_lookup(u64 seq, int *idx)
+{
+	int i;
+
+	for (i = 0; i < CUBB2_PEND_MAX; i++) {
+		if (cubb_pend[i].valid && cubb_pend[i].seq == seq) {
+			if (idx)
+				*idx = i;
+			return &cubb_pend[i];
+		}
+	}
+	return NULL;
+}
+
+static int cubb_pend_slot(void)
+{
+	int i;
+
+	for (i = 0; i < CUBB2_PEND_MAX; i++)
+		if (!cubb_pend[i].valid)
+			return i;
+	return -1;
+}
+
+/* ------------------------------------------------------------------ */
+/* launch decode -> request queue                                      */
+/* ------------------------------------------------------------------ */
+
+bool gsp_cuda_bridge_enabled(void)
+{
+	return cubb_enable && cubb_debugfs_ok;
+}
+
+/*
+ * Called from svc_decode_stream on the service thread when a complete inline
+ * QMD has been accumulated. Only queues the launch: marshalling guest memory
+ * happens on the next service pass so the push pump is never held.
+ */
+void gsp_cuda_bridge_launch(struct nv_vgpu_device *vgpu, u32 h_object,
+			    const u32 qmd[64], u64 qmd_dev_va, u64 push_va,
+			    const u64 *recent_dsts, u32 n_recent_dsts)
+{
+	unsigned long flags;
+	u64 hash = 1469598103934665603ULL;
+	int i;
+
+	if (!gsp_cuda_bridge_enabled())
+		return;
+	(void)vgpu;
+
+	for (i = 0; i < 64; i++) {
+		hash ^= qmd[i];
+		hash *= 1099511628211ULL;
+	}
+	hash ^= (u64)h_object << 32;
+	hash *= 1099511628211ULL;
+	hash ^= qmd_dev_va;
+
+	spin_lock_irqsave(&cubb_req_lock, flags);
+	for (i = 0; i < CUBB2_DEDUP; i++) {
+		if (cubb_dedup[i].hash == hash &&
+		    time_before(jiffies,
+				cubb_dedup[i].when +
+				msecs_to_jiffies(CUBB2_DEDUP_MS))) {
+			spin_unlock_irqrestore(&cubb_req_lock, flags);
+			cubb_dedup_hits++;
+			return;
+		}
+	}
+	cubb_dedup[cubb_dedup_next % CUBB2_DEDUP].hash = hash;
+	cubb_dedup[cubb_dedup_next % CUBB2_DEDUP].when = jiffies;
+	cubb_dedup_next++;
+
+	if (cubb_req_head - cubb_req_tail >= CUBB2_REQ_MAX) {
+		spin_unlock_irqrestore(&cubb_req_lock, flags);
+		cubb_dropped++;
+		pr_info_ratelimited("request ring full, launch dropped (push=0x%llx)\n",
+				    push_va);
+		return;
+	}
+	{
+		struct cubb_req *r =
+			&cubb_req[cubb_req_head % CUBB2_REQ_MAX];
+
+		r->h_object = h_object;
+		r->qmd_dev_va = qmd_dev_va;
+		r->push_va = push_va;
+		r->attempts = 0;
+		memcpy(r->qmd, qmd, sizeof(r->qmd));
+		r->n_recent_dsts = 0;
+		if (recent_dsts && n_recent_dsts) {
+			if (n_recent_dsts > CUBB2_INLINE_MAX)
+				n_recent_dsts = CUBB2_INLINE_MAX;
+			memcpy(r->recent_dsts, recent_dsts,
+			       n_recent_dsts * sizeof(u64));
+			r->n_recent_dsts = n_recent_dsts;
+		}
+	}
+	cubb_req_head++;
+	spin_unlock_irqrestore(&cubb_req_lock, flags);
+}
+
+/* Consume the request the marshaller was working on (service thread only). */
+static void cubb_req_drop_head(void)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&cubb_req_lock, flags);
+	if (cubb_req_tail != cubb_req_head)
+		cubb_req_tail++;
+	spin_unlock_irqrestore(&cubb_req_lock, flags);
+}
+
+/*
+ * A guest VA read can miss because the targeted rm_mem scan for that page is
+ * still in progress (it walks the RAM a slice per call and reports -EAGAIN ->
+ * -ENOENT). Keep the request at the queue head and retry on later passes.
+ */
+#define CUBB2_REQ_TRIES	300
+
+static void cubb_req_note_attempt(const struct cubb_req *r)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&cubb_req_lock, flags);
+	if (cubb_req_tail != cubb_req_head)
+		cubb_req[cubb_req_tail % CUBB2_REQ_MAX].attempts = r->attempts;
+	spin_unlock_irqrestore(&cubb_req_lock, flags);
+}
+
+static bool cubb_req_retry(const struct cubb_req *r, int rc)
+{
+	if ((rc == -ENOENT || rc == -EAGAIN) &&
+	    r->attempts < CUBB2_REQ_TRIES) {
+		cubb_req_note_attempt(r);
+		return true;
+	}
+	return false;
+}
+
+/* ------------------------------------------------------------------ */
+/* request -> pending job (marshalling)                                */
+/* ------------------------------------------------------------------ */
+
+static void cubb_log_raw_qmd(const u32 *qmd)
+{
+	int i;
+
+	pr_info("qmd raw (64 words):\n");
+	for (i = 0; i < 64; i += 4)
+		pr_info("qmd[%02d] %08x %08x %08x %08x\n", i,
+			qmd[i], qmd[i + 1], qmd[i + 2], qmd[i + 3]);
+}
+
+static int cubb_marshal_one(struct nv_vgpu_device *vgpu)
+{
+	struct cubb_req r;
+	struct cubb2_job job;
+	unsigned long flags;
+	u32 gx, gy, gz, bx, by, bz, shared;
+	u64 param_va, sem_va, prog_va;
+	u32 param_size, sem_payload, sem_flags, red_en, one_word, enable;
+	u8 params[CUBB2_PARAM_BLOB];
+	u32 plen, n = 0, opcode = 0, param_count = 0;
+	u32 hint = 0, found_kind = 0;
+	u64 params_off = 0;
+	const char *params_src = "bank+0";
+	u64 dst_va = 0, in_bytes = 0, out_bytes = 0;
+	u64 dst_pa = 0;
+	u32 dst_ap = 0;
+	u8 *in = NULL;
+	int rc, slot;
+
+	spin_lock_irqsave(&cubb_req_lock, flags);
+	if (cubb_req_tail == cubb_req_head) {
+		spin_unlock_irqrestore(&cubb_req_lock, flags);
+		return 0;
+	}
+	r = cubb_req[cubb_req_tail % CUBB2_REQ_MAX];
+	r.attempts++;
+	spin_unlock_irqrestore(&cubb_req_lock, flags);
+
+	gx = r.qmd[12];
+	gy = r.qmd[13] & 0xffff;
+	gz = r.qmd[14] & 0xffff;
+	shared = r.qmd[17] & 0x3ffff;
+	bx = r.qmd[18] >> 16;
+	by = r.qmd[19] & 0xffff;
+	bz = r.qmd[19] >> 16;
+	enable = (r.qmd[4] >> 10) & 1;
+	sem_va = r.qmd[23] | ((u64)(r.qmd[24] & 0xff) << 32);
+	red_en = (r.qmd[24] >> 26) & 1;
+	one_word = (r.qmd[24] >> 31) & 1;
+	sem_payload = r.qmd[25];
+	param_va = r.qmd[32] | ((u64)(r.qmd[33] & 0x1ffff) << 32);
+	param_size = (r.qmd[33] >> 19) << 4;
+	prog_va = r.qmd[48] | ((u64)(r.qmd[49] & 0x1ffff) << 32);
+	sem_flags = (red_en ? 1u : 0u) | (one_word ? 2u : 0u) |
+		    (enable ? 4u : 0u);
+
+	/*
+	 * The parameter bank lives in the client's VA space, and CUDA writes
+	 * the arguments into it with GRCE inline writes at an arbitrary
+	 * offset (observed: mail args at bank+0x160, bank+0 unrelated). The
+	 * frozen grid/block shapes hint at the kernel; locate the actual
+	 * argument block by recent inline-write destination, then by shape
+	 * scan of the bank.
+	 */
+	plen = sizeof(params);
+	if (gx == 1 && gy == 1 && gz == 1 && bx == 64 && by == 1 && bz == 1)
+		hint = 1;
+	else if (gx == 4096 && gy == 1 && gz == 1 && bx == 256 && by == 1 &&
+		 bz == 1)
+		hint = 2;
+
+	rc = cubb_find_params(vgpu, param_va, param_size, r.recent_dsts,
+			      r.n_recent_dsts, hint, &params_off, &found_kind,
+			      params, plen, &params_src);
+	if (rc) {
+		if (cubb_req_retry(&r, rc))
+			return 0;
+		if (rc != -ENODATA) {
+			cubb_req_drop_head();
+			cubb_failed++;
+			cubb_last_err = rc;
+			pr_info_ratelimited("failed seq=0 param locate va=0x%llx size=0x%x rc=%d push=0x%llx h=0x%x\n",
+					    param_va, param_size, rc, r.push_va,
+					    r.h_object);
+			return 0;
+		}
+		/* No argument shape found: legacy bank+0 read. */
+		rc = gsp_sema_service_va_read(vgpu, param_va, params, plen);
+		if (rc) {
+			if (cubb_req_retry(&r, rc))
+				return 0;
+			cubb_req_drop_head();
+			cubb_failed++;
+			cubb_last_err = rc;
+			pr_info_ratelimited("failed seq=0 op=? param bank va=0x%llx size=0x%x rc=%d push=0x%llx h=0x%x\n",
+					    param_va, param_size, rc, r.push_va,
+					    r.h_object);
+			return 0;
+		}
+		params_off = 0;
+		found_kind = 0;
+		params_src = "bank+0";
+	}
+
+	/*
+	 * Classify: the two frozen launch shapes first; then the located
+	 * argument block; otherwise the old content heuristic.
+	 */
+	if (hint)
+		opcode = hint;
+	else if (found_kind)
+		opcode = found_kind;
+	else if (cubb_va_plausible(cubb_rd64(params)) &&
+		 cubb_va_plausible(cubb_rd64(params + 8)) &&
+		 cubb_va_plausible(cubb_rd64(params + 16)) &&
+		 cubb_rd32(params + 24) &&
+		 cubb_rd32(params + 24) <= (64u << 20))
+		opcode = 2;
+	else
+		opcode = 1;
+
+	if (opcode == 1) {
+		/* mail: {ptr, len} */
+		dst_va = cubb_rd64(params);
+		n = cubb_rd32(params + 8);
+		out_bytes = n;
+		in_bytes = 0;
+		param_count = 2;
+		if (n > 4096) {
+			cubb_req_drop_head();
+			cubb_failed++;
+			cubb_last_err = -E2BIG;
+			pr_info_ratelimited("failed seq=0 op=mail len %u too large (dst=0x%llx)\n",
+					    n, dst_va);
+			return 0;
+		}
+	} else {
+		/* vadd: {a, b, c, n} (+ 5th slot) */
+		u64 a = cubb_rd64(params);
+		u64 b = cubb_rd64(params + 8);
+
+		dst_va = cubb_rd64(params + 16);
+		n = cubb_rd32(params + 24);
+		out_bytes = 4ULL * n;
+		in_bytes = 8ULL * n;
+		param_count = 5;
+		if (!n || in_bytes > CUBB2_IN_MAX ||
+		    out_bytes > CUBB2_OUT_MAX) {
+			cubb_req_drop_head();
+			cubb_failed++;
+			cubb_last_err = -E2BIG;
+			pr_info_ratelimited("failed seq=0 op=vadd n=%u in=%llu out=%llu too large\n",
+					    n, in_bytes, out_bytes);
+			return 0;
+		}
+		in = vmalloc(in_bytes);
+		if (!in) {
+			cubb_req_drop_head();
+			cubb_failed++;
+			cubb_last_err = -ENOMEM;
+			pr_info_ratelimited("failed seq=0 op=vadd in alloc %llu bytes\n",
+					    in_bytes);
+			return 0;
+		}
+		rc = gsp_sema_service_va_read(vgpu, a, in, in_bytes / 2);
+		if (!rc)
+			rc = gsp_sema_service_va_read(vgpu, b,
+						      in + in_bytes / 2,
+						      in_bytes / 2);
+		if (rc) {
+			vfree(in);
+			if (cubb_req_retry(&r, rc))
+				return 0;
+			cubb_req_drop_head();
+			cubb_failed++;
+			cubb_last_err = rc;
+			pr_info_ratelimited("failed seq=0 op=vadd src a=0x%llx b=0x%llx rc=%d\n",
+					    a, b, rc);
+			return 0;
+		}
+	}
+
+	if (!n || !out_bytes || out_bytes > CUBB2_OUT_MAX) {
+		vfree(in);
+		cubb_req_drop_head();
+		cubb_failed++;
+		cubb_last_err = -EINVAL;
+		pr_info_ratelimited("failed seq=0 op=%u bad sizes n=%u out=%llu\n",
+				    opcode, n, out_bytes);
+		return 0;
+	}
+
+	if (opcode == 1)
+		pr_info_ratelimited("params off=0x%llx kind=mail ptr=0x%llx len/n=%u\n",
+				    params_off, dst_va, n);
+	else
+		pr_info_ratelimited("params off=0x%llx kind=vadd ptr=0x%llx len/n=%u\n",
+				    params_off, dst_va, n);
+
+	rc = gsp_sema_service_va_resolve(vgpu, dst_va, &dst_pa, &dst_ap);
+
+	memset(&job, 0, sizeof(job));
+	job.magic = CUBB2_JOB_MAGIC;
+	job.version = CUBB2_VERSION;
+	job.opcode = opcode;
+	job.grid[0] = gx;
+	job.grid[1] = gy;
+	job.grid[2] = gz;
+	job.block[0] = bx;
+	job.block[1] = by;
+	job.block[2] = bz;
+	job.shared = shared;
+	job.n = n;
+	job.in_bytes = in_bytes;
+	job.out_bytes = out_bytes;
+	job.sem_va = sem_va;
+	job.sem_payload = sem_payload;
+	job.sem_flags = sem_flags;
+	job.prog_va = prog_va;
+	job.param_count = param_count;
+
+	mutex_lock(&cubb_lock);
+	slot = cubb_pend_slot();
+	if (slot < 0) {
+		mutex_unlock(&cubb_lock);
+		vfree(in);
+		cubb_req_drop_head();
+		cubb_dropped++;
+		pr_info_ratelimited("pending queue full, launch dropped\n");
+		return 0;
+	}
+	job.seq = ++cubb_seq;
+	cubb_pend[slot].valid = true;
+	cubb_pend[slot].seq = job.seq;
+	cubb_pend[slot].h_object = r.h_object;
+	cubb_pend[slot].opcode = opcode;
+	cubb_pend[slot].dst_va = dst_va;
+	cubb_pend[slot].out_bytes = out_bytes;
+	cubb_pend[slot].sem_va = sem_va;
+	cubb_pend[slot].sem_payload = sem_payload;
+	cubb_pend[slot].sem_flags = sem_flags;
+	cubb_pend[slot].job = job;
+	cubb_pend[slot].in = in;
+	cubb_jobs++;
+	cubb_last_seq = job.seq;
+	mutex_unlock(&cubb_lock);
+	cubb_req_drop_head();
+
+	if (cubb_launch_log < 8) {
+		cubb_launch_log++;
+		pr_info("launch seq=%llu op=%s grid=%u,%u,%u block=%u,%u,%u shared=%u n=%u in=%llu out=%llu sem=0x%llx pay=%u flags=0x%x param=0x%llx psize=0x%x poff=0x%llx psrc=%s prog=0x%llx dst=0x%llx dstpa=0x%llx dstap=%u h=0x%x push=0x%llx qmddev=0x%llx\n",
+			job.seq, opcode == 1 ? "mail" : "vadd",
+			gx, gy, gz, bx, by, bz, shared, n, in_bytes,
+			out_bytes, sem_va, sem_payload, sem_flags, param_va,
+			param_size, params_off, params_src, prog_va, dst_va,
+			dst_pa, dst_ap, r.h_object, r.push_va, r.qmd_dev_va);
+		if (gsp_sema_dbg) {
+			cubb_log_raw_qmd(r.qmd);
+			print_hex_dump(KERN_INFO, "cuda-bridge: params ",
+				       DUMP_PREFIX_OFFSET, 16, 1, params,
+				       plen, false);
+		}
+	} else {
+		pr_info_ratelimited("launch seq=%llu op=%s n=%u out=%llu\n",
+				    job.seq, opcode == 1 ? "mail" : "vadd",
+				    n, out_bytes);
+	}
+	return 0;
+}
+
+/* ------------------------------------------------------------------ */
+/* done records -> completion                                          */
+/* ------------------------------------------------------------------ */
+
+static bool cubb_done_pop(struct cubb2_done *out)
+{
+	unsigned long flags;
+	bool have = false;
+
+	spin_lock_irqsave(&cubb_done_lock, flags);
+	if (cubb_done_tail != cubb_done_head) {
+		*out = cubb_done_ring[cubb_done_tail % CUBB2_DONE_RING];
+		cubb_done_tail++;
+		have = true;
+	}
+	spin_unlock_irqrestore(&cubb_done_lock, flags);
+	return have;
+}
+
+static void cubb_finish_one(struct nv_vgpu_device *vgpu,
+			    const struct cubb2_done *d)
+{
+	struct cubb_pend *p;
+	int idx = -1;
+	int rc = 0;
+	u32 sem_val = 0;
+	u32 sem_rc = 0;
+	u64 wrote;
+
+	mutex_lock(&cubb_lock);
+	p = cubb_pend_lookup(d->seq, &idx);
+	if (!p) {
+		mutex_unlock(&cubb_lock);
+		cubb_stale++;
+		pr_info_ratelimited("stale done seq=%llu (no pending job)\n",
+				    d->seq);
+		return;
+	}
+
+	if (d->status != 0) {
+		cubb_failed++;
+		cubb_last_err = (int)d->err_code;
+		pr_info("failed seq=%llu err=%u status=%u op=%u\n",
+			d->seq, d->err_code, d->status, p->opcode);
+		goto out;
+	}
+
+	wrote = min_t(u64, p->out_bytes, cubb_out_len);
+	if (wrote < p->out_bytes) {
+		cubb_failed++;
+		cubb_last_err = -ENODATA;
+		pr_info("failed seq=%llu err=%d (daemon wrote %llu of %llu out bytes)\n",
+			d->seq, -ENODATA, wrote, p->out_bytes);
+		goto out;
+	}
+
+	rc = gsp_sema_service_va_write(vgpu, p->dst_va, cubb_out_buf, wrote);
+	if (rc) {
+		cubb_failed++;
+		cubb_last_err = rc;
+		pr_info("failed seq=%llu err=%d (output write va=0x%llx len=%llu)\n",
+			d->seq, rc, p->dst_va, wrote);
+		goto out;
+	}
+	/*
+	 * Read-back check over the whole output: proves the bytes landed
+	 * through the translation the guest will read (vidmem shadow or
+	 * guest RAM). A missing/mismatching copy must NOT be signalled
+	 * complete, or the guest reads stale data and runs on.
+	 */
+	{
+		u8 check[256];
+		size_t off = 0;
+
+		while (off < wrote) {
+			size_t clen = min_t(size_t, wrote - off,
+					    sizeof(check));
+
+			if (gsp_sema_service_va_read(vgpu, p->dst_va + off,
+						     check, clen) ||
+			    memcmp(check, cubb_out_buf + off, clen)) {
+				cubb_failed++;
+				cubb_last_err = -EIO;
+				pr_info("failed seq=%llu err=%d (output read-back mismatch va=0x%llx off=%zu len=%zu)\n",
+					d->seq, -EIO, p->dst_va, off, clen);
+				goto out;
+			}
+			off += clen;
+		}
+	}
+
+	if ((p->sem_flags & 4u) && p->sem_va) {
+		sem_val = p->sem_payload;
+		if (p->sem_flags & 1u) {
+			u32 cur = 0;
+
+			if (!gsp_sema_service_sem_read(vgpu, p->sem_va, &cur))
+				sem_val = cur + p->sem_payload;
+		}
+		sem_rc = gsp_sema_service_sem_write(vgpu, p->h_object,
+						    p->sem_va, sem_val);
+		if (sem_rc) {
+			cubb_failed++;
+			cubb_last_err = sem_rc;
+			pr_info("failed seq=%llu err=%d sem_va=0x%llx pay=%u\n",
+				d->seq, sem_rc, p->sem_va, sem_val);
+			goto out;
+		}
+	}
+
+	{
+		u64 dpa = 0;
+		u32 dap = 0;
+		u64 sempa = 0;
+		u32 semap = 0;
+
+		(void)gsp_sema_service_va_resolve(vgpu, p->dst_va, &dpa, &dap);
+		if (p->sem_va)
+			(void)gsp_sema_service_va_resolve(vgpu, p->sem_va,
+							  &sempa, &semap);
+		cubb_completed++;
+		pr_info("completed seq=%llu op=%u dst=0x%llx dstpa=0x%llx dstap=%u out_bytes=%llu/%llu sem_va=0x%llx sempa=0x%llx pay=%u bad=%u kernel_ns=%llu\n",
+			d->seq, p->opcode, p->dst_va, dpa, dap, wrote,
+			p->out_bytes, p->sem_va, sempa, sem_val, d->bad,
+			d->kernel_ns);
+	}
+
+out:
+	vfree(p->in);
+	p->in = NULL;
+	p->valid = false;
+	cubb_out_len = 0;
+	if (idx >= 0) {
+		/* compact: shift the younger entries down one slot */
+		int j = idx;
+
+		while (j + 1 < CUBB2_PEND_MAX && cubb_pend[j + 1].valid) {
+			cubb_pend[j] = cubb_pend[j + 1];
+			j++;
+		}
+		cubb_pend[j].in = NULL;
+		cubb_pend[j].valid = false;
+	}
+	mutex_unlock(&cubb_lock);
+}
+
+/*
+ * Service-thread hook: marshal at most one queued launch into a daemon-visible
+ * job, then resolve any completed jobs. Called every service pass; a no-op
+ * when no launch has been seen.
+ */
+void gsp_cuda_bridge_service(struct nv_vgpu_device *vgpu)
+{
+	struct cubb2_done d;
+	int guard = 0;
+
+	if (!gsp_cuda_bridge_enabled())
+		return;
+
+	(void)cubb_marshal_one(vgpu);
+	while (guard++ < CUBB2_DONE_RING && cubb_done_pop(&d)) {
+		if (d.magic != CUBB2_DONE_MAGIC ||
+		    d.version != CUBB2_VERSION) {
+			cubb_malformed++;
+			continue;
+		}
+		cubb_finish_one(vgpu, &d);
+	}
+}
+
+/* ------------------------------------------------------------------ */
+/* debugfs handlers                                                    */
+/* ------------------------------------------------------------------ */
+
+static ssize_t cubb_job_read(struct file *f, char __user *ubuf, size_t count,
+			     loff_t *ppos)
+{
+	u8 rec[CUBB2_RECORD_SIZE];
+	int idx = -1;
+	struct cubb_pend *p = NULL;
+	size_t off = *ppos;
+	size_t n;
+
+	if (!cubb_debugfs_ok)
+		return 0;
+	mutex_lock(&cubb_lock);
+	for (idx = 0; idx < CUBB2_PEND_MAX; idx++) {
+		if (cubb_pend[idx].valid) {
+			p = &cubb_pend[idx];
+			break;
+		}
+	}
+	if (!p) {
+		mutex_unlock(&cubb_lock);
+		return 0;
+	}
+	memcpy(rec, &p->job, sizeof(rec));
+	mutex_unlock(&cubb_lock);
+
+	if (off >= sizeof(rec))
+		return 0;
+	n = min_t(size_t, count, sizeof(rec) - off);
+	if (copy_to_user(ubuf, rec + off, n))
+		return -EFAULT;
+	*ppos = off + n;
+	return n;
+}
+
+static ssize_t cubb_in_read(struct file *f, char __user *ubuf, size_t count,
+			    loff_t *ppos)
+{
+	struct cubb_pend *p = NULL;
+	u64 in_bytes = 0;
+	u8 *in = NULL;
+	size_t off = *ppos;
+	size_t n;
+	int rc = 0;
+	int idx;
+
+	if (!cubb_debugfs_ok)
+		return 0;
+	mutex_lock(&cubb_lock);
+	for (idx = 0; idx < CUBB2_PEND_MAX; idx++) {
+		if (cubb_pend[idx].valid) {
+			p = &cubb_pend[idx];
+			break;
+		}
+	}
+	if (!p) {
+		mutex_unlock(&cubb_lock);
+		return 0;
+	}
+	in = p->in;
+	in_bytes = p->job.in_bytes;
+	if (!in || off >= in_bytes) {
+		mutex_unlock(&cubb_lock);
+		return 0;
+	}
+	n = min_t(size_t, count, in_bytes - off);
+	rc = copy_to_user(ubuf, in + off, n);
+	mutex_unlock(&cubb_lock);
+	if (rc)
+		return -EFAULT;
+	*ppos = off + n;
+	return n;
+}
+
+/*
+ * Accumulate the daemon's output blob. ppos 0 starts a new blob; growth is
+ * amortised (double until the record fits, capped at CUBB2_OUT_MAX).
+ */
+static ssize_t cubb_out_write(struct file *f, const char __user *ubuf,
+			      size_t count, loff_t *ppos)
+{
+	u64 off = *ppos;
+	u64 end;
+	int rc = 0;
+
+	if (!cubb_debugfs_ok)
+		return -ENODEV;
+	if (!count)
+		return 0;
+	if (*ppos < 0)
+		return -EINVAL;
+	end = off + count;
+	if (end > CUBB2_OUT_MAX)
+		return -E2BIG;
+
+	mutex_lock(&cubb_lock);
+	if (off == 0)
+		cubb_out_len = 0;
+	if (end > cubb_out_alloc) {
+		unsigned int want = cubb_out_alloc ? cubb_out_alloc : 4096;
+		u8 *nb;
+
+		while (want < end && want < CUBB2_OUT_MAX)
+			want *= 2;
+		if (want > CUBB2_OUT_MAX)
+			want = CUBB2_OUT_MAX;
+		nb = vmalloc(want);
+		if (!nb) {
+			mutex_unlock(&cubb_lock);
+			return -ENOMEM;
+		}
+		if (cubb_out_len)
+			memcpy(nb, cubb_out_buf, cubb_out_len);
+		vfree(cubb_out_buf);
+		cubb_out_buf = nb;
+		cubb_out_alloc = want;
+	}
+	rc = copy_from_user(cubb_out_buf + off, ubuf, count);
+	if (!rc && end > cubb_out_len)
+		cubb_out_len = end;
+	mutex_unlock(&cubb_lock);
+	if (rc)
+		return -EFAULT;
+	*ppos = end;
+	return count;
+}
+
+/* Accumulate a 256-byte CBD2 record (single write or ppos-chunked). */
+static u8 cubb_done_stage[CUBB2_RECORD_SIZE];
+static size_t cubb_done_stage_len;
+
+static ssize_t cubb_done_write(struct file *f, const char __user *ubuf,
+			       size_t count, loff_t *ppos)
+{
+	/*
+	 * Accept both pwrite(fd, rec, 256, 0) and a long-lived fd writing
+	 * records back to back: a write starting on a record boundary starts
+	 * a fresh record.
+	 */
+	size_t off = (size_t)(*ppos % CUBB2_RECORD_SIZE);
+	unsigned long flags;
+	int rc;
+
+	if (!cubb_debugfs_ok)
+		return -ENODEV;
+	if (!count)
+		return 0;
+	if (count > CUBB2_RECORD_SIZE - off)
+		count = CUBB2_RECORD_SIZE - off;
+
+	mutex_lock(&cubb_lock);
+	if (off == 0)
+		cubb_done_stage_len = 0;
+	rc = copy_from_user(cubb_done_stage + off, ubuf, count);
+	if (rc) {
+		mutex_unlock(&cubb_lock);
+		return -EFAULT;
+	}
+	if (off + count > cubb_done_stage_len)
+		cubb_done_stage_len = off + count;
+
+	if (cubb_done_stage_len >= CUBB2_RECORD_SIZE) {
+		struct cubb2_done *d = (struct cubb2_done *)cubb_done_stage;
+
+		if (d->magic == CUBB2_DONE_MAGIC &&
+		    d->version == CUBB2_VERSION) {
+			spin_lock_irqsave(&cubb_done_lock, flags);
+			if (cubb_done_head - cubb_done_tail <
+			    CUBB2_DONE_RING) {
+				cubb_done_ring[cubb_done_head %
+					       CUBB2_DONE_RING] = *d;
+				cubb_done_head++;
+			} else {
+				cubb_dropped++;
+			}
+			spin_unlock_irqrestore(&cubb_done_lock, flags);
+		} else {
+			cubb_malformed++;
+		}
+		memset(cubb_done_stage, 0, sizeof(cubb_done_stage));
+		cubb_done_stage_len = 0;
+	}
+	mutex_unlock(&cubb_lock);
+	*ppos = off + count;
+	return count;
+}
+
+static ssize_t cubb_status_read(struct file *f, char __user *ubuf,
+				size_t count, loff_t *ppos)
+{
+	char buf[256];
+	int len;
+
+	len = scnprintf(buf, sizeof(buf),
+			"jobs=%llu completed=%llu failed=%llu last_seq=%llu last_err=%d dropped=%llu stale=%llu malformed=%llu dedup=%llu\n",
+			cubb_jobs, cubb_completed, cubb_failed,
+			cubb_last_seq, cubb_last_err, cubb_dropped,
+			cubb_stale, cubb_malformed, cubb_dedup_hits);
+	return simple_read_from_buffer(ubuf, count, ppos, buf, len);
+}
+
+static const struct file_operations cubb_job_fops = {
+	.owner = THIS_MODULE,
+	.open = simple_open,
+	.read = cubb_job_read,
+	.llseek = default_llseek,
+};
+
+static const struct file_operations cubb_in_fops = {
+	.owner = THIS_MODULE,
+	.open = simple_open,
+	.read = cubb_in_read,
+	.llseek = default_llseek,
+};
+
+static const struct file_operations cubb_out_fops = {
+	.owner = THIS_MODULE,
+	.open = simple_open,
+	.write = cubb_out_write,
+	.llseek = default_llseek,
+};
+
+static const struct file_operations cubb_done_fops = {
+	.owner = THIS_MODULE,
+	.open = simple_open,
+	.write = cubb_done_write,
+	.llseek = default_llseek,
+};
+
+static const struct file_operations cubb_status_fops = {
+	.owner = THIS_MODULE,
+	.open = simple_open,
+	.read = cubb_status_read,
+	.llseek = default_llseek,
+};
+
+/* ------------------------------------------------------------------ */
+/* lifecycle                                                           */
+/* ------------------------------------------------------------------ */
+
+int gsp_cuda_bridge_init(void)
+{
+	cubb_debugfs_init();
+	return cubb_debugfs_ok ? 0 : -ENODEV;
+}
+
+void gsp_cuda_bridge_fini(void)
+{
+	int i;
+
+	if (cubb_out_buf) {
+		vfree(cubb_out_buf);
+		cubb_out_buf = NULL;
+		cubb_out_alloc = 0;
+		cubb_out_len = 0;
+	}
+	mutex_lock(&cubb_lock);
+	for (i = 0; i < CUBB2_PEND_MAX; i++) {
+		vfree(cubb_pend[i].in);
+		cubb_pend[i].in = NULL;
+		cubb_pend[i].valid = false;
+	}
+	mutex_unlock(&cubb_lock);
+	cubb_req_head = cubb_req_tail = 0;
+	cubb_done_head = cubb_done_tail = 0;
+	cubb_debugfs_fini();
+}