path: root/kernel/gsp_cuda_bridge.c

/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Host CUDA bridge transport (v2): module side of the debugfs protocol under
 * /sys/kernel/debug/vgsp/. The guest's queued kernel launches (inline-QMD
 * method runs on the GR compute channel, see gsp_sema_service.c) are turned
 * into CUBB2 job records; a userspace daemon executes them on the physical
 * GPU and returns a CBD2 done record plus the output payload. Because the
 * guest's CUDA buffers live in this module's emulated vidmem shadow, the
 * daemon cannot reach them: the module marshals the inputs out and copies
 * the daemon's output back into the resolved destination, then writes the
 * QMD release semaphore so the guest's cudaDeviceSynchronize completes.
 *
 * Files (all under the existing vgsp debugfs directory):
 *   cuda_bridge_job    0444  module serves the oldest pending 256-byte job
 *   cuda_bridge_in     0444  module serves that job's input blob (ppos)
 *   cuda_bridge_out    0200  daemon writes its output blob (ppos)
 *   cuda_bridge_done   0200  daemon writes a 256-byte CBD2 record
 *   cuda_bridge_status 0444  text counters
 *
 * Everything is inert until a launch is detected; the whole path can be
 * switched off with the cuda_bridge module parameter.
 */

#define pr_fmt(fmt) "cuda-bridge: " fmt

#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/debugfs.h>
#include <linux/fs.h>
#include <linux/mutex.h>
#include <linux/spinlock.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/uaccess.h>
#include <linux/jiffies.h>
#include <linux/string.h>

#include "nvidia-vgpu-vfio.h"
#include "gsp_cuda_bridge.h"

/* Records and limits (see tools/cuda-bridge/PROTOCOL.md, v2). */
#define CUBB2_JOB_MAGIC		0x32424243u	/* 'CBB2' */
#define CUBB2_DONE_MAGIC	0x32444243u	/* 'CBD2' */
#define CUBB2_VERSION		2
#define CUBB2_RECORD_SIZE	256
#define CUBB2_IN_MAX		(32u * 1024u * 1024u)
#define CUBB2_OUT_MAX		(32u * 1024u * 1024u)
#define CUBB2_REQ_MAX		16
#define CUBB2_PEND_MAX		4
#define CUBB2_DONE_RING		16
#define CUBB2_NAME_MAX		64

/*
 * Parameter-bank location. CUDA writes the launch arguments into the QMD's
 * parameter bank with GRCE LOAD_INLINE_DATA at an arbitrary offset (observed:
 * mail args at bank+0x160 while bank+0 held unrelated constants), so the
 * bridge searches the recorded recent inline-write destinations inside the
 * bank, then scans the bank itself for a mail/vadd argument shape.
 */
#define CUBB2_INLINE_MAX	16
#define CUBB2_PARAM_BLOB	40
#define CUBB2_PARAM_SCAN_MAX	(128u * 1024u)
#define CUBB2_PAGE		4096

/*
 * Natural-alignment layout (x86_64 kernel and daemon agree):
 * in_bytes at 0x38, sem_flags 0x58, prog_va 0x60, rsv at 0x70.
 */
struct cubb2_job {
	u32 magic;
	u32 version;
	u64 seq;
	u32 opcode;
	u32 grid[3];
	u32 block[3];
	u32 shared;
	u32 n;
	u64 in_bytes;
	u64 out_bytes;
	u64 sem_va;
	u64 sem_payload;
	u32 sem_flags;
	u32 pad;
	u64 prog_va;
	u32 param_count;
	u32 pad2;
	u8  rsv[0x100 - 0x70];
};

/* status 0 = ok, 1 = bad job, 2 = CUDA error, 3 = RAM/mapping error. */
struct cubb2_done {
	u32 magic;
	u32 version;
	u64 seq;
	u32 status;
	u32 bad;
	u64 checksum;
	u64 kernel_ns;
	u32 out_bytes_written;
	u32 err_code;
	char device_name[CUBB2_NAME_MAX];
	u64 host_ns;
	u8  rsv[0x100 - 0x78];
};

static_assert(sizeof(struct cubb2_job) == CUBB2_RECORD_SIZE,
	      "cubb2_job must be 256 bytes");
static_assert(sizeof(struct cubb2_done) == CUBB2_RECORD_SIZE,
	      "cubb2_done must be 256 bytes");
static_assert(offsetof(struct cubb2_job, prog_va) == 0x60,
	      "cubb2_job natural layout changed");
static_assert(offsetof(struct cubb2_done, host_ns) == 0x70,
	      "cubb2_done natural layout changed");

/* Entry points are declared in gsp_cuda_bridge.h. */

/* Helpers exported by gsp_sema_service.c (declared here, not in the header,
 * so the bridge stays a leaf of the sema service). */
int gsp_sema_service_va_read(struct nv_vgpu_device *vgpu, u64 va, void *buf,
			     size_t len);
int gsp_sema_service_va_write(struct nv_vgpu_device *vgpu, u64 va,
			      const void *buf, size_t len);
int gsp_sema_service_va_resolve(struct nv_vgpu_device *vgpu, u64 va, u64 *pa,
				u32 *ap);
int gsp_sema_service_va_translate(struct nv_vgpu_device *vgpu, u64 va,
				  u64 *pa, u32 *ap);
void gsp_sema_service_va_explain(struct nv_vgpu_device *vgpu, u64 va);
void gsp_sema_service_trace_window(u32 ms);
int gsp_sema_service_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val);
int gsp_sema_service_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va,
			       u32 val);
extern bool gsp_sema_dbg;

static bool cubb_enable = true;
module_param_named(cuda_bridge, cubb_enable, bool, 0644);
MODULE_PARM_DESC(cuda_bridge,
		 "Enable the host CUDA bridge launch transport (default 1)");

/* ------------------------------------------------------------------ */
/* launch requests: decode-thread producer, service-thread consumer     */
/* ------------------------------------------------------------------ */

struct cubb_req {
	u32 h_object;
	u64 qmd_dev_va;
	u64 push_va;
	u32 qmd[64];
	u32 attempts;
	u32 n_inl;
	struct gsp_bridge_inline inl[GSP_BRIDGE_INLINE_MAX];
};

static struct cubb_req cubb_req[CUBB2_REQ_MAX];
static u32 cubb_req_head;	/* producer */
static u32 cubb_req_tail;	/* consumer */
static DEFINE_SPINLOCK(cubb_req_lock);

/*
 * A pin-batch rewind replays recently consumed GPFIFO entries, so the same
 * launch push is decoded again and would queue a duplicate job (victory24
 * showed the same push three times). Dedupe on a content hash of the QMD:
 * replaying one push hashes identically, a genuine relaunch has a different
 * completion semaphore/payload inside the QMD.
 */
#define CUBB2_DEDUP		8
#define CUBB2_DEDUP_MS		5000
struct cubb_dedup {
	u64 hash;
	unsigned long when;
};
static struct cubb_dedup cubb_dedup[CUBB2_DEDUP];
static u32 cubb_dedup_next;
static u64 cubb_dedup_hits;

/* ------------------------------------------------------------------ */
/* pending jobs (daemon-visible) and daemon write-back state            */
/* ------------------------------------------------------------------ */

struct cubb_pend {
	bool valid;
	u64 seq;
	u32 h_object;
	u32 opcode;
	u64 dst_va;
	u64 out_bytes;
	u64 sem_va;
	u32 sem_payload;
	u32 sem_flags;
	struct cubb2_job job;
	u8 *in;			/* vmalloc'd input blob */
	/*
	 * Deferred completion: a done record whose output write could not
	 * resolve yet (the UVM device VA's translation is learned
	 * asynchronously) is staged here and retried on later service passes
	 * for a bounded time instead of failing the job immediately.
	 */
	bool have_done;
	u32 done_tries;
	unsigned long done_at;
	u64 done_out_len;
	u8 *done_out;		/* vmalloc'd output blob */
	struct cubb2_done done;
};

static struct cubb_pend cubb_pend[CUBB2_PEND_MAX];
static u64 cubb_seq;
static DEFINE_MUTEX(cubb_lock);

/* daemon out blob, accumulated by cuda_bridge_out writes */
static u8 *cubb_out_buf;
static unsigned int cubb_out_alloc;
static size_t cubb_out_len;

/* done ring, producer = daemon write handler, consumer = service thread */
static struct cubb2_done cubb_done_ring[CUBB2_DONE_RING];
static u32 cubb_done_head;
static u32 cubb_done_tail;
static DEFINE_SPINLOCK(cubb_done_lock);

/* counters (plain reads from the status file) */
static u64 cubb_jobs;
static u64 cubb_completed;
static u64 cubb_failed;
static u64 cubb_dropped;	/* request ring/marshal/allocation drops */
static u64 cubb_stale;		/* done records for unknown seq */
static u64 cubb_malformed;	/* done records with bad magic/version */
static u64 cubb_last_seq;
static bool cubb_explained;	/* one-shot walk explainer fired */
static int cubb_last_err;
static u64 cubb_launch_log;	/* verbose header budget */

/* ------------------------------------------------------------------ */
/* debugfs                                                             */
/* ------------------------------------------------------------------ */

static bool cubb_root_owned;
static bool cubb_debugfs_ok;

static void cubb_debugfs_fini(void);
static const struct file_operations cubb_job_fops;
static const struct file_operations cubb_in_fops;
static const struct file_operations cubb_out_fops;
static const struct file_operations cubb_done_fops;
static const struct file_operations cubb_status_fops;

static struct dentry *cubb_find_root(void)
{
	struct dentry *root = debugfs_lookup("vgsp", NULL);

	if (IS_ERR(root))
		return NULL;
	return root;
}

static void cubb_debugfs_init(void)
{
	struct dentry *root, *f;

	cubb_root_owned = false;
	root = cubb_find_root();
	if (!root) {
		root = debugfs_create_dir("vgsp", NULL);
		if (IS_ERR_OR_NULL(root)) {
			pr_warn("debugfs dir vgsp unavailable\n");
			return;
		}
		cubb_root_owned = true;
	}

	f = debugfs_create_file("cuda_bridge_job", 0444, root, NULL,
				&cubb_job_fops);
	if (IS_ERR_OR_NULL(f))
		goto fail;
	f = debugfs_create_file("cuda_bridge_in", 0444, root, NULL,
				&cubb_in_fops);
	if (IS_ERR_OR_NULL(f))
		goto fail;
	f = debugfs_create_file("cuda_bridge_out", 0200, root, NULL,
				&cubb_out_fops);
	if (IS_ERR_OR_NULL(f))
		goto fail;
	f = debugfs_create_file("cuda_bridge_done", 0200, root, NULL,
				&cubb_done_fops);
	if (IS_ERR_OR_NULL(f))
		goto fail;
	f = debugfs_create_file("cuda_bridge_status", 0444, root, NULL,
				&cubb_status_fops);
	if (IS_ERR_OR_NULL(f))
		goto fail;

	cubb_debugfs_ok = true;
	dput(root);
	pr_info("debugfs ready (/sys/kernel/debug/vgsp/cuda_bridge_*)\n");
	return;

fail:
	dput(root);
	cubb_debugfs_fini();
	pr_warn("debugfs file creation failed\n");
}

static void cubb_debugfs_fini(void)
{
	struct dentry *root;

	cubb_debugfs_ok = false;
	root = cubb_find_root();
	if (!root)
		return;
	debugfs_lookup_and_remove("cuda_bridge_job", root);
	debugfs_lookup_and_remove("cuda_bridge_in", root);
	debugfs_lookup_and_remove("cuda_bridge_out", root);
	debugfs_lookup_and_remove("cuda_bridge_done", root);
	debugfs_lookup_and_remove("cuda_bridge_status", root);
	dput(root);
	if (cubb_root_owned) {
		debugfs_lookup_and_remove("vgsp", NULL);
		cubb_root_owned = false;
	}
}

/* ------------------------------------------------------------------ */
/* staging helpers                                                     */
/* ------------------------------------------------------------------ */

static u32 cubb_rd32(const u8 *p)
{
	u32 v;

	memcpy(&v, p, 4);
	return v;
}

static u64 cubb_rd64(const u8 *p)
{
	u64 v;

	memcpy(&v, p, 8);
	return v;
}

/* A pointer the guest could have given the GPU (user/device VA range). */
static bool cubb_va_plausible(u64 v)
{
	return v >= 0x10000 && v < (1ULL << 49);
}

/* mail arguments: {u64 output ptr, u32 len}, len 1..4096. */
static bool cubb_mail_shape(const u8 *p, size_t len, u64 *ptr, u32 *n)
{
	u64 v;
	u32 l;

	if (len < 12)
		return false;
	v = cubb_rd64(p);
	l = cubb_rd32(p + 8);
	if (!cubb_va_plausible(v) || !l || l > 4096)
		return false;
	if (ptr)
		*ptr = v;
	if (n)
		*n = l;
	return true;
}

/* vadd arguments: {u64 a, b, c, u32 n}, n == 1 MiB (frozen launch shape). */
static bool cubb_vadd_shape(const u8 *p, size_t len, u64 *out, u32 *n)
{
	u64 a, b, c;
	u32 k;

	if (len < 32)
		return false;
	a = cubb_rd64(p);
	b = cubb_rd64(p + 8);
	c = cubb_rd64(p + 16);
	k = cubb_rd32(p + 24);
	if (!cubb_va_plausible(a) || !cubb_va_plausible(b) ||
	    !cubb_va_plausible(c) || k != (1u << 20))
		return false;
	if (out)
		*out = c;
	if (n)
		*n = k;
	return true;
}

static int cubb_shape_kind(const u8 *p, size_t len)
{
	if (cubb_mail_shape(p, len, NULL, NULL))
		return 1;
	if (cubb_vadd_shape(p, len, NULL, NULL))
		return 2;
	return 0;
}

/*
 * Locate the launch's argument block inside the QMD parameter bank. CUDA
 * writes the arguments with GRCE LOAD_INLINE_DATA at an arbitrary bank
 * offset, so never assume +0: first match the payload bytes the sema service
 * cached for the recent inline-write destinations that fall inside
 * [bank_va, bank_va + bank_size), newest first (no VA read needed: these
 * banks are UVM-managed VAs with no reliable translation). If the writes
 * were split, VA-adjacent cached records are merged and shape-scanned. Then
 * scan the bank itself for a mail/vadd argument shape as a fallback. `want`
 * names the shape implied by grid/block (0 = unknown). On a match returns 0
 * and fills off/kind/params/src; -ENODATA means nothing matched and no read
 * failed (caller may fall back to bank+0); -ENOENT means a bank read failed
 * (the page resolve may still be in progress) and nothing matched, so the
 * caller should retry.
 */
static int cubb_find_params(struct nv_vgpu_device *vgpu, u64 bank_va,
			    u32 bank_size,
			    const struct gsp_bridge_inline *recs, u32 n_recs,
			    u32 want, u64 *off, u32 *kind, u8 *params,
			    size_t plen, const char **src)
{
	u8 cand[CUBB2_PARAM_BLOB];
	u8 *page;
	u64 hi = bank_va + bank_size;
	u32 i, pgoff, o, scan;
	bool read_fail = false;

	*off = 0;
	*kind = 0;
	if (bank_size < 16)
		return -ENODATA;

	/*
	 * (a) cached inline-write payloads, newest first. The bytes are in
	 * hand from the push decode, so a shape match needs no translation.
	 */
	for (i = 0; i < n_recs; i++) {
		const struct gsp_bridge_inline *r = &recs[i];
		size_t need = want == 2 ? 32 : 12;
		size_t got;
		int k;

		if (!r->dst_va || !r->len || r->dst_va < bank_va ||
		    r->dst_va + need > hi)
			continue;
		got = min_t(size_t, r->len, sizeof(cand));
		if (r->dst_va + got > hi)
			got = hi - r->dst_va;
		k = cubb_shape_kind(r->data, got);
		if (!k || (want && k != want))
			continue;
		*off = r->dst_va - bank_va;
		*kind = k;
		memcpy(params, r->data, min_t(size_t, got, plen));
		*src = "cache";
		return 0;
	}

	/*
	 * (a2) a large argument block can arrive as several consecutive
	 * LOAD_INLINE_DATA runs: merge VA-adjacent cached records and scan the
	 * merged bytes for a shape.
	 */
	for (i = 0; i < n_recs; i++) {
		u8 merged[CUBB2_PARAM_BLOB];
		u32 mlen, j;
		u64 base;
		int k;

		base = recs[i].dst_va;
		if (!base || !recs[i].len || base < bank_va || base + 12 > hi)
			continue;
		mlen = min_t(u32, recs[i].len, sizeof(merged));
		memcpy(merged, recs[i].data, mlen);
		for (j = 0; j < n_recs && mlen < sizeof(merged); j++) {
			u32 take;

			if (!recs[j].len ||
			    recs[j].dst_va != base + mlen)
				continue;
			take = min_t(u32, recs[j].len,
				     sizeof(merged) - mlen);
			memcpy(merged + mlen, recs[j].data, take);
			mlen += take;
		}
		for (o = 0; o + 12 <= mlen; o += 4) {
			k = cubb_shape_kind(merged + o, mlen - o);
			if (!k || (want && k != want))
				continue;
			*off = base + o - bank_va;
			*kind = k;
			memcpy(params, merged + o,
			       min_t(u32, mlen - o, plen));
			*src = "cache-merge";
			return 0;
		}
	}

	/* (b) scan the bank for an argument shape (straddle-safe per page) */
	scan = bank_size > CUBB2_PARAM_SCAN_MAX ? CUBB2_PARAM_SCAN_MAX :
						 bank_size;
	page = vmalloc(CUBB2_PAGE + CUBB2_PARAM_BLOB);
	if (!page)
		return read_fail ? -ENOENT : -ENODATA;
	for (pgoff = 0; pgoff + 32 <= scan; pgoff += CUBB2_PAGE) {
		u32 available = scan - pgoff;
		u32 read_len = available > CUBB2_PAGE + CUBB2_PARAM_BLOB ?
			       CUBB2_PAGE + CUBB2_PARAM_BLOB : available;

		if (gsp_sema_service_va_read(vgpu, bank_va + pgoff, page,
					     read_len)) {
			read_fail = true;
			continue;
		}
		for (o = 0; o < available && o < CUBB2_PAGE &&
			    o + 32 <= read_len; o += 8) {
			int k = cubb_shape_kind(page + o, read_len - o);

			if (!k || (want && k != want))
				continue;
			*off = pgoff + o;
			*kind = k;
			memcpy(params, page + o,
			       min_t(size_t, read_len - o, plen));
			*src = "scan";
			vfree(page);
			return 0;
		}
	}
	vfree(page);
	return read_fail ? -ENOENT : -ENODATA;
}

static struct cubb_pend *cubb_pend_lookup(u64 seq, int *idx)
{
	int i;

	for (i = 0; i < CUBB2_PEND_MAX; i++) {
		if (cubb_pend[i].valid && cubb_pend[i].seq == seq) {
			if (idx)
				*idx = i;
			return &cubb_pend[i];
		}
	}
	return NULL;
}

static int cubb_pend_slot(void)
{
	int i;

	for (i = 0; i < CUBB2_PEND_MAX; i++)
		if (!cubb_pend[i].valid)
			return i;
	return -1;
}

/* ------------------------------------------------------------------ */
/* launch decode -> request queue                                      */
/* ------------------------------------------------------------------ */

bool gsp_cuda_bridge_enabled(void)
{
	return cubb_enable && cubb_debugfs_ok;
}

/*
 * Called from svc_decode_stream on the service thread when a complete inline
 * QMD has been accumulated. Only queues the launch: marshalling guest memory
 * happens on the next service pass so the push pump is never held.
 */
void gsp_cuda_bridge_launch(struct nv_vgpu_device *vgpu, u32 h_object,
			    const u32 qmd[64], u64 qmd_dev_va, u64 push_va,
			    const struct gsp_bridge_inline *recs, u32 n_recs)
{
	unsigned long flags;
	u64 hash = 1469598103934665603ULL;
	int i;

	if (!gsp_cuda_bridge_enabled())
		return;
	(void)vgpu;

	for (i = 0; i < 64; i++) {
		hash ^= qmd[i];
		hash *= 1099511628211ULL;
	}
	hash ^= (u64)h_object << 32;
	hash *= 1099511628211ULL;
	hash ^= qmd_dev_va;

	spin_lock_irqsave(&cubb_req_lock, flags);
	for (i = 0; i < CUBB2_DEDUP; i++) {
		if (cubb_dedup[i].hash == hash &&
		    time_before(jiffies,
				cubb_dedup[i].when +
				msecs_to_jiffies(CUBB2_DEDUP_MS))) {
			spin_unlock_irqrestore(&cubb_req_lock, flags);
			cubb_dedup_hits++;
			return;
		}
	}
	cubb_dedup[cubb_dedup_next % CUBB2_DEDUP].hash = hash;
	cubb_dedup[cubb_dedup_next % CUBB2_DEDUP].when = jiffies;
	cubb_dedup_next++;

	if (cubb_req_head - cubb_req_tail >= CUBB2_REQ_MAX) {
		spin_unlock_irqrestore(&cubb_req_lock, flags);
		cubb_dropped++;
		pr_info_ratelimited("request ring full, launch dropped (push=0x%llx)\n",
				    push_va);
		return;
	}
	{
		struct cubb_req *r =
			&cubb_req[cubb_req_head % CUBB2_REQ_MAX];

		r->h_object = h_object;
		r->qmd_dev_va = qmd_dev_va;
		r->push_va = push_va;
		r->attempts = 0;
		memcpy(r->qmd, qmd, sizeof(r->qmd));
		r->n_inl = 0;
		if (recs && n_recs) {
			if (n_recs > GSP_BRIDGE_INLINE_MAX)
				n_recs = GSP_BRIDGE_INLINE_MAX;
			memcpy(r->inl, recs,
			       n_recs * sizeof(struct gsp_bridge_inline));
			r->n_inl = n_recs;
		}
	}
	cubb_req_head++;
	spin_unlock_irqrestore(&cubb_req_lock, flags);
}

/* Consume the request the marshaller was working on (service thread only). */
static void cubb_req_drop_head(void)
{
	unsigned long flags;

	spin_lock_irqsave(&cubb_req_lock, flags);
	if (cubb_req_tail != cubb_req_head)
		cubb_req_tail++;
	spin_unlock_irqrestore(&cubb_req_lock, flags);
}

/*
 * A guest VA read can miss because the targeted rm_mem scan for that page is
 * still in progress (it walks the RAM a slice per call and reports -EAGAIN ->
 * -ENOENT). Keep the request at the queue head and retry on later passes.
 */
#define CUBB2_REQ_TRIES	300

static void cubb_req_note_attempt(const struct cubb_req *r)
{
	unsigned long flags;

	spin_lock_irqsave(&cubb_req_lock, flags);
	if (cubb_req_tail != cubb_req_head)
		cubb_req[cubb_req_tail % CUBB2_REQ_MAX].attempts = r->attempts;
	spin_unlock_irqrestore(&cubb_req_lock, flags);
}

static bool cubb_req_retry(const struct cubb_req *r, int rc)
{
	if ((rc == -ENOENT || rc == -EAGAIN) &&
	    r->attempts < CUBB2_REQ_TRIES) {
		cubb_req_note_attempt(r);
		return true;
	}
	return false;
}

/* ------------------------------------------------------------------ */
/* request -> pending job (marshalling)                                */
/* ------------------------------------------------------------------ */

static void cubb_log_raw_qmd(const u32 *qmd)
{
	int i;

	pr_info("qmd raw (64 words):\n");
	for (i = 0; i < 64; i += 4)
		pr_info("qmd[%02d] %08x %08x %08x %08x\n", i,
			qmd[i], qmd[i + 1], qmd[i + 2], qmd[i + 3]);
}

/*
 * One-shot evidence dump when a launch's argument block cannot be located:
 * the cached inline-write records the search had to work with (destination,
 * length, first bytes) and the QMD's declared parameter-bank window. Decides
 * on the next run whether the cache missed the write or the shape check was
 * too strict.
 */
static u64 cubb_inline_log;

static void cubb_dump_inline(const struct cubb_req *r, u64 param_va,
			     u32 param_size)
{
	u32 i;

	if (cubb_inline_log >= 8)
		return;
	cubb_inline_log++;
	pr_info("param miss: bank=0x%llx size=0x%x push=0x%llx h=0x%x records=%u\n",
		param_va, param_size, r->push_va, r->h_object, r->n_inl);
	for (i = 0; i < r->n_inl; i++) {
		const struct gsp_bridge_inline *rec = &r->inl[i];
		u32 show = min_t(u32, rec->len, 16);

		pr_info("param miss: [%u] dst=0x%llx len=%u data16=%*phN\n",
			i, rec->dst_va, rec->len, show, rec->data);
	}
}

static int cubb_marshal_one(struct nv_vgpu_device *vgpu)
{
	/*
	 * Static: the request carries 16 inline-payload records (~1.2 KB) and
	 * this runs on the single service thread only (frame-size audit).
	 */
	static struct cubb_req r;
	struct cubb2_job job;
	unsigned long flags;
	u32 gx, gy, gz, bx, by, bz, shared;
	u64 param_va, sem_va, prog_va;
	u32 param_size, sem_payload, sem_flags, red_en, one_word, enable;
	u8 params[CUBB2_PARAM_BLOB];
	u32 plen, n = 0, opcode = 0, param_count = 0;
	u32 hint = 0, found_kind = 0;
	u64 params_off = 0;
	const char *params_src = "bank+0";
	u64 dst_va = 0, in_bytes = 0, out_bytes = 0;
	u64 dst_pa = 0;
	u32 dst_ap = 0;
	u8 *in = NULL;
	int rc, slot;

	spin_lock_irqsave(&cubb_req_lock, flags);
	if (cubb_req_tail == cubb_req_head) {
		spin_unlock_irqrestore(&cubb_req_lock, flags);
		return 0;
	}
	r = cubb_req[cubb_req_tail % CUBB2_REQ_MAX];
	r.attempts++;
	spin_unlock_irqrestore(&cubb_req_lock, flags);

	gx = r.qmd[12];
	gy = r.qmd[13] & 0xffff;
	gz = r.qmd[14] & 0xffff;
	shared = r.qmd[17] & 0x3ffff;
	bx = r.qmd[18] >> 16;
	by = r.qmd[19] & 0xffff;
	bz = r.qmd[19] >> 16;
	enable = (r.qmd[4] >> 10) & 1;
	sem_va = r.qmd[23] | ((u64)(r.qmd[24] & 0xff) << 32);
	red_en = (r.qmd[24] >> 26) & 1;
	one_word = (r.qmd[24] >> 31) & 1;
	sem_payload = r.qmd[25];
	param_va = r.qmd[32] | ((u64)(r.qmd[33] & 0x1ffff) << 32);
	param_size = (r.qmd[33] >> 19) << 4;
	prog_va = r.qmd[48] | ((u64)(r.qmd[49] & 0x1ffff) << 32);
	sem_flags = (red_en ? 1u : 0u) | (one_word ? 2u : 0u) |
		    (enable ? 4u : 0u);

	/*
	 * The parameter bank lives in the client's VA space, and CUDA writes
	 * the arguments into it with GRCE inline writes at an arbitrary
	 * offset (observed: mail args at bank+0x160, bank+0 unrelated). The
	 * frozen grid/block shapes hint at the kernel; locate the actual
	 * argument block by recent inline-write destination, then by shape
	 * scan of the bank.
	 */
	plen = sizeof(params);
	if (gx == 1 && gy == 1 && gz == 1 && bx == 64 && by == 1 && bz == 1)
		hint = 1;
	else if (gx == 4096 && gy == 1 && gz == 1 && bx == 256 && by == 1 &&
		 bz == 1)
		hint = 2;

	rc = cubb_find_params(vgpu, param_va, param_size, r.inl,
			      r.n_inl, hint, &params_off, &found_kind,
			      params, plen, &params_src);
	if (rc) {
		if (cubb_req_retry(&r, rc))
			return 0;
		cubb_dump_inline(&r, param_va, param_size);
		if (rc != -ENODATA) {
			cubb_req_drop_head();
			cubb_failed++;
			cubb_last_err = rc;
			pr_info_ratelimited("failed seq=0 param locate va=0x%llx size=0x%x rc=%d push=0x%llx h=0x%x\n",
					    param_va, param_size, rc, r.push_va,
					    r.h_object);
			return 0;
		}
		/* No argument shape found: legacy bank+0 read. */
		rc = gsp_sema_service_va_read(vgpu, param_va, params, plen);
		if (rc) {
			if (cubb_req_retry(&r, rc))
				return 0;
			cubb_req_drop_head();
			cubb_failed++;
			cubb_last_err = rc;
			pr_info_ratelimited("failed seq=0 op=? param bank va=0x%llx size=0x%x rc=%d push=0x%llx h=0x%x\n",
					    param_va, param_size, rc, r.push_va,
					    r.h_object);
			return 0;
		}
		params_off = 0;
		found_kind = 0;
		params_src = "bank+0";
	}

	/*
	 * Classify: the two frozen launch shapes first; then the located
	 * argument block; otherwise the old content heuristic.
	 */
	if (hint)
		opcode = hint;
	else if (found_kind)
		opcode = found_kind;
	else if (cubb_va_plausible(cubb_rd64(params)) &&
		 cubb_va_plausible(cubb_rd64(params + 8)) &&
		 cubb_va_plausible(cubb_rd64(params + 16)) &&
		 cubb_rd32(params + 24) &&
		 cubb_rd32(params + 24) <= (64u << 20))
		opcode = 2;
	else
		opcode = 1;

	if (opcode == 1) {
		/* mail: {ptr, len} */
		dst_va = cubb_rd64(params);
		n = cubb_rd32(params + 8);
		out_bytes = n;
		in_bytes = 0;
		param_count = 2;
		if (n > 4096) {
			cubb_req_drop_head();
			cubb_failed++;
			cubb_last_err = -E2BIG;
			pr_info_ratelimited("failed seq=0 op=mail len %u too large (dst=0x%llx)\n",
					    n, dst_va);
			return 0;
		}
	} else {
		/* vadd: {a, b, c, n} (+ 5th slot) */
		u64 a = cubb_rd64(params);
		u64 b = cubb_rd64(params + 8);

		dst_va = cubb_rd64(params + 16);
		n = cubb_rd32(params + 24);
		out_bytes = 4ULL * n;
		in_bytes = 8ULL * n;
		param_count = 5;
		if (!n || in_bytes > CUBB2_IN_MAX ||
		    out_bytes > CUBB2_OUT_MAX) {
			cubb_req_drop_head();
			cubb_failed++;
			cubb_last_err = -E2BIG;
			pr_info_ratelimited("failed seq=0 op=vadd n=%u in=%llu out=%llu too large\n",
					    n, in_bytes, out_bytes);
			return 0;
		}
		in = vmalloc(in_bytes);
		if (!in) {
			cubb_req_drop_head();
			cubb_failed++;
			cubb_last_err = -ENOMEM;
			pr_info_ratelimited("failed seq=0 op=vadd in alloc %llu bytes\n",
					    in_bytes);
			return 0;
		}
		rc = gsp_sema_service_va_read(vgpu, a, in, in_bytes / 2);
		if (!rc)
			rc = gsp_sema_service_va_read(vgpu, b,
						      in + in_bytes / 2,
						      in_bytes / 2);
		if (rc) {
			vfree(in);
			if (cubb_req_retry(&r, rc))
				return 0;
			cubb_req_drop_head();
			cubb_failed++;
			cubb_last_err = rc;
			pr_info_ratelimited("failed seq=0 op=vadd src a=0x%llx b=0x%llx rc=%d\n",
					    a, b, rc);
			return 0;
		}
		/*
		 * Input-path proof: hash exactly what the bridge read from the
		 * guest's device VAs (FNV-1a over the raw bytes) plus a float
		 * NaN count per half, so the guest-side data and the daemon's
		 * received data can be compared without guessing.
		 */
		{
			uint64_t h = 1469598103934665603ULL;
			u32 k, nan_a = 0, nan_b = 0;
			const u32 *w = (const u32 *)in;

			for (k = 0; k < in_bytes / 4; k++) {
				h ^= w[k];
				h *= 1099511628211ULL;
			}
			for (k = 0; k < in_bytes / 8; k++) {
				u32 wa = w[k];
				u32 wb = w[in_bytes / 8 + k];

				if ((wa & 0x7f800000u) == 0x7f800000u &&
				    (wa & 0x007fffffu))
					nan_a++;
				if ((wb & 0x7f800000u) == 0x7f800000u &&
				    (wb & 0x007fffffu))
					nan_b++;
			}
			pr_info("bridge-in: a=0x%llx b=0x%llx bytes=%llu hash=%016llx nan_a=%u nan_b=%u\n",
				a, b, in_bytes, (unsigned long long)h,
				nan_a, nan_b);
		}
	}

	if (!n || !out_bytes || out_bytes > CUBB2_OUT_MAX) {
		vfree(in);
		cubb_req_drop_head();
		cubb_failed++;
		cubb_last_err = -EINVAL;
		pr_info_ratelimited("failed seq=0 op=%u bad sizes n=%u out=%llu\n",
				    opcode, n, out_bytes);
		return 0;
	}

	if (opcode == 1)
		pr_info_ratelimited("params off=0x%llx kind=mail ptr=0x%llx len/n=%u\n",
				    params_off, dst_va, n);
	else
		pr_info_ratelimited("params off=0x%llx kind=vadd ptr=0x%llx len/n=%u\n",
				    params_off, dst_va, n);

	rc = gsp_sema_service_va_resolve(vgpu, dst_va, &dst_pa, &dst_ap);
	if (rc && !cubb_explained) {
		cubb_explained = true;
		gsp_sema_service_va_explain(vgpu, dst_va);
	}

	memset(&job, 0, sizeof(job));
	job.magic = CUBB2_JOB_MAGIC;
	job.version = CUBB2_VERSION;
	job.opcode = opcode;
	job.grid[0] = gx;
	job.grid[1] = gy;
	job.grid[2] = gz;
	job.block[0] = bx;
	job.block[1] = by;
	job.block[2] = bz;
	job.shared = shared;
	job.n = n;
	job.in_bytes = in_bytes;
	job.out_bytes = out_bytes;
	job.sem_va = sem_va;
	job.sem_payload = sem_payload;
	job.sem_flags = sem_flags;
	job.prog_va = prog_va;
	job.param_count = param_count;

	mutex_lock(&cubb_lock);
	slot = cubb_pend_slot();
	if (slot < 0) {
		mutex_unlock(&cubb_lock);
		vfree(in);
		cubb_req_drop_head();
		cubb_dropped++;
		pr_info_ratelimited("pending queue full, launch dropped\n");
		return 0;
	}
	job.seq = ++cubb_seq;
	cubb_pend[slot].valid = true;
	cubb_pend[slot].seq = job.seq;
	cubb_pend[slot].h_object = r.h_object;
	cubb_pend[slot].opcode = opcode;
	cubb_pend[slot].dst_va = dst_va;
	cubb_pend[slot].out_bytes = out_bytes;
	cubb_pend[slot].sem_va = sem_va;
	cubb_pend[slot].sem_payload = sem_payload;
	cubb_pend[slot].sem_flags = sem_flags;
	cubb_pend[slot].job = job;
	cubb_pend[slot].in = in;
	/* A reused slot must never inherit the previous job's staged done. */
	cubb_pend[slot].have_done = false;
	cubb_pend[slot].done_out = NULL;
	cubb_pend[slot].done_out_len = 0;
	cubb_pend[slot].done_tries = 0;
	memset(&cubb_pend[slot].done, 0, sizeof(cubb_pend[slot].done));
	cubb_jobs++;
	cubb_last_seq = job.seq;
	mutex_unlock(&cubb_lock);
	cubb_req_drop_head();

	if (cubb_launch_log < 8) {
		cubb_launch_log++;
		pr_info("launch seq=%llu op=%s grid=%u,%u,%u block=%u,%u,%u shared=%u n=%u in=%llu out=%llu sem=0x%llx pay=%u flags=0x%x param=0x%llx psize=0x%x poff=0x%llx psrc=%s prog=0x%llx dst=0x%llx dstpa=0x%llx dstap=%u h=0x%x push=0x%llx qmddev=0x%llx\n",
			job.seq, opcode == 1 ? "mail" : "vadd",
			gx, gy, gz, bx, by, bz, shared, n, in_bytes,
			out_bytes, sem_va, sem_payload, sem_flags, param_va,
			param_size, params_off, params_src, prog_va, dst_va,
			dst_pa, dst_ap, r.h_object, r.push_va, r.qmd_dev_va);
		if (gsp_sema_dbg) {
			cubb_log_raw_qmd(r.qmd);
			print_hex_dump(KERN_INFO, "cuda-bridge: params ",
				       DUMP_PREFIX_OFFSET, 16, 1, params,
				       plen, false);
		}
	} else {
		pr_info_ratelimited("launch seq=%llu op=%s n=%u out=%llu\n",
				    job.seq, opcode == 1 ? "mail" : "vadd",
				    n, out_bytes);
	}
	return 0;
}

/* ------------------------------------------------------------------ */
/* done records -> completion                                          */
/* ------------------------------------------------------------------ */

static bool cubb_done_pop(struct cubb2_done *out)
{
	unsigned long flags;
	bool have = false;

	spin_lock_irqsave(&cubb_done_lock, flags);
	if (cubb_done_tail != cubb_done_head) {
		*out = cubb_done_ring[cubb_done_tail % CUBB2_DONE_RING];
		cubb_done_tail++;
		have = true;
	}
	spin_unlock_irqrestore(&cubb_done_lock, flags);
	return have;
}

/* How long a staged done record waits for its destination translation (and a
 * consistent read-back) before the job is declared failed. The service thread
 * passes frequently, so a 300 s bound is tens of thousands of attempts. */
#define CUBB2_FINISH_RETRY_MS	300000

/* How long completion waits for a VA translation to appear (the guest's
 * sysmem-backed device buffers are published asynchronously by the pin
 * stream) before falling back to shadow-only delivery. */
#define CUBB2_XLATE_GRACE_MS	5000

/* Release a pending slot and compact the younger entries down one slot. */
static void cubb_pend_release(struct cubb_pend *p, int idx)
{
	vfree(p->in);
	p->in = NULL;
	vfree(p->done_out);
	p->done_out = NULL;
	/*
	 * Clear the deferred-completion state together with the blob. Leaving
	 * have_done/done_out_len set on a freed slot is fatal: the next job
	 * staged into the same slot then looks like it already has a staged
	 * done record and the completion path copies from a NULL blob
	 * (observed: svc_vas_write NULL deref on the second launch, killing
	 * the service thread while it held cubb_lock).
	 */
	p->have_done = false;
	p->done_out_len = 0;
	p->done_tries = 0;
	p->valid = false;
	cubb_out_len = 0;
	if (idx >= 0) {
		int j = idx;

		while (j + 1 < CUBB2_PEND_MAX && cubb_pend[j + 1].valid) {
			cubb_pend[j] = cubb_pend[j + 1];
			j++;
		}
		cubb_pend[j].in = NULL;
		cubb_pend[j].done_out = NULL;
		cubb_pend[j].have_done = false;
		cubb_pend[j].done_out_len = 0;
		cubb_pend[j].done_tries = 0;
		cubb_pend[j].valid = false;
	}
}

/*
 * Stage a daemon done record: validate it against the pending job and copy the
 * output blob out of the shared accumulation buffer (so the next job's writes
 * cannot overwrite it while completion retries). Called under cubb_lock.
 */
static void cubb_stage_done(const struct cubb2_done *d)
{
	struct cubb_pend *p;
	int idx = -1;
	u64 wrote;

	p = cubb_pend_lookup(d->seq, &idx);
	if (!p) {
		cubb_stale++;
		pr_info_ratelimited("stale done seq=%llu (no pending job)\n",
				    d->seq);
		return;
	}

	if (d->status != 0) {
		cubb_failed++;
		cubb_last_err = (int)d->err_code;
		pr_info("failed seq=%llu err=%u status=%u op=%u\n",
			d->seq, d->err_code, d->status, p->opcode);
		cubb_pend_release(p, idx);
		return;
	}

	wrote = min_t(u64, p->out_bytes, cubb_out_len);
	if (wrote < p->out_bytes) {
		cubb_failed++;
		cubb_last_err = -ENODATA;
		pr_info("failed seq=%llu err=%d (daemon wrote %llu of %llu out bytes)\n",
			d->seq, -ENODATA, wrote, p->out_bytes);
		cubb_pend_release(p, idx);
		return;
	}

	p->done_out = vmalloc(wrote);
	if (!p->done_out) {
		cubb_failed++;
		cubb_last_err = -ENOMEM;
		pr_info("failed seq=%llu err=%d (done out alloc %llu bytes)\n",
			d->seq, -ENOMEM, wrote);
		cubb_pend_release(p, idx);
		return;
	}
	memcpy(p->done_out, cubb_out_buf, wrote);
	p->done_out_len = wrote;
	/*
	 * Output-path proof: hash the blob the daemon produced exactly as it
	 * arrives (before any delivery translation) so the daemon's OUTCHECK
	 * hash and the guest's read-back can be compared end to end.
	 */
	if (wrote >= (256u << 10)) {
		uint64_t h = 1469598103934665603ULL;
		u64 k;
		const u32 *w = (const u32 *)p->done_out;

		for (k = 0; k < wrote / 4; k++) {
			h ^= w[k];
			h *= 1099511628211ULL;
		}
		pr_info("bridge-out: seq=%llu bytes=%llu hash=%016llx\n",
			d->seq, wrote, (unsigned long long)h);
	}
	p->done = *d;
	p->have_done = true;
	p->done_tries = 0;
	p->done_at = jiffies;
	cubb_out_len = 0;
}

/*
 * Try to complete one staged job: write the output through the VA
 * translation, read it back, then apply the release semaphore. A translation
 * miss (-ENOENT/-EAGAIN) or a read-back mismatch (-EIO) is retried until the
 * bounded deadline (the UVM device VA's translation is learned
 * asynchronously); every other error fails immediately. Called under
 * cubb_lock; returns true when the job was released.
 */
static bool cubb_finish_try(struct nv_vgpu_device *vgpu, struct cubb_pend *p,
			    int idx)
{
	const struct cubb2_done *d = &p->done;
	u32 sem_val = 0;
	u32 sem_rc = 0;
	bool retryable;
	int rc;

	/*
	 * Defensive: a staged done record with output bytes but no blob is
	 * invalid state (a slot reused before this was fixed). Never copy
	 * from a NULL source - release the job instead of oopsing the
	 * service thread while it holds cubb_lock.
	 */
	if (!p->done_out && p->done_out_len) {
		cubb_failed++;
		cubb_last_err = -EINVAL;
		pr_info("seq=%llu invalid staged done (out=%llu, no blob), dropping\n",
			d->seq, p->done_out_len);
		cubb_pend_release(p, idx);
		return true;
	}

	/*
	 * The guest's read-back may be a CPU read of a guest page named by the
	 * VA's translation (UVM sysmem-backed device buffers), so prefer a
	 * translated delivery. While no translation exists, stage the bytes in
	 * the VA shadow and retry for a bounded grace window (the pin stream
	 * publishes guest mappings asynchronously); after the window fall back
	 * to shadow-only delivery so a shadow-served read still works and the
	 * failure stays visible.
	 */
	{
		u64 xpa = 0;
		u32 xap = 0;

		if (gsp_sema_service_va_translate(vgpu, p->dst_va, &xpa, &xap)) {
			(void)gsp_sema_service_va_write(vgpu, p->dst_va,
							p->done_out,
							p->done_out_len);
			if (time_before(jiffies,
					p->done_at +
					msecs_to_jiffies(CUBB2_XLATE_GRACE_MS))) {
				p->done_tries++;
				if ((p->done_tries & 0x7f) == 1)
					pr_info("seq=%llu no translation yet, staged in shadow (retry=%u)\n",
						d->seq, p->done_tries);
				return false;
			}
		}
	}

	rc = gsp_sema_service_va_write(vgpu, p->dst_va, p->done_out,
				       p->done_out_len);
	if (!rc) {
		/*
		 * Read-back check over the whole output: proves the bytes
		 * landed through the translation the guest will read (vidmem
		 * shadow or guest RAM). A missing/mismatching copy must NOT
		 * be signalled complete, or the guest reads stale data and
		 * runs on.
		 */
		u8 check[256];
		size_t off = 0;

		while (off < p->done_out_len) {
			size_t clen = min_t(size_t,
					    p->done_out_len - off,
					    sizeof(check));

			if (gsp_sema_service_va_read(vgpu, p->dst_va + off,
						     check, clen) ||
			    memcmp(check, p->done_out + off, clen)) {
				rc = -EIO;
				break;
			}
			off += clen;
		}
	}
	if (!rc && (p->sem_flags & 4u) && p->sem_va) {
		sem_val = p->sem_payload;
		if (p->sem_flags & 1u) {
			u32 cur = 0;

			if (!gsp_sema_service_sem_read(vgpu, p->sem_va, &cur))
				sem_val = cur + p->sem_payload;
		}
		sem_rc = gsp_sema_service_sem_write(vgpu, p->h_object,
						    p->sem_va, sem_val);
		if (sem_rc)
			rc = sem_rc;
	}

	retryable = (rc == -ENOENT || rc == -EAGAIN || rc == -EIO);
	p->done_tries++;
	if (rc && retryable &&
	    time_before(jiffies,
			p->done_at + msecs_to_jiffies(CUBB2_FINISH_RETRY_MS))) {
		/* Periodic resolve diagnostic: proves whether the
		 * translation ever becomes available while we retry. */
		if ((p->done_tries & 0x7f) == 1) {
			u64 dpa = 0;
			u32 dap = 0;
			int rrc = gsp_sema_service_va_resolve(vgpu,
							      p->dst_va,
							      &dpa, &dap);

			pr_info("seq=%llu dst resolve retry=%u rc=%d pa=0x%llx ap=%u\n",
				d->seq, p->done_tries, rrc, dpa, dap);
		}
		return false;
	}

	if (rc) {
		cubb_failed++;
		cubb_last_err = rc;
		pr_info("failed seq=%llu err=%d (completion va=0x%llx len=%llu tries=%u)\n",
			d->seq, rc, p->dst_va, p->done_out_len, p->done_tries);
	} else {
		u64 dpa = 0;
		u32 dap = 0;
		u64 sempa = 0;
		u32 semap = 0;

		(void)gsp_sema_service_va_resolve(vgpu, p->dst_va, &dpa, &dap);
		if (p->sem_va)
			(void)gsp_sema_service_va_resolve(vgpu, p->sem_va,
							  &sempa, &semap);
		cubb_completed++;
		pr_info("completed seq=%llu op=%u dst=0x%llx dstpa=0x%llx dstap=%u out_bytes=%llu/%llu sem_va=0x%llx sempa=0x%llx pay=%u bad=%u kernel_ns=%llu tries=%u\n",
			d->seq, p->opcode, p->dst_va, dpa, dap,
			p->done_out_len, p->out_bytes, p->sem_va, sempa,
			sem_val, d->bad, d->kernel_ns, p->done_tries);
		/* Record how the guest reads the delivered buffer back. */
		gsp_sema_service_trace_window(10000);
	}
	cubb_pend_release(p, idx);
	return true;
}

/*
 * Service-thread hook: marshal at most one queued launch into a daemon-visible
 * job, stage any returned done records, then drive staged completions. Called
 * every service pass; a no-op when no launch has been seen.
 */
void gsp_cuda_bridge_service(struct nv_vgpu_device *vgpu)
{
	struct cubb2_done d;
	int guard = 0;
	int i;

	if (!gsp_cuda_bridge_enabled())
		return;

	(void)cubb_marshal_one(vgpu);
	while (guard++ < CUBB2_DONE_RING && cubb_done_pop(&d)) {
		if (d.magic != CUBB2_DONE_MAGIC ||
		    d.version != CUBB2_VERSION) {
			cubb_malformed++;
			continue;
		}
		mutex_lock(&cubb_lock);
		cubb_stage_done(&d);
		mutex_unlock(&cubb_lock);
	}

	mutex_lock(&cubb_lock);
	for (i = 0; i < CUBB2_PEND_MAX; i++) {
		if (cubb_pend[i].valid && cubb_pend[i].have_done &&
		    cubb_finish_try(vgpu, &cubb_pend[i], i))
			i = -1;	/* compaction shifted the entries; rescan */
	}
	mutex_unlock(&cubb_lock);
}

/* ------------------------------------------------------------------ */
/* debugfs handlers                                                    */
/* ------------------------------------------------------------------ */

static ssize_t cubb_job_read(struct file *f, char __user *ubuf, size_t count,
			     loff_t *ppos)
{
	u8 rec[CUBB2_RECORD_SIZE];
	int idx = -1;
	struct cubb_pend *p = NULL;
	size_t off = *ppos;
	size_t n;

	if (!cubb_debugfs_ok)
		return 0;
	mutex_lock(&cubb_lock);
	for (idx = 0; idx < CUBB2_PEND_MAX; idx++) {
		if (cubb_pend[idx].valid) {
			p = &cubb_pend[idx];
			break;
		}
	}
	if (!p) {
		mutex_unlock(&cubb_lock);
		return 0;
	}
	memcpy(rec, &p->job, sizeof(rec));
	mutex_unlock(&cubb_lock);

	if (off >= sizeof(rec))
		return 0;
	n = min_t(size_t, count, sizeof(rec) - off);
	if (copy_to_user(ubuf, rec + off, n))
		return -EFAULT;
	*ppos = off + n;
	return n;
}

static ssize_t cubb_in_read(struct file *f, char __user *ubuf, size_t count,
			    loff_t *ppos)
{
	struct cubb_pend *p = NULL;
	u64 in_bytes = 0;
	u8 *in = NULL;
	size_t off = *ppos;
	size_t n;
	int rc = 0;
	int idx;

	if (!cubb_debugfs_ok)
		return 0;
	mutex_lock(&cubb_lock);
	for (idx = 0; idx < CUBB2_PEND_MAX; idx++) {
		if (cubb_pend[idx].valid) {
			p = &cubb_pend[idx];
			break;
		}
	}
	if (!p) {
		mutex_unlock(&cubb_lock);
		return 0;
	}
	in = p->in;
	in_bytes = p->job.in_bytes;
	if (!in || off >= in_bytes) {
		mutex_unlock(&cubb_lock);
		return 0;
	}
	n = min_t(size_t, count, in_bytes - off);
	rc = copy_to_user(ubuf, in + off, n);
	mutex_unlock(&cubb_lock);
	if (rc)
		return -EFAULT;
	*ppos = off + n;
	return n;
}

/*
 * Accumulate the daemon's output blob. ppos 0 starts a new blob; growth is
 * amortised (double until the record fits, capped at CUBB2_OUT_MAX).
 */
static ssize_t cubb_out_write(struct file *f, const char __user *ubuf,
			      size_t count, loff_t *ppos)
{
	u64 off = *ppos;
	u64 end;
	int rc = 0;

	if (!cubb_debugfs_ok)
		return -ENODEV;
	if (!count)
		return 0;
	if (*ppos < 0)
		return -EINVAL;
	end = off + count;
	if (end > CUBB2_OUT_MAX)
		return -E2BIG;

	mutex_lock(&cubb_lock);
	if (off == 0)
		cubb_out_len = 0;
	if (end > cubb_out_alloc) {
		unsigned int want = cubb_out_alloc ? cubb_out_alloc : 4096;
		u8 *nb;

		while (want < end && want < CUBB2_OUT_MAX)
			want *= 2;
		if (want > CUBB2_OUT_MAX)
			want = CUBB2_OUT_MAX;
		nb = vmalloc(want);
		if (!nb) {
			mutex_unlock(&cubb_lock);
			return -ENOMEM;
		}
		if (cubb_out_len)
			memcpy(nb, cubb_out_buf, cubb_out_len);
		vfree(cubb_out_buf);
		cubb_out_buf = nb;
		cubb_out_alloc = want;
	}
	rc = copy_from_user(cubb_out_buf + off, ubuf, count);
	if (!rc && end > cubb_out_len)
		cubb_out_len = end;
	mutex_unlock(&cubb_lock);
	if (rc)
		return -EFAULT;
	*ppos = end;
	return count;
}

/* Accumulate a 256-byte CBD2 record (single write or ppos-chunked). */
static u8 cubb_done_stage[CUBB2_RECORD_SIZE];
static size_t cubb_done_stage_len;

static ssize_t cubb_done_write(struct file *f, const char __user *ubuf,
			       size_t count, loff_t *ppos)
{
	/*
	 * Accept both pwrite(fd, rec, 256, 0) and a long-lived fd writing
	 * records back to back: a write starting on a record boundary starts
	 * a fresh record.
	 */
	size_t off = (size_t)(*ppos % CUBB2_RECORD_SIZE);
	unsigned long flags;
	int rc;

	if (!cubb_debugfs_ok)
		return -ENODEV;
	if (!count)
		return 0;
	if (count > CUBB2_RECORD_SIZE - off)
		count = CUBB2_RECORD_SIZE - off;

	mutex_lock(&cubb_lock);
	if (off == 0)
		cubb_done_stage_len = 0;
	rc = copy_from_user(cubb_done_stage + off, ubuf, count);
	if (rc) {
		mutex_unlock(&cubb_lock);
		return -EFAULT;
	}
	if (off + count > cubb_done_stage_len)
		cubb_done_stage_len = off + count;

	if (cubb_done_stage_len >= CUBB2_RECORD_SIZE) {
		struct cubb2_done *d = (struct cubb2_done *)cubb_done_stage;

		if (d->magic == CUBB2_DONE_MAGIC &&
		    d->version == CUBB2_VERSION) {
			spin_lock_irqsave(&cubb_done_lock, flags);
			if (cubb_done_head - cubb_done_tail <
			    CUBB2_DONE_RING) {
				cubb_done_ring[cubb_done_head %
					       CUBB2_DONE_RING] = *d;
				cubb_done_head++;
			} else {
				cubb_dropped++;
			}
			spin_unlock_irqrestore(&cubb_done_lock, flags);
		} else {
			cubb_malformed++;
		}
		memset(cubb_done_stage, 0, sizeof(cubb_done_stage));
		cubb_done_stage_len = 0;
	}
	mutex_unlock(&cubb_lock);
	*ppos = off + count;
	return count;
}

static ssize_t cubb_status_read(struct file *f, char __user *ubuf,
				size_t count, loff_t *ppos)
{
	char buf[256];
	int len;

	len = scnprintf(buf, sizeof(buf),
			"jobs=%llu completed=%llu failed=%llu last_seq=%llu last_err=%d dropped=%llu stale=%llu malformed=%llu dedup=%llu\n",
			cubb_jobs, cubb_completed, cubb_failed,
			cubb_last_seq, cubb_last_err, cubb_dropped,
			cubb_stale, cubb_malformed, cubb_dedup_hits);
	return simple_read_from_buffer(ubuf, count, ppos, buf, len);
}

static const struct file_operations cubb_job_fops = {
	.owner = THIS_MODULE,
	.open = simple_open,
	.read = cubb_job_read,
	.llseek = default_llseek,
};

static const struct file_operations cubb_in_fops = {
	.owner = THIS_MODULE,
	.open = simple_open,
	.read = cubb_in_read,
	.llseek = default_llseek,
};

static const struct file_operations cubb_out_fops = {
	.owner = THIS_MODULE,
	.open = simple_open,
	.write = cubb_out_write,
	.llseek = default_llseek,
};

static const struct file_operations cubb_done_fops = {
	.owner = THIS_MODULE,
	.open = simple_open,
	.write = cubb_done_write,
	.llseek = default_llseek,
};

static const struct file_operations cubb_status_fops = {
	.owner = THIS_MODULE,
	.open = simple_open,
	.read = cubb_status_read,
	.llseek = default_llseek,
};

/* ------------------------------------------------------------------ */
/* lifecycle                                                           */
/* ------------------------------------------------------------------ */

int gsp_cuda_bridge_init(void)
{
	cubb_debugfs_init();
	return cubb_debugfs_ok ? 0 : -ENODEV;
}

void gsp_cuda_bridge_fini(void)
{
	int i;

	if (cubb_out_buf) {
		vfree(cubb_out_buf);
		cubb_out_buf = NULL;
		cubb_out_alloc = 0;
		cubb_out_len = 0;
	}
	mutex_lock(&cubb_lock);
	for (i = 0; i < CUBB2_PEND_MAX; i++) {
		vfree(cubb_pend[i].in);
		cubb_pend[i].in = NULL;
		vfree(cubb_pend[i].done_out);
		cubb_pend[i].done_out = NULL;
		cubb_pend[i].valid = false;
	}
	mutex_unlock(&cubb_lock);
	cubb_req_head = cubb_req_tail = 0;
	cubb_done_head = cubb_done_tail = 0;
	cubb_debugfs_fini();
}