diff options
Diffstat (limited to 'kernel/gsp_cuda_bridge.c')
| -rw-r--r-- | kernel/gsp_cuda_bridge.c | 1328 |
1 files changed, 1328 insertions, 0 deletions
diff --git a/kernel/gsp_cuda_bridge.c b/kernel/gsp_cuda_bridge.c new file mode 100644 index 0000000..aaad5ee --- /dev/null +++ b/kernel/gsp_cuda_bridge.c @@ -0,0 +1,1328 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Host CUDA bridge transport (v2): module side of the debugfs protocol under + * /sys/kernel/debug/vgsp/. The guest's queued kernel launches (inline-QMD + * method runs on the GR compute channel, see gsp_sema_service.c) are turned + * into CUBB2 job records; a userspace daemon executes them on the physical + * GPU and returns a CBD2 done record plus the output payload. Because the + * guest's CUDA buffers live in this module's emulated vidmem shadow, the + * daemon cannot reach them: the module marshals the inputs out and copies + * the daemon's output back into the resolved destination, then writes the + * QMD release semaphore so the guest's cudaDeviceSynchronize completes. + * + * Files (all under the existing vgsp debugfs directory): + * cuda_bridge_job 0444 module serves the oldest pending 256-byte job + * cuda_bridge_in 0444 module serves that job's input blob (ppos) + * cuda_bridge_out 0200 daemon writes its output blob (ppos) + * cuda_bridge_done 0200 daemon writes a 256-byte CBD2 record + * cuda_bridge_status 0444 text counters + * + * Everything is inert until a launch is detected; the whole path can be + * switched off with the cuda_bridge module parameter. + */ + +#define pr_fmt(fmt) "cuda-bridge: " fmt + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/debugfs.h> +#include <linux/fs.h> +#include <linux/mutex.h> +#include <linux/spinlock.h> +#include <linux/slab.h> +#include <linux/vmalloc.h> +#include <linux/uaccess.h> +#include <linux/jiffies.h> +#include <linux/string.h> + +#include "nvidia-vgpu-vfio.h" + +/* Records and limits (see tools/cuda-bridge/PROTOCOL.md, v2). */ +#define CUBB2_JOB_MAGIC 0x32424243u /* 'CBB2' */ +#define CUBB2_DONE_MAGIC 0x32444243u /* 'CBD2' */ +#define CUBB2_VERSION 2 +#define CUBB2_RECORD_SIZE 256 +#define CUBB2_IN_MAX (32u * 1024u * 1024u) +#define CUBB2_OUT_MAX (32u * 1024u * 1024u) +#define CUBB2_REQ_MAX 16 +#define CUBB2_PEND_MAX 4 +#define CUBB2_DONE_RING 16 +#define CUBB2_NAME_MAX 64 + +/* + * Parameter-bank location. CUDA writes the launch arguments into the QMD's + * parameter bank with GRCE LOAD_INLINE_DATA at an arbitrary offset (observed: + * mail args at bank+0x160 while bank+0 held unrelated constants), so the + * bridge searches the recorded recent inline-write destinations inside the + * bank, then scans the bank itself for a mail/vadd argument shape. + */ +#define CUBB2_INLINE_MAX 16 +#define CUBB2_PARAM_BLOB 40 +#define CUBB2_PARAM_SCAN_MAX (128u * 1024u) +#define CUBB2_PAGE 4096 + +/* + * Natural-alignment layout (x86_64 kernel and daemon agree): + * in_bytes at 0x38, sem_flags 0x58, prog_va 0x60, rsv at 0x70. + */ +struct cubb2_job { + u32 magic; + u32 version; + u64 seq; + u32 opcode; + u32 grid[3]; + u32 block[3]; + u32 shared; + u32 n; + u64 in_bytes; + u64 out_bytes; + u64 sem_va; + u64 sem_payload; + u32 sem_flags; + u32 pad; + u64 prog_va; + u32 param_count; + u32 pad2; + u8 rsv[0x100 - 0x70]; +}; + +/* status 0 = ok, 1 = bad job, 2 = CUDA error, 3 = RAM/mapping error. */ +struct cubb2_done { + u32 magic; + u32 version; + u64 seq; + u32 status; + u32 bad; + u64 checksum; + u64 kernel_ns; + u32 out_bytes_written; + u32 err_code; + char device_name[CUBB2_NAME_MAX]; + u64 host_ns; + u8 rsv[0x100 - 0x78]; +}; + +static_assert(sizeof(struct cubb2_job) == CUBB2_RECORD_SIZE, + "cubb2_job must be 256 bytes"); +static_assert(sizeof(struct cubb2_done) == CUBB2_RECORD_SIZE, + "cubb2_done must be 256 bytes"); +static_assert(offsetof(struct cubb2_job, prog_va) == 0x60, + "cubb2_job natural layout changed"); +static_assert(offsetof(struct cubb2_done, host_ns) == 0x70, + "cubb2_done natural layout changed"); + +/* Entry points (also declared in gsp_sema_service.c). */ +bool gsp_cuda_bridge_enabled(void); +void gsp_cuda_bridge_launch(struct nv_vgpu_device *vgpu, u32 h_object, + const u32 qmd[64], u64 qmd_dev_va, u64 push_va, + const u64 *recent_dsts, u32 n_recent_dsts); +void gsp_cuda_bridge_service(struct nv_vgpu_device *vgpu); +int gsp_cuda_bridge_init(void); +void gsp_cuda_bridge_fini(void); + +/* Helpers exported by gsp_sema_service.c (declared here, not in the header, + * so the bridge stays a leaf of the sema service). */ +int gsp_sema_service_va_read(struct nv_vgpu_device *vgpu, u64 va, void *buf, + size_t len); +int gsp_sema_service_va_write(struct nv_vgpu_device *vgpu, u64 va, + const void *buf, size_t len); +int gsp_sema_service_va_resolve(struct nv_vgpu_device *vgpu, u64 va, u64 *pa, + u32 *ap); +int gsp_sema_service_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val); +int gsp_sema_service_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, + u32 val); +extern bool gsp_sema_dbg; + +static bool cubb_enable = true; +module_param_named(cuda_bridge, cubb_enable, bool, 0644); +MODULE_PARM_DESC(cuda_bridge, + "Enable the host CUDA bridge launch transport (default 1)"); + +/* ------------------------------------------------------------------ */ +/* launch requests: decode-thread producer, service-thread consumer */ +/* ------------------------------------------------------------------ */ + +struct cubb_req { + u32 h_object; + u64 qmd_dev_va; + u64 push_va; + u32 qmd[64]; + u32 attempts; + u32 n_recent_dsts; + u64 recent_dsts[CUBB2_INLINE_MAX]; +}; + +static struct cubb_req cubb_req[CUBB2_REQ_MAX]; +static u32 cubb_req_head; /* producer */ +static u32 cubb_req_tail; /* consumer */ +static DEFINE_SPINLOCK(cubb_req_lock); + +/* + * A pin-batch rewind replays recently consumed GPFIFO entries, so the same + * launch push is decoded again and would queue a duplicate job (victory24 + * showed the same push three times). Dedupe on a content hash of the QMD: + * replaying one push hashes identically, a genuine relaunch has a different + * completion semaphore/payload inside the QMD. + */ +#define CUBB2_DEDUP 8 +#define CUBB2_DEDUP_MS 5000 +struct cubb_dedup { + u64 hash; + unsigned long when; +}; +static struct cubb_dedup cubb_dedup[CUBB2_DEDUP]; +static u32 cubb_dedup_next; +static u64 cubb_dedup_hits; + +/* ------------------------------------------------------------------ */ +/* pending jobs (daemon-visible) and daemon write-back state */ +/* ------------------------------------------------------------------ */ + +struct cubb_pend { + bool valid; + u64 seq; + u32 h_object; + u32 opcode; + u64 dst_va; + u64 out_bytes; + u64 sem_va; + u32 sem_payload; + u32 sem_flags; + struct cubb2_job job; + u8 *in; /* vmalloc'd input blob */ +}; + +static struct cubb_pend cubb_pend[CUBB2_PEND_MAX]; +static u64 cubb_seq; +static DEFINE_MUTEX(cubb_lock); + +/* daemon out blob, accumulated by cuda_bridge_out writes */ +static u8 *cubb_out_buf; +static unsigned int cubb_out_alloc; +static size_t cubb_out_len; + +/* done ring, producer = daemon write handler, consumer = service thread */ +static struct cubb2_done cubb_done_ring[CUBB2_DONE_RING]; +static u32 cubb_done_head; +static u32 cubb_done_tail; +static DEFINE_SPINLOCK(cubb_done_lock); + +/* counters (plain reads from the status file) */ +static u64 cubb_jobs; +static u64 cubb_completed; +static u64 cubb_failed; +static u64 cubb_dropped; /* request ring/marshal/allocation drops */ +static u64 cubb_stale; /* done records for unknown seq */ +static u64 cubb_malformed; /* done records with bad magic/version */ +static u64 cubb_last_seq; +static int cubb_last_err; +static u64 cubb_launch_log; /* verbose header budget */ + +/* ------------------------------------------------------------------ */ +/* debugfs */ +/* ------------------------------------------------------------------ */ + +static bool cubb_root_owned; +static bool cubb_debugfs_ok; + +static void cubb_debugfs_fini(void); +static const struct file_operations cubb_job_fops; +static const struct file_operations cubb_in_fops; +static const struct file_operations cubb_out_fops; +static const struct file_operations cubb_done_fops; +static const struct file_operations cubb_status_fops; + +static struct dentry *cubb_find_root(void) +{ + struct dentry *root = debugfs_lookup("vgsp", NULL); + + if (IS_ERR(root)) + return NULL; + return root; +} + +static void cubb_debugfs_init(void) +{ + struct dentry *root, *f; + + cubb_root_owned = false; + root = cubb_find_root(); + if (!root) { + root = debugfs_create_dir("vgsp", NULL); + if (IS_ERR_OR_NULL(root)) { + pr_warn("debugfs dir vgsp unavailable\n"); + return; + } + cubb_root_owned = true; + } + + f = debugfs_create_file("cuda_bridge_job", 0444, root, NULL, + &cubb_job_fops); + if (IS_ERR_OR_NULL(f)) + goto fail; + f = debugfs_create_file("cuda_bridge_in", 0444, root, NULL, + &cubb_in_fops); + if (IS_ERR_OR_NULL(f)) + goto fail; + f = debugfs_create_file("cuda_bridge_out", 0200, root, NULL, + &cubb_out_fops); + if (IS_ERR_OR_NULL(f)) + goto fail; + f = debugfs_create_file("cuda_bridge_done", 0200, root, NULL, + &cubb_done_fops); + if (IS_ERR_OR_NULL(f)) + goto fail; + f = debugfs_create_file("cuda_bridge_status", 0444, root, NULL, + &cubb_status_fops); + if (IS_ERR_OR_NULL(f)) + goto fail; + + cubb_debugfs_ok = true; + dput(root); + pr_info("debugfs ready (/sys/kernel/debug/vgsp/cuda_bridge_*)\n"); + return; + +fail: + dput(root); + cubb_debugfs_fini(); + pr_warn("debugfs file creation failed\n"); +} + +static void cubb_debugfs_fini(void) +{ + struct dentry *root; + + cubb_debugfs_ok = false; + root = cubb_find_root(); + if (!root) + return; + debugfs_lookup_and_remove("cuda_bridge_job", root); + debugfs_lookup_and_remove("cuda_bridge_in", root); + debugfs_lookup_and_remove("cuda_bridge_out", root); + debugfs_lookup_and_remove("cuda_bridge_done", root); + debugfs_lookup_and_remove("cuda_bridge_status", root); + dput(root); + if (cubb_root_owned) { + debugfs_lookup_and_remove("vgsp", NULL); + cubb_root_owned = false; + } +} + +/* ------------------------------------------------------------------ */ +/* staging helpers */ +/* ------------------------------------------------------------------ */ + +static u32 cubb_rd32(const u8 *p) +{ + u32 v; + + memcpy(&v, p, 4); + return v; +} + +static u64 cubb_rd64(const u8 *p) +{ + u64 v; + + memcpy(&v, p, 8); + return v; +} + +/* A pointer the guest could have given the GPU (user/device VA range). */ +static bool cubb_va_plausible(u64 v) +{ + return v >= 0x10000 && v < (1ULL << 49); +} + +/* mail arguments: {u64 output ptr, u32 len}, len 1..4096. */ +static bool cubb_mail_shape(const u8 *p, size_t len, u64 *ptr, u32 *n) +{ + u64 v; + u32 l; + + if (len < 12) + return false; + v = cubb_rd64(p); + l = cubb_rd32(p + 8); + if (!cubb_va_plausible(v) || !l || l > 4096) + return false; + if (ptr) + *ptr = v; + if (n) + *n = l; + return true; +} + +/* vadd arguments: {u64 a, b, c, u32 n}, n == 1 MiB (frozen launch shape). */ +static bool cubb_vadd_shape(const u8 *p, size_t len, u64 *out, u32 *n) +{ + u64 a, b, c; + u32 k; + + if (len < 32) + return false; + a = cubb_rd64(p); + b = cubb_rd64(p + 8); + c = cubb_rd64(p + 16); + k = cubb_rd32(p + 24); + if (!cubb_va_plausible(a) || !cubb_va_plausible(b) || + !cubb_va_plausible(c) || k != (1u << 20)) + return false; + if (out) + *out = c; + if (n) + *n = k; + return true; +} + +static int cubb_shape_kind(const u8 *p, size_t len) +{ + if (cubb_mail_shape(p, len, NULL, NULL)) + return 1; + if (cubb_vadd_shape(p, len, NULL, NULL)) + return 2; + return 0; +} + +/* + * Locate the launch's argument block inside the QMD parameter bank. CUDA + * writes the arguments with GRCE LOAD_INLINE_DATA at an arbitrary bank + * offset, so never assume +0: first try the recorded recent inline-write + * destinations that fall inside [bank_va, bank_va + bank_size), newest + * first, then scan the bank for a mail/vadd argument shape. `want` names the + * shape implied by grid/block (0 = unknown). On a match returns 0 and fills + * off/kind/params/src; -ENODATA means the bank was readable but no shape was + * found (caller may fall back to bank+0); -ENOENT means a read failed (the + * page resolve may still be in progress) and nothing matched, so the caller + * should retry. + */ +static int cubb_find_params(struct nv_vgpu_device *vgpu, u64 bank_va, + u32 bank_size, const u64 *dsts, u32 n_dsts, + u32 want, u64 *off, u32 *kind, u8 *params, + size_t plen, const char **src) +{ + u8 cand[CUBB2_PARAM_BLOB]; + u8 *page; + u64 hi = bank_va + bank_size; + u32 i, pgoff, o, scan; + bool read_fail = false; + + *off = 0; + *kind = 0; + if (bank_size < 16) + return -ENODATA; + + /* (a) most recently written inline destinations inside the bank */ + for (i = 0; i < n_dsts; i++) { + u64 va = dsts[i]; + size_t need = want == 2 ? 32 : 12; + size_t got = min_t(size_t, plen, sizeof(cand)); + int k; + + if (!va || va < bank_va || va + need > hi) + continue; + if (va + got > hi) + got = hi - va; + if (gsp_sema_service_va_read(vgpu, va, cand, got)) { + read_fail = true; + continue; + } + k = cubb_shape_kind(cand, got); + if (!k || (want && k != want)) + continue; + *off = va - bank_va; + *kind = k; + memcpy(params, cand, min_t(size_t, got, plen)); + *src = "inline"; + return 0; + } + + /* (b) scan the bank for an argument shape (straddle-safe per page) */ + scan = bank_size > CUBB2_PARAM_SCAN_MAX ? CUBB2_PARAM_SCAN_MAX : + bank_size; + page = vmalloc(CUBB2_PAGE + CUBB2_PARAM_BLOB); + if (!page) + return read_fail ? -ENOENT : -ENODATA; + for (pgoff = 0; pgoff + 32 <= scan; pgoff += CUBB2_PAGE) { + u32 available = scan - pgoff; + u32 read_len = available > CUBB2_PAGE + CUBB2_PARAM_BLOB ? + CUBB2_PAGE + CUBB2_PARAM_BLOB : available; + + if (gsp_sema_service_va_read(vgpu, bank_va + pgoff, page, + read_len)) { + read_fail = true; + continue; + } + for (o = 0; o < available && o < CUBB2_PAGE && + o + 32 <= read_len; o += 8) { + int k = cubb_shape_kind(page + o, read_len - o); + + if (!k || (want && k != want)) + continue; + *off = pgoff + o; + *kind = k; + memcpy(params, page + o, + min_t(size_t, read_len - o, plen)); + *src = "scan"; + vfree(page); + return 0; + } + } + vfree(page); + return read_fail ? -ENOENT : -ENODATA; +} + +static struct cubb_pend *cubb_pend_lookup(u64 seq, int *idx) +{ + int i; + + for (i = 0; i < CUBB2_PEND_MAX; i++) { + if (cubb_pend[i].valid && cubb_pend[i].seq == seq) { + if (idx) + *idx = i; + return &cubb_pend[i]; + } + } + return NULL; +} + +static int cubb_pend_slot(void) +{ + int i; + + for (i = 0; i < CUBB2_PEND_MAX; i++) + if (!cubb_pend[i].valid) + return i; + return -1; +} + +/* ------------------------------------------------------------------ */ +/* launch decode -> request queue */ +/* ------------------------------------------------------------------ */ + +bool gsp_cuda_bridge_enabled(void) +{ + return cubb_enable && cubb_debugfs_ok; +} + +/* + * Called from svc_decode_stream on the service thread when a complete inline + * QMD has been accumulated. Only queues the launch: marshalling guest memory + * happens on the next service pass so the push pump is never held. + */ +void gsp_cuda_bridge_launch(struct nv_vgpu_device *vgpu, u32 h_object, + const u32 qmd[64], u64 qmd_dev_va, u64 push_va, + const u64 *recent_dsts, u32 n_recent_dsts) +{ + unsigned long flags; + u64 hash = 1469598103934665603ULL; + int i; + + if (!gsp_cuda_bridge_enabled()) + return; + (void)vgpu; + + for (i = 0; i < 64; i++) { + hash ^= qmd[i]; + hash *= 1099511628211ULL; + } + hash ^= (u64)h_object << 32; + hash *= 1099511628211ULL; + hash ^= qmd_dev_va; + + spin_lock_irqsave(&cubb_req_lock, flags); + for (i = 0; i < CUBB2_DEDUP; i++) { + if (cubb_dedup[i].hash == hash && + time_before(jiffies, + cubb_dedup[i].when + + msecs_to_jiffies(CUBB2_DEDUP_MS))) { + spin_unlock_irqrestore(&cubb_req_lock, flags); + cubb_dedup_hits++; + return; + } + } + cubb_dedup[cubb_dedup_next % CUBB2_DEDUP].hash = hash; + cubb_dedup[cubb_dedup_next % CUBB2_DEDUP].when = jiffies; + cubb_dedup_next++; + + if (cubb_req_head - cubb_req_tail >= CUBB2_REQ_MAX) { + spin_unlock_irqrestore(&cubb_req_lock, flags); + cubb_dropped++; + pr_info_ratelimited("request ring full, launch dropped (push=0x%llx)\n", + push_va); + return; + } + { + struct cubb_req *r = + &cubb_req[cubb_req_head % CUBB2_REQ_MAX]; + + r->h_object = h_object; + r->qmd_dev_va = qmd_dev_va; + r->push_va = push_va; + r->attempts = 0; + memcpy(r->qmd, qmd, sizeof(r->qmd)); + r->n_recent_dsts = 0; + if (recent_dsts && n_recent_dsts) { + if (n_recent_dsts > CUBB2_INLINE_MAX) + n_recent_dsts = CUBB2_INLINE_MAX; + memcpy(r->recent_dsts, recent_dsts, + n_recent_dsts * sizeof(u64)); + r->n_recent_dsts = n_recent_dsts; + } + } + cubb_req_head++; + spin_unlock_irqrestore(&cubb_req_lock, flags); +} + +/* Consume the request the marshaller was working on (service thread only). */ +static void cubb_req_drop_head(void) +{ + unsigned long flags; + + spin_lock_irqsave(&cubb_req_lock, flags); + if (cubb_req_tail != cubb_req_head) + cubb_req_tail++; + spin_unlock_irqrestore(&cubb_req_lock, flags); +} + +/* + * A guest VA read can miss because the targeted rm_mem scan for that page is + * still in progress (it walks the RAM a slice per call and reports -EAGAIN -> + * -ENOENT). Keep the request at the queue head and retry on later passes. + */ +#define CUBB2_REQ_TRIES 300 + +static void cubb_req_note_attempt(const struct cubb_req *r) +{ + unsigned long flags; + + spin_lock_irqsave(&cubb_req_lock, flags); + if (cubb_req_tail != cubb_req_head) + cubb_req[cubb_req_tail % CUBB2_REQ_MAX].attempts = r->attempts; + spin_unlock_irqrestore(&cubb_req_lock, flags); +} + +static bool cubb_req_retry(const struct cubb_req *r, int rc) +{ + if ((rc == -ENOENT || rc == -EAGAIN) && + r->attempts < CUBB2_REQ_TRIES) { + cubb_req_note_attempt(r); + return true; + } + return false; +} + +/* ------------------------------------------------------------------ */ +/* request -> pending job (marshalling) */ +/* ------------------------------------------------------------------ */ + +static void cubb_log_raw_qmd(const u32 *qmd) +{ + int i; + + pr_info("qmd raw (64 words):\n"); + for (i = 0; i < 64; i += 4) + pr_info("qmd[%02d] %08x %08x %08x %08x\n", i, + qmd[i], qmd[i + 1], qmd[i + 2], qmd[i + 3]); +} + +static int cubb_marshal_one(struct nv_vgpu_device *vgpu) +{ + struct cubb_req r; + struct cubb2_job job; + unsigned long flags; + u32 gx, gy, gz, bx, by, bz, shared; + u64 param_va, sem_va, prog_va; + u32 param_size, sem_payload, sem_flags, red_en, one_word, enable; + u8 params[CUBB2_PARAM_BLOB]; + u32 plen, n = 0, opcode = 0, param_count = 0; + u32 hint = 0, found_kind = 0; + u64 params_off = 0; + const char *params_src = "bank+0"; + u64 dst_va = 0, in_bytes = 0, out_bytes = 0; + u64 dst_pa = 0; + u32 dst_ap = 0; + u8 *in = NULL; + int rc, slot; + + spin_lock_irqsave(&cubb_req_lock, flags); + if (cubb_req_tail == cubb_req_head) { + spin_unlock_irqrestore(&cubb_req_lock, flags); + return 0; + } + r = cubb_req[cubb_req_tail % CUBB2_REQ_MAX]; + r.attempts++; + spin_unlock_irqrestore(&cubb_req_lock, flags); + + gx = r.qmd[12]; + gy = r.qmd[13] & 0xffff; + gz = r.qmd[14] & 0xffff; + shared = r.qmd[17] & 0x3ffff; + bx = r.qmd[18] >> 16; + by = r.qmd[19] & 0xffff; + bz = r.qmd[19] >> 16; + enable = (r.qmd[4] >> 10) & 1; + sem_va = r.qmd[23] | ((u64)(r.qmd[24] & 0xff) << 32); + red_en = (r.qmd[24] >> 26) & 1; + one_word = (r.qmd[24] >> 31) & 1; + sem_payload = r.qmd[25]; + param_va = r.qmd[32] | ((u64)(r.qmd[33] & 0x1ffff) << 32); + param_size = (r.qmd[33] >> 19) << 4; + prog_va = r.qmd[48] | ((u64)(r.qmd[49] & 0x1ffff) << 32); + sem_flags = (red_en ? 1u : 0u) | (one_word ? 2u : 0u) | + (enable ? 4u : 0u); + + /* + * The parameter bank lives in the client's VA space, and CUDA writes + * the arguments into it with GRCE inline writes at an arbitrary + * offset (observed: mail args at bank+0x160, bank+0 unrelated). The + * frozen grid/block shapes hint at the kernel; locate the actual + * argument block by recent inline-write destination, then by shape + * scan of the bank. + */ + plen = sizeof(params); + if (gx == 1 && gy == 1 && gz == 1 && bx == 64 && by == 1 && bz == 1) + hint = 1; + else if (gx == 4096 && gy == 1 && gz == 1 && bx == 256 && by == 1 && + bz == 1) + hint = 2; + + rc = cubb_find_params(vgpu, param_va, param_size, r.recent_dsts, + r.n_recent_dsts, hint, ¶ms_off, &found_kind, + params, plen, ¶ms_src); + if (rc) { + if (cubb_req_retry(&r, rc)) + return 0; + if (rc != -ENODATA) { + cubb_req_drop_head(); + cubb_failed++; + cubb_last_err = rc; + pr_info_ratelimited("failed seq=0 param locate va=0x%llx size=0x%x rc=%d push=0x%llx h=0x%x\n", + param_va, param_size, rc, r.push_va, + r.h_object); + return 0; + } + /* No argument shape found: legacy bank+0 read. */ + rc = gsp_sema_service_va_read(vgpu, param_va, params, plen); + if (rc) { + if (cubb_req_retry(&r, rc)) + return 0; + cubb_req_drop_head(); + cubb_failed++; + cubb_last_err = rc; + pr_info_ratelimited("failed seq=0 op=? param bank va=0x%llx size=0x%x rc=%d push=0x%llx h=0x%x\n", + param_va, param_size, rc, r.push_va, + r.h_object); + return 0; + } + params_off = 0; + found_kind = 0; + params_src = "bank+0"; + } + + /* + * Classify: the two frozen launch shapes first; then the located + * argument block; otherwise the old content heuristic. + */ + if (hint) + opcode = hint; + else if (found_kind) + opcode = found_kind; + else if (cubb_va_plausible(cubb_rd64(params)) && + cubb_va_plausible(cubb_rd64(params + 8)) && + cubb_va_plausible(cubb_rd64(params + 16)) && + cubb_rd32(params + 24) && + cubb_rd32(params + 24) <= (64u << 20)) + opcode = 2; + else + opcode = 1; + + if (opcode == 1) { + /* mail: {ptr, len} */ + dst_va = cubb_rd64(params); + n = cubb_rd32(params + 8); + out_bytes = n; + in_bytes = 0; + param_count = 2; + if (n > 4096) { + cubb_req_drop_head(); + cubb_failed++; + cubb_last_err = -E2BIG; + pr_info_ratelimited("failed seq=0 op=mail len %u too large (dst=0x%llx)\n", + n, dst_va); + return 0; + } + } else { + /* vadd: {a, b, c, n} (+ 5th slot) */ + u64 a = cubb_rd64(params); + u64 b = cubb_rd64(params + 8); + + dst_va = cubb_rd64(params + 16); + n = cubb_rd32(params + 24); + out_bytes = 4ULL * n; + in_bytes = 8ULL * n; + param_count = 5; + if (!n || in_bytes > CUBB2_IN_MAX || + out_bytes > CUBB2_OUT_MAX) { + cubb_req_drop_head(); + cubb_failed++; + cubb_last_err = -E2BIG; + pr_info_ratelimited("failed seq=0 op=vadd n=%u in=%llu out=%llu too large\n", + n, in_bytes, out_bytes); + return 0; + } + in = vmalloc(in_bytes); + if (!in) { + cubb_req_drop_head(); + cubb_failed++; + cubb_last_err = -ENOMEM; + pr_info_ratelimited("failed seq=0 op=vadd in alloc %llu bytes\n", + in_bytes); + return 0; + } + rc = gsp_sema_service_va_read(vgpu, a, in, in_bytes / 2); + if (!rc) + rc = gsp_sema_service_va_read(vgpu, b, + in + in_bytes / 2, + in_bytes / 2); + if (rc) { + vfree(in); + if (cubb_req_retry(&r, rc)) + return 0; + cubb_req_drop_head(); + cubb_failed++; + cubb_last_err = rc; + pr_info_ratelimited("failed seq=0 op=vadd src a=0x%llx b=0x%llx rc=%d\n", + a, b, rc); + return 0; + } + } + + if (!n || !out_bytes || out_bytes > CUBB2_OUT_MAX) { + vfree(in); + cubb_req_drop_head(); + cubb_failed++; + cubb_last_err = -EINVAL; + pr_info_ratelimited("failed seq=0 op=%u bad sizes n=%u out=%llu\n", + opcode, n, out_bytes); + return 0; + } + + if (opcode == 1) + pr_info_ratelimited("params off=0x%llx kind=mail ptr=0x%llx len/n=%u\n", + params_off, dst_va, n); + else + pr_info_ratelimited("params off=0x%llx kind=vadd ptr=0x%llx len/n=%u\n", + params_off, dst_va, n); + + rc = gsp_sema_service_va_resolve(vgpu, dst_va, &dst_pa, &dst_ap); + + memset(&job, 0, sizeof(job)); + job.magic = CUBB2_JOB_MAGIC; + job.version = CUBB2_VERSION; + job.opcode = opcode; + job.grid[0] = gx; + job.grid[1] = gy; + job.grid[2] = gz; + job.block[0] = bx; + job.block[1] = by; + job.block[2] = bz; + job.shared = shared; + job.n = n; + job.in_bytes = in_bytes; + job.out_bytes = out_bytes; + job.sem_va = sem_va; + job.sem_payload = sem_payload; + job.sem_flags = sem_flags; + job.prog_va = prog_va; + job.param_count = param_count; + + mutex_lock(&cubb_lock); + slot = cubb_pend_slot(); + if (slot < 0) { + mutex_unlock(&cubb_lock); + vfree(in); + cubb_req_drop_head(); + cubb_dropped++; + pr_info_ratelimited("pending queue full, launch dropped\n"); + return 0; + } + job.seq = ++cubb_seq; + cubb_pend[slot].valid = true; + cubb_pend[slot].seq = job.seq; + cubb_pend[slot].h_object = r.h_object; + cubb_pend[slot].opcode = opcode; + cubb_pend[slot].dst_va = dst_va; + cubb_pend[slot].out_bytes = out_bytes; + cubb_pend[slot].sem_va = sem_va; + cubb_pend[slot].sem_payload = sem_payload; + cubb_pend[slot].sem_flags = sem_flags; + cubb_pend[slot].job = job; + cubb_pend[slot].in = in; + cubb_jobs++; + cubb_last_seq = job.seq; + mutex_unlock(&cubb_lock); + cubb_req_drop_head(); + + if (cubb_launch_log < 8) { + cubb_launch_log++; + pr_info("launch seq=%llu op=%s grid=%u,%u,%u block=%u,%u,%u shared=%u n=%u in=%llu out=%llu sem=0x%llx pay=%u flags=0x%x param=0x%llx psize=0x%x poff=0x%llx psrc=%s prog=0x%llx dst=0x%llx dstpa=0x%llx dstap=%u h=0x%x push=0x%llx qmddev=0x%llx\n", + job.seq, opcode == 1 ? "mail" : "vadd", + gx, gy, gz, bx, by, bz, shared, n, in_bytes, + out_bytes, sem_va, sem_payload, sem_flags, param_va, + param_size, params_off, params_src, prog_va, dst_va, + dst_pa, dst_ap, r.h_object, r.push_va, r.qmd_dev_va); + if (gsp_sema_dbg) { + cubb_log_raw_qmd(r.qmd); + print_hex_dump(KERN_INFO, "cuda-bridge: params ", + DUMP_PREFIX_OFFSET, 16, 1, params, + plen, false); + } + } else { + pr_info_ratelimited("launch seq=%llu op=%s n=%u out=%llu\n", + job.seq, opcode == 1 ? "mail" : "vadd", + n, out_bytes); + } + return 0; +} + +/* ------------------------------------------------------------------ */ +/* done records -> completion */ +/* ------------------------------------------------------------------ */ + +static bool cubb_done_pop(struct cubb2_done *out) +{ + unsigned long flags; + bool have = false; + + spin_lock_irqsave(&cubb_done_lock, flags); + if (cubb_done_tail != cubb_done_head) { + *out = cubb_done_ring[cubb_done_tail % CUBB2_DONE_RING]; + cubb_done_tail++; + have = true; + } + spin_unlock_irqrestore(&cubb_done_lock, flags); + return have; +} + +static void cubb_finish_one(struct nv_vgpu_device *vgpu, + const struct cubb2_done *d) +{ + struct cubb_pend *p; + int idx = -1; + int rc = 0; + u32 sem_val = 0; + u32 sem_rc = 0; + u64 wrote; + + mutex_lock(&cubb_lock); + p = cubb_pend_lookup(d->seq, &idx); + if (!p) { + mutex_unlock(&cubb_lock); + cubb_stale++; + pr_info_ratelimited("stale done seq=%llu (no pending job)\n", + d->seq); + return; + } + + if (d->status != 0) { + cubb_failed++; + cubb_last_err = (int)d->err_code; + pr_info("failed seq=%llu err=%u status=%u op=%u\n", + d->seq, d->err_code, d->status, p->opcode); + goto out; + } + + wrote = min_t(u64, p->out_bytes, cubb_out_len); + if (wrote < p->out_bytes) { + cubb_failed++; + cubb_last_err = -ENODATA; + pr_info("failed seq=%llu err=%d (daemon wrote %llu of %llu out bytes)\n", + d->seq, -ENODATA, wrote, p->out_bytes); + goto out; + } + + rc = gsp_sema_service_va_write(vgpu, p->dst_va, cubb_out_buf, wrote); + if (rc) { + cubb_failed++; + cubb_last_err = rc; + pr_info("failed seq=%llu err=%d (output write va=0x%llx len=%llu)\n", + d->seq, rc, p->dst_va, wrote); + goto out; + } + /* + * Read-back check over the whole output: proves the bytes landed + * through the translation the guest will read (vidmem shadow or + * guest RAM). A missing/mismatching copy must NOT be signalled + * complete, or the guest reads stale data and runs on. + */ + { + u8 check[256]; + size_t off = 0; + + while (off < wrote) { + size_t clen = min_t(size_t, wrote - off, + sizeof(check)); + + if (gsp_sema_service_va_read(vgpu, p->dst_va + off, + check, clen) || + memcmp(check, cubb_out_buf + off, clen)) { + cubb_failed++; + cubb_last_err = -EIO; + pr_info("failed seq=%llu err=%d (output read-back mismatch va=0x%llx off=%zu len=%zu)\n", + d->seq, -EIO, p->dst_va, off, clen); + goto out; + } + off += clen; + } + } + + if ((p->sem_flags & 4u) && p->sem_va) { + sem_val = p->sem_payload; + if (p->sem_flags & 1u) { + u32 cur = 0; + + if (!gsp_sema_service_sem_read(vgpu, p->sem_va, &cur)) + sem_val = cur + p->sem_payload; + } + sem_rc = gsp_sema_service_sem_write(vgpu, p->h_object, + p->sem_va, sem_val); + if (sem_rc) { + cubb_failed++; + cubb_last_err = sem_rc; + pr_info("failed seq=%llu err=%d sem_va=0x%llx pay=%u\n", + d->seq, sem_rc, p->sem_va, sem_val); + goto out; + } + } + + { + u64 dpa = 0; + u32 dap = 0; + u64 sempa = 0; + u32 semap = 0; + + (void)gsp_sema_service_va_resolve(vgpu, p->dst_va, &dpa, &dap); + if (p->sem_va) + (void)gsp_sema_service_va_resolve(vgpu, p->sem_va, + &sempa, &semap); + cubb_completed++; + pr_info("completed seq=%llu op=%u dst=0x%llx dstpa=0x%llx dstap=%u out_bytes=%llu/%llu sem_va=0x%llx sempa=0x%llx pay=%u bad=%u kernel_ns=%llu\n", + d->seq, p->opcode, p->dst_va, dpa, dap, wrote, + p->out_bytes, p->sem_va, sempa, sem_val, d->bad, + d->kernel_ns); + } + +out: + vfree(p->in); + p->in = NULL; + p->valid = false; + cubb_out_len = 0; + if (idx >= 0) { + /* compact: shift the younger entries down one slot */ + int j = idx; + + while (j + 1 < CUBB2_PEND_MAX && cubb_pend[j + 1].valid) { + cubb_pend[j] = cubb_pend[j + 1]; + j++; + } + cubb_pend[j].in = NULL; + cubb_pend[j].valid = false; + } + mutex_unlock(&cubb_lock); +} + +/* + * Service-thread hook: marshal at most one queued launch into a daemon-visible + * job, then resolve any completed jobs. Called every service pass; a no-op + * when no launch has been seen. + */ +void gsp_cuda_bridge_service(struct nv_vgpu_device *vgpu) +{ + struct cubb2_done d; + int guard = 0; + + if (!gsp_cuda_bridge_enabled()) + return; + + (void)cubb_marshal_one(vgpu); + while (guard++ < CUBB2_DONE_RING && cubb_done_pop(&d)) { + if (d.magic != CUBB2_DONE_MAGIC || + d.version != CUBB2_VERSION) { + cubb_malformed++; + continue; + } + cubb_finish_one(vgpu, &d); + } +} + +/* ------------------------------------------------------------------ */ +/* debugfs handlers */ +/* ------------------------------------------------------------------ */ + +static ssize_t cubb_job_read(struct file *f, char __user *ubuf, size_t count, + loff_t *ppos) +{ + u8 rec[CUBB2_RECORD_SIZE]; + int idx = -1; + struct cubb_pend *p = NULL; + size_t off = *ppos; + size_t n; + + if (!cubb_debugfs_ok) + return 0; + mutex_lock(&cubb_lock); + for (idx = 0; idx < CUBB2_PEND_MAX; idx++) { + if (cubb_pend[idx].valid) { + p = &cubb_pend[idx]; + break; + } + } + if (!p) { + mutex_unlock(&cubb_lock); + return 0; + } + memcpy(rec, &p->job, sizeof(rec)); + mutex_unlock(&cubb_lock); + + if (off >= sizeof(rec)) + return 0; + n = min_t(size_t, count, sizeof(rec) - off); + if (copy_to_user(ubuf, rec + off, n)) + return -EFAULT; + *ppos = off + n; + return n; +} + +static ssize_t cubb_in_read(struct file *f, char __user *ubuf, size_t count, + loff_t *ppos) +{ + struct cubb_pend *p = NULL; + u64 in_bytes = 0; + u8 *in = NULL; + size_t off = *ppos; + size_t n; + int rc = 0; + int idx; + + if (!cubb_debugfs_ok) + return 0; + mutex_lock(&cubb_lock); + for (idx = 0; idx < CUBB2_PEND_MAX; idx++) { + if (cubb_pend[idx].valid) { + p = &cubb_pend[idx]; + break; + } + } + if (!p) { + mutex_unlock(&cubb_lock); + return 0; + } + in = p->in; + in_bytes = p->job.in_bytes; + if (!in || off >= in_bytes) { + mutex_unlock(&cubb_lock); + return 0; + } + n = min_t(size_t, count, in_bytes - off); + rc = copy_to_user(ubuf, in + off, n); + mutex_unlock(&cubb_lock); + if (rc) + return -EFAULT; + *ppos = off + n; + return n; +} + +/* + * Accumulate the daemon's output blob. ppos 0 starts a new blob; growth is + * amortised (double until the record fits, capped at CUBB2_OUT_MAX). + */ +static ssize_t cubb_out_write(struct file *f, const char __user *ubuf, + size_t count, loff_t *ppos) +{ + u64 off = *ppos; + u64 end; + int rc = 0; + + if (!cubb_debugfs_ok) + return -ENODEV; + if (!count) + return 0; + if (*ppos < 0) + return -EINVAL; + end = off + count; + if (end > CUBB2_OUT_MAX) + return -E2BIG; + + mutex_lock(&cubb_lock); + if (off == 0) + cubb_out_len = 0; + if (end > cubb_out_alloc) { + unsigned int want = cubb_out_alloc ? cubb_out_alloc : 4096; + u8 *nb; + + while (want < end && want < CUBB2_OUT_MAX) + want *= 2; + if (want > CUBB2_OUT_MAX) + want = CUBB2_OUT_MAX; + nb = vmalloc(want); + if (!nb) { + mutex_unlock(&cubb_lock); + return -ENOMEM; + } + if (cubb_out_len) + memcpy(nb, cubb_out_buf, cubb_out_len); + vfree(cubb_out_buf); + cubb_out_buf = nb; + cubb_out_alloc = want; + } + rc = copy_from_user(cubb_out_buf + off, ubuf, count); + if (!rc && end > cubb_out_len) + cubb_out_len = end; + mutex_unlock(&cubb_lock); + if (rc) + return -EFAULT; + *ppos = end; + return count; +} + +/* Accumulate a 256-byte CBD2 record (single write or ppos-chunked). */ +static u8 cubb_done_stage[CUBB2_RECORD_SIZE]; +static size_t cubb_done_stage_len; + +static ssize_t cubb_done_write(struct file *f, const char __user *ubuf, + size_t count, loff_t *ppos) +{ + /* + * Accept both pwrite(fd, rec, 256, 0) and a long-lived fd writing + * records back to back: a write starting on a record boundary starts + * a fresh record. + */ + size_t off = (size_t)(*ppos % CUBB2_RECORD_SIZE); + unsigned long flags; + int rc; + + if (!cubb_debugfs_ok) + return -ENODEV; + if (!count) + return 0; + if (count > CUBB2_RECORD_SIZE - off) + count = CUBB2_RECORD_SIZE - off; + + mutex_lock(&cubb_lock); + if (off == 0) + cubb_done_stage_len = 0; + rc = copy_from_user(cubb_done_stage + off, ubuf, count); + if (rc) { + mutex_unlock(&cubb_lock); + return -EFAULT; + } + if (off + count > cubb_done_stage_len) + cubb_done_stage_len = off + count; + + if (cubb_done_stage_len >= CUBB2_RECORD_SIZE) { + struct cubb2_done *d = (struct cubb2_done *)cubb_done_stage; + + if (d->magic == CUBB2_DONE_MAGIC && + d->version == CUBB2_VERSION) { + spin_lock_irqsave(&cubb_done_lock, flags); + if (cubb_done_head - cubb_done_tail < + CUBB2_DONE_RING) { + cubb_done_ring[cubb_done_head % + CUBB2_DONE_RING] = *d; + cubb_done_head++; + } else { + cubb_dropped++; + } + spin_unlock_irqrestore(&cubb_done_lock, flags); + } else { + cubb_malformed++; + } + memset(cubb_done_stage, 0, sizeof(cubb_done_stage)); + cubb_done_stage_len = 0; + } + mutex_unlock(&cubb_lock); + *ppos = off + count; + return count; +} + +static ssize_t cubb_status_read(struct file *f, char __user *ubuf, + size_t count, loff_t *ppos) +{ + char buf[256]; + int len; + + len = scnprintf(buf, sizeof(buf), + "jobs=%llu completed=%llu failed=%llu last_seq=%llu last_err=%d dropped=%llu stale=%llu malformed=%llu dedup=%llu\n", + cubb_jobs, cubb_completed, cubb_failed, + cubb_last_seq, cubb_last_err, cubb_dropped, + cubb_stale, cubb_malformed, cubb_dedup_hits); + return simple_read_from_buffer(ubuf, count, ppos, buf, len); +} + +static const struct file_operations cubb_job_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = cubb_job_read, + .llseek = default_llseek, +}; + +static const struct file_operations cubb_in_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = cubb_in_read, + .llseek = default_llseek, +}; + +static const struct file_operations cubb_out_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = cubb_out_write, + .llseek = default_llseek, +}; + +static const struct file_operations cubb_done_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = cubb_done_write, + .llseek = default_llseek, +}; + +static const struct file_operations cubb_status_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = cubb_status_read, + .llseek = default_llseek, +}; + +/* ------------------------------------------------------------------ */ +/* lifecycle */ +/* ------------------------------------------------------------------ */ + +int gsp_cuda_bridge_init(void) +{ + cubb_debugfs_init(); + return cubb_debugfs_ok ? 0 : -ENODEV; +} + +void gsp_cuda_bridge_fini(void) +{ + int i; + + if (cubb_out_buf) { + vfree(cubb_out_buf); + cubb_out_buf = NULL; + cubb_out_alloc = 0; + cubb_out_len = 0; + } + mutex_lock(&cubb_lock); + for (i = 0; i < CUBB2_PEND_MAX; i++) { + vfree(cubb_pend[i].in); + cubb_pend[i].in = NULL; + cubb_pend[i].valid = false; + } + mutex_unlock(&cubb_lock); + cubb_req_head = cubb_req_tail = 0; + cubb_done_head = cubb_done_tail = 0; + cubb_debugfs_fini(); +} |