/* SPDX-License-Identifier: GPL-2.0 */ /* * Host CUDA bridge transport (v2): module side of the debugfs protocol under * /sys/kernel/debug/vgsp/. The guest's queued kernel launches (inline-QMD * method runs on the GR compute channel, see gsp_sema_service.c) are turned * into CUBB2 job records; a userspace daemon executes them on the physical * GPU and returns a CBD2 done record plus the output payload. Because the * guest's CUDA buffers live in this module's emulated vidmem shadow, the * daemon cannot reach them: the module marshals the inputs out and copies * the daemon's output back into the resolved destination, then writes the * QMD release semaphore so the guest's cudaDeviceSynchronize completes. * * Files (all under the existing vgsp debugfs directory): * cuda_bridge_job 0444 module serves the oldest pending 256-byte job * cuda_bridge_in 0444 module serves that job's input blob (ppos) * cuda_bridge_out 0200 daemon writes its output blob (ppos) * cuda_bridge_done 0200 daemon writes a 256-byte CBD2 record * cuda_bridge_status 0444 text counters * * Everything is inert until a launch is detected; the whole path can be * switched off with the cuda_bridge module parameter. */ #define pr_fmt(fmt) "cuda-bridge: " fmt #include #include #include #include #include #include #include #include #include #include #include #include "nvidia-vgpu-vfio.h" #include "gsp_cuda_bridge.h" /* Records and limits (see tools/cuda-bridge/PROTOCOL.md, v2). */ #define CUBB2_JOB_MAGIC 0x32424243u /* 'CBB2' */ #define CUBB2_DONE_MAGIC 0x32444243u /* 'CBD2' */ #define CUBB2_VERSION 2 #define CUBB2_RECORD_SIZE 256 #define CUBB2_IN_MAX (32u * 1024u * 1024u) #define CUBB2_OUT_MAX (32u * 1024u * 1024u) #define CUBB2_REQ_MAX 16 #define CUBB2_PEND_MAX 4 #define CUBB2_DONE_RING 16 #define CUBB2_NAME_MAX 64 /* * Parameter-bank location. CUDA writes the launch arguments into the QMD's * parameter bank with GRCE LOAD_INLINE_DATA at an arbitrary offset (observed: * mail args at bank+0x160 while bank+0 held unrelated constants), so the * bridge searches the recorded recent inline-write destinations inside the * bank, then scans the bank itself for a mail/vadd argument shape. */ #define CUBB2_INLINE_MAX 16 #define CUBB2_PARAM_BLOB 40 #define CUBB2_PARAM_SCAN_MAX (128u * 1024u) #define CUBB2_PAGE 4096 /* * Natural-alignment layout (x86_64 kernel and daemon agree): * in_bytes at 0x38, sem_flags 0x58, prog_va 0x60, rsv at 0x70. */ struct cubb2_job { u32 magic; u32 version; u64 seq; u32 opcode; u32 grid[3]; u32 block[3]; u32 shared; u32 n; u64 in_bytes; u64 out_bytes; u64 sem_va; u64 sem_payload; u32 sem_flags; u32 pad; u64 prog_va; u32 param_count; u32 pad2; u8 rsv[0x100 - 0x70]; }; /* status 0 = ok, 1 = bad job, 2 = CUDA error, 3 = RAM/mapping error. */ struct cubb2_done { u32 magic; u32 version; u64 seq; u32 status; u32 bad; u64 checksum; u64 kernel_ns; u32 out_bytes_written; u32 err_code; char device_name[CUBB2_NAME_MAX]; u64 host_ns; u8 rsv[0x100 - 0x78]; }; static_assert(sizeof(struct cubb2_job) == CUBB2_RECORD_SIZE, "cubb2_job must be 256 bytes"); static_assert(sizeof(struct cubb2_done) == CUBB2_RECORD_SIZE, "cubb2_done must be 256 bytes"); static_assert(offsetof(struct cubb2_job, prog_va) == 0x60, "cubb2_job natural layout changed"); static_assert(offsetof(struct cubb2_done, host_ns) == 0x70, "cubb2_done natural layout changed"); /* Entry points are declared in gsp_cuda_bridge.h. */ /* Helpers exported by gsp_sema_service.c (declared here, not in the header, * so the bridge stays a leaf of the sema service). */ int gsp_sema_service_va_read(struct nv_vgpu_device *vgpu, u64 va, void *buf, size_t len); int gsp_sema_service_va_write(struct nv_vgpu_device *vgpu, u64 va, const void *buf, size_t len); int gsp_sema_service_va_resolve(struct nv_vgpu_device *vgpu, u64 va, u64 *pa, u32 *ap); int gsp_sema_service_va_translate(struct nv_vgpu_device *vgpu, u64 va, u64 *pa, u32 *ap); void gsp_sema_service_va_explain(struct nv_vgpu_device *vgpu, u64 va); void gsp_sema_service_trace_window(u32 ms); int gsp_sema_service_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val); int gsp_sema_service_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val); extern bool gsp_sema_dbg; static bool cubb_enable = true; module_param_named(cuda_bridge, cubb_enable, bool, 0644); MODULE_PARM_DESC(cuda_bridge, "Enable the host CUDA bridge launch transport (default 1)"); /* ------------------------------------------------------------------ */ /* launch requests: decode-thread producer, service-thread consumer */ /* ------------------------------------------------------------------ */ struct cubb_req { u32 h_object; u64 qmd_dev_va; u64 push_va; u32 qmd[64]; u32 attempts; u32 n_inl; struct gsp_bridge_inline inl[GSP_BRIDGE_INLINE_MAX]; }; static struct cubb_req cubb_req[CUBB2_REQ_MAX]; static u32 cubb_req_head; /* producer */ static u32 cubb_req_tail; /* consumer */ static DEFINE_SPINLOCK(cubb_req_lock); /* * A pin-batch rewind replays recently consumed GPFIFO entries, so the same * launch push is decoded again and would queue a duplicate job (victory24 * showed the same push three times). Dedupe on a content hash of the QMD: * replaying one push hashes identically, a genuine relaunch has a different * completion semaphore/payload inside the QMD. */ #define CUBB2_DEDUP 8 #define CUBB2_DEDUP_MS 5000 struct cubb_dedup { u64 hash; unsigned long when; }; static struct cubb_dedup cubb_dedup[CUBB2_DEDUP]; static u32 cubb_dedup_next; static u64 cubb_dedup_hits; /* ------------------------------------------------------------------ */ /* pending jobs (daemon-visible) and daemon write-back state */ /* ------------------------------------------------------------------ */ struct cubb_pend { bool valid; u64 seq; u32 h_object; u32 opcode; u64 dst_va; u64 out_bytes; u64 sem_va; u32 sem_payload; u32 sem_flags; struct cubb2_job job; u8 *in; /* vmalloc'd input blob */ /* * Deferred completion: a done record whose output write could not * resolve yet (the UVM device VA's translation is learned * asynchronously) is staged here and retried on later service passes * for a bounded time instead of failing the job immediately. */ bool have_done; u32 done_tries; unsigned long done_at; u64 done_out_len; u8 *done_out; /* vmalloc'd output blob */ struct cubb2_done done; }; static struct cubb_pend cubb_pend[CUBB2_PEND_MAX]; static u64 cubb_seq; static DEFINE_MUTEX(cubb_lock); /* daemon out blob, accumulated by cuda_bridge_out writes */ static u8 *cubb_out_buf; static unsigned int cubb_out_alloc; static size_t cubb_out_len; /* done ring, producer = daemon write handler, consumer = service thread */ static struct cubb2_done cubb_done_ring[CUBB2_DONE_RING]; static u32 cubb_done_head; static u32 cubb_done_tail; static DEFINE_SPINLOCK(cubb_done_lock); /* counters (plain reads from the status file) */ static u64 cubb_jobs; static u64 cubb_completed; static u64 cubb_failed; static u64 cubb_dropped; /* request ring/marshal/allocation drops */ static u64 cubb_stale; /* done records for unknown seq */ static u64 cubb_malformed; /* done records with bad magic/version */ static u64 cubb_last_seq; static bool cubb_explained; /* one-shot walk explainer fired */ static int cubb_last_err; static u64 cubb_launch_log; /* verbose header budget */ /* ------------------------------------------------------------------ */ /* debugfs */ /* ------------------------------------------------------------------ */ static bool cubb_root_owned; static bool cubb_debugfs_ok; static void cubb_debugfs_fini(void); static const struct file_operations cubb_job_fops; static const struct file_operations cubb_in_fops; static const struct file_operations cubb_out_fops; static const struct file_operations cubb_done_fops; static const struct file_operations cubb_status_fops; static struct dentry *cubb_find_root(void) { struct dentry *root = debugfs_lookup("vgsp", NULL); if (IS_ERR(root)) return NULL; return root; } static void cubb_debugfs_init(void) { struct dentry *root, *f; cubb_root_owned = false; root = cubb_find_root(); if (!root) { root = debugfs_create_dir("vgsp", NULL); if (IS_ERR_OR_NULL(root)) { pr_warn("debugfs dir vgsp unavailable\n"); return; } cubb_root_owned = true; } f = debugfs_create_file("cuda_bridge_job", 0444, root, NULL, &cubb_job_fops); if (IS_ERR_OR_NULL(f)) goto fail; f = debugfs_create_file("cuda_bridge_in", 0444, root, NULL, &cubb_in_fops); if (IS_ERR_OR_NULL(f)) goto fail; f = debugfs_create_file("cuda_bridge_out", 0200, root, NULL, &cubb_out_fops); if (IS_ERR_OR_NULL(f)) goto fail; f = debugfs_create_file("cuda_bridge_done", 0200, root, NULL, &cubb_done_fops); if (IS_ERR_OR_NULL(f)) goto fail; f = debugfs_create_file("cuda_bridge_status", 0444, root, NULL, &cubb_status_fops); if (IS_ERR_OR_NULL(f)) goto fail; cubb_debugfs_ok = true; dput(root); pr_info("debugfs ready (/sys/kernel/debug/vgsp/cuda_bridge_*)\n"); return; fail: dput(root); cubb_debugfs_fini(); pr_warn("debugfs file creation failed\n"); } static void cubb_debugfs_fini(void) { struct dentry *root; cubb_debugfs_ok = false; root = cubb_find_root(); if (!root) return; debugfs_lookup_and_remove("cuda_bridge_job", root); debugfs_lookup_and_remove("cuda_bridge_in", root); debugfs_lookup_and_remove("cuda_bridge_out", root); debugfs_lookup_and_remove("cuda_bridge_done", root); debugfs_lookup_and_remove("cuda_bridge_status", root); dput(root); if (cubb_root_owned) { debugfs_lookup_and_remove("vgsp", NULL); cubb_root_owned = false; } } /* ------------------------------------------------------------------ */ /* staging helpers */ /* ------------------------------------------------------------------ */ static u32 cubb_rd32(const u8 *p) { u32 v; memcpy(&v, p, 4); return v; } static u64 cubb_rd64(const u8 *p) { u64 v; memcpy(&v, p, 8); return v; } /* A pointer the guest could have given the GPU (user/device VA range). */ static bool cubb_va_plausible(u64 v) { return v >= 0x10000 && v < (1ULL << 49); } /* mail arguments: {u64 output ptr, u32 len}, len 1..4096. */ static bool cubb_mail_shape(const u8 *p, size_t len, u64 *ptr, u32 *n) { u64 v; u32 l; if (len < 12) return false; v = cubb_rd64(p); l = cubb_rd32(p + 8); if (!cubb_va_plausible(v) || !l || l > 4096) return false; if (ptr) *ptr = v; if (n) *n = l; return true; } /* vadd arguments: {u64 a, b, c, u32 n}, n == 1 MiB (frozen launch shape). */ static bool cubb_vadd_shape(const u8 *p, size_t len, u64 *out, u32 *n) { u64 a, b, c; u32 k; if (len < 32) return false; a = cubb_rd64(p); b = cubb_rd64(p + 8); c = cubb_rd64(p + 16); k = cubb_rd32(p + 24); if (!cubb_va_plausible(a) || !cubb_va_plausible(b) || !cubb_va_plausible(c) || k != (1u << 20)) return false; if (out) *out = c; if (n) *n = k; return true; } static int cubb_shape_kind(const u8 *p, size_t len) { if (cubb_mail_shape(p, len, NULL, NULL)) return 1; if (cubb_vadd_shape(p, len, NULL, NULL)) return 2; return 0; } /* * Locate the launch's argument block inside the QMD parameter bank. CUDA * writes the arguments with GRCE LOAD_INLINE_DATA at an arbitrary bank * offset, so never assume +0: first match the payload bytes the sema service * cached for the recent inline-write destinations that fall inside * [bank_va, bank_va + bank_size), newest first (no VA read needed: these * banks are UVM-managed VAs with no reliable translation). If the writes * were split, VA-adjacent cached records are merged and shape-scanned. Then * scan the bank itself for a mail/vadd argument shape as a fallback. `want` * names the shape implied by grid/block (0 = unknown). On a match returns 0 * and fills off/kind/params/src; -ENODATA means nothing matched and no read * failed (caller may fall back to bank+0); -ENOENT means a bank read failed * (the page resolve may still be in progress) and nothing matched, so the * caller should retry. */ static int cubb_find_params(struct nv_vgpu_device *vgpu, u64 bank_va, u32 bank_size, const struct gsp_bridge_inline *recs, u32 n_recs, u32 want, u64 *off, u32 *kind, u8 *params, size_t plen, const char **src) { u8 cand[CUBB2_PARAM_BLOB]; u8 *page; u64 hi = bank_va + bank_size; u32 i, pgoff, o, scan; bool read_fail = false; *off = 0; *kind = 0; if (bank_size < 16) return -ENODATA; /* * (a) cached inline-write payloads, newest first. The bytes are in * hand from the push decode, so a shape match needs no translation. */ for (i = 0; i < n_recs; i++) { const struct gsp_bridge_inline *r = &recs[i]; size_t need = want == 2 ? 32 : 12; size_t got; int k; if (!r->dst_va || !r->len || r->dst_va < bank_va || r->dst_va + need > hi) continue; got = min_t(size_t, r->len, sizeof(cand)); if (r->dst_va + got > hi) got = hi - r->dst_va; k = cubb_shape_kind(r->data, got); if (!k || (want && k != want)) continue; *off = r->dst_va - bank_va; *kind = k; memcpy(params, r->data, min_t(size_t, got, plen)); *src = "cache"; return 0; } /* * (a2) a large argument block can arrive as several consecutive * LOAD_INLINE_DATA runs: merge VA-adjacent cached records and scan the * merged bytes for a shape. */ for (i = 0; i < n_recs; i++) { u8 merged[CUBB2_PARAM_BLOB]; u32 mlen, j; u64 base; int k; base = recs[i].dst_va; if (!base || !recs[i].len || base < bank_va || base + 12 > hi) continue; mlen = min_t(u32, recs[i].len, sizeof(merged)); memcpy(merged, recs[i].data, mlen); for (j = 0; j < n_recs && mlen < sizeof(merged); j++) { u32 take; if (!recs[j].len || recs[j].dst_va != base + mlen) continue; take = min_t(u32, recs[j].len, sizeof(merged) - mlen); memcpy(merged + mlen, recs[j].data, take); mlen += take; } for (o = 0; o + 12 <= mlen; o += 4) { k = cubb_shape_kind(merged + o, mlen - o); if (!k || (want && k != want)) continue; *off = base + o - bank_va; *kind = k; memcpy(params, merged + o, min_t(u32, mlen - o, plen)); *src = "cache-merge"; return 0; } } /* (b) scan the bank for an argument shape (straddle-safe per page) */ scan = bank_size > CUBB2_PARAM_SCAN_MAX ? CUBB2_PARAM_SCAN_MAX : bank_size; page = vmalloc(CUBB2_PAGE + CUBB2_PARAM_BLOB); if (!page) return read_fail ? -ENOENT : -ENODATA; for (pgoff = 0; pgoff + 32 <= scan; pgoff += CUBB2_PAGE) { u32 available = scan - pgoff; u32 read_len = available > CUBB2_PAGE + CUBB2_PARAM_BLOB ? CUBB2_PAGE + CUBB2_PARAM_BLOB : available; if (gsp_sema_service_va_read(vgpu, bank_va + pgoff, page, read_len)) { read_fail = true; continue; } for (o = 0; o < available && o < CUBB2_PAGE && o + 32 <= read_len; o += 8) { int k = cubb_shape_kind(page + o, read_len - o); if (!k || (want && k != want)) continue; *off = pgoff + o; *kind = k; memcpy(params, page + o, min_t(size_t, read_len - o, plen)); *src = "scan"; vfree(page); return 0; } } vfree(page); return read_fail ? -ENOENT : -ENODATA; } static struct cubb_pend *cubb_pend_lookup(u64 seq, int *idx) { int i; for (i = 0; i < CUBB2_PEND_MAX; i++) { if (cubb_pend[i].valid && cubb_pend[i].seq == seq) { if (idx) *idx = i; return &cubb_pend[i]; } } return NULL; } static int cubb_pend_slot(void) { int i; for (i = 0; i < CUBB2_PEND_MAX; i++) if (!cubb_pend[i].valid) return i; return -1; } /* ------------------------------------------------------------------ */ /* launch decode -> request queue */ /* ------------------------------------------------------------------ */ bool gsp_cuda_bridge_enabled(void) { return cubb_enable && cubb_debugfs_ok; } /* * Called from svc_decode_stream on the service thread when a complete inline * QMD has been accumulated. Only queues the launch: marshalling guest memory * happens on the next service pass so the push pump is never held. */ void gsp_cuda_bridge_launch(struct nv_vgpu_device *vgpu, u32 h_object, const u32 qmd[64], u64 qmd_dev_va, u64 push_va, const struct gsp_bridge_inline *recs, u32 n_recs) { unsigned long flags; u64 hash = 1469598103934665603ULL; int i; if (!gsp_cuda_bridge_enabled()) return; (void)vgpu; for (i = 0; i < 64; i++) { hash ^= qmd[i]; hash *= 1099511628211ULL; } hash ^= (u64)h_object << 32; hash *= 1099511628211ULL; hash ^= qmd_dev_va; spin_lock_irqsave(&cubb_req_lock, flags); for (i = 0; i < CUBB2_DEDUP; i++) { if (cubb_dedup[i].hash == hash && time_before(jiffies, cubb_dedup[i].when + msecs_to_jiffies(CUBB2_DEDUP_MS))) { spin_unlock_irqrestore(&cubb_req_lock, flags); cubb_dedup_hits++; return; } } cubb_dedup[cubb_dedup_next % CUBB2_DEDUP].hash = hash; cubb_dedup[cubb_dedup_next % CUBB2_DEDUP].when = jiffies; cubb_dedup_next++; if (cubb_req_head - cubb_req_tail >= CUBB2_REQ_MAX) { spin_unlock_irqrestore(&cubb_req_lock, flags); cubb_dropped++; pr_info_ratelimited("request ring full, launch dropped (push=0x%llx)\n", push_va); return; } { struct cubb_req *r = &cubb_req[cubb_req_head % CUBB2_REQ_MAX]; r->h_object = h_object; r->qmd_dev_va = qmd_dev_va; r->push_va = push_va; r->attempts = 0; memcpy(r->qmd, qmd, sizeof(r->qmd)); r->n_inl = 0; if (recs && n_recs) { if (n_recs > GSP_BRIDGE_INLINE_MAX) n_recs = GSP_BRIDGE_INLINE_MAX; memcpy(r->inl, recs, n_recs * sizeof(struct gsp_bridge_inline)); r->n_inl = n_recs; } } cubb_req_head++; spin_unlock_irqrestore(&cubb_req_lock, flags); } /* Consume the request the marshaller was working on (service thread only). */ static void cubb_req_drop_head(void) { unsigned long flags; spin_lock_irqsave(&cubb_req_lock, flags); if (cubb_req_tail != cubb_req_head) cubb_req_tail++; spin_unlock_irqrestore(&cubb_req_lock, flags); } /* * A guest VA read can miss because the targeted rm_mem scan for that page is * still in progress (it walks the RAM a slice per call and reports -EAGAIN -> * -ENOENT). Keep the request at the queue head and retry on later passes. */ #define CUBB2_REQ_TRIES 300 static void cubb_req_note_attempt(const struct cubb_req *r) { unsigned long flags; spin_lock_irqsave(&cubb_req_lock, flags); if (cubb_req_tail != cubb_req_head) cubb_req[cubb_req_tail % CUBB2_REQ_MAX].attempts = r->attempts; spin_unlock_irqrestore(&cubb_req_lock, flags); } static bool cubb_req_retry(const struct cubb_req *r, int rc) { if ((rc == -ENOENT || rc == -EAGAIN) && r->attempts < CUBB2_REQ_TRIES) { cubb_req_note_attempt(r); return true; } return false; } /* ------------------------------------------------------------------ */ /* request -> pending job (marshalling) */ /* ------------------------------------------------------------------ */ static void cubb_log_raw_qmd(const u32 *qmd) { int i; pr_info("qmd raw (64 words):\n"); for (i = 0; i < 64; i += 4) pr_info("qmd[%02d] %08x %08x %08x %08x\n", i, qmd[i], qmd[i + 1], qmd[i + 2], qmd[i + 3]); } /* * One-shot evidence dump when a launch's argument block cannot be located: * the cached inline-write records the search had to work with (destination, * length, first bytes) and the QMD's declared parameter-bank window. Decides * on the next run whether the cache missed the write or the shape check was * too strict. */ static u64 cubb_inline_log; static void cubb_dump_inline(const struct cubb_req *r, u64 param_va, u32 param_size) { u32 i; if (cubb_inline_log >= 8) return; cubb_inline_log++; pr_info("param miss: bank=0x%llx size=0x%x push=0x%llx h=0x%x records=%u\n", param_va, param_size, r->push_va, r->h_object, r->n_inl); for (i = 0; i < r->n_inl; i++) { const struct gsp_bridge_inline *rec = &r->inl[i]; u32 show = min_t(u32, rec->len, 16); pr_info("param miss: [%u] dst=0x%llx len=%u data16=%*phN\n", i, rec->dst_va, rec->len, show, rec->data); } } static int cubb_marshal_one(struct nv_vgpu_device *vgpu) { /* * Static: the request carries 16 inline-payload records (~1.2 KB) and * this runs on the single service thread only (frame-size audit). */ static struct cubb_req r; struct cubb2_job job; unsigned long flags; u32 gx, gy, gz, bx, by, bz, shared; u64 param_va, sem_va, prog_va; u32 param_size, sem_payload, sem_flags, red_en, one_word, enable; u8 params[CUBB2_PARAM_BLOB]; u32 plen, n = 0, opcode = 0, param_count = 0; u32 hint = 0, found_kind = 0; u64 params_off = 0; const char *params_src = "bank+0"; u64 dst_va = 0, in_bytes = 0, out_bytes = 0; u64 dst_pa = 0; u32 dst_ap = 0; u8 *in = NULL; int rc, slot; spin_lock_irqsave(&cubb_req_lock, flags); if (cubb_req_tail == cubb_req_head) { spin_unlock_irqrestore(&cubb_req_lock, flags); return 0; } r = cubb_req[cubb_req_tail % CUBB2_REQ_MAX]; r.attempts++; spin_unlock_irqrestore(&cubb_req_lock, flags); gx = r.qmd[12]; gy = r.qmd[13] & 0xffff; gz = r.qmd[14] & 0xffff; shared = r.qmd[17] & 0x3ffff; bx = r.qmd[18] >> 16; by = r.qmd[19] & 0xffff; bz = r.qmd[19] >> 16; enable = (r.qmd[4] >> 10) & 1; sem_va = r.qmd[23] | ((u64)(r.qmd[24] & 0xff) << 32); red_en = (r.qmd[24] >> 26) & 1; one_word = (r.qmd[24] >> 31) & 1; sem_payload = r.qmd[25]; param_va = r.qmd[32] | ((u64)(r.qmd[33] & 0x1ffff) << 32); param_size = (r.qmd[33] >> 19) << 4; prog_va = r.qmd[48] | ((u64)(r.qmd[49] & 0x1ffff) << 32); sem_flags = (red_en ? 1u : 0u) | (one_word ? 2u : 0u) | (enable ? 4u : 0u); /* * The parameter bank lives in the client's VA space, and CUDA writes * the arguments into it with GRCE inline writes at an arbitrary * offset (observed: mail args at bank+0x160, bank+0 unrelated). The * frozen grid/block shapes hint at the kernel; locate the actual * argument block by recent inline-write destination, then by shape * scan of the bank. */ plen = sizeof(params); if (gx == 1 && gy == 1 && gz == 1 && bx == 64 && by == 1 && bz == 1) hint = 1; else if (gx == 4096 && gy == 1 && gz == 1 && bx == 256 && by == 1 && bz == 1) hint = 2; rc = cubb_find_params(vgpu, param_va, param_size, r.inl, r.n_inl, hint, ¶ms_off, &found_kind, params, plen, ¶ms_src); if (rc) { if (cubb_req_retry(&r, rc)) return 0; cubb_dump_inline(&r, param_va, param_size); if (rc != -ENODATA) { cubb_req_drop_head(); cubb_failed++; cubb_last_err = rc; pr_info_ratelimited("failed seq=0 param locate va=0x%llx size=0x%x rc=%d push=0x%llx h=0x%x\n", param_va, param_size, rc, r.push_va, r.h_object); return 0; } /* No argument shape found: legacy bank+0 read. */ rc = gsp_sema_service_va_read(vgpu, param_va, params, plen); if (rc) { if (cubb_req_retry(&r, rc)) return 0; cubb_req_drop_head(); cubb_failed++; cubb_last_err = rc; pr_info_ratelimited("failed seq=0 op=? param bank va=0x%llx size=0x%x rc=%d push=0x%llx h=0x%x\n", param_va, param_size, rc, r.push_va, r.h_object); return 0; } params_off = 0; found_kind = 0; params_src = "bank+0"; } /* * Classify: the two frozen launch shapes first; then the located * argument block; otherwise the old content heuristic. */ if (hint) opcode = hint; else if (found_kind) opcode = found_kind; else if (cubb_va_plausible(cubb_rd64(params)) && cubb_va_plausible(cubb_rd64(params + 8)) && cubb_va_plausible(cubb_rd64(params + 16)) && cubb_rd32(params + 24) && cubb_rd32(params + 24) <= (64u << 20)) opcode = 2; else opcode = 1; if (opcode == 1) { /* mail: {ptr, len} */ dst_va = cubb_rd64(params); n = cubb_rd32(params + 8); out_bytes = n; in_bytes = 0; param_count = 2; if (n > 4096) { cubb_req_drop_head(); cubb_failed++; cubb_last_err = -E2BIG; pr_info_ratelimited("failed seq=0 op=mail len %u too large (dst=0x%llx)\n", n, dst_va); return 0; } } else { /* vadd: {a, b, c, n} (+ 5th slot) */ u64 a = cubb_rd64(params); u64 b = cubb_rd64(params + 8); dst_va = cubb_rd64(params + 16); n = cubb_rd32(params + 24); out_bytes = 4ULL * n; in_bytes = 8ULL * n; param_count = 5; if (!n || in_bytes > CUBB2_IN_MAX || out_bytes > CUBB2_OUT_MAX) { cubb_req_drop_head(); cubb_failed++; cubb_last_err = -E2BIG; pr_info_ratelimited("failed seq=0 op=vadd n=%u in=%llu out=%llu too large\n", n, in_bytes, out_bytes); return 0; } in = vmalloc(in_bytes); if (!in) { cubb_req_drop_head(); cubb_failed++; cubb_last_err = -ENOMEM; pr_info_ratelimited("failed seq=0 op=vadd in alloc %llu bytes\n", in_bytes); return 0; } rc = gsp_sema_service_va_read(vgpu, a, in, in_bytes / 2); if (!rc) rc = gsp_sema_service_va_read(vgpu, b, in + in_bytes / 2, in_bytes / 2); if (rc) { vfree(in); if (cubb_req_retry(&r, rc)) return 0; cubb_req_drop_head(); cubb_failed++; cubb_last_err = rc; pr_info_ratelimited("failed seq=0 op=vadd src a=0x%llx b=0x%llx rc=%d\n", a, b, rc); return 0; } /* * Input-path proof: hash exactly what the bridge read from the * guest's device VAs (FNV-1a over the raw bytes) plus a float * NaN count per half, so the guest-side data and the daemon's * received data can be compared without guessing. */ { uint64_t h = 1469598103934665603ULL; u32 k, nan_a = 0, nan_b = 0; const u32 *w = (const u32 *)in; for (k = 0; k < in_bytes / 4; k++) { h ^= w[k]; h *= 1099511628211ULL; } for (k = 0; k < in_bytes / 8; k++) { u32 wa = w[k]; u32 wb = w[in_bytes / 8 + k]; if ((wa & 0x7f800000u) == 0x7f800000u && (wa & 0x007fffffu)) nan_a++; if ((wb & 0x7f800000u) == 0x7f800000u && (wb & 0x007fffffu)) nan_b++; } pr_info("bridge-in: a=0x%llx b=0x%llx bytes=%llu hash=%016llx nan_a=%u nan_b=%u\n", a, b, in_bytes, (unsigned long long)h, nan_a, nan_b); } } if (!n || !out_bytes || out_bytes > CUBB2_OUT_MAX) { vfree(in); cubb_req_drop_head(); cubb_failed++; cubb_last_err = -EINVAL; pr_info_ratelimited("failed seq=0 op=%u bad sizes n=%u out=%llu\n", opcode, n, out_bytes); return 0; } if (opcode == 1) pr_info_ratelimited("params off=0x%llx kind=mail ptr=0x%llx len/n=%u\n", params_off, dst_va, n); else pr_info_ratelimited("params off=0x%llx kind=vadd ptr=0x%llx len/n=%u\n", params_off, dst_va, n); rc = gsp_sema_service_va_resolve(vgpu, dst_va, &dst_pa, &dst_ap); if (rc && !cubb_explained) { cubb_explained = true; gsp_sema_service_va_explain(vgpu, dst_va); } memset(&job, 0, sizeof(job)); job.magic = CUBB2_JOB_MAGIC; job.version = CUBB2_VERSION; job.opcode = opcode; job.grid[0] = gx; job.grid[1] = gy; job.grid[2] = gz; job.block[0] = bx; job.block[1] = by; job.block[2] = bz; job.shared = shared; job.n = n; job.in_bytes = in_bytes; job.out_bytes = out_bytes; job.sem_va = sem_va; job.sem_payload = sem_payload; job.sem_flags = sem_flags; job.prog_va = prog_va; job.param_count = param_count; mutex_lock(&cubb_lock); slot = cubb_pend_slot(); if (slot < 0) { mutex_unlock(&cubb_lock); vfree(in); cubb_req_drop_head(); cubb_dropped++; pr_info_ratelimited("pending queue full, launch dropped\n"); return 0; } job.seq = ++cubb_seq; cubb_pend[slot].valid = true; cubb_pend[slot].seq = job.seq; cubb_pend[slot].h_object = r.h_object; cubb_pend[slot].opcode = opcode; cubb_pend[slot].dst_va = dst_va; cubb_pend[slot].out_bytes = out_bytes; cubb_pend[slot].sem_va = sem_va; cubb_pend[slot].sem_payload = sem_payload; cubb_pend[slot].sem_flags = sem_flags; cubb_pend[slot].job = job; cubb_pend[slot].in = in; /* A reused slot must never inherit the previous job's staged done. */ cubb_pend[slot].have_done = false; cubb_pend[slot].done_out = NULL; cubb_pend[slot].done_out_len = 0; cubb_pend[slot].done_tries = 0; memset(&cubb_pend[slot].done, 0, sizeof(cubb_pend[slot].done)); cubb_jobs++; cubb_last_seq = job.seq; mutex_unlock(&cubb_lock); cubb_req_drop_head(); if (cubb_launch_log < 8) { cubb_launch_log++; pr_info("launch seq=%llu op=%s grid=%u,%u,%u block=%u,%u,%u shared=%u n=%u in=%llu out=%llu sem=0x%llx pay=%u flags=0x%x param=0x%llx psize=0x%x poff=0x%llx psrc=%s prog=0x%llx dst=0x%llx dstpa=0x%llx dstap=%u h=0x%x push=0x%llx qmddev=0x%llx\n", job.seq, opcode == 1 ? "mail" : "vadd", gx, gy, gz, bx, by, bz, shared, n, in_bytes, out_bytes, sem_va, sem_payload, sem_flags, param_va, param_size, params_off, params_src, prog_va, dst_va, dst_pa, dst_ap, r.h_object, r.push_va, r.qmd_dev_va); if (gsp_sema_dbg) { cubb_log_raw_qmd(r.qmd); print_hex_dump(KERN_INFO, "cuda-bridge: params ", DUMP_PREFIX_OFFSET, 16, 1, params, plen, false); } } else { pr_info_ratelimited("launch seq=%llu op=%s n=%u out=%llu\n", job.seq, opcode == 1 ? "mail" : "vadd", n, out_bytes); } return 0; } /* ------------------------------------------------------------------ */ /* done records -> completion */ /* ------------------------------------------------------------------ */ static bool cubb_done_pop(struct cubb2_done *out) { unsigned long flags; bool have = false; spin_lock_irqsave(&cubb_done_lock, flags); if (cubb_done_tail != cubb_done_head) { *out = cubb_done_ring[cubb_done_tail % CUBB2_DONE_RING]; cubb_done_tail++; have = true; } spin_unlock_irqrestore(&cubb_done_lock, flags); return have; } /* How long a staged done record waits for its destination translation (and a * consistent read-back) before the job is declared failed. The service thread * passes frequently, so a 300 s bound is tens of thousands of attempts. */ #define CUBB2_FINISH_RETRY_MS 300000 /* How long completion waits for a VA translation to appear (the guest's * sysmem-backed device buffers are published asynchronously by the pin * stream) before falling back to shadow-only delivery. */ #define CUBB2_XLATE_GRACE_MS 5000 /* Release a pending slot and compact the younger entries down one slot. */ static void cubb_pend_release(struct cubb_pend *p, int idx) { vfree(p->in); p->in = NULL; vfree(p->done_out); p->done_out = NULL; /* * Clear the deferred-completion state together with the blob. Leaving * have_done/done_out_len set on a freed slot is fatal: the next job * staged into the same slot then looks like it already has a staged * done record and the completion path copies from a NULL blob * (observed: svc_vas_write NULL deref on the second launch, killing * the service thread while it held cubb_lock). */ p->have_done = false; p->done_out_len = 0; p->done_tries = 0; p->valid = false; cubb_out_len = 0; if (idx >= 0) { int j = idx; while (j + 1 < CUBB2_PEND_MAX && cubb_pend[j + 1].valid) { cubb_pend[j] = cubb_pend[j + 1]; j++; } cubb_pend[j].in = NULL; cubb_pend[j].done_out = NULL; cubb_pend[j].have_done = false; cubb_pend[j].done_out_len = 0; cubb_pend[j].done_tries = 0; cubb_pend[j].valid = false; } } /* * Stage a daemon done record: validate it against the pending job and copy the * output blob out of the shared accumulation buffer (so the next job's writes * cannot overwrite it while completion retries). Called under cubb_lock. */ static void cubb_stage_done(const struct cubb2_done *d) { struct cubb_pend *p; int idx = -1; u64 wrote; p = cubb_pend_lookup(d->seq, &idx); if (!p) { cubb_stale++; pr_info_ratelimited("stale done seq=%llu (no pending job)\n", d->seq); return; } if (d->status != 0) { cubb_failed++; cubb_last_err = (int)d->err_code; pr_info("failed seq=%llu err=%u status=%u op=%u\n", d->seq, d->err_code, d->status, p->opcode); cubb_pend_release(p, idx); return; } wrote = min_t(u64, p->out_bytes, cubb_out_len); if (wrote < p->out_bytes) { cubb_failed++; cubb_last_err = -ENODATA; pr_info("failed seq=%llu err=%d (daemon wrote %llu of %llu out bytes)\n", d->seq, -ENODATA, wrote, p->out_bytes); cubb_pend_release(p, idx); return; } p->done_out = vmalloc(wrote); if (!p->done_out) { cubb_failed++; cubb_last_err = -ENOMEM; pr_info("failed seq=%llu err=%d (done out alloc %llu bytes)\n", d->seq, -ENOMEM, wrote); cubb_pend_release(p, idx); return; } memcpy(p->done_out, cubb_out_buf, wrote); p->done_out_len = wrote; /* * Output-path proof: hash the blob the daemon produced exactly as it * arrives (before any delivery translation) so the daemon's OUTCHECK * hash and the guest's read-back can be compared end to end. */ if (wrote >= (256u << 10)) { uint64_t h = 1469598103934665603ULL; u64 k; const u32 *w = (const u32 *)p->done_out; for (k = 0; k < wrote / 4; k++) { h ^= w[k]; h *= 1099511628211ULL; } pr_info("bridge-out: seq=%llu bytes=%llu hash=%016llx\n", d->seq, wrote, (unsigned long long)h); } p->done = *d; p->have_done = true; p->done_tries = 0; p->done_at = jiffies; cubb_out_len = 0; } /* * Try to complete one staged job: write the output through the VA * translation, read it back, then apply the release semaphore. A translation * miss (-ENOENT/-EAGAIN) or a read-back mismatch (-EIO) is retried until the * bounded deadline (the UVM device VA's translation is learned * asynchronously); every other error fails immediately. Called under * cubb_lock; returns true when the job was released. */ static bool cubb_finish_try(struct nv_vgpu_device *vgpu, struct cubb_pend *p, int idx) { const struct cubb2_done *d = &p->done; u32 sem_val = 0; u32 sem_rc = 0; bool retryable; int rc; /* * Defensive: a staged done record with output bytes but no blob is * invalid state (a slot reused before this was fixed). Never copy * from a NULL source - release the job instead of oopsing the * service thread while it holds cubb_lock. */ if (!p->done_out && p->done_out_len) { cubb_failed++; cubb_last_err = -EINVAL; pr_info("seq=%llu invalid staged done (out=%llu, no blob), dropping\n", d->seq, p->done_out_len); cubb_pend_release(p, idx); return true; } /* * The guest's read-back may be a CPU read of a guest page named by the * VA's translation (UVM sysmem-backed device buffers), so prefer a * translated delivery. While no translation exists, stage the bytes in * the VA shadow and retry for a bounded grace window (the pin stream * publishes guest mappings asynchronously); after the window fall back * to shadow-only delivery so a shadow-served read still works and the * failure stays visible. */ { u64 xpa = 0; u32 xap = 0; if (gsp_sema_service_va_translate(vgpu, p->dst_va, &xpa, &xap)) { (void)gsp_sema_service_va_write(vgpu, p->dst_va, p->done_out, p->done_out_len); if (time_before(jiffies, p->done_at + msecs_to_jiffies(CUBB2_XLATE_GRACE_MS))) { p->done_tries++; if ((p->done_tries & 0x7f) == 1) pr_info("seq=%llu no translation yet, staged in shadow (retry=%u)\n", d->seq, p->done_tries); return false; } } } rc = gsp_sema_service_va_write(vgpu, p->dst_va, p->done_out, p->done_out_len); if (!rc) { /* * Read-back check over the whole output: proves the bytes * landed through the translation the guest will read (vidmem * shadow or guest RAM). A missing/mismatching copy must NOT * be signalled complete, or the guest reads stale data and * runs on. */ u8 check[256]; size_t off = 0; while (off < p->done_out_len) { size_t clen = min_t(size_t, p->done_out_len - off, sizeof(check)); if (gsp_sema_service_va_read(vgpu, p->dst_va + off, check, clen) || memcmp(check, p->done_out + off, clen)) { rc = -EIO; break; } off += clen; } } if (!rc && (p->sem_flags & 4u) && p->sem_va) { sem_val = p->sem_payload; if (p->sem_flags & 1u) { u32 cur = 0; if (!gsp_sema_service_sem_read(vgpu, p->sem_va, &cur)) sem_val = cur + p->sem_payload; } sem_rc = gsp_sema_service_sem_write(vgpu, p->h_object, p->sem_va, sem_val); if (sem_rc) rc = sem_rc; } retryable = (rc == -ENOENT || rc == -EAGAIN || rc == -EIO); p->done_tries++; if (rc && retryable && time_before(jiffies, p->done_at + msecs_to_jiffies(CUBB2_FINISH_RETRY_MS))) { /* Periodic resolve diagnostic: proves whether the * translation ever becomes available while we retry. */ if ((p->done_tries & 0x7f) == 1) { u64 dpa = 0; u32 dap = 0; int rrc = gsp_sema_service_va_resolve(vgpu, p->dst_va, &dpa, &dap); pr_info("seq=%llu dst resolve retry=%u rc=%d pa=0x%llx ap=%u\n", d->seq, p->done_tries, rrc, dpa, dap); } return false; } if (rc) { cubb_failed++; cubb_last_err = rc; pr_info("failed seq=%llu err=%d (completion va=0x%llx len=%llu tries=%u)\n", d->seq, rc, p->dst_va, p->done_out_len, p->done_tries); } else { u64 dpa = 0; u32 dap = 0; u64 sempa = 0; u32 semap = 0; (void)gsp_sema_service_va_resolve(vgpu, p->dst_va, &dpa, &dap); if (p->sem_va) (void)gsp_sema_service_va_resolve(vgpu, p->sem_va, &sempa, &semap); cubb_completed++; pr_info("completed seq=%llu op=%u dst=0x%llx dstpa=0x%llx dstap=%u out_bytes=%llu/%llu sem_va=0x%llx sempa=0x%llx pay=%u bad=%u kernel_ns=%llu tries=%u\n", d->seq, p->opcode, p->dst_va, dpa, dap, p->done_out_len, p->out_bytes, p->sem_va, sempa, sem_val, d->bad, d->kernel_ns, p->done_tries); /* Record how the guest reads the delivered buffer back. */ gsp_sema_service_trace_window(10000); } cubb_pend_release(p, idx); return true; } /* * Service-thread hook: marshal at most one queued launch into a daemon-visible * job, stage any returned done records, then drive staged completions. Called * every service pass; a no-op when no launch has been seen. */ void gsp_cuda_bridge_service(struct nv_vgpu_device *vgpu) { struct cubb2_done d; int guard = 0; int i; if (!gsp_cuda_bridge_enabled()) return; (void)cubb_marshal_one(vgpu); while (guard++ < CUBB2_DONE_RING && cubb_done_pop(&d)) { if (d.magic != CUBB2_DONE_MAGIC || d.version != CUBB2_VERSION) { cubb_malformed++; continue; } mutex_lock(&cubb_lock); cubb_stage_done(&d); mutex_unlock(&cubb_lock); } mutex_lock(&cubb_lock); for (i = 0; i < CUBB2_PEND_MAX; i++) { if (cubb_pend[i].valid && cubb_pend[i].have_done && cubb_finish_try(vgpu, &cubb_pend[i], i)) i = -1; /* compaction shifted the entries; rescan */ } mutex_unlock(&cubb_lock); } /* ------------------------------------------------------------------ */ /* debugfs handlers */ /* ------------------------------------------------------------------ */ static ssize_t cubb_job_read(struct file *f, char __user *ubuf, size_t count, loff_t *ppos) { u8 rec[CUBB2_RECORD_SIZE]; int idx = -1; struct cubb_pend *p = NULL; size_t off = *ppos; size_t n; if (!cubb_debugfs_ok) return 0; mutex_lock(&cubb_lock); for (idx = 0; idx < CUBB2_PEND_MAX; idx++) { if (cubb_pend[idx].valid) { p = &cubb_pend[idx]; break; } } if (!p) { mutex_unlock(&cubb_lock); return 0; } memcpy(rec, &p->job, sizeof(rec)); mutex_unlock(&cubb_lock); if (off >= sizeof(rec)) return 0; n = min_t(size_t, count, sizeof(rec) - off); if (copy_to_user(ubuf, rec + off, n)) return -EFAULT; *ppos = off + n; return n; } static ssize_t cubb_in_read(struct file *f, char __user *ubuf, size_t count, loff_t *ppos) { struct cubb_pend *p = NULL; u64 in_bytes = 0; u8 *in = NULL; size_t off = *ppos; size_t n; int rc = 0; int idx; if (!cubb_debugfs_ok) return 0; mutex_lock(&cubb_lock); for (idx = 0; idx < CUBB2_PEND_MAX; idx++) { if (cubb_pend[idx].valid) { p = &cubb_pend[idx]; break; } } if (!p) { mutex_unlock(&cubb_lock); return 0; } in = p->in; in_bytes = p->job.in_bytes; if (!in || off >= in_bytes) { mutex_unlock(&cubb_lock); return 0; } n = min_t(size_t, count, in_bytes - off); rc = copy_to_user(ubuf, in + off, n); mutex_unlock(&cubb_lock); if (rc) return -EFAULT; *ppos = off + n; return n; } /* * Accumulate the daemon's output blob. ppos 0 starts a new blob; growth is * amortised (double until the record fits, capped at CUBB2_OUT_MAX). */ static ssize_t cubb_out_write(struct file *f, const char __user *ubuf, size_t count, loff_t *ppos) { u64 off = *ppos; u64 end; int rc = 0; if (!cubb_debugfs_ok) return -ENODEV; if (!count) return 0; if (*ppos < 0) return -EINVAL; end = off + count; if (end > CUBB2_OUT_MAX) return -E2BIG; mutex_lock(&cubb_lock); if (off == 0) cubb_out_len = 0; if (end > cubb_out_alloc) { unsigned int want = cubb_out_alloc ? cubb_out_alloc : 4096; u8 *nb; while (want < end && want < CUBB2_OUT_MAX) want *= 2; if (want > CUBB2_OUT_MAX) want = CUBB2_OUT_MAX; nb = vmalloc(want); if (!nb) { mutex_unlock(&cubb_lock); return -ENOMEM; } if (cubb_out_len) memcpy(nb, cubb_out_buf, cubb_out_len); vfree(cubb_out_buf); cubb_out_buf = nb; cubb_out_alloc = want; } rc = copy_from_user(cubb_out_buf + off, ubuf, count); if (!rc && end > cubb_out_len) cubb_out_len = end; mutex_unlock(&cubb_lock); if (rc) return -EFAULT; *ppos = end; return count; } /* Accumulate a 256-byte CBD2 record (single write or ppos-chunked). */ static u8 cubb_done_stage[CUBB2_RECORD_SIZE]; static size_t cubb_done_stage_len; static ssize_t cubb_done_write(struct file *f, const char __user *ubuf, size_t count, loff_t *ppos) { /* * Accept both pwrite(fd, rec, 256, 0) and a long-lived fd writing * records back to back: a write starting on a record boundary starts * a fresh record. */ size_t off = (size_t)(*ppos % CUBB2_RECORD_SIZE); unsigned long flags; int rc; if (!cubb_debugfs_ok) return -ENODEV; if (!count) return 0; if (count > CUBB2_RECORD_SIZE - off) count = CUBB2_RECORD_SIZE - off; mutex_lock(&cubb_lock); if (off == 0) cubb_done_stage_len = 0; rc = copy_from_user(cubb_done_stage + off, ubuf, count); if (rc) { mutex_unlock(&cubb_lock); return -EFAULT; } if (off + count > cubb_done_stage_len) cubb_done_stage_len = off + count; if (cubb_done_stage_len >= CUBB2_RECORD_SIZE) { struct cubb2_done *d = (struct cubb2_done *)cubb_done_stage; if (d->magic == CUBB2_DONE_MAGIC && d->version == CUBB2_VERSION) { spin_lock_irqsave(&cubb_done_lock, flags); if (cubb_done_head - cubb_done_tail < CUBB2_DONE_RING) { cubb_done_ring[cubb_done_head % CUBB2_DONE_RING] = *d; cubb_done_head++; } else { cubb_dropped++; } spin_unlock_irqrestore(&cubb_done_lock, flags); } else { cubb_malformed++; } memset(cubb_done_stage, 0, sizeof(cubb_done_stage)); cubb_done_stage_len = 0; } mutex_unlock(&cubb_lock); *ppos = off + count; return count; } static ssize_t cubb_status_read(struct file *f, char __user *ubuf, size_t count, loff_t *ppos) { char buf[256]; int len; len = scnprintf(buf, sizeof(buf), "jobs=%llu completed=%llu failed=%llu last_seq=%llu last_err=%d dropped=%llu stale=%llu malformed=%llu dedup=%llu\n", cubb_jobs, cubb_completed, cubb_failed, cubb_last_seq, cubb_last_err, cubb_dropped, cubb_stale, cubb_malformed, cubb_dedup_hits); return simple_read_from_buffer(ubuf, count, ppos, buf, len); } static const struct file_operations cubb_job_fops = { .owner = THIS_MODULE, .open = simple_open, .read = cubb_job_read, .llseek = default_llseek, }; static const struct file_operations cubb_in_fops = { .owner = THIS_MODULE, .open = simple_open, .read = cubb_in_read, .llseek = default_llseek, }; static const struct file_operations cubb_out_fops = { .owner = THIS_MODULE, .open = simple_open, .write = cubb_out_write, .llseek = default_llseek, }; static const struct file_operations cubb_done_fops = { .owner = THIS_MODULE, .open = simple_open, .write = cubb_done_write, .llseek = default_llseek, }; static const struct file_operations cubb_status_fops = { .owner = THIS_MODULE, .open = simple_open, .read = cubb_status_read, .llseek = default_llseek, }; /* ------------------------------------------------------------------ */ /* lifecycle */ /* ------------------------------------------------------------------ */ int gsp_cuda_bridge_init(void) { cubb_debugfs_init(); return cubb_debugfs_ok ? 0 : -ENODEV; } void gsp_cuda_bridge_fini(void) { int i; if (cubb_out_buf) { vfree(cubb_out_buf); cubb_out_buf = NULL; cubb_out_alloc = 0; cubb_out_len = 0; } mutex_lock(&cubb_lock); for (i = 0; i < CUBB2_PEND_MAX; i++) { vfree(cubb_pend[i].in); cubb_pend[i].in = NULL; vfree(cubb_pend[i].done_out); cubb_pend[i].done_out = NULL; cubb_pend[i].valid = false; } mutex_unlock(&cubb_lock); cubb_req_head = cubb_req_tail = 0; cubb_done_head = cubb_done_tail = 0; cubb_debugfs_fini(); }