diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/Makefile | 1 | ||||
| -rw-r--r-- | kernel/gsp_cuda_bridge.c | 1328 | ||||
| -rw-r--r-- | kernel/gsp_phys_booter.c | 186 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.c | 34 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.h | 13 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 1731 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.h | 10 | ||||
| -rw-r--r-- | kernel/main.c | 10 |
8 files changed, 3044 insertions, 269 deletions
diff --git a/kernel/Makefile b/kernel/Makefile index c035e41..da8f240 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -8,6 +8,7 @@ vgsp-y := \ gsp_emulation.o \ gsp_rm_emulation.o \ gsp_sema_service.o \ + gsp_cuda_bridge.o \ gsp_rpc_client.o \ gsp_rpc_integration.o \ memory.o \ diff --git a/kernel/gsp_cuda_bridge.c b/kernel/gsp_cuda_bridge.c new file mode 100644 index 0000000..aaad5ee --- /dev/null +++ b/kernel/gsp_cuda_bridge.c @@ -0,0 +1,1328 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Host CUDA bridge transport (v2): module side of the debugfs protocol under + * /sys/kernel/debug/vgsp/. The guest's queued kernel launches (inline-QMD + * method runs on the GR compute channel, see gsp_sema_service.c) are turned + * into CUBB2 job records; a userspace daemon executes them on the physical + * GPU and returns a CBD2 done record plus the output payload. Because the + * guest's CUDA buffers live in this module's emulated vidmem shadow, the + * daemon cannot reach them: the module marshals the inputs out and copies + * the daemon's output back into the resolved destination, then writes the + * QMD release semaphore so the guest's cudaDeviceSynchronize completes. + * + * Files (all under the existing vgsp debugfs directory): + * cuda_bridge_job 0444 module serves the oldest pending 256-byte job + * cuda_bridge_in 0444 module serves that job's input blob (ppos) + * cuda_bridge_out 0200 daemon writes its output blob (ppos) + * cuda_bridge_done 0200 daemon writes a 256-byte CBD2 record + * cuda_bridge_status 0444 text counters + * + * Everything is inert until a launch is detected; the whole path can be + * switched off with the cuda_bridge module parameter. + */ + +#define pr_fmt(fmt) "cuda-bridge: " fmt + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/debugfs.h> +#include <linux/fs.h> +#include <linux/mutex.h> +#include <linux/spinlock.h> +#include <linux/slab.h> +#include <linux/vmalloc.h> +#include <linux/uaccess.h> +#include <linux/jiffies.h> +#include <linux/string.h> + +#include "nvidia-vgpu-vfio.h" + +/* Records and limits (see tools/cuda-bridge/PROTOCOL.md, v2). */ +#define CUBB2_JOB_MAGIC 0x32424243u /* 'CBB2' */ +#define CUBB2_DONE_MAGIC 0x32444243u /* 'CBD2' */ +#define CUBB2_VERSION 2 +#define CUBB2_RECORD_SIZE 256 +#define CUBB2_IN_MAX (32u * 1024u * 1024u) +#define CUBB2_OUT_MAX (32u * 1024u * 1024u) +#define CUBB2_REQ_MAX 16 +#define CUBB2_PEND_MAX 4 +#define CUBB2_DONE_RING 16 +#define CUBB2_NAME_MAX 64 + +/* + * Parameter-bank location. CUDA writes the launch arguments into the QMD's + * parameter bank with GRCE LOAD_INLINE_DATA at an arbitrary offset (observed: + * mail args at bank+0x160 while bank+0 held unrelated constants), so the + * bridge searches the recorded recent inline-write destinations inside the + * bank, then scans the bank itself for a mail/vadd argument shape. + */ +#define CUBB2_INLINE_MAX 16 +#define CUBB2_PARAM_BLOB 40 +#define CUBB2_PARAM_SCAN_MAX (128u * 1024u) +#define CUBB2_PAGE 4096 + +/* + * Natural-alignment layout (x86_64 kernel and daemon agree): + * in_bytes at 0x38, sem_flags 0x58, prog_va 0x60, rsv at 0x70. + */ +struct cubb2_job { + u32 magic; + u32 version; + u64 seq; + u32 opcode; + u32 grid[3]; + u32 block[3]; + u32 shared; + u32 n; + u64 in_bytes; + u64 out_bytes; + u64 sem_va; + u64 sem_payload; + u32 sem_flags; + u32 pad; + u64 prog_va; + u32 param_count; + u32 pad2; + u8 rsv[0x100 - 0x70]; +}; + +/* status 0 = ok, 1 = bad job, 2 = CUDA error, 3 = RAM/mapping error. */ +struct cubb2_done { + u32 magic; + u32 version; + u64 seq; + u32 status; + u32 bad; + u64 checksum; + u64 kernel_ns; + u32 out_bytes_written; + u32 err_code; + char device_name[CUBB2_NAME_MAX]; + u64 host_ns; + u8 rsv[0x100 - 0x78]; +}; + +static_assert(sizeof(struct cubb2_job) == CUBB2_RECORD_SIZE, + "cubb2_job must be 256 bytes"); +static_assert(sizeof(struct cubb2_done) == CUBB2_RECORD_SIZE, + "cubb2_done must be 256 bytes"); +static_assert(offsetof(struct cubb2_job, prog_va) == 0x60, + "cubb2_job natural layout changed"); +static_assert(offsetof(struct cubb2_done, host_ns) == 0x70, + "cubb2_done natural layout changed"); + +/* Entry points (also declared in gsp_sema_service.c). */ +bool gsp_cuda_bridge_enabled(void); +void gsp_cuda_bridge_launch(struct nv_vgpu_device *vgpu, u32 h_object, + const u32 qmd[64], u64 qmd_dev_va, u64 push_va, + const u64 *recent_dsts, u32 n_recent_dsts); +void gsp_cuda_bridge_service(struct nv_vgpu_device *vgpu); +int gsp_cuda_bridge_init(void); +void gsp_cuda_bridge_fini(void); + +/* Helpers exported by gsp_sema_service.c (declared here, not in the header, + * so the bridge stays a leaf of the sema service). */ +int gsp_sema_service_va_read(struct nv_vgpu_device *vgpu, u64 va, void *buf, + size_t len); +int gsp_sema_service_va_write(struct nv_vgpu_device *vgpu, u64 va, + const void *buf, size_t len); +int gsp_sema_service_va_resolve(struct nv_vgpu_device *vgpu, u64 va, u64 *pa, + u32 *ap); +int gsp_sema_service_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val); +int gsp_sema_service_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, + u32 val); +extern bool gsp_sema_dbg; + +static bool cubb_enable = true; +module_param_named(cuda_bridge, cubb_enable, bool, 0644); +MODULE_PARM_DESC(cuda_bridge, + "Enable the host CUDA bridge launch transport (default 1)"); + +/* ------------------------------------------------------------------ */ +/* launch requests: decode-thread producer, service-thread consumer */ +/* ------------------------------------------------------------------ */ + +struct cubb_req { + u32 h_object; + u64 qmd_dev_va; + u64 push_va; + u32 qmd[64]; + u32 attempts; + u32 n_recent_dsts; + u64 recent_dsts[CUBB2_INLINE_MAX]; +}; + +static struct cubb_req cubb_req[CUBB2_REQ_MAX]; +static u32 cubb_req_head; /* producer */ +static u32 cubb_req_tail; /* consumer */ +static DEFINE_SPINLOCK(cubb_req_lock); + +/* + * A pin-batch rewind replays recently consumed GPFIFO entries, so the same + * launch push is decoded again and would queue a duplicate job (victory24 + * showed the same push three times). Dedupe on a content hash of the QMD: + * replaying one push hashes identically, a genuine relaunch has a different + * completion semaphore/payload inside the QMD. + */ +#define CUBB2_DEDUP 8 +#define CUBB2_DEDUP_MS 5000 +struct cubb_dedup { + u64 hash; + unsigned long when; +}; +static struct cubb_dedup cubb_dedup[CUBB2_DEDUP]; +static u32 cubb_dedup_next; +static u64 cubb_dedup_hits; + +/* ------------------------------------------------------------------ */ +/* pending jobs (daemon-visible) and daemon write-back state */ +/* ------------------------------------------------------------------ */ + +struct cubb_pend { + bool valid; + u64 seq; + u32 h_object; + u32 opcode; + u64 dst_va; + u64 out_bytes; + u64 sem_va; + u32 sem_payload; + u32 sem_flags; + struct cubb2_job job; + u8 *in; /* vmalloc'd input blob */ +}; + +static struct cubb_pend cubb_pend[CUBB2_PEND_MAX]; +static u64 cubb_seq; +static DEFINE_MUTEX(cubb_lock); + +/* daemon out blob, accumulated by cuda_bridge_out writes */ +static u8 *cubb_out_buf; +static unsigned int cubb_out_alloc; +static size_t cubb_out_len; + +/* done ring, producer = daemon write handler, consumer = service thread */ +static struct cubb2_done cubb_done_ring[CUBB2_DONE_RING]; +static u32 cubb_done_head; +static u32 cubb_done_tail; +static DEFINE_SPINLOCK(cubb_done_lock); + +/* counters (plain reads from the status file) */ +static u64 cubb_jobs; +static u64 cubb_completed; +static u64 cubb_failed; +static u64 cubb_dropped; /* request ring/marshal/allocation drops */ +static u64 cubb_stale; /* done records for unknown seq */ +static u64 cubb_malformed; /* done records with bad magic/version */ +static u64 cubb_last_seq; +static int cubb_last_err; +static u64 cubb_launch_log; /* verbose header budget */ + +/* ------------------------------------------------------------------ */ +/* debugfs */ +/* ------------------------------------------------------------------ */ + +static bool cubb_root_owned; +static bool cubb_debugfs_ok; + +static void cubb_debugfs_fini(void); +static const struct file_operations cubb_job_fops; +static const struct file_operations cubb_in_fops; +static const struct file_operations cubb_out_fops; +static const struct file_operations cubb_done_fops; +static const struct file_operations cubb_status_fops; + +static struct dentry *cubb_find_root(void) +{ + struct dentry *root = debugfs_lookup("vgsp", NULL); + + if (IS_ERR(root)) + return NULL; + return root; +} + +static void cubb_debugfs_init(void) +{ + struct dentry *root, *f; + + cubb_root_owned = false; + root = cubb_find_root(); + if (!root) { + root = debugfs_create_dir("vgsp", NULL); + if (IS_ERR_OR_NULL(root)) { + pr_warn("debugfs dir vgsp unavailable\n"); + return; + } + cubb_root_owned = true; + } + + f = debugfs_create_file("cuda_bridge_job", 0444, root, NULL, + &cubb_job_fops); + if (IS_ERR_OR_NULL(f)) + goto fail; + f = debugfs_create_file("cuda_bridge_in", 0444, root, NULL, + &cubb_in_fops); + if (IS_ERR_OR_NULL(f)) + goto fail; + f = debugfs_create_file("cuda_bridge_out", 0200, root, NULL, + &cubb_out_fops); + if (IS_ERR_OR_NULL(f)) + goto fail; + f = debugfs_create_file("cuda_bridge_done", 0200, root, NULL, + &cubb_done_fops); + if (IS_ERR_OR_NULL(f)) + goto fail; + f = debugfs_create_file("cuda_bridge_status", 0444, root, NULL, + &cubb_status_fops); + if (IS_ERR_OR_NULL(f)) + goto fail; + + cubb_debugfs_ok = true; + dput(root); + pr_info("debugfs ready (/sys/kernel/debug/vgsp/cuda_bridge_*)\n"); + return; + +fail: + dput(root); + cubb_debugfs_fini(); + pr_warn("debugfs file creation failed\n"); +} + +static void cubb_debugfs_fini(void) +{ + struct dentry *root; + + cubb_debugfs_ok = false; + root = cubb_find_root(); + if (!root) + return; + debugfs_lookup_and_remove("cuda_bridge_job", root); + debugfs_lookup_and_remove("cuda_bridge_in", root); + debugfs_lookup_and_remove("cuda_bridge_out", root); + debugfs_lookup_and_remove("cuda_bridge_done", root); + debugfs_lookup_and_remove("cuda_bridge_status", root); + dput(root); + if (cubb_root_owned) { + debugfs_lookup_and_remove("vgsp", NULL); + cubb_root_owned = false; + } +} + +/* ------------------------------------------------------------------ */ +/* staging helpers */ +/* ------------------------------------------------------------------ */ + +static u32 cubb_rd32(const u8 *p) +{ + u32 v; + + memcpy(&v, p, 4); + return v; +} + +static u64 cubb_rd64(const u8 *p) +{ + u64 v; + + memcpy(&v, p, 8); + return v; +} + +/* A pointer the guest could have given the GPU (user/device VA range). */ +static bool cubb_va_plausible(u64 v) +{ + return v >= 0x10000 && v < (1ULL << 49); +} + +/* mail arguments: {u64 output ptr, u32 len}, len 1..4096. */ +static bool cubb_mail_shape(const u8 *p, size_t len, u64 *ptr, u32 *n) +{ + u64 v; + u32 l; + + if (len < 12) + return false; + v = cubb_rd64(p); + l = cubb_rd32(p + 8); + if (!cubb_va_plausible(v) || !l || l > 4096) + return false; + if (ptr) + *ptr = v; + if (n) + *n = l; + return true; +} + +/* vadd arguments: {u64 a, b, c, u32 n}, n == 1 MiB (frozen launch shape). */ +static bool cubb_vadd_shape(const u8 *p, size_t len, u64 *out, u32 *n) +{ + u64 a, b, c; + u32 k; + + if (len < 32) + return false; + a = cubb_rd64(p); + b = cubb_rd64(p + 8); + c = cubb_rd64(p + 16); + k = cubb_rd32(p + 24); + if (!cubb_va_plausible(a) || !cubb_va_plausible(b) || + !cubb_va_plausible(c) || k != (1u << 20)) + return false; + if (out) + *out = c; + if (n) + *n = k; + return true; +} + +static int cubb_shape_kind(const u8 *p, size_t len) +{ + if (cubb_mail_shape(p, len, NULL, NULL)) + return 1; + if (cubb_vadd_shape(p, len, NULL, NULL)) + return 2; + return 0; +} + +/* + * Locate the launch's argument block inside the QMD parameter bank. CUDA + * writes the arguments with GRCE LOAD_INLINE_DATA at an arbitrary bank + * offset, so never assume +0: first try the recorded recent inline-write + * destinations that fall inside [bank_va, bank_va + bank_size), newest + * first, then scan the bank for a mail/vadd argument shape. `want` names the + * shape implied by grid/block (0 = unknown). On a match returns 0 and fills + * off/kind/params/src; -ENODATA means the bank was readable but no shape was + * found (caller may fall back to bank+0); -ENOENT means a read failed (the + * page resolve may still be in progress) and nothing matched, so the caller + * should retry. + */ +static int cubb_find_params(struct nv_vgpu_device *vgpu, u64 bank_va, + u32 bank_size, const u64 *dsts, u32 n_dsts, + u32 want, u64 *off, u32 *kind, u8 *params, + size_t plen, const char **src) +{ + u8 cand[CUBB2_PARAM_BLOB]; + u8 *page; + u64 hi = bank_va + bank_size; + u32 i, pgoff, o, scan; + bool read_fail = false; + + *off = 0; + *kind = 0; + if (bank_size < 16) + return -ENODATA; + + /* (a) most recently written inline destinations inside the bank */ + for (i = 0; i < n_dsts; i++) { + u64 va = dsts[i]; + size_t need = want == 2 ? 32 : 12; + size_t got = min_t(size_t, plen, sizeof(cand)); + int k; + + if (!va || va < bank_va || va + need > hi) + continue; + if (va + got > hi) + got = hi - va; + if (gsp_sema_service_va_read(vgpu, va, cand, got)) { + read_fail = true; + continue; + } + k = cubb_shape_kind(cand, got); + if (!k || (want && k != want)) + continue; + *off = va - bank_va; + *kind = k; + memcpy(params, cand, min_t(size_t, got, plen)); + *src = "inline"; + return 0; + } + + /* (b) scan the bank for an argument shape (straddle-safe per page) */ + scan = bank_size > CUBB2_PARAM_SCAN_MAX ? CUBB2_PARAM_SCAN_MAX : + bank_size; + page = vmalloc(CUBB2_PAGE + CUBB2_PARAM_BLOB); + if (!page) + return read_fail ? -ENOENT : -ENODATA; + for (pgoff = 0; pgoff + 32 <= scan; pgoff += CUBB2_PAGE) { + u32 available = scan - pgoff; + u32 read_len = available > CUBB2_PAGE + CUBB2_PARAM_BLOB ? + CUBB2_PAGE + CUBB2_PARAM_BLOB : available; + + if (gsp_sema_service_va_read(vgpu, bank_va + pgoff, page, + read_len)) { + read_fail = true; + continue; + } + for (o = 0; o < available && o < CUBB2_PAGE && + o + 32 <= read_len; o += 8) { + int k = cubb_shape_kind(page + o, read_len - o); + + if (!k || (want && k != want)) + continue; + *off = pgoff + o; + *kind = k; + memcpy(params, page + o, + min_t(size_t, read_len - o, plen)); + *src = "scan"; + vfree(page); + return 0; + } + } + vfree(page); + return read_fail ? -ENOENT : -ENODATA; +} + +static struct cubb_pend *cubb_pend_lookup(u64 seq, int *idx) +{ + int i; + + for (i = 0; i < CUBB2_PEND_MAX; i++) { + if (cubb_pend[i].valid && cubb_pend[i].seq == seq) { + if (idx) + *idx = i; + return &cubb_pend[i]; + } + } + return NULL; +} + +static int cubb_pend_slot(void) +{ + int i; + + for (i = 0; i < CUBB2_PEND_MAX; i++) + if (!cubb_pend[i].valid) + return i; + return -1; +} + +/* ------------------------------------------------------------------ */ +/* launch decode -> request queue */ +/* ------------------------------------------------------------------ */ + +bool gsp_cuda_bridge_enabled(void) +{ + return cubb_enable && cubb_debugfs_ok; +} + +/* + * Called from svc_decode_stream on the service thread when a complete inline + * QMD has been accumulated. Only queues the launch: marshalling guest memory + * happens on the next service pass so the push pump is never held. + */ +void gsp_cuda_bridge_launch(struct nv_vgpu_device *vgpu, u32 h_object, + const u32 qmd[64], u64 qmd_dev_va, u64 push_va, + const u64 *recent_dsts, u32 n_recent_dsts) +{ + unsigned long flags; + u64 hash = 1469598103934665603ULL; + int i; + + if (!gsp_cuda_bridge_enabled()) + return; + (void)vgpu; + + for (i = 0; i < 64; i++) { + hash ^= qmd[i]; + hash *= 1099511628211ULL; + } + hash ^= (u64)h_object << 32; + hash *= 1099511628211ULL; + hash ^= qmd_dev_va; + + spin_lock_irqsave(&cubb_req_lock, flags); + for (i = 0; i < CUBB2_DEDUP; i++) { + if (cubb_dedup[i].hash == hash && + time_before(jiffies, + cubb_dedup[i].when + + msecs_to_jiffies(CUBB2_DEDUP_MS))) { + spin_unlock_irqrestore(&cubb_req_lock, flags); + cubb_dedup_hits++; + return; + } + } + cubb_dedup[cubb_dedup_next % CUBB2_DEDUP].hash = hash; + cubb_dedup[cubb_dedup_next % CUBB2_DEDUP].when = jiffies; + cubb_dedup_next++; + + if (cubb_req_head - cubb_req_tail >= CUBB2_REQ_MAX) { + spin_unlock_irqrestore(&cubb_req_lock, flags); + cubb_dropped++; + pr_info_ratelimited("request ring full, launch dropped (push=0x%llx)\n", + push_va); + return; + } + { + struct cubb_req *r = + &cubb_req[cubb_req_head % CUBB2_REQ_MAX]; + + r->h_object = h_object; + r->qmd_dev_va = qmd_dev_va; + r->push_va = push_va; + r->attempts = 0; + memcpy(r->qmd, qmd, sizeof(r->qmd)); + r->n_recent_dsts = 0; + if (recent_dsts && n_recent_dsts) { + if (n_recent_dsts > CUBB2_INLINE_MAX) + n_recent_dsts = CUBB2_INLINE_MAX; + memcpy(r->recent_dsts, recent_dsts, + n_recent_dsts * sizeof(u64)); + r->n_recent_dsts = n_recent_dsts; + } + } + cubb_req_head++; + spin_unlock_irqrestore(&cubb_req_lock, flags); +} + +/* Consume the request the marshaller was working on (service thread only). */ +static void cubb_req_drop_head(void) +{ + unsigned long flags; + + spin_lock_irqsave(&cubb_req_lock, flags); + if (cubb_req_tail != cubb_req_head) + cubb_req_tail++; + spin_unlock_irqrestore(&cubb_req_lock, flags); +} + +/* + * A guest VA read can miss because the targeted rm_mem scan for that page is + * still in progress (it walks the RAM a slice per call and reports -EAGAIN -> + * -ENOENT). Keep the request at the queue head and retry on later passes. + */ +#define CUBB2_REQ_TRIES 300 + +static void cubb_req_note_attempt(const struct cubb_req *r) +{ + unsigned long flags; + + spin_lock_irqsave(&cubb_req_lock, flags); + if (cubb_req_tail != cubb_req_head) + cubb_req[cubb_req_tail % CUBB2_REQ_MAX].attempts = r->attempts; + spin_unlock_irqrestore(&cubb_req_lock, flags); +} + +static bool cubb_req_retry(const struct cubb_req *r, int rc) +{ + if ((rc == -ENOENT || rc == -EAGAIN) && + r->attempts < CUBB2_REQ_TRIES) { + cubb_req_note_attempt(r); + return true; + } + return false; +} + +/* ------------------------------------------------------------------ */ +/* request -> pending job (marshalling) */ +/* ------------------------------------------------------------------ */ + +static void cubb_log_raw_qmd(const u32 *qmd) +{ + int i; + + pr_info("qmd raw (64 words):\n"); + for (i = 0; i < 64; i += 4) + pr_info("qmd[%02d] %08x %08x %08x %08x\n", i, + qmd[i], qmd[i + 1], qmd[i + 2], qmd[i + 3]); +} + +static int cubb_marshal_one(struct nv_vgpu_device *vgpu) +{ + struct cubb_req r; + struct cubb2_job job; + unsigned long flags; + u32 gx, gy, gz, bx, by, bz, shared; + u64 param_va, sem_va, prog_va; + u32 param_size, sem_payload, sem_flags, red_en, one_word, enable; + u8 params[CUBB2_PARAM_BLOB]; + u32 plen, n = 0, opcode = 0, param_count = 0; + u32 hint = 0, found_kind = 0; + u64 params_off = 0; + const char *params_src = "bank+0"; + u64 dst_va = 0, in_bytes = 0, out_bytes = 0; + u64 dst_pa = 0; + u32 dst_ap = 0; + u8 *in = NULL; + int rc, slot; + + spin_lock_irqsave(&cubb_req_lock, flags); + if (cubb_req_tail == cubb_req_head) { + spin_unlock_irqrestore(&cubb_req_lock, flags); + return 0; + } + r = cubb_req[cubb_req_tail % CUBB2_REQ_MAX]; + r.attempts++; + spin_unlock_irqrestore(&cubb_req_lock, flags); + + gx = r.qmd[12]; + gy = r.qmd[13] & 0xffff; + gz = r.qmd[14] & 0xffff; + shared = r.qmd[17] & 0x3ffff; + bx = r.qmd[18] >> 16; + by = r.qmd[19] & 0xffff; + bz = r.qmd[19] >> 16; + enable = (r.qmd[4] >> 10) & 1; + sem_va = r.qmd[23] | ((u64)(r.qmd[24] & 0xff) << 32); + red_en = (r.qmd[24] >> 26) & 1; + one_word = (r.qmd[24] >> 31) & 1; + sem_payload = r.qmd[25]; + param_va = r.qmd[32] | ((u64)(r.qmd[33] & 0x1ffff) << 32); + param_size = (r.qmd[33] >> 19) << 4; + prog_va = r.qmd[48] | ((u64)(r.qmd[49] & 0x1ffff) << 32); + sem_flags = (red_en ? 1u : 0u) | (one_word ? 2u : 0u) | + (enable ? 4u : 0u); + + /* + * The parameter bank lives in the client's VA space, and CUDA writes + * the arguments into it with GRCE inline writes at an arbitrary + * offset (observed: mail args at bank+0x160, bank+0 unrelated). The + * frozen grid/block shapes hint at the kernel; locate the actual + * argument block by recent inline-write destination, then by shape + * scan of the bank. + */ + plen = sizeof(params); + if (gx == 1 && gy == 1 && gz == 1 && bx == 64 && by == 1 && bz == 1) + hint = 1; + else if (gx == 4096 && gy == 1 && gz == 1 && bx == 256 && by == 1 && + bz == 1) + hint = 2; + + rc = cubb_find_params(vgpu, param_va, param_size, r.recent_dsts, + r.n_recent_dsts, hint, ¶ms_off, &found_kind, + params, plen, ¶ms_src); + if (rc) { + if (cubb_req_retry(&r, rc)) + return 0; + if (rc != -ENODATA) { + cubb_req_drop_head(); + cubb_failed++; + cubb_last_err = rc; + pr_info_ratelimited("failed seq=0 param locate va=0x%llx size=0x%x rc=%d push=0x%llx h=0x%x\n", + param_va, param_size, rc, r.push_va, + r.h_object); + return 0; + } + /* No argument shape found: legacy bank+0 read. */ + rc = gsp_sema_service_va_read(vgpu, param_va, params, plen); + if (rc) { + if (cubb_req_retry(&r, rc)) + return 0; + cubb_req_drop_head(); + cubb_failed++; + cubb_last_err = rc; + pr_info_ratelimited("failed seq=0 op=? param bank va=0x%llx size=0x%x rc=%d push=0x%llx h=0x%x\n", + param_va, param_size, rc, r.push_va, + r.h_object); + return 0; + } + params_off = 0; + found_kind = 0; + params_src = "bank+0"; + } + + /* + * Classify: the two frozen launch shapes first; then the located + * argument block; otherwise the old content heuristic. + */ + if (hint) + opcode = hint; + else if (found_kind) + opcode = found_kind; + else if (cubb_va_plausible(cubb_rd64(params)) && + cubb_va_plausible(cubb_rd64(params + 8)) && + cubb_va_plausible(cubb_rd64(params + 16)) && + cubb_rd32(params + 24) && + cubb_rd32(params + 24) <= (64u << 20)) + opcode = 2; + else + opcode = 1; + + if (opcode == 1) { + /* mail: {ptr, len} */ + dst_va = cubb_rd64(params); + n = cubb_rd32(params + 8); + out_bytes = n; + in_bytes = 0; + param_count = 2; + if (n > 4096) { + cubb_req_drop_head(); + cubb_failed++; + cubb_last_err = -E2BIG; + pr_info_ratelimited("failed seq=0 op=mail len %u too large (dst=0x%llx)\n", + n, dst_va); + return 0; + } + } else { + /* vadd: {a, b, c, n} (+ 5th slot) */ + u64 a = cubb_rd64(params); + u64 b = cubb_rd64(params + 8); + + dst_va = cubb_rd64(params + 16); + n = cubb_rd32(params + 24); + out_bytes = 4ULL * n; + in_bytes = 8ULL * n; + param_count = 5; + if (!n || in_bytes > CUBB2_IN_MAX || + out_bytes > CUBB2_OUT_MAX) { + cubb_req_drop_head(); + cubb_failed++; + cubb_last_err = -E2BIG; + pr_info_ratelimited("failed seq=0 op=vadd n=%u in=%llu out=%llu too large\n", + n, in_bytes, out_bytes); + return 0; + } + in = vmalloc(in_bytes); + if (!in) { + cubb_req_drop_head(); + cubb_failed++; + cubb_last_err = -ENOMEM; + pr_info_ratelimited("failed seq=0 op=vadd in alloc %llu bytes\n", + in_bytes); + return 0; + } + rc = gsp_sema_service_va_read(vgpu, a, in, in_bytes / 2); + if (!rc) + rc = gsp_sema_service_va_read(vgpu, b, + in + in_bytes / 2, + in_bytes / 2); + if (rc) { + vfree(in); + if (cubb_req_retry(&r, rc)) + return 0; + cubb_req_drop_head(); + cubb_failed++; + cubb_last_err = rc; + pr_info_ratelimited("failed seq=0 op=vadd src a=0x%llx b=0x%llx rc=%d\n", + a, b, rc); + return 0; + } + } + + if (!n || !out_bytes || out_bytes > CUBB2_OUT_MAX) { + vfree(in); + cubb_req_drop_head(); + cubb_failed++; + cubb_last_err = -EINVAL; + pr_info_ratelimited("failed seq=0 op=%u bad sizes n=%u out=%llu\n", + opcode, n, out_bytes); + return 0; + } + + if (opcode == 1) + pr_info_ratelimited("params off=0x%llx kind=mail ptr=0x%llx len/n=%u\n", + params_off, dst_va, n); + else + pr_info_ratelimited("params off=0x%llx kind=vadd ptr=0x%llx len/n=%u\n", + params_off, dst_va, n); + + rc = gsp_sema_service_va_resolve(vgpu, dst_va, &dst_pa, &dst_ap); + + memset(&job, 0, sizeof(job)); + job.magic = CUBB2_JOB_MAGIC; + job.version = CUBB2_VERSION; + job.opcode = opcode; + job.grid[0] = gx; + job.grid[1] = gy; + job.grid[2] = gz; + job.block[0] = bx; + job.block[1] = by; + job.block[2] = bz; + job.shared = shared; + job.n = n; + job.in_bytes = in_bytes; + job.out_bytes = out_bytes; + job.sem_va = sem_va; + job.sem_payload = sem_payload; + job.sem_flags = sem_flags; + job.prog_va = prog_va; + job.param_count = param_count; + + mutex_lock(&cubb_lock); + slot = cubb_pend_slot(); + if (slot < 0) { + mutex_unlock(&cubb_lock); + vfree(in); + cubb_req_drop_head(); + cubb_dropped++; + pr_info_ratelimited("pending queue full, launch dropped\n"); + return 0; + } + job.seq = ++cubb_seq; + cubb_pend[slot].valid = true; + cubb_pend[slot].seq = job.seq; + cubb_pend[slot].h_object = r.h_object; + cubb_pend[slot].opcode = opcode; + cubb_pend[slot].dst_va = dst_va; + cubb_pend[slot].out_bytes = out_bytes; + cubb_pend[slot].sem_va = sem_va; + cubb_pend[slot].sem_payload = sem_payload; + cubb_pend[slot].sem_flags = sem_flags; + cubb_pend[slot].job = job; + cubb_pend[slot].in = in; + cubb_jobs++; + cubb_last_seq = job.seq; + mutex_unlock(&cubb_lock); + cubb_req_drop_head(); + + if (cubb_launch_log < 8) { + cubb_launch_log++; + pr_info("launch seq=%llu op=%s grid=%u,%u,%u block=%u,%u,%u shared=%u n=%u in=%llu out=%llu sem=0x%llx pay=%u flags=0x%x param=0x%llx psize=0x%x poff=0x%llx psrc=%s prog=0x%llx dst=0x%llx dstpa=0x%llx dstap=%u h=0x%x push=0x%llx qmddev=0x%llx\n", + job.seq, opcode == 1 ? "mail" : "vadd", + gx, gy, gz, bx, by, bz, shared, n, in_bytes, + out_bytes, sem_va, sem_payload, sem_flags, param_va, + param_size, params_off, params_src, prog_va, dst_va, + dst_pa, dst_ap, r.h_object, r.push_va, r.qmd_dev_va); + if (gsp_sema_dbg) { + cubb_log_raw_qmd(r.qmd); + print_hex_dump(KERN_INFO, "cuda-bridge: params ", + DUMP_PREFIX_OFFSET, 16, 1, params, + plen, false); + } + } else { + pr_info_ratelimited("launch seq=%llu op=%s n=%u out=%llu\n", + job.seq, opcode == 1 ? "mail" : "vadd", + n, out_bytes); + } + return 0; +} + +/* ------------------------------------------------------------------ */ +/* done records -> completion */ +/* ------------------------------------------------------------------ */ + +static bool cubb_done_pop(struct cubb2_done *out) +{ + unsigned long flags; + bool have = false; + + spin_lock_irqsave(&cubb_done_lock, flags); + if (cubb_done_tail != cubb_done_head) { + *out = cubb_done_ring[cubb_done_tail % CUBB2_DONE_RING]; + cubb_done_tail++; + have = true; + } + spin_unlock_irqrestore(&cubb_done_lock, flags); + return have; +} + +static void cubb_finish_one(struct nv_vgpu_device *vgpu, + const struct cubb2_done *d) +{ + struct cubb_pend *p; + int idx = -1; + int rc = 0; + u32 sem_val = 0; + u32 sem_rc = 0; + u64 wrote; + + mutex_lock(&cubb_lock); + p = cubb_pend_lookup(d->seq, &idx); + if (!p) { + mutex_unlock(&cubb_lock); + cubb_stale++; + pr_info_ratelimited("stale done seq=%llu (no pending job)\n", + d->seq); + return; + } + + if (d->status != 0) { + cubb_failed++; + cubb_last_err = (int)d->err_code; + pr_info("failed seq=%llu err=%u status=%u op=%u\n", + d->seq, d->err_code, d->status, p->opcode); + goto out; + } + + wrote = min_t(u64, p->out_bytes, cubb_out_len); + if (wrote < p->out_bytes) { + cubb_failed++; + cubb_last_err = -ENODATA; + pr_info("failed seq=%llu err=%d (daemon wrote %llu of %llu out bytes)\n", + d->seq, -ENODATA, wrote, p->out_bytes); + goto out; + } + + rc = gsp_sema_service_va_write(vgpu, p->dst_va, cubb_out_buf, wrote); + if (rc) { + cubb_failed++; + cubb_last_err = rc; + pr_info("failed seq=%llu err=%d (output write va=0x%llx len=%llu)\n", + d->seq, rc, p->dst_va, wrote); + goto out; + } + /* + * Read-back check over the whole output: proves the bytes landed + * through the translation the guest will read (vidmem shadow or + * guest RAM). A missing/mismatching copy must NOT be signalled + * complete, or the guest reads stale data and runs on. + */ + { + u8 check[256]; + size_t off = 0; + + while (off < wrote) { + size_t clen = min_t(size_t, wrote - off, + sizeof(check)); + + if (gsp_sema_service_va_read(vgpu, p->dst_va + off, + check, clen) || + memcmp(check, cubb_out_buf + off, clen)) { + cubb_failed++; + cubb_last_err = -EIO; + pr_info("failed seq=%llu err=%d (output read-back mismatch va=0x%llx off=%zu len=%zu)\n", + d->seq, -EIO, p->dst_va, off, clen); + goto out; + } + off += clen; + } + } + + if ((p->sem_flags & 4u) && p->sem_va) { + sem_val = p->sem_payload; + if (p->sem_flags & 1u) { + u32 cur = 0; + + if (!gsp_sema_service_sem_read(vgpu, p->sem_va, &cur)) + sem_val = cur + p->sem_payload; + } + sem_rc = gsp_sema_service_sem_write(vgpu, p->h_object, + p->sem_va, sem_val); + if (sem_rc) { + cubb_failed++; + cubb_last_err = sem_rc; + pr_info("failed seq=%llu err=%d sem_va=0x%llx pay=%u\n", + d->seq, sem_rc, p->sem_va, sem_val); + goto out; + } + } + + { + u64 dpa = 0; + u32 dap = 0; + u64 sempa = 0; + u32 semap = 0; + + (void)gsp_sema_service_va_resolve(vgpu, p->dst_va, &dpa, &dap); + if (p->sem_va) + (void)gsp_sema_service_va_resolve(vgpu, p->sem_va, + &sempa, &semap); + cubb_completed++; + pr_info("completed seq=%llu op=%u dst=0x%llx dstpa=0x%llx dstap=%u out_bytes=%llu/%llu sem_va=0x%llx sempa=0x%llx pay=%u bad=%u kernel_ns=%llu\n", + d->seq, p->opcode, p->dst_va, dpa, dap, wrote, + p->out_bytes, p->sem_va, sempa, sem_val, d->bad, + d->kernel_ns); + } + +out: + vfree(p->in); + p->in = NULL; + p->valid = false; + cubb_out_len = 0; + if (idx >= 0) { + /* compact: shift the younger entries down one slot */ + int j = idx; + + while (j + 1 < CUBB2_PEND_MAX && cubb_pend[j + 1].valid) { + cubb_pend[j] = cubb_pend[j + 1]; + j++; + } + cubb_pend[j].in = NULL; + cubb_pend[j].valid = false; + } + mutex_unlock(&cubb_lock); +} + +/* + * Service-thread hook: marshal at most one queued launch into a daemon-visible + * job, then resolve any completed jobs. Called every service pass; a no-op + * when no launch has been seen. + */ +void gsp_cuda_bridge_service(struct nv_vgpu_device *vgpu) +{ + struct cubb2_done d; + int guard = 0; + + if (!gsp_cuda_bridge_enabled()) + return; + + (void)cubb_marshal_one(vgpu); + while (guard++ < CUBB2_DONE_RING && cubb_done_pop(&d)) { + if (d.magic != CUBB2_DONE_MAGIC || + d.version != CUBB2_VERSION) { + cubb_malformed++; + continue; + } + cubb_finish_one(vgpu, &d); + } +} + +/* ------------------------------------------------------------------ */ +/* debugfs handlers */ +/* ------------------------------------------------------------------ */ + +static ssize_t cubb_job_read(struct file *f, char __user *ubuf, size_t count, + loff_t *ppos) +{ + u8 rec[CUBB2_RECORD_SIZE]; + int idx = -1; + struct cubb_pend *p = NULL; + size_t off = *ppos; + size_t n; + + if (!cubb_debugfs_ok) + return 0; + mutex_lock(&cubb_lock); + for (idx = 0; idx < CUBB2_PEND_MAX; idx++) { + if (cubb_pend[idx].valid) { + p = &cubb_pend[idx]; + break; + } + } + if (!p) { + mutex_unlock(&cubb_lock); + return 0; + } + memcpy(rec, &p->job, sizeof(rec)); + mutex_unlock(&cubb_lock); + + if (off >= sizeof(rec)) + return 0; + n = min_t(size_t, count, sizeof(rec) - off); + if (copy_to_user(ubuf, rec + off, n)) + return -EFAULT; + *ppos = off + n; + return n; +} + +static ssize_t cubb_in_read(struct file *f, char __user *ubuf, size_t count, + loff_t *ppos) +{ + struct cubb_pend *p = NULL; + u64 in_bytes = 0; + u8 *in = NULL; + size_t off = *ppos; + size_t n; + int rc = 0; + int idx; + + if (!cubb_debugfs_ok) + return 0; + mutex_lock(&cubb_lock); + for (idx = 0; idx < CUBB2_PEND_MAX; idx++) { + if (cubb_pend[idx].valid) { + p = &cubb_pend[idx]; + break; + } + } + if (!p) { + mutex_unlock(&cubb_lock); + return 0; + } + in = p->in; + in_bytes = p->job.in_bytes; + if (!in || off >= in_bytes) { + mutex_unlock(&cubb_lock); + return 0; + } + n = min_t(size_t, count, in_bytes - off); + rc = copy_to_user(ubuf, in + off, n); + mutex_unlock(&cubb_lock); + if (rc) + return -EFAULT; + *ppos = off + n; + return n; +} + +/* + * Accumulate the daemon's output blob. ppos 0 starts a new blob; growth is + * amortised (double until the record fits, capped at CUBB2_OUT_MAX). + */ +static ssize_t cubb_out_write(struct file *f, const char __user *ubuf, + size_t count, loff_t *ppos) +{ + u64 off = *ppos; + u64 end; + int rc = 0; + + if (!cubb_debugfs_ok) + return -ENODEV; + if (!count) + return 0; + if (*ppos < 0) + return -EINVAL; + end = off + count; + if (end > CUBB2_OUT_MAX) + return -E2BIG; + + mutex_lock(&cubb_lock); + if (off == 0) + cubb_out_len = 0; + if (end > cubb_out_alloc) { + unsigned int want = cubb_out_alloc ? cubb_out_alloc : 4096; + u8 *nb; + + while (want < end && want < CUBB2_OUT_MAX) + want *= 2; + if (want > CUBB2_OUT_MAX) + want = CUBB2_OUT_MAX; + nb = vmalloc(want); + if (!nb) { + mutex_unlock(&cubb_lock); + return -ENOMEM; + } + if (cubb_out_len) + memcpy(nb, cubb_out_buf, cubb_out_len); + vfree(cubb_out_buf); + cubb_out_buf = nb; + cubb_out_alloc = want; + } + rc = copy_from_user(cubb_out_buf + off, ubuf, count); + if (!rc && end > cubb_out_len) + cubb_out_len = end; + mutex_unlock(&cubb_lock); + if (rc) + return -EFAULT; + *ppos = end; + return count; +} + +/* Accumulate a 256-byte CBD2 record (single write or ppos-chunked). */ +static u8 cubb_done_stage[CUBB2_RECORD_SIZE]; +static size_t cubb_done_stage_len; + +static ssize_t cubb_done_write(struct file *f, const char __user *ubuf, + size_t count, loff_t *ppos) +{ + /* + * Accept both pwrite(fd, rec, 256, 0) and a long-lived fd writing + * records back to back: a write starting on a record boundary starts + * a fresh record. + */ + size_t off = (size_t)(*ppos % CUBB2_RECORD_SIZE); + unsigned long flags; + int rc; + + if (!cubb_debugfs_ok) + return -ENODEV; + if (!count) + return 0; + if (count > CUBB2_RECORD_SIZE - off) + count = CUBB2_RECORD_SIZE - off; + + mutex_lock(&cubb_lock); + if (off == 0) + cubb_done_stage_len = 0; + rc = copy_from_user(cubb_done_stage + off, ubuf, count); + if (rc) { + mutex_unlock(&cubb_lock); + return -EFAULT; + } + if (off + count > cubb_done_stage_len) + cubb_done_stage_len = off + count; + + if (cubb_done_stage_len >= CUBB2_RECORD_SIZE) { + struct cubb2_done *d = (struct cubb2_done *)cubb_done_stage; + + if (d->magic == CUBB2_DONE_MAGIC && + d->version == CUBB2_VERSION) { + spin_lock_irqsave(&cubb_done_lock, flags); + if (cubb_done_head - cubb_done_tail < + CUBB2_DONE_RING) { + cubb_done_ring[cubb_done_head % + CUBB2_DONE_RING] = *d; + cubb_done_head++; + } else { + cubb_dropped++; + } + spin_unlock_irqrestore(&cubb_done_lock, flags); + } else { + cubb_malformed++; + } + memset(cubb_done_stage, 0, sizeof(cubb_done_stage)); + cubb_done_stage_len = 0; + } + mutex_unlock(&cubb_lock); + *ppos = off + count; + return count; +} + +static ssize_t cubb_status_read(struct file *f, char __user *ubuf, + size_t count, loff_t *ppos) +{ + char buf[256]; + int len; + + len = scnprintf(buf, sizeof(buf), + "jobs=%llu completed=%llu failed=%llu last_seq=%llu last_err=%d dropped=%llu stale=%llu malformed=%llu dedup=%llu\n", + cubb_jobs, cubb_completed, cubb_failed, + cubb_last_seq, cubb_last_err, cubb_dropped, + cubb_stale, cubb_malformed, cubb_dedup_hits); + return simple_read_from_buffer(ubuf, count, ppos, buf, len); +} + +static const struct file_operations cubb_job_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = cubb_job_read, + .llseek = default_llseek, +}; + +static const struct file_operations cubb_in_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = cubb_in_read, + .llseek = default_llseek, +}; + +static const struct file_operations cubb_out_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = cubb_out_write, + .llseek = default_llseek, +}; + +static const struct file_operations cubb_done_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = cubb_done_write, + .llseek = default_llseek, +}; + +static const struct file_operations cubb_status_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = cubb_status_read, + .llseek = default_llseek, +}; + +/* ------------------------------------------------------------------ */ +/* lifecycle */ +/* ------------------------------------------------------------------ */ + +int gsp_cuda_bridge_init(void) +{ + cubb_debugfs_init(); + return cubb_debugfs_ok ? 0 : -ENODEV; +} + +void gsp_cuda_bridge_fini(void) +{ + int i; + + if (cubb_out_buf) { + vfree(cubb_out_buf); + cubb_out_buf = NULL; + cubb_out_alloc = 0; + cubb_out_len = 0; + } + mutex_lock(&cubb_lock); + for (i = 0; i < CUBB2_PEND_MAX; i++) { + vfree(cubb_pend[i].in); + cubb_pend[i].in = NULL; + cubb_pend[i].valid = false; + } + mutex_unlock(&cubb_lock); + cubb_req_head = cubb_req_tail = 0; + cubb_done_head = cubb_done_tail = 0; + cubb_debugfs_fini(); +} diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 3d1ccea..bead218 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -440,8 +440,34 @@ bool gsp_phys_boot_requested(void) * carries both PA and VA in one entry, on the non-KMD V5 channel. */ #define GSP_PHYS_RMCALL_GR_PROMOTE_GUEST_SET 19u +/* + * Mode 20 (2026-10-02, untested): the mode-6 no-entry hVirtMemory promote on + * the V6 KMD-managed channel (SKIP_CTXBUFFER_ALLOC). Mode 6 was refused with + * NV_ERR_INVALID_ARGUMENT (0x1f) on the V5 channel because the server's + * client-promote path rejects the hVirtMemory (pVirtMemDesc) branch unless the + * channel is KMD-managed; V6 sets exactly that flag. The object alloc stays on + * the default external VAS, so the shared-management ctx-buffer map path that + * wedged grsvas2/grsvas3 never runs. The promote asks the server to create the + * virtual-context memdesc/vaList entry for the context window itself, after + * which the KMD commit path can validate ENG_CTX_VALID. + */ +#define GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT 20u +/* + * Mode 21 (2026-10-02, untested): the mode-7 no-entry bare (virtAddress, size) + * promote on the V6 KMD-managed channel - the counterpart of mode 20 without + * the hVirtMemory object. + */ +#define GSP_PHYS_RMCALL_GR_PROMOTE_KMD_BARE_VA 21u +/* + * Mode 22 (2026-10-02, untested): install the physical context memdesc with + * GPU_INITIALIZE_CTX (accepted alone in modes 10/11), then send mode 20's + * no-entry hVirtMemory promote. If mode 20's plain 0x40 was the promote's + * physical-memdesc precondition, this combination satisfies it before the + * virtual-memdesc promote. + */ +#define GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT 22u #define GSP_PHYS_RMCALL_GR_PROMOTE_MAX \ - (GSP_PHYS_RMCALL_GR_PROMOTE_GUEST_SET) + (GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT) /* * Work-submit token notifier slot. The default index is @@ -483,9 +509,17 @@ bool gsp_phys_boot_requested(void) */ #define GSP_PHYS_RMCALL_VAS_INDEX_GPU_NEW 0u #define GSP_PHYS_RMCALL_VAS_FLAG_IS_EXTERNALLY_OWNED BIT(3) +#define GSP_PHYS_RMCALL_VAS_FLAG_SHARED_MANAGEMENT BIT(2) #define GSP_PHYS_RMCALL_VAS_FLAG_ALLOW_ZERO_ADDRESS BIT(8) #define GSP_PHYS_RMCALL_VAS_SIZE 0x10000000ULL /* + * Shared-management VAS size: the normal KMD/UVM client passes vaBase=0 and + * rm_va_size = 128 TB on Volta (uvm_volta.c:48-49); the split-VAS server-RM + * carveout math (gpu_vaspace.c:440-470) underflows with a small (256 MB) + * size and asserts the alloc with NV_ERR_INVALID_ARGUMENT. + */ +#define GSP_PHYS_RMCALL_VAS_SIZE_SHARED 0x200000000000ULL +/* * Turing/TU104 uses the GP10X GMMU root format: the root level has 2 index * bits, i.e. exactly 4 entries covering 2^47 bytes each, and the server caps * the external root at vaLimitMax = 2^49-1 with vaLimitNew = @@ -2969,6 +3003,16 @@ static void gsp_phys_fault_corroborate(struct gsp_phys_fault_info *out); #define GSP_PHYS_NOCAT_OFF_DIAGBUF 180 #define GSP_PHYS_NOCAT_MIN_PAYLOAD GSP_PHYS_NOCAT_OFF_DIAGBUF #define GSP_PHYS_NOCAT_FULL_PAYLOAD 1208 +/* + * Assert diagnostic buffer (RM_NOCAT_ASSERT_DIAG_BUFFER, g_journal_nvoc.h): + * ten u32 call-stack entries (each already relative to the NOCAT load + * address) followed by a u32 repeat count. The stack entries are the exact + * call chain of the assert, so a dump makes the failing path nameable + * offline without another run. + */ +#define GSP_PHYS_NOCAT_DIAG_STACK 10 +#define GSP_PHYS_NOCAT_OFF_DIAGCOUNT (GSP_PHYS_NOCAT_OFF_DIAGBUF + 4 * GSP_PHYS_NOCAT_DIAG_STACK) +#define GSP_PHYS_NOCAT_DIAG_NEEDED (GSP_PHYS_NOCAT_OFF_DIAGCOUNT + 4) /* Local re-declaration of the reference record, only for the offset asserts. */ struct gsp_phys_nocat_journal_record { @@ -3116,10 +3160,33 @@ static void gsp_phys_nocat_record(const struct gsp_phys_rx_record *rec) gsp_phys_nocat_accumulate(rec_type, error_code, subsystem, bugcheck, source, engine, rec->seq_num); - if (gsp_phys_nocat_param) + if (gsp_phys_nocat_param) { + u32 diag_len = 0; + u32 count = 0; + u32 i; + char stack[GSP_PHYS_NOCAT_DIAG_STACK * 12]; + size_t pos = 0; + pr_info("vgsp-nocat: rec seq=%u recType=%u errorCode=0x%llx subsystem=0x%x bugcheck=0x%x payload=%u source=\"%s\" faultingEngine=\"%s\"\n", rec->seq_num, rec_type, error_code, subsystem, bugcheck, len, source, engine); + if (len >= GSP_PHYS_NOCAT_OFF_DIAGBUFLEN + 4) + diag_len = get_unaligned_le32(p + + GSP_PHYS_NOCAT_OFF_DIAGBUFLEN); + if (rec_type == 5 && diag_len >= 4 * GSP_PHYS_NOCAT_DIAG_STACK + 4 && + len >= GSP_PHYS_NOCAT_DIAG_NEEDED) { + for (i = 0; i < GSP_PHYS_NOCAT_DIAG_STACK; i++) { + u32 v = get_unaligned_le32(p + + GSP_PHYS_NOCAT_OFF_DIAGBUF + 4 * i); + pos += scnprintf(stack + pos, sizeof(stack) - pos, + "%s0x%x", i ? "," : "", v); + } + count = get_unaligned_le32(p + + GSP_PHYS_NOCAT_OFF_DIAGCOUNT); + pr_info("vgsp-nocat: rec seq=%u assertCallStack count=%u [%s] (offsets from load base 0x4000000)\n", + rec->seq_num, count, stack); + } + } } static void gsp_phys_nocat_summary_dump(void) @@ -6732,6 +6799,12 @@ static const char *gsp_phys_rmcall_gr_promote_name(void) return "mode 8 shape plus a client FERMI_CONTEXT_SHARE_A (hContextShare) that allocates the GraphicsSubcontext"; case GSP_PHYS_RMCALL_GR_PROMOTE_GUEST_SET: return "stock guest's 9-entry promote set (MAIN 2 MB PA+VA, PATCH/global/FECS/priv maps) on V5"; + case GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT: + return "KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC) + no-entry hVirtMemory promote"; + case GSP_PHYS_RMCALL_GR_PROMOTE_KMD_BARE_VA: + return "KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC) + no-entry bare VA+size promote"; + case GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT: + return "KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC) + GPU_INITIALIZE_CTX then no-entry hVirtMemory promote"; default: return "off (PROMOTE_CTX never sent)"; } @@ -6907,7 +6980,13 @@ static int gsp_phys_rmcall_alloc_channel(void __iomem *bar0) boot_stage.rmcall_gr_promote == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_PM || boot_stage.rmcall_gr_promote == - GSP_PHYS_RMCALL_GR_PROMOTE_CTXSHARE)) { + GSP_PHYS_RMCALL_GR_PROMOTE_CTXSHARE || + boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT || + boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_KMD_BARE_VA || + boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT)) { vars = &gsp_phys_rmcall_channel_variant_skipctx; attempts = 1; } @@ -7843,7 +7922,10 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0) mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD || mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_FB || mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_PM || - mode == GSP_PHYS_RMCALL_GR_PROMOTE_CTXSHARE) && + mode == GSP_PHYS_RMCALL_GR_PROMOTE_CTXSHARE || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_BARE_VA || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT) && boot_stage.rmcall_channel_variant != gsp_phys_rmcall_channel_variant_skipctx.id) { pr_err("vgsp-boot: rm-call: mode %u requires the SKIP_CTXBUFFER_ALLOC channel (V%u %s); winning variant is V%u; no PROMOTE_CTX is sent and the ladder aborts with no retry\n", @@ -7981,17 +8063,21 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0) } boot_stage.rmcall_gr_promote_step = 2; } else if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT || - mode == GSP_PHYS_RMCALL_GR_PROMOTE_BARE_VA) { + mode == GSP_PHYS_RMCALL_GR_PROMOTE_BARE_VA || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_BARE_VA) { /* - * Modes 6/7: the documented no-entry branches. Mode 6 first - * reserves the context window with an NV01_MEMORY_VIRTUAL - * object on the ladder VAS and passes it as hVirtMemory + * Modes 6/7/20: the documented no-entry branches. Modes 6/20 + * first reserve the context window with an NV01_MEMORY_VIRTUAL + * object on the ladder VAS and pass it as hVirtMemory * (hClient is its owner); mode 7 sends the bare * (virtAddress, size) pair with no hVirtMemory. entryCount=0 - * in both, so the server creates the memdesc/vaList entry - * instead of resolving one of our raw-PTE entries. + * in all, so the server creates the memdesc/vaList entry + * instead of resolving one of our raw-PTE entries. Mode 20 is + * the same shape as mode 6 but on the V6 KMD-managed channel. */ - if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT) { + if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT) { ret = gsp_phys_rmcall_alloc_gr_hvirt(bar0); if (ret) { kvfree(p); @@ -8009,11 +8095,13 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0) GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX, p, sizeof(*p), 24, NULL, 0, &rpc_result, &status, - mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT ? + (mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT) ? "GPU_PROMOTE_CTX (hVirtMemory)" : "GPU_PROMOTE_CTX (bare VA+size)"); pr_info("vgsp-boot: rm-call: PROMOTE_CTX %s: entryCount=0 hVirtMemory=0x%x virtAddress=0x%llx size=0x%x status=0x%x rpc_result=0x%x (hClient=0x%x hObject=0x%x hChanClient=0x%x engineType=0x%x gr_promote_param=%u)\n", - mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT ? + (mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT) ? "hVirtMemory" : "bare VA+size", le32_to_cpu(p->h_virt_memory), boot_stage.rmcall_ctx_va, boot_stage.rmcall_ctx_size, @@ -8022,7 +8110,8 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0) GSP_PHYS_RMCALL_ENGINE_GR0, mode); if (ret) { pr_err("vgsp-boot: rm-call: PROMOTE_CTX %s failed (status=0x%x rpc_result=0x%x, %s, ret=%d); the GR context is not committed and the ladder aborts before BIND with no retry\n", - mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT ? + (mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT) ? "hVirtMemory" : "bare VA+size", status, rpc_result, gsp_phys_rmcall_status_str(status), ret); @@ -8063,9 +8152,10 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0) } boot_stage.rmcall_gr_promote_step = 1; } else if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_OLDIFACE_VA || - mode == GSP_PHYS_RMCALL_GR_PROMOTE_OLDIFACE_FB) { + mode == GSP_PHYS_RMCALL_GR_PROMOTE_OLDIFACE_FB || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT) { /* - * Modes 10/11: the old-interface client-managed flow. Step 1 + * Modes 10/11/22: the old-interface client-managed flow. Step 1 * sends GPU_INITIALIZE_CTX (0x2080012d) with the physical * context buffer, which installs the PA/size/physAttr * memdesc on the GraphicsContext (grctxInitializeAPI) that @@ -8076,7 +8166,8 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0) * entryCount=0 and the bare (virtAddress, size) pair; the * server builds the virtual memdesc itself and, on the V6 * KMD-managed channel, binds it without the VA/bookkeeping - * resolve. + * resolve. Mode 22 is mode 10's step 1 followed by mode 20's + * hVirtMemory no-entry promote instead of the bare pair. */ struct gsp_phys_rmcall_init_ctx_params *ic; u32 ic_phys_attr = (mode == @@ -8126,6 +8217,16 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0) p->virt_address = cpu_to_le64(boot_stage.rmcall_ctx_va); p->size = cpu_to_le64(boot_stage.rmcall_ctx_size); p->entry_count = cpu_to_le32(0); + if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT) { + ret = gsp_phys_rmcall_alloc_gr_hvirt(bar0); + if (ret) { + kvfree(p); + return ret; + } + p->h_client = cpu_to_le32(boot_stage.rmcall_client); + p->h_virt_memory = + cpu_to_le32(boot_stage.rmcall_gr_hvirt); + } rpc_result = 0; status = 0; ret = gsp_phys_rmcall_control(bar0, @@ -8133,14 +8234,21 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0) GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX, p, sizeof(*p), 24, NULL, 0, &rpc_result, &status, + (mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT) ? + "GPU_PROMOTE_CTX (hVirtMemory after init)" : "GPU_PROMOTE_CTX (KMD bare VA+size after init)"); - pr_info("vgsp-boot: rm-call: PROMOTE_CTX bare VA+size after init: entryCount=0 virtAddress=0x%llx size=0x%x status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x gr_promote_param=%u)\n", + pr_info("vgsp-boot: rm-call: PROMOTE_CTX %s: entryCount=0 hVirtMemory=0x%x virtAddress=0x%llx size=0x%x status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x gr_promote_param=%u)\n", + (mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT) ? + "hVirtMemory after init" : "bare VA+size after init", + le32_to_cpu(p->h_virt_memory), boot_stage.rmcall_ctx_va, boot_stage.rmcall_ctx_size, status, rpc_result, boot_stage.rmcall_channel, boot_stage.rmcall_client, GSP_PHYS_RMCALL_ENGINE_GR0, mode); if (ret) { - pr_err("vgsp-boot: rm-call: PROMOTE_CTX bare VA+size failed (status=0x%x rpc_result=0x%x, %s, ret=%d); last completed promote step: INITIALIZE_CTX accepted (1/2); the GR context is not committed and the ladder aborts before BIND with no retry\n", + pr_err("vgsp-boot: rm-call: PROMOTE_CTX %s failed (status=0x%x rpc_result=0x%x, %s, ret=%d); last completed promote step: INITIALIZE_CTX accepted (1/2); the GR context is not committed and the ladder aborts before BIND with no retry\n", + (mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT) ? + "hVirtMemory after init" : "bare VA+size", status, rpc_result, gsp_phys_rmcall_status_str(status), ret); kvfree(p); @@ -8632,17 +8740,41 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) gsp_phys_rmcall_gr_promote_snapshot(); /* - * The TSG below carries this externally-owned FERMI_VASPACE_A, whose - * root page directory is committed from the zeroed VASPDB chunk before - * any channel alloc (the instance-block setup programs that base and - * fails 0x40 when it is NULL, kern_gmmu_gp100.c:150-152). Either call - * aborts the ladder: the variant walk cannot succeed without them. + * The TSG below carries this explicitly allocated FERMI_VASPACE_A, + * whose root page directory is committed from the zeroed VASPDB chunk + * before any channel alloc (the instance-block setup programs that + * base and fails 0x40 when it is NULL, kern_gmmu_gp100.c:150-152). + * Either call aborts the ladder: the variant walk cannot succeed + * without them. + * + * History (2026-10-02): the VAS was allocated IS_EXTERNALLY_OWNED to + * mirror the guest's UVM-owned address space. That flag tells the RM + * that the client owns all page tables and VA/context commitment, so + * the server never books the GR engine-context VA itself and every + * promote either asserts ("Externally owned object not found", + * kchannelMapEngineCtxBuf) or silently leaves the channel + * CTXNOTVALID. The normal KMD client instead allocates its VAS with + * SHARED_MANAGEMENT (nv_gpu_ops.c:2495): the client provides the root + * PDB (SET_PAGE_DIRECTORY still legal, gpu_vaspace.c:3108) while the + * RM manages the lower page tables and maps/commits engine contexts. + * gsp_phys_rmcall_vas_shared=1 selects that variant; the default 0 + * keeps the byte-identical external path. */ memset(&vas, 0, sizeof(vas)); vas.index = cpu_to_le32(GSP_PHYS_RMCALL_VAS_INDEX_GPU_NEW); - vas.flags = cpu_to_le32(GSP_PHYS_RMCALL_VAS_FLAG_IS_EXTERNALLY_OWNED | - GSP_PHYS_RMCALL_VAS_FLAG_ALLOW_ZERO_ADDRESS); - vas.va_size = cpu_to_le64(GSP_PHYS_RMCALL_VAS_SIZE); + vas.flags = cpu_to_le32(gsp_phys_rmcall_vas_shared_value() ? + (GSP_PHYS_RMCALL_VAS_FLAG_SHARED_MANAGEMENT | + GSP_PHYS_RMCALL_VAS_FLAG_ALLOW_ZERO_ADDRESS) : + (GSP_PHYS_RMCALL_VAS_FLAG_IS_EXTERNALLY_OWNED | + GSP_PHYS_RMCALL_VAS_FLAG_ALLOW_ZERO_ADDRESS)); + vas.va_size = cpu_to_le64(gsp_phys_rmcall_vas_shared_value() ? + GSP_PHYS_RMCALL_VAS_SIZE_SHARED : + GSP_PHYS_RMCALL_VAS_SIZE); + pr_info("vgsp-boot: rm-call: FERMI_VASPACE_A flags=0x%x va_size=0x%llx (%s)\n", + le32_to_cpu(vas.flags), le64_to_cpu(vas.va_size), + gsp_phys_rmcall_vas_shared_value() ? + "shared management (RM owns page tables/commit)" : + "externally owned (client owns page tables/commit)"); ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client, boot_stage.rmcall_device, GSP_PHYS_RMCALL_HANDLE_VAS, diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c index 1fa1670..91f7a77 100644 --- a/kernel/gsp_phys_gpu.c +++ b/kernel/gsp_phys_gpu.c @@ -424,8 +424,8 @@ static int gsp_phys_rmcall_gr_promote_set(const char *val, return ret; if (gsp_phys_rmcall_gr_promote_param < 0 || - gsp_phys_rmcall_gr_promote_param > 19) { - pr_warn("vgsp: gsp_phys_rmcall_gr_promote=%d is outside 0..19; clamping to 0 (no PROMOTE_CTX)\n", + gsp_phys_rmcall_gr_promote_param > 22) { + pr_warn("vgsp: gsp_phys_rmcall_gr_promote=%d is outside 0..22; clamping to 0 (no PROMOTE_CTX)\n", gsp_phys_rmcall_gr_promote_param); gsp_phys_rmcall_gr_promote_param = 0; } @@ -444,7 +444,7 @@ module_param_cb(gsp_phys_rmcall_gr_promote, &gsp_phys_rmcall_gr_promote_ops, &gsp_phys_rmcall_gr_promote_param, 0644); MODULE_PARM_DESC(gsp_phys_rmcall_gr_promote, - "GR context-promote bisection (default 0: never send PROMOTE_CTX; 1: single entry init+nonmapped=0 size=raw+subctx, the p6h1b shape; 2: two-phase PA-only nonmapped then VA-only with the same fields; 3: single entry size=raw; 4: two-phase with a zero-field VA-only step; 5: batched entryCount=2 PA-only init + zero-field VA-only; 6: no-entry hVirtMemory, NV01_MEMORY_VIRTUAL (class 0x70) on the VAS at the context window + PROMOTE_CTX entryCount=0; 7: no-entry bare virtAddress+size, entryCount=0; 8: single MAIN entry like 1 on the KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC), server creates the virtual context memdesc itself; 9: same V6 channel with the stock client's zero-field VA-only MAIN entry, bInitialize=0; 10: same V6 channel, GPU_INITIALIZE_CTX (0x2080012d) installs the physical context memdesc then PROMOTE_CTX entryCount=0 bare virtAddress+size; 11: mode 10 with a server FBMEM context buffer, VIDMEM physAttr 0x4; 12: V5 UVM/vGPU-faithful PA-only MAIN entry (bInitialize=1 bNonmapped=1, no VA) and continue to BIND/SCHEDULE/submit; 13: mode 12 plus a PATCH PA-only promote first, sized from GET_ENGINE_CONTEXT_PROPERTIES engineId 0x10; 14: KMD channel (V6) + PATCH PA-only then the single-entry MAIN init+VA (the flow that books the context VA and commits ENG_CTX_VALID); 15: mode 14 with a server FBMEM MAIN context buffer and VIDMEM physAttr 0x4; 16: mode 14 plus a PM ctxsw PA-only promote first; 17: no client PROMOTE_CTX, paired with gsp_phys_srv_ctxbuf=1 so the GSP-RM server-manages and commits the context; 18: mode 8 shape plus a client FERMI_CONTEXT_SHARE_A on the TSG (hContextShare) whose construct allocates the GraphicsSubcontext; 19: the stock guest's exact 9-entry promote set (MAIN 2 MB PA+VA, PATCH, global buffers, FECS event, private access maps) on V5); values outside 0..19 clamp to 0 with a warning"); + "GR context-promote bisection (default 0: never send PROMOTE_CTX; 1: single entry init+nonmapped=0 size=raw+subctx, the p6h1b shape; 2: two-phase PA-only nonmapped then VA-only with the same fields; 3: single entry size=raw; 4: two-phase with a zero-field VA-only step; 5: batched entryCount=2 PA-only init + zero-field VA-only; 6: no-entry hVirtMemory, NV01_MEMORY_VIRTUAL (class 0x70) on the VAS at the context window + PROMOTE_CTX entryCount=0; 7: no-entry bare virtAddress+size, entryCount=0; 8: single MAIN entry like 1 on the KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC), server creates the virtual context memdesc itself; 9: same V6 channel with the stock client's zero-field VA-only MAIN entry, bInitialize=0; 10: same V6 channel, GPU_INITIALIZE_CTX (0x2080012d) installs the physical context memdesc then PROMOTE_CTX entryCount=0 bare virtAddress+size; 11: mode 10 with a server FBMEM context buffer, VIDMEM physAttr 0x4; 12: V5 UVM/vGPU-faithful PA-only MAIN entry (bInitialize=1 bNonmapped=1, no VA) and continue to BIND/SCHEDULE/submit; 13: mode 12 plus a PATCH PA-only promote first, sized from GET_ENGINE_CONTEXT_PROPERTIES engineId 0x10; 14: KMD channel (V6) + PATCH PA-only then the single-entry MAIN init+VA (the flow that books the context VA and commits ENG_CTX_VALID); 15: mode 14 with a server FBMEM MAIN context buffer and VIDMEM physAttr 0x4; 16: mode 14 plus a PM ctxsw PA-only promote first; 17: no client PROMOTE_CTX, paired with gsp_phys_srv_ctxbuf=1 so the GSP-RM server-manages and commits the context; 18: mode 8 shape plus a client FERMI_CONTEXT_SHARE_A on the TSG (hContextShare) whose construct allocates the GraphicsSubcontext; 19: the stock guest's exact 9-entry promote set (MAIN 2 MB PA+VA, PATCH, global buffers, FECS event, private access maps) on V5; 20: mode 6's no-entry hVirtMemory promote on the KMD-managed V6 channel (SKIP_CTXBUFFER_ALLOC), untested - the combination the V5 refusal (0x1f) points at); values outside 0..20 clamp to 0 with a warning"); int gsp_phys_rmcall_gr_promote_value(void) { @@ -453,6 +453,34 @@ int gsp_phys_rmcall_gr_promote_value(void) EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_promote_value); /* + * Ladder VAS ownership selector (gsp_phys_rmcall_vas_shared; default 0). + * The H1 GR promote wall is a direct consequence of the ladder's + * externally-owned FERMI_VASPACE_A: with IS_EXTERNALLY_OWNED the RM treats + * the client (UVM) as the owner of all page tables, VA bookings and engine + * context commitment, so the server never creates the context VA booking + * itself and the kickoff either asserts in kchannelMapEngineCtxBuf + * ("Externally owned object not found") or leaves the channel CTXNOTVALID. + * The normal KMD client instead allocates the VAS with SHARED_MANAGEMENT + * (nv_gpu_ops.c:2495): SET_PAGE_DIRECTORY is still legal (gpu_vaspace.c: + * 3108) and migrates the ladder's committed root PDB into the RM page + * walker, while the RM allocates the lower tables and maps/commits engine + * contexts. 1 selects the shared-management VAS; 0 keeps the byte-identical + * external path used by every previous cycle. Read once, at the stage-3 + * VAS alloc; a mid-session change is ignored (the VAS already exists). + */ +static bool gsp_phys_rmcall_vas_shared; +module_param_named(gsp_phys_rmcall_vas_shared, gsp_phys_rmcall_vas_shared, + bool, 0644); +MODULE_PARM_DESC(gsp_phys_rmcall_vas_shared, + "Ladder FERMI_VASPACE_A ownership (default 0: IS_EXTERNALLY_OWNED, the client owns page tables and VA/context commitment - the state under which every GR promote failed; 1: SHARED_MANAGEMENT, the client supplies the root PDB but the RM owns the lower page tables and maps/commits engine contexts, like the normal KMD client)"); + +bool gsp_phys_rmcall_vas_shared_value(void) +{ + return gsp_phys_rmcall_vas_shared; +} +EXPORT_SYMBOL_GPL(gsp_phys_rmcall_vas_shared_value); + +/* * One-way power latch (default). Re-enabling runtime PM and D3cold after a * claim lets the board power-gate and the host hard-freeze on resume * (observed 2026-10-01); keep the pre-claim D0 policy instead and require diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h index 7d9b695..74e3103 100644 --- a/kernel/gsp_phys_gpu.h +++ b/kernel/gsp_phys_gpu.h @@ -210,6 +210,19 @@ int gsp_phys_rmcall_engine_value(void); int gsp_phys_rmcall_gr_promote_value(void); /* + * Ladder VAS ownership selector (gsp_phys_rmcall_vas_shared; default 0). + * 0 = FERMI_VASPACE_A is allocated IS_EXTERNALLY_OWNED (historical ladder + * behaviour: the client owns page tables, VA bookings and engine-context + * commitment, which is why every host GR promote failed or wedged). + * 1 = SHARED_MANAGEMENT: the ladder still supplies and commits its root PDB + * via SET_PAGE_DIRECTORY (legal for shared management, gpu_vaspace.c:3108) + * while the RM owns the lower page tables and maps/commits engine contexts + * itself, like the normal KMD client (nv_gpu_ops.c:2495). Read at the + * stage-3 VAS alloc; a mid-session change is ignored. + */ +bool gsp_phys_rmcall_vas_shared_value(void); + +/* * Translate a guest GPA to the pinned host PA (gsp_phys_guest_mem ladder * support). The mdev is registered with vfio_register_emulated_iommu_dev(): * its VFIO type-1 container keeps the guest DMA mappings privately, so diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index ae74881..8b8708d 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -198,6 +198,9 @@ struct svc_channel { * (registered via the sema_counter debug param); * 0 = not registered */ u32 counter_last; /* last value published to counter_pa */ + u64 pushpg_va; /* last push page logged as pin-vs-window + * divergent (rate limit: one line per page) */ + u8 pushpg_n; /* pushpg lines emitted for this channel (cap) */ u32 pending_rel_pay; /* release payload decoded while counter_pa * was still unknown (the injected pin for the * counter page can land a moment after the @@ -246,6 +249,10 @@ struct svc_pin { u64 va_page; u64 pa_page; int ap; /* SVC_APERTURE_* of the mapped page */ + u8 win; /* linear push-window guess (svc_pb_pair): the page + * was NOT read back, only its window base was. Push + * reads skip these and resolve each page themselves; + * every other path still uses them as before. */ }; struct svc_vaspace { @@ -405,6 +412,8 @@ struct sema_svc { u64 entry_log; /* per-push summary budget */ u64 ops_denied; /* work refused: implausible stream */ u64 false_trans; /* resolvable VA mapped to non-push */ + u64 sem_write_reresolve; /* writes re-resolved after read-back */ + u64 pushpg_fallback; /* injected pin rejected by content */ u64 gpfifo_skipped; /* stale entries consumed */ u64 releases_seen; u64 releases_applied; @@ -415,6 +424,12 @@ struct sema_svc { u64 releases_stale; /* payload <= current */ u64 releases_jumped; /* applied with gap > 1 */ u64 releases_faked; /* skipped-entry fallback */ + u64 releases_forwarded; /* exact slot known, not translatable: + * expectation advanced, write queued */ + u64 gpfifo_held; /* passes an entry was left pending + * for a translation/grace window */ + u64 gpfifo_starved; /* entries consumed without a + * deliverable exact slot */ u64 counter_rel_applied; /* releases to registered counters */ u64 counter_rel_dbg; /* rel-apply diagnostic budget */ u64 counter_pub_dbg; /* counter-publish diagnostic budget */ @@ -480,6 +495,7 @@ struct sema_svc { u64 fwd_hold_engaged; u64 fwd_hold_timeouts; u64 fwd_hold_canceled; + u64 rel_hold_direct; /* decoded releases applied without a forwarded copy */ /* * P6c completion relay (gsp_phys_relay=1 only): relay selections armed * on a deferred guest request (fwd_relay_requested), relays whose @@ -568,6 +584,19 @@ static int svc_write(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len #define SVC_APERTURE_FB 2 /* + * Aperture selectors are codes (1=SYS, 2=FB), never booleans. The idiom + * `ap ? SVC_APERTURE_FB : SVC_APERTURE_SYS` turns a valid SYS result (1) + * into an FB access into the 256 MiB BAR1 vidmem aperture: for a guest PA + * above the aperture the access fails, and the pool-semaphore read path then + * erased the verified cache entry it had just stored. Normalise every + * resolver output through this (0 also normalises to SYS). + */ +static inline u32 svc_ap_of(int ap) +{ + return ap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS; +} + +/* * Aperture-aware access: channels describe their USERD/instance blocks with an * address-space tag, and "frame buffer" addresses are offsets into the * emulated vidmem (BAR1 backing), not guest RAM. @@ -1261,7 +1290,7 @@ static void svc_lat_store(u64 va, u64 addr, int ap) } static int pin_lookup_tab(const struct svc_pin *tab, u32 slots, u32 probe_max, - u64 va, u64 *pa, int *ap) + u64 va, u64 *pa, int *ap, bool skip_win) { u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); u32 mask = slots - 1; @@ -1274,6 +1303,13 @@ static int pin_lookup_tab(const struct svc_pin *tab, u32 slots, u32 probe_max, if (!p->va_page) return -ENOENT; if (p->va_page == page) { + /* + * The push read path asks for validated pins only: + * a window guess must not shadow the per-page + * sources it would otherwise resolve from. + */ + if (skip_win && p->win) + return -ENOENT; *pa = p->pa_page | (va & (SEMA_PAGE_SIZE - 1)); *ap = p->ap; return 0; @@ -1283,7 +1319,7 @@ static int pin_lookup_tab(const struct svc_pin *tab, u32 slots, u32 probe_max, } static void pin_store_tab(struct svc_pin *tab, u32 slots, u32 probe_max, - u32 *n, u64 *drops, u64 va, u64 pa, int ap) + u32 *n, u64 *drops, u64 va, u64 pa, int ap, u8 win) { u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); u32 mask = slots - 1; @@ -1299,6 +1335,7 @@ static void pin_store_tab(struct svc_pin *tab, u32 slots, u32 probe_max, p->va_page = page; p->pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); p->ap = ap; + p->win = win; return; } } @@ -1307,13 +1344,32 @@ static void pin_store_tab(struct svc_pin *tab, u32 slots, u32 probe_max, static int svc_pin_lookup(u64 va, u64 *pa, int *ap) { - return pin_lookup_tab(svc.pin, SVC_PIN_SLOTS, SVC_PIN_PROBE, va, pa, ap); + return pin_lookup_tab(svc.pin, SVC_PIN_SLOTS, SVC_PIN_PROBE, va, pa, ap, + false); +} + +/* + * Validated self-pins only (skip_win): what the push read path may use + * without shadowing a better per-page source. Window guesses are served by + * svc_pb_win_lookup() as the explicit last-resort fallback instead. + */ +static int svc_pin_lookup_valid(u64 va, u64 *pa, int *ap) +{ + return pin_lookup_tab(svc.pin, SVC_PIN_SLOTS, SVC_PIN_PROBE, va, pa, ap, + true); } static void svc_pin_store(u64 va, u64 pa, int ap) { pin_store_tab(svc.pin, SVC_PIN_SLOTS, SVC_PIN_PROBE, &svc.pin_n, - &svc.pin_drop, va, pa, ap); + &svc.pin_drop, va, pa, ap, 0); +} + +/* The linear window convention of svc_pb_pair: a guess, never validated. */ +static void svc_pin_store_win(u64 va, u64 pa, int ap) +{ + pin_store_tab(svc.pin, SVC_PIN_SLOTS, SVC_PIN_PROBE, &svc.pin_n, + &svc.pin_drop, va, pa, ap, 1); } /* @@ -1327,13 +1383,14 @@ static void svc_pin_store(u64 va, u64 pa, int ap) */ static int svc_ipin_lookup(u64 va, u64 *pa, int *ap) { - return pin_lookup_tab(svc.ipin, SVC_IPIN_SLOTS, SVC_IPIN_PROBE, va, pa, ap); + return pin_lookup_tab(svc.ipin, SVC_IPIN_SLOTS, SVC_IPIN_PROBE, va, pa, + ap, false); } static void svc_ipin_store(u64 va, u64 pa, int ap) { pin_store_tab(svc.ipin, SVC_IPIN_SLOTS, SVC_IPIN_PROBE, &svc.ipin_n, - &svc.ipin_drop, va, pa, ap); + &svc.ipin_drop, va, pa, ap, 0); } /* Any pinned translation: injected truth first, then self-discovered pins. */ @@ -1393,6 +1450,22 @@ static void svc_tr_note_failure(u64 page_va) tr_fail_next = (tr_fail_next + 1) % SVC_TR_FAIL_MAX; } +/* + * A successful resolve invalidates any earlier failure hold for the page: + * the hold exists to stop rescanning a VA that was not resolvable, and a + * fresh verified translation proves the guest has since mapped it. Without + * this, a page resolved through a pin/cache after a failed scan stayed + * blocked from the targeted scan for the rest of the 60 s window. + */ +static void svc_tr_clear_failure(u64 page_va) +{ + u32 i; + + for (i = 0; i < SVC_TR_FAIL_MAX; i++) + if (tr_failed[i].va == page_va) + tr_failed[i].va = 0; +} + static int svc_targeted_resolve(struct nv_vgpu_device *vgpu, u64 gpu_va, u64 *pa_out) { @@ -1475,6 +1548,7 @@ static int svc_targeted_resolve(struct nv_vgpu_device *vgpu, u64 gpu_va, *pa_out = pa | (gpu_va & (SEMA_PAGE_SIZE - 1)); svc_pin_store(page_va, pa, SVC_APERTURE_SYS); svc_lat_store(page_va, pa, SVC_APERTURE_SYS); + svc_tr_clear_failure(page_va); tr_va = 0; tr_scans++; pr_info("sema-svc: targeted resolve va=0x%llx -> pa=0x%llx (gpa=0x%llx off=%u, scan #%llu)\n", @@ -1608,34 +1682,369 @@ static int svc_resolve_va(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, * resolvable stays resolvable for the boot. */ svc_pin_store(va, *addr, ap); - *aperture = (ap == SVC_APERTURE_FB) ? 1 : 0; + /* + * Raw aperture code, matching svc_lookup_trusted/svc_pool_pa_resolve: + * a boolean 0/1 encoding here invited truthiness conversions + * downstream that mapped SYS (1) to FB. + */ + *aperture = svc_ap_of(ap); return 0; } +/* + * CE channel-pool semaphores: the completion words UVM allocates for the CE + * channels (observed at 0x121000000 + 4*ce_index inside the CE window + * 0x120000000..0x121400000). They are sysmem objects the client polls + * directly, and the learned cache / content-scanned VA map can hold a + * stale/wrong PA for them: a release written through such a translation + * lands in a page the client never reads (observed: the pool semaphore + * stayed at 1 while the guest spun in + * uvm_page_table_range_vec_init -> uvm_tracker_wait). Pool releases resolve + * pin -> verified cache -> targeted rm_mem record, never the learned + * sources first; the GR/CE completion counters (0x2040xxxxx) are outside + * this range and keep their exact-slot eager logic untouched. + */ +#define SVC_POOL_SEM_VA_LO 0x121000000ULL +#define SVC_POOL_SEM_VA_HI 0x122000000ULL + +static inline bool svc_pool_sem_va(u64 va) +{ + return va >= SVC_POOL_SEM_VA_LO && va < SVC_POOL_SEM_VA_HI; +} + +/* + * Verified pool-semaphore translations, keyed by page VA. An entry is only + * created from an injected pin or from a targeted rm_mem resolve whose page + * was read back successfully; the learned cache and the discovered map are + * never cached here. Plain per-page state on the service thread, mirroring + * counter_pa / svc_cnt_der lifetime; a failed write or an unreadable cached + * page forgets the entry so the next pass re-resolves from the live record. + */ +#define SVC_POOL_PA_MAX 64 +#define SVC_POOL_PA_LOG_MAX 32 +#define SVC_POOL_PA_HIT_LOG_MAX 32 +#define SVC_POOL_WR_LOG_MAX 64 + +struct svc_pool_pa { + u64 va_page; + u64 pa_page; + int ap; +}; + +static struct svc_pool_pa svc_pool_pa[SVC_POOL_PA_MAX]; +static u32 svc_pool_pa_n; +static u32 svc_pool_pa_next; +static u64 svc_pool_pa_logged[SVC_POOL_PA_LOG_MAX]; +static u32 svc_pool_pa_logged_n; +static u64 svc_pool_pa_miss_log; +static u32 svc_pool_pa_hit_log; +static u32 svc_pool_wr_log; + +/* First-use-per-page proof line; bounded for the boot. */ +static void svc_pool_pa_note(u32 h, u64 va, u64 pa, const char *src) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 i; + + for (i = 0; i < svc_pool_pa_logged_n; i++) + if (svc_pool_pa_logged[i] == page) + return; + if (svc_pool_pa_logged_n >= SVC_POOL_PA_LOG_MAX) + return; + svc_pool_pa_logged[svc_pool_pa_logged_n++] = page; + pr_info("vgsp-sema: pool-pa ch=0x%x va=0x%llx pa=0x%llx src=%s\n", + h, va, pa, src); +} + +static void svc_pool_pa_miss_note(u64 va) +{ + if (svc_pool_pa_miss_log >= 16) + return; + svc_pool_pa_miss_log++; + pr_info("vgsp-sema: sem-pa-miss va=0x%llx (using fallback)\n", va); +} + +/* + * Bounded per-write proof line: the first 64 pool writes always print, later + * ones are rate-limited. Every release apply funnels through svc_sem_write + * (direct release, deferred pending apply with h=0, and hold-expired apply), + * so this covers all three with the handle of the originating call site. + */ +static void svc_pool_wr_note(u32 h, u64 va, u64 pa, const char *src, + u32 cur, u32 val, const char *res) +{ + if (svc_pool_wr_log < SVC_POOL_WR_LOG_MAX) { + svc_pool_wr_log++; + pr_info("vgsp-sema: pool-wr ch=0x%x va=0x%llx pa=0x%llx src=%s cur=%u -> val=%u %s\n", + h, va, pa, src, cur, val, res); + } else { + pr_info_ratelimited("vgsp-sema: pool-wr ch=0x%x va=0x%llx pa=0x%llx src=%s cur=%u -> val=%u %s\n", + h, va, pa, src, cur, val, res); + } +} + +static int svc_pool_pa_lookup(u64 va, u64 *pa, int *ap) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 i; + + for (i = 0; i < svc_pool_pa_n; i++) { + if (svc_pool_pa[i].va_page == page) { + *pa = svc_pool_pa[i].pa_page | + (va & (SEMA_PAGE_SIZE - 1)); + *ap = svc_pool_pa[i].ap; + return 0; + } + } + return -ENOENT; +} + +static void svc_pool_pa_store(u64 va, u64 pa, int ap) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 i; + + for (i = 0; i < svc_pool_pa_n; i++) { + if (svc_pool_pa[i].va_page == page) { + svc_pool_pa[i].pa_page = + pa & ~(u64)(SEMA_PAGE_SIZE - 1); + svc_pool_pa[i].ap = ap; + return; + } + } + if (svc_pool_pa_n < SVC_POOL_PA_MAX) { + i = svc_pool_pa_n++; + } else { + i = svc_pool_pa_next++ % SVC_POOL_PA_MAX; + } + svc_pool_pa[i].va_page = page; + svc_pool_pa[i].pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); + svc_pool_pa[i].ap = ap; +} + +static void svc_pool_pa_forget(u64 va) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 i; + + for (i = 0; i < svc_pool_pa_n; i++) + if (svc_pool_pa[i].va_page == page) + svc_pool_pa[i].va_page = 0; +} + +/* + * Resolve one pool-semaphore target the way the successful deferred apply did: + * injected pin (host-verified ground truth), the verified cache, then the + * targeted rm_mem record scan, whose result is read back before it is cached. + * The learned cache, the discovered map and the page-directory walks are + * deliberately skipped: each can answer with a stale page for a pool + * semaphore, and a write there is silently lost. Returns -ENOENT when no + * source applies; the caller then uses the historical order. The cache is + * keyed by VA page only and shared by every caller: the handle is logging + * only, never part of the key. `src`, when non-NULL, names the source that + * answered (for the write proof line). + */ +static int svc_pool_pa_resolve(struct nv_vgpu_device *vgpu, u32 h, u64 va, + u64 *addr, int *ap, const char **src) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 pa = 0; + int a = SVC_APERTURE_SYS; + u32 probe = 0; + u64 wpa = 0; + int wap = SVC_APERTURE_SYS; + + if (!svc_ipin_lookup(va, &pa, &a)) { + svc_tr_clear_failure(page); + if (src) + *src = "pin"; + svc_pool_pa_note(h, va, pa, "pin"); + *addr = pa; + *ap = a; + return 0; + } + if (!svc_pool_pa_lookup(va, &pa, &a)) { + svc_tr_clear_failure(page); + if (svc_pool_pa_hit_log < SVC_POOL_PA_HIT_LOG_MAX) { + svc_pool_pa_hit_log++; + pr_info("vgsp-sema: pool-pa hit ch=0x%x va=0x%llx pa=0x%llx\n", + h, va, pa); + } + if (src) + *src = "cache"; + *addr = pa; + *ap = a; + return 0; + } + /* + * Structural ground truth before the content scan: the client's own + * page tables (the self-discovered directory; the driver-published + * directories are tried by svc_lookup_trusted). A walk result is + * deterministic and cannot pick a stale rm_mem duplicate. It is only + * accepted when the exact release offset is readable through it, the + * same read-back proof every other source must pass. On every boot + * where the directory does not contain this VA the walk misses and + * the sequence falls through unchanged. + */ + if (!svc_walk_discovered(vgpu, page, &wpa, &wap) && + wap == SVC_APERTURE_SYS) { + u64 exact = (wpa & ~(u64)(SEMA_PAGE_SIZE - 1)) | + (va & (SEMA_PAGE_SIZE - 1)); + + if (!svc_read_ap(vgpu, exact, SVC_APERTURE_SYS, &probe, + sizeof(probe))) { + svc_pool_pa_store(va, exact, SVC_APERTURE_SYS); + svc_tr_clear_failure(page); + if (src) + *src = "walk"; + svc_pool_pa_note(h, va, exact, "walk"); + *addr = exact; + *ap = SVC_APERTURE_SYS; + return 0; + } + } + if (!svc_targeted_resolve(vgpu, va, &pa)) { + /* Read back before trusting: an unreadable page is not kept. */ + if (!svc_read_ap(vgpu, pa, SVC_APERTURE_SYS, &probe, + sizeof(probe))) { + svc_pool_pa_store(va, pa, SVC_APERTURE_SYS); + svc_tr_clear_failure(page); + if (src) + *src = "targeted"; + svc_pool_pa_note(h, va, pa, "targeted"); + *addr = pa; + *ap = SVC_APERTURE_SYS; + return 0; + } + } + return -ENOENT; +} + static int svc_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val) { u64 addr = 0; u32 ap = 0; + if (svc_pool_sem_va(va)) { + if (!svc_pool_pa_resolve(vgpu, 0, va, &addr, &ap, NULL)) { + if (!svc_read_ap(vgpu, addr, svc_ap_of(ap), val, + sizeof(*val))) + return 0; + /* + * The verified translation became unreadable (the + * guest reallocated the pool). Re-resolve the live + * record once and retry: only a targeted resolve that + * also fails drops the entry, so one transient read + * failure cannot evict a translation the following + * write depends on (this was the v17 bypass). + */ + if (!svc_targeted_resolve(vgpu, va, &addr)) { + svc_pool_pa_store(va, addr, SVC_APERTURE_SYS); + if (!svc_read_ap(vgpu, addr, + SVC_APERTURE_SYS, val, + sizeof(*val))) + return 0; + } + svc_pool_pa_forget(va); + } + } if (svc_resolve_va(vgpu, va, &addr, &ap)) return -ENOENT; - return svc_read_ap(vgpu, addr, - ap ? SVC_APERTURE_FB : SVC_APERTURE_SYS, - val, sizeof(*val)); + return svc_read_ap(vgpu, addr, svc_ap_of(ap), val, sizeof(*val)); } -static int svc_sem_write(struct nv_vgpu_device *vgpu, u64 va, u32 val) +static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) { - u64 addr = 0; + u64 addr = 0, old_addr; u32 ap = 0; + u32 back = 0; + u32 cur = 0; + const char *src = "fallback"; + bool is_pool = svc_pool_sem_va(va); + int wap, rc; + bool verified = false; - if (svc_resolve_va(vgpu, va, &addr, &ap)) - return -ENOENT; - if (ap) + /* + * Pool semaphores resolve pin -> verified cache -> targeted rm_mem + * record before any learned translation; the learned cache and the + * discovered map may hold a stale PA for them and a write through it + * is lost (see svc_pool_pa_resolve). Every other VA keeps the + * historical order below. + */ + if (is_pool) { + if (!svc_pool_pa_resolve(vgpu, h, va, &addr, &ap, &src)) + verified = true; + else + svc_pool_pa_miss_note(va); + } + if (!verified && svc_resolve_va(vgpu, va, &addr, &ap)) { + /* + * No trusted translation at all. One last exact resolve: the + * targeted rm_mem scan validates the live record header for + * this VA and pins the page. Bounded: one slice per call and a + * full-scan miss is cached for SVC_TR_FAIL_HOLD, so a genuinely + * unmapped semaphore is not rescanned on every pass. + */ + if (svc_targeted_resolve(vgpu, va, &addr)) + return -ENOENT; + ap = SVC_APERTURE_SYS; + src = "targeted-direct"; + } + /* + * Raw aperture code, never a truthiness test: a SYS resolve (1) used + * to turn into FB here and the release landed in the vidmem shadow. + */ + wap = svc_ap_of(ap); + if (wap == SVC_APERTURE_FB) svc.vidmem_writes++; - return svc_write_ap(vgpu, addr, - ap ? SVC_APERTURE_FB : SVC_APERTURE_SYS, - &val, sizeof(val)); + old_addr = addr; + if (is_pool) + (void)svc_read_ap(vgpu, addr, wap, &cur, sizeof(cur)); + rc = svc_write_ap(vgpu, addr, wap, &val, sizeof(val)); + if (!rc && !svc_read_ap(vgpu, addr, wap, &back, sizeof(back)) && + back == val) { + if (is_pool) + svc_pool_wr_note(h, va, addr, src, cur, val, "ok"); + return 0; + } + + /* + * The value is not visible through the translation we just wrote: + * the entry can be a stale content-scanned cache line or an + * unvalidated self-pin whose page is not the one the client polls. + * Re-resolve the exact VA through the targeted rm_mem record scan + * (the live record names the client's current PA) and rewrite once. + * Bounded: exactly one re-resolve; its result is read back but never + * re-resolved again, so a hopeless VA costs one extra attempt. + */ + if (is_pool) + svc_pool_pa_forget(va); /* stale: rebuild from the live record */ + if (!svc_targeted_resolve(vgpu, va, &addr)) { + back = 0; + if (!svc_write_ap(vgpu, addr, SVC_APERTURE_SYS, &val, + sizeof(val)) && + !svc_read_ap(vgpu, addr, SVC_APERTURE_SYS, &back, + sizeof(back)) && back == val) { + if (is_pool) { + svc_pool_pa_store(va, addr, SVC_APERTURE_SYS); + svc_pool_wr_note(h, va, addr, "targeted", cur, + val, "reresolve"); + } + svc.sem_write_reresolve++; + pr_info_ratelimited("vgsp-sema: sem-write re-resolve va=0x%llx pa=0x%llx old=0x%llx new=0x%x\n", + va, addr, old_addr, val); + return 0; + } + } + if (!rc) { + if (is_pool) + svc_pool_wr_note(h, va, old_addr, src, cur, val, + "advisory"); + return 0; /* the original write landed; read-back is advisory */ + } + if (is_pool) + svc_pool_wr_note(h, va, old_addr, src, cur, val, "fail"); + return rc; } /* ------------------------------------------------------------------ */ @@ -1981,8 +2390,12 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) svc.memsets_fail, svc.ring_ops, svc.ops_denied, svc.false_trans, svc.gpfifo_skipped, svc.releases_faked, svc.releases_recovered, svc.counter_rel_applied); - pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu\n", - svc.counter_pub_calls, svc.gr_rel_applied); + pr_info("sema-svc: drain held=%llu starved=%llu forwarded=%llu faked=%llu rec=%llu\n", + svc.gpfifo_held, svc.gpfifo_starved, svc.releases_forwarded, + svc.releases_faked, svc.releases_recovered); + pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu\n", + svc.counter_pub_calls, svc.gr_rel_applied, + svc.sem_write_reresolve, svc.pushpg_fallback); pr_info("sema-svc: gpfifo done=%llu nomap=%llu vidmem_writes=%llu ring_auto=%u pb=%u/%u pins=%u/%llu ipins=%u/%llu\n", svc.gpfifo_entries_done, svc.gpfifo_nomap, svc.vidmem_writes, svc_ring_n, svc_pb_n, svc_pb_passes, @@ -2010,13 +2423,13 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) svc.fwd_rel_fallback || svc.fwd_rel_pending || svc.fwd_rel_overflow || svc.fwd_hold_engaged || svc.fwd_hold_timeouts || svc.fwd_hold_canceled) - pr_info("sema-svc: work_guard deferred=%llu applied=%llu fallback=%llu pending=%llu overflow=%llu engaged=%llu timeouts=%llu canceled=%llu copy_skipped=%llu copy_soft=%llu copy_fail=%llu\n", + pr_info("sema-svc: work_guard deferred=%llu applied=%llu fallback=%llu pending=%llu overflow=%llu engaged=%llu timeouts=%llu canceled=%llu copy_skipped=%llu copy_soft=%llu copy_fail=%llu direct=%llu\n", svc.fwd_rel_deferred, svc.fwd_rel_applied, svc.fwd_rel_fallback, svc.fwd_rel_pending, svc.fwd_rel_overflow, svc.fwd_hold_engaged, svc.fwd_hold_timeouts, svc.fwd_hold_canceled, svc.fwd_copy_skipped, svc.fwd_copy_soft, - svc.fwd_copy_fail); + svc.fwd_copy_fail, svc.rel_hold_direct); /* show a few entries so mappings can be verified from dmesg */ if (gsp_sema_dbg) { u32 shown; @@ -2063,6 +2476,28 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) #define GRCE_M_LAUNCH_DMA 0x1b0 #define GRCE_M_LOAD_INLINE_DATA 0x1b4 +/* + * C5C0 (compute class) inline-QMD launch methods: SET_INLINE_QMD_ADDRESS_A/B + * carry the shifted QMD device address, LOAD_INLINE_QMD_DATA the 64 QMD words. + * The driver emits them as one INC run covering 0x318..0x41c (observed: + * op=1 count=66 at 0x318) or as several runs; the host CUDA bridge + * (gsp_cuda_bridge.c) accumulates the words and turns a completed QMD into a + * daemon job. + */ +#define SVC_QMD_ADDR_A 0x318 +#define SVC_QMD_ADDR_B 0x31c +#define SVC_QMD_DATA 0x320 +#define SVC_QMD_WORDS 64 + +/* + * Recent GRCE LOAD_INLINE_DATA destinations, newest at (next - 1). CUDA + * writes the launch arguments into the QMD's parameter bank at an arbitrary + * offset with these writes (observed: mail args at bank+0x160), so the host + * CUDA bridge searches this ring for the argument block instead of reading + * bank+0. Service thread only, so no lock. + */ +#define SVC_GRCE_DST_MAX 32 + /* LAUNCH_DMA fields. */ #define DMA_DATA_TRANSFER_TYPE(v) ((v) & 0x3) #define DMA_SEMAPHORE_TYPE(v) (((v) >> 3) & 0x3) @@ -2116,8 +2551,37 @@ struct svc_push_state { u64 rel_va; /* last release decoded in this push */ u32 rel_payload; u32 have_rel; + /* inline-QMD launch accumulation (GR compute channel) */ + u64 qmd_dev_va; /* devVA decoded from ADDRESS_A/B */ + u32 qmd_hi, qmd_lo; /* ADDRESS_A / ADDRESS_B payloads */ + u32 qmd_have; /* A/B seen bitmap */ + u32 qmd_n; /* QMD words collected so far */ + u64 qmd_bits; /* received-word bitmap */ + u64 push_va; /* push VA, for launch logging */ + u32 qmd[SVC_QMD_WORDS]; }; +/* Host CUDA bridge (gsp_cuda_bridge.c): launch transport + service hook. */ +void gsp_cuda_bridge_launch(struct nv_vgpu_device *vgpu, u32 h_object, + const u32 qmd[SVC_QMD_WORDS], u64 qmd_dev_va, + u64 push_va, const u64 *recent_dsts, + u32 n_recent_dsts); +bool gsp_cuda_bridge_enabled(void); +void gsp_cuda_bridge_service(struct nv_vgpu_device *vgpu); +int gsp_cuda_bridge_init(void); +void gsp_cuda_bridge_fini(void); + +/* VA/semaphore helpers exported to gsp_cuda_bridge.c. */ +int gsp_sema_service_va_read(struct nv_vgpu_device *vgpu, u64 va, void *buf, + size_t len); +int gsp_sema_service_va_write(struct nv_vgpu_device *vgpu, u64 va, + const void *buf, size_t len); +int gsp_sema_service_va_resolve(struct nv_vgpu_device *vgpu, u64 va, u64 *pa, + u32 *ap); +int gsp_sema_service_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val); +int gsp_sema_service_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, + u32 val); + /* ------------------------------------------------------------------ */ /* CE execution */ /* ------------------------------------------------------------------ */ @@ -2782,15 +3246,35 @@ static bool svc_stream_parse(const u8 *buf, u32 len) if (off + 4 > len) break; memcpy(&hdr, buf + off, 4); - if (hdr == 0) + if (hdr == 0) { + /* + * UVM's large page-table pushes begin with a + * zero-filled header region (observed 0x1004 + * bytes) before the first method, so zeros before + * the first method are leading padding; zeros + * after one are trailing padding and end the + * stream. + */ + if (!methods) { + off += 4; + continue; + } break; /* trailing zero padding */ + } op = (hdr >> 29) & 0x7; count = (hdr >> 16) & 0xfff; switch (op) { case 1: /* INC_METHOD */ case 3: /* NON_INC_METHOD */ case 5: /* ONE_INC */ - if (!count || count > 64 || + /* + * The count field is 12 bits, and UVM emits + * NON_INC runs of 1024 words (the page-table + * pushes); the old 64 cap rejected exactly those, + * which made the content rescue unable to ever + * validate a large push. + */ + if (!count || count > 0xfff || off + 4 + count * 4 > len) return false; off += 4 + count * 4; @@ -2851,7 +3335,15 @@ static u64 svc_rel_va_seen; static u32 svc_rel_count; static u32 svc_dbg_streams; -static void svc_counter_learn(u32 h, u64 sem_va); +/* + * Ring of recent GRCE inline-write destinations (newest at next-1), consumed + * by the CUDA bridge to locate a launch's argument block inside the QMD + * parameter bank. Written and read on the service thread only. + */ +static u64 svc_grce_dst_ring[SVC_GRCE_DST_MAX]; +static u32 svc_grce_dst_ring_next; + +static void svc_counter_learn(struct nv_vgpu_device *vgpu, u32 h, u64 sem_va); static void svc_pending_add(u64 va, u32 payload, u32 op); static int svc_rescue_push_page(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 entry_va, @@ -2902,7 +3394,7 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h, * it from the pin table first. */ if (sem_va >= 0x204000000ULL && sem_va < 0x204100000ULL) - svc_counter_learn(h, sem_va); + svc_counter_learn(vgpu, h, sem_va); spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { @@ -3019,12 +3511,88 @@ static void svc_counter_flush_pending(struct nv_vgpu_device *vgpu, } /* + * In-kernel counter-slot derivation. + * + * The authoritative counter translation is the external injected pin, but the + * pin watcher can land tens of seconds after the first release of a channel + * (observed: a GR release at 7586.5 s whose counter was only learned at + * 7609.9 s), and every release queued in that window leaves the client's + * tracker polling a value that never arrives. On an injected-pin miss, + * resolve the slot from the module's own translations and register it - but + * only translations this service already treats as structural truth: + * + * - the page directories the driver itself published (svc_walk_vaspaces), + * - the self-discovered directory (svc_walk_discovered; accepted only after + * it reproduced several already-known VA->PA pairs), and + * - the targeted rm_mem resolve (svc_targeted_resolve; validates the rm_mem + * record header for this exact VA). + * + * The content-scanned VA map and the learned cache are deliberately NOT + * consulted: both can hold a stale translation for a counter page, and a + * write through one lands in a page the client never polls (see + * svc_counter_autoreg). A successful derivation is proven readable before it + * is returned, cached per page so the next release does not rescan, and never + * overrides an injected pin: svc_counter_learn re-checks the pin table first + * and re-registers when the injected PA differs. + * + * Service-thread only (like every other release/decode path); the cache is + * plain per-page state, mirroring counter_pa's lifetime. + */ +#define SVC_CNT_DER_MAX 16 +static struct { + u64 va_page; + u64 pa_page; +} svc_cnt_der[SVC_CNT_DER_MAX]; +static u32 svc_cnt_der_n; + +static int svc_counter_derive(struct nv_vgpu_device *vgpu, u64 va, u64 *pa_out) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 off = va & (SEMA_PAGE_SIZE - 1); + u64 pa = 0; + int ap = SVC_APERTURE_SYS; + u32 probe = 0; + u32 i; + bool fresh = true; + + for (i = 0; i < svc_cnt_der_n; i++) { + if (svc_cnt_der[i].va_page == page) { + pa = svc_cnt_der[i].pa_page | off; + fresh = false; + break; + } + } + if (fresh) { + if (svc_walk_vaspaces(vgpu, page, &pa, &ap) && + svc_walk_discovered(vgpu, page, &pa, &ap) && + svc_targeted_resolve(vgpu, page, &pa)) + return -ENOENT; + if (ap != SVC_APERTURE_SYS) + return -ENOENT; + pa = (pa & ~(u64)(SEMA_PAGE_SIZE - 1)) | off; + } + /* Read the slot before trusting the mapping: an unreadable page is + * not registered, so the caller keeps the "slot pending" fallback. */ + if (svc_read_ap(vgpu, pa, SVC_APERTURE_SYS, &probe, sizeof(probe))) + return -EIO; + if (fresh && svc_cnt_der_n < SVC_CNT_DER_MAX) { + svc_cnt_der[svc_cnt_der_n].va_page = page; + svc_cnt_der[svc_cnt_der_n].pa_page = + pa & ~(u64)(SEMA_PAGE_SIZE - 1); + svc_cnt_der_n++; + } + *pa_out = pa; + return 0; +} + +/* * Auto-register a GR channel's completion-counter slot from the observed * layout, without waiting for a readable push: the client zips GR channels * in ascending handle order with descending counter slots, so with n_gr - * channels the rank-r channel owns slot 0x20400fec0 + (n_gr-1-r)*0x10. Only - * a host-verified (pinned) translation is used, so the write target is the - * page the client actually polls. This lets the publish path advance every + * channels the rank-r channel owns slot 0x20400fec0 + (n_gr-1-r)*0x10. The + * slot PA comes from a host-verified pin or from the in-kernel structural + * derivation (svc_counter_derive), so the write target is a page the client + * actually polls. This lets the publish path advance every * channel's counter even when some GR pushes are unreadable (observed: the * lower ranks' 676-byte context-init pushes stayed unresolved while the * other 13 channels' counters advanced, and the client timed out waiting). @@ -3034,6 +3602,7 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu, { u64 va, pa = 0; int ap = SVC_APERTURE_SYS; + bool derived = false; if (ch->engine_type || n_gr <= 0) return; @@ -3045,52 +3614,76 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu, return; va = 0x20400fec0ULL + (u64)(n_gr - 1 - ch->gr_index) * 0x10ULL; /* - * Only the authoritative injected-pin table may register a counter - * slot: the self-discovered map can hold a stale translation for this - * VA (content-scanned page), and a write to that page never reaches - * the client. If no injected pin exists yet, leave counter_pa zero so - * a later pass (once the pin lands) registers it. + * The injected-pin table is the authoritative source; only it may + * shadow a self-discovered map entry (which can hold a stale + * translation for this VA - a content-scanned page - and a write + * there never reaches the client). On a miss, derive the slot + * in-kernel from the structural translations and the read-back proof + * instead of waiting for the pin watcher; the layout formula only + * names which slot belongs to this rank. If no translation exists, + * leave counter_pa zero so a later pass registers it. */ - if (svc_ipin_lookup(va, &pa, &ap) || ap != SVC_APERTURE_SYS) - return; + if (svc_ipin_lookup(va, &pa, &ap) || ap != SVC_APERTURE_SYS) { + if (!gsp_sema_counter_derive || + svc_counter_derive(vgpu, va, &pa)) + return; + derived = true; + } ch->counter_pa = pa & ~0x3ULL; - pr_info("sema-chan: counter auto h=0x%x rank=%u pa=0x%llx (slot va=0x%llx)\n", - ch->h_object, ch->gr_index, ch->counter_pa, va); + if (derived) + pr_info("sema-chan: counter derived h=0x%x rank=%u pa=0x%llx (slot va=0x%llx)\n", + ch->h_object, ch->gr_index, ch->counter_pa, va); + else + pr_info("sema-chan: counter auto h=0x%x rank=%u pa=0x%llx (slot va=0x%llx)\n", + ch->h_object, ch->gr_index, ch->counter_pa, va); svc_counter_flush_pending(vgpu, ch); } /* * Learn a GR channel's completion-counter guest PA from a release VA the * push carried, so the publish path can advance the exact slot the client - * polls without any guest-side enumeration. Only a host-verified translation - * (pin/ipin) is trusted; the release VA of a GR push is the counter slot - * itself (0x20400fXX observed). No-op once registered. + * polls without any guest-side enumeration. The release VA of a GR push is + * the counter slot itself (0x20400fXX observed). An injected pin is the + * authoritative source; on a miss the slot is derived in-kernel + * (svc_counter_derive) so the release is authored immediately rather than + * waiting for the external pin watcher. No-op once registered at the same PA. */ -static void svc_counter_learn(u32 h, u64 sem_va) +static void svc_counter_learn(struct nv_vgpu_device *vgpu, u32 h, u64 sem_va) { unsigned long flags; u64 pa = 0; int ap = SVC_APERTURE_SYS; + bool derived = false; int i; if (!h || sem_va < 0x204000000ULL || sem_va >= 0x204100000ULL) return; /* - * Authoritative injected pins only (see svc_counter_autoreg): a stale - * self-discovered entry would make every release land on a page the - * client never polls. Re-learn when the injected PA differs, so a pin - * that lands after an earlier stale registration still wins. + * Injected pins first (see svc_counter_autoreg): they stay + * authoritative and re-learn when their PA differs, so a pin that + * lands after a derived/stale registration still wins. On a miss, + * derive from the structural translations only - never from the + * content-scanned map or the learned cache, which can hold a stale + * page for this VA. */ - if (svc_ipin_lookup(sem_va, &pa, &ap) || ap != SVC_APERTURE_SYS) - return; + if (svc_ipin_lookup(sem_va, &pa, &ap) || ap != SVC_APERTURE_SYS) { + if (!gsp_sema_counter_derive || + svc_counter_derive(vgpu, sem_va, &pa)) + return; + derived = true; + } spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { if (svc.chan[i].valid && svc.chan[i].h_object == h) { if (svc.chan[i].counter_pa != (pa & ~0x3ULL)) { svc.chan[i].counter_pa = pa & ~0x3ULL; - pr_info("sema-chan: counter learned h=0x%x pa=0x%llx (from release va=0x%llx)\n", - h, pa & ~0x3ULL, sem_va); + if (derived) + pr_info("sema-chan: counter derived h=0x%x pa=0x%llx (from release va=0x%llx)\n", + h, pa & ~0x3ULL, sem_va); + else + pr_info("sema-chan: counter learned h=0x%x pa=0x%llx (from release va=0x%llx)\n", + h, pa & ~0x3ULL, sem_va); } break; } @@ -3114,14 +3707,14 @@ static bool svc_rel_apply_906f(struct nv_vgpu_device *vgpu, u32 cur = 0; if (ch->h_object && svc_counter_va(va)) { - svc_counter_learn(ch->h_object, va); + svc_counter_learn(vgpu, ch->h_object, va); svc_counter_release_apply(vgpu, ch->h_object, va, payload); return false; } if (!svc_sem_read(vgpu, va, &cur) && payload > cur) { svc.pushes_pending++; cur = (op == HOST_SEM_OP_REDUCTION) ? cur + payload : payload; - if (!svc_sem_write(vgpu, va, cur)) { + if (!svc_sem_write(vgpu, ch->h_object, va, cur)) { svc.releases_applied++; svc.post_hint_until = jiffies + 10 * HZ; return true; @@ -3137,7 +3730,7 @@ static bool svc_rel_apply_reportsem(struct nv_vgpu_device *vgpu, u32 cur = 0; if (ch->h_object && svc_counter_va(va)) { - svc_counter_learn(ch->h_object, va); + svc_counter_learn(vgpu, ch->h_object, va); svc_counter_release_apply(vgpu, ch->h_object, va, payload); return false; } @@ -3148,7 +3741,7 @@ static bool svc_rel_apply_reportsem(struct nv_vgpu_device *vgpu, if (payload <= cur) return false; svc.pushes_pending++; - if (!svc_sem_write(vgpu, va, payload)) { + if (!svc_sem_write(vgpu, ch->h_object, va, payload)) { svc.releases_applied++; svc.post_hint_until = jiffies + 10 * HZ; return true; @@ -3175,14 +3768,14 @@ static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu, */ if (ch->engine_type == 0 && svc_counter_va(sem_va)) { /* handled by the exact-slot path */ - } else if (!svc_sem_write(vgpu, sem_va, cur)) { + } else if (!svc_sem_write(vgpu, ch->h_object, sem_va, cur)) { svc.releases_applied++; svc.post_hint_until = jiffies + 10 * HZ; return true; } } if (ch->engine_type == 0) - svc_counter_learn(ch->h_object, sem_va); + svc_counter_learn(vgpu, ch->h_object, sem_va); svc_counter_release_apply(vgpu, ch->h_object, sem_va, payload); return false; } @@ -3235,19 +3828,109 @@ static void svc_fwd_relay_note(struct nv_vgpu_device *vgpu, DMA_SEM_REDUCTION(launch_dma) == SEM_REDUCTION_INC) return; } - if (svc_resolve_va(vgpu, r->va, &addr, &ap)) + if (svc_pool_sem_va(r->va)) { + if (svc_pool_pa_resolve(vgpu, ch->h_object, r->va, &addr, + &ap, NULL)) + return; + } else if (svc_resolve_va(vgpu, r->va, &addr, &ap)) { return; + } if (svc_sem_read(vgpu, r->va, &cur)) return; /* Forward-only pre-check: only a word behind the payload is relayed. */ if (cur >= r->payload) return; r->tgt_ok = true; - r->tgt_kind = ap ? SVC_APERTURE_FB : SVC_APERTURE_SYS; + r->tgt_kind = svc_ap_of(ap); r->tgt_page = addr & ~(u64)(SEMA_PAGE_SIZE - 1); r->tgt_off = (u32)(addr & (SEMA_PAGE_SIZE - 1)); } +/* Whole-boot budget for the "no forwarded copy" proof line. */ +static u32 svc_fwd_rel_direct_log; + +/* + * One rate-limited line per decoded release that was applied directly because + * no forwarded hardware copy ever engaged (or the bounded hold window expired + * without one). A decoded release is never left to a copy that will not run; + * this line is the next-run proof of that path. + */ +static void svc_fwd_rel_direct_note(u32 h, u64 va, u32 payload) +{ + svc.rel_hold_direct++; + if (svc_fwd_rel_direct_log >= 32) + return; + svc_fwd_rel_direct_log++; + pr_info("vgsp-work: rel-hold direct h=0x%x va=0x%llx pay=%u (no work)\n", + h, va, payload); +} + +/* + * Accumulate one method run's inline-QMD words; hand a complete 64-word QMD + * to the host CUDA bridge. The run may cover the address registers and the + * data registers as a single INC/NON_INC/ONE_INC method (observed: op=1 + * count=66 starting at 0x318), so every data word is mapped to its register + * offset. Each completed QMD resets the accumulator, so a push may carry + * more than one launch. + */ +static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, + struct svc_push_state *st, u32 addr, u32 count, + const u8 *data) +{ + u32 i; + + for (i = 0; i < count; i++) { + u32 a = addr + 4 * i; + u32 v; + + memcpy(&v, data + 4 * i, 4); + if (a == SVC_QMD_ADDR_A) { + st->qmd_hi = v; + st->qmd_have |= 1; + } else if (a == SVC_QMD_ADDR_B) { + st->qmd_lo = v; + st->qmd_have |= 2; + } else if (a >= SVC_QMD_DATA && + a < SVC_QMD_DATA + SVC_QMD_WORDS * 4) { + u32 idx = (a - SVC_QMD_DATA) / 4; + + if (!(st->qmd_bits & (1ULL << idx))) { + st->qmd_bits |= 1ULL << idx; + st->qmd[idx] = v; + st->qmd_n++; + } + } + } + if (st->qmd_have == 3) + st->qmd_dev_va = (((u64)st->qmd_hi << 32) | st->qmd_lo) << 8; + if (st->qmd_n == SVC_QMD_WORDS) { + u64 dsts[SVC_GRCE_DST_MAX]; + u32 have, i, n = 0; + + /* + * Hand the bridge the recent inline-write destinations, + * newest first, so it can find the argument block CUDA + * placed inside the QMD parameter bank (never assume + * bank+0). The current push's writes are already in the + * ring because the QMD run follows them. + */ + have = svc_grce_dst_ring_next < SVC_GRCE_DST_MAX ? + svc_grce_dst_ring_next : SVC_GRCE_DST_MAX; + for (i = 0; i < have; i++) { + u32 idx = (svc_grce_dst_ring_next - 1 - i) % + SVC_GRCE_DST_MAX; + + dsts[n++] = svc_grce_dst_ring[idx]; + } + gsp_cuda_bridge_launch(vgpu, ch->h_object, st->qmd, + st->qmd_dev_va, st->push_va, + dsts, n); + st->qmd_n = 0; + st->qmd_bits = 0; + st->qmd_have = 0; + } +} + static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u32 len, u64 page_pa, bool allow_ops, struct svc_channel *ch, u32 entry_idx, @@ -3279,6 +3962,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, } memset(&st, 0, sizeof(st)); + st.push_va = push_va; /* * P6b wave 1: clear the guest-forwarding candidate at the start of * every decode pass; the first eligible copy captured below is the one @@ -3364,6 +4048,19 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, trusted, allow_ops, st.have_out, st.line_length, ch->h_object); } + /* + * Record the destination for the CUDA bridge's + * argument-block search, whether or not the write was + * executed: the launch QMD later in this push needs it + * to find where CUDA placed the kernel arguments in + * the QMD parameter bank. + */ + if (st.have_out && st.offset_out) { + svc_grce_dst_ring[svc_grce_dst_ring_next % + SVC_GRCE_DST_MAX] = + st.offset_out; + svc_grce_dst_ring_next++; + } off += 4 + count * 4; continue; } @@ -3375,12 +4072,28 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, continue; } - if ((op != 1 && op != 3 && op != 5) || !count || count > 64 || + if ((op != 1 && op != 3 && op != 5) || !count || count > 0xfff || off + 4 + count * 4 > len) { off += 4; /* resync */ continue; } + /* + * CUDA launch (C5C0 compute class, subchannel >= 1): capture + * the inline-QMD method run for the host CUDA bridge. The run + * is skipped whole exactly as before (release/decode behaviour + * is untouched); a completed 64-word QMD queues a daemon job. + */ + if (ch->engine_type == 0 && subch >= 1 && + gsp_cuda_bridge_enabled() && + addr >= SVC_QMD_ADDR_A && + addr <= SVC_QMD_DATA + (SVC_QMD_WORDS - 1) * 4) { + svc_qmd_accum(vgpu, ch, &st, addr, count, + buf + off + 4); + off += 4 + count * 4; + continue; + } + if (op == 3 || op == 5) { /* * NON_INC / ONE_INC methods carry their own data @@ -3502,7 +4215,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * only the apply moves. Overflow applies inline. */ if (gsp_phys_work_defer_release_enabled() && - svc_fwd_cand.valid) { + svc_fwd_cand.valid && + gsp_phys_guest_mem_enabled()) { if (svc_fwd_rel_n < SVC_FWD_REL_MAX) { struct svc_fwd_rel *r = &svc_fwd_rel[svc_fwd_rel_n++]; @@ -3617,7 +4331,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * Overflow applies inline. */ if (gsp_phys_work_defer_release_enabled() && - svc_fwd_cand.valid) { + svc_fwd_cand.valid && + gsp_phys_guest_mem_enabled()) { if (svc_fwd_rel_n < SVC_FWD_REL_MAX) { struct svc_fwd_rel *r = &svc_fwd_rel[svc_fwd_rel_n++]; @@ -3952,7 +4667,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * release as decoded. Overflow applies inline. */ if (gsp_phys_work_defer_release_enabled() && - svc_fwd_cand.valid) { + svc_fwd_cand.valid && + gsp_phys_guest_mem_enabled()) { if (svc_fwd_rel_n < SVC_FWD_REL_MAX) { struct svc_fwd_rel *r = &svc_fwd_rel[svc_fwd_rel_n++]; @@ -3996,12 +4712,29 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, svc_rel_count++; } } - } else if (svc_rel_apply_launch(vgpu, ch, sem_va, - st.sem_payload, - cur, pending, inc, - inc_apply)) { - svc_rel_va_seen = sem_va; - svc_rel_count++; + } else { + /* + * Deferral is enabled but no forwarded + * hardware copy can run for this push + * (guest-memory copy path off, or no + * candidate): apply the decoded release + * directly, exactly once. The note is + * the proof line that a decoded release + * is never left to a copy that will + * not engage. + */ + if (gsp_phys_work_defer_release_enabled() && + svc_fwd_cand.valid) + svc_fwd_rel_direct_note(ch->h_object, + sem_va, + st.sem_payload); + if (svc_rel_apply_launch(vgpu, ch, sem_va, + st.sem_payload, + cur, pending, inc, + inc_apply)) { + svc_rel_va_seen = sem_va; + svc_rel_count++; + } } /* release ends the push */ n_ops = 0; @@ -4096,7 +4829,7 @@ static void svc_apply_pending(struct nv_vgpu_device *vgpu) svc_pending[i] = svc_pending[--svc_pending_n]; continue; } - if (!svc_sem_write(vgpu, p->va, + if (!svc_sem_write(vgpu, 0, p->va, p->op ? cur + p->payload : p->payload)) { svc.releases_applied++; svc.post_hint_until = jiffies + 10 * HZ; @@ -4239,7 +4972,7 @@ static u32 __maybe_unused __svc_apply_page_data_disabled(struct nv_vgpu_device * (lo & 0xfff) < 0x20 && svc_pb_first_push(buf)) svc_pb_add(pa_page, lo & 0xfff); - if (svc_sem_write(vgpu, va, payload) == 0) { + if (svc_sem_write(vgpu, 0, va, payload) == 0) { svc.releases_applied++; svc.post_hint_until = jiffies + 10 * HZ; applied++; @@ -4595,6 +5328,223 @@ static int svc_read_va_trusted(struct nv_vgpu_device *vgpu, u64 va, void *buf, } /* + * Linear fallback for a push-buffer window page. + * + * svc_pb_pair() pairs the window base VA with the base PA it found by content + * and pins page N at base_pa + N*0x1000. That holds only while the guest's + * push-buffer allocation is physically contiguous, which is not guaranteed + * and was observed to change run-to-run. This returns the guess for a page in + * the channel's window; callers consult it only after every validated + * per-page source missed. Returns 0 with the page-aligned PA, -ENOENT when + * the page is outside every window known for the channel's rank. + */ +static int svc_pb_win_lookup(const struct svc_channel *ch, u64 va_page, + u64 *pa_page) +{ + u32 slot, i; + + if (!ch || !ch->engine_type || ch->ce_index >= 32) + return -ENOENT; + slot = 4 * ch->ce_index; + for (i = 0; i < svc_pb_n; i++) { + u64 pa; + + if (svc_pb[i].slot != slot || !svc_pb[i].va || + !svc_pb[i].pa_page) + continue; + if (va_page < svc_pb[i].va || + va_page >= svc_pb[i].va + 0x100000ULL) + continue; + pa = svc_pb[i].pa_page + (va_page - svc_pb[i].va); + if (pa >= SEMA_PA_MAX) + return -ENOENT; + *pa_page = pa; + return 0; + } + return -ENOENT; +} + +#define SVC_PUSH_PG_LOG_MAX 4 + +/* + * One line when a push page resolved from a validated pin while the channel's + * linear window guess would have answered differently - the non-contiguous + * case the window convention gets wrong. Capped per channel and one line per + * page so a boot log shows whether pins carried the read without per-push + * spam. Behind sema_dbg like the other decode diagnostics. + */ +static void svc_pushpg_note(struct svc_channel *ch, u64 page_va, u64 pin_pa, + u64 win_pa) +{ + if (!gsp_sema_dbg || ch->pushpg_n >= SVC_PUSH_PG_LOG_MAX) + return; + if ((pin_pa & ~0xfffULL) == (win_pa & ~0xfffULL)) + return; /* the window guess already agrees */ + if (ch->pushpg_va == page_va) + return; + ch->pushpg_va = page_va; + ch->pushpg_n++; + pr_info("sema-pushpg: h=0x%x va=0x%llx pin=1 pa=0x%llx win=0x%llx\n", + ch->h_object, page_va, pin_pa & ~0xfffULL, + win_pa & ~0xfffULL); +} + +/* + * Cheap candidate-arbitration prefix: a real push starts with a method header + * (op 1/3/5 with a non-zero count, or a header-only 4/7), optionally after + * the zero-filled header region UVM emits before large page-table pushes + * (observed up to 0x1004 bytes). A zero page or unrelated data practically + * never starts with one, so a wrong injected pin can be rejected before the + * pump's full-stream parse. An all-zero chunk cannot refute the pin and is + * accepted; the full parse is the final arbiter. The run length is + * deliberately not bounded by the first chunk: a valid method run may span + * into the next page and the full parse settles that. + */ +static bool svc_push_prefix_ok(const u8 *buf, size_t len) +{ + u32 hdr, op, count; + size_t off = 0; + + while (off + 4 <= len) { + memcpy(&hdr, buf + off, 4); + if (hdr != 0) + break; + off += 4; + } + if (off + 4 > len) + return true; /* all zeros: not refutable here */ + op = (hdr >> 29) & 0x7; + if (op == 4 || op == 7) /* header-only methods */ + return true; + if (op != 1 && op != 3 && op != 5) + return false; + count = (hdr >> 16) & 0xfff; + return count != 0; +} + +/* Fallback-line budget for the whole boot. */ +static u32 svc_pushpg_fb_log; + +/* + * Read a push the way the engine does: every 4 KB page of the range is + * translated independently. + * + * The old push read trusted the window's linear mapping for the whole range + * (through the pins svc_pb_pair installed). When the guest's push-buffer + * allocation is not physically contiguous, the continuation page answered + * with the wrong PA, the method-stream plausibility check failed, and the + * release the client waited on was never applied (observed: push #25 on + * h=0xcaf00012 at va=0x120000f58 crossing into a page whose PA was not + * base+0x1000; the pool semaphore stalled at 24 and the guest UVM hung). + * + * Order per page: + * 1. injected pin (svc_ipin_lookup): host-verified ground truth, but the + * content arbitrates - a pin whose first chunk does not start like a + * method stream is rejected and the remaining candidates are tried + * (observed: a wrong injected pin for the push page made every push + * decode no release, starving the client's tracker); + * 2. validated self-pin (svc_pin_lookup_valid): read-back-validated pages + * (content rescue, first-push pairing), window guesses excluded; + * 3. the channel's linear window mapping (svc_pb_win_lookup): last resort + * for a page no validated source knows; + * 4. outside any window: the remaining trusted sources + * (svc_lookup_trusted), exactly the old behaviour for GR/other pushes. + * + * For a contiguous allocation every path resolves to the same PA the old + * read returned, so the normal case is byte-for-byte unchanged; only a + * non-contiguous or wrongly-pinned page changes behaviour (to its next + * validated candidate). Reads stay defensively copied through + * svc_read_ap()/vfio_dma_rw(), which enforce the aperture bounds. + */ +static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 va, void *buf, + size_t len, bool skip_ipin) +{ + u8 *p = buf; + + while (len) { + u64 page_va = va & ~(u64)(SEMA_PAGE_SIZE - 1); + size_t chunk = min_t(size_t, len, + SEMA_PAGE_SIZE - (va & (SEMA_PAGE_SIZE - 1))); + u64 pa = 0, win_pa = 0; + int ap = SVC_APERTURE_SYS; + bool have_win, done = false; + bool first = (p == buf); + int last_rc = -ENOENT; + u32 src; + + have_win = !svc_pb_win_lookup(ch, page_va, &win_pa); + + for (src = 0; src < 4 && !done; src++) { + bool via_ipin = false; + + pa = 0; + ap = SVC_APERTURE_SYS; + switch (src) { + case 0: + if (skip_ipin || + svc_ipin_lookup(va, &pa, &ap)) + continue; + via_ipin = true; + if (have_win) + svc_pushpg_note(ch, page_va, pa, + win_pa); + break; + case 1: + if (svc_pin_lookup_valid(va, &pa, &ap)) + continue; + if (have_win) + svc_pushpg_note(ch, page_va, pa, + win_pa); + break; + case 2: + if (!have_win) + continue; + pa = win_pa + (va & (SEMA_PAGE_SIZE - 1)); + ap = SVC_APERTURE_SYS; + break; + default: + /* learned cache / discovered map / page + * directories / targeted rm_mem resolve */ + if (svc_lookup_trusted(vgpu, va, &pa, &ap)) + continue; + break; + } + + if (ap == SVC_APERTURE_SYS && pa >= SEMA_PA_MAX) { + last_rc = -EIO; + continue; + } + if (svc_read_ap(vgpu, pa, + ap == SVC_APERTURE_FB ? + SVC_APERTURE_FB : + SVC_APERTURE_SYS, + p, chunk)) { + last_rc = -EIO; + continue; + } + if (via_ipin && first && + !svc_push_prefix_ok(p, chunk)) + continue; /* content says: wrong pin */ + done = true; + } + if (!done) + return last_rc; + p += chunk; + va += chunk; + len -= chunk; + } + return 0; +} + +static int svc_read_va_push(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 va, void *buf, + size_t len) +{ + return svc_read_va_push_ex(vgpu, ch, va, buf, len, false); +} + +/* * Read the GPFIFO get/put pair from a candidate USERD. Returns 0 when the * values are plausible for a channel with `entries` GPFIFO slots. */ @@ -4815,8 +5765,18 @@ static int svc_chan_entry0(struct nv_vgpu_device *vgpu, struct svc_channel *ch, */ #define SVC_GR_RING_STRIDE 0x3000ULL #define SVC_GR_USERD_OFF 0x2000ULL - bool gsp_sema_gr_rings = true; + +/* + * In-kernel GR completion-counter slot derivation (svc_counter_derive): on an + * injected-pin miss, resolve a release's counter slot from the structural + * translations (published page directories, self-discovered directory, + * targeted rm_mem record) instead of holding the release until the external + * pin watcher lands. Default on; set 0 to restore the injected-pin-only + * behaviour for an A/B without a rebuild. + */ +bool gsp_sema_counter_derive = true; + /* * GR (compute) entries whose push page is not translatable are left pending * for this long before the pump consumes them as stale. The push pages are @@ -4968,12 +5928,54 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, /* * The candidate chunk did not validate. If another source already - * translates this ring VA (bridge or manual pin), trust it. + * translates this ring VA (bridge or manual pin), do not just trust + * it: read the first GPFIFO entry through that translation exactly + * like the FB candidate above and validate it with the same checks. + * Previously this branch latched the rank without storing the + * translation or the USERD, so for a sysmem pin svc_chan_userd() had + * no candidate (its FB-only ring offset path is skipped) and the + * pump stalled before ever draining the ring: the GR push decode, + * its release and the client's completion counter all never + * happened. Only a validated read may latch the rank, so a stale or + * wrong pin still falls through to the "not at fb" retry. */ if (!svc_pin_lookup_any(ch->gpfifo_va, &pin_pa, &pin_ap)) { - set_bit(ch->gr_index, &svc_gr_ring_done); - svc_ring_n++; - return; + if (!svc_read_ap(vgpu, pin_pa, svc_ap_of(pin_ap), e, + sizeof(e))) { + memcpy(&e0, e, 4); + memcpy(&e1, e + 4, 4); + va = ((u64)(e1 & 0xff) << 32) | + (u64)(e0 & 0xfffffffc); + len = ((e1 >> 10) & 0x1fffff) << 2; + if (!(va & 0xfffff) && va >= SEMA_VA_LO && + va < SEMA_VA_HI && len >= 4 && len <= 0x10000) { + /* + * Best effort only: a wrapped GP_PUT + * legitimately exceeds the entry count, so a + * failed probe must not reject the ring; the + * pump reads the raw put/get itself and + * reduces put modulo the entry count. + */ + (void)svc_userd_probe(vgpu, + pin_pa + SVC_GR_USERD_OFF, + svc_ap_of(pin_ap), + ch->gpfifo_entries, &put, + &get); + svc_pin_store(ch->gpfifo_va, pin_pa, pin_ap); + svc_lat_store(ch->gpfifo_va, pin_pa, pin_ap); + ch->userd_pa = pin_pa + SVC_GR_USERD_OFF; + ch->userd_as = svc_ap_of(pin_ap); + ch->userd_ok = true; + svc_gr_ring_base = base; + set_bit(ch->gr_index, &svc_gr_ring_done); + svc_ring_n++; + pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx (entry0 va=0x%llx len=%u put=%u get=%u)\n", + ch->h_object, ch->gr_index, + ch->gpfifo_va, pin_pa, va, len, put, + get); + return; + } + } } if (!test_and_set_bit(ch->gr_index, &svc_gr_ring_tried)) @@ -5053,8 +6055,19 @@ static void svc_pb_pair(struct nv_vgpu_device *vgpu, struct svc_channel *ch) len < 4 || len > 0x10000) return; + /* + * Install the window mapping as tagged *guesses*: only the base page + * was paired against real content (svc_chan_entry0 read the first + * push there). The rest of the 1 MB window is NOT assumed physically + * contiguous - a push may cross into a page whose PA is not + * base+N*0x1000 (observed: a push at window offset 0xf58 whose + * continuation page lived one page *below* the base). Push reads + * skip these guesses and resolve each page on its own + * (svc_read_va_push), using this window mapping only as a last-resort + * fallback; every other consumer still sees the pins as before. + */ for (off = 0; off < 0x100000; off += SEMA_PAGE_SIZE) { - svc_pin_store(va + off, hit_pa + off, SVC_APERTURE_SYS); + svc_pin_store_win(va + off, hit_pa + off, SVC_APERTURE_SYS); svc_lat_store(va + off, hit_pa + off, SVC_APERTURE_SYS); svc_mark_hot(hit_pa + off); } @@ -5433,14 +6446,47 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu, static void svc_init_expected_payload(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { - u64 pool_va = 0x121000000ULL + 4ULL * ch->ce_index; + /* + * The exact slot is the release VA this channel's last decoded push + * carried. Until one is decoded, the rank-derived VA is a guess (the + * UVM pool index is allocated in channel creation order, not by + * rank); it is only used to seed the expectation early. Once a push + * decodes, last_rel_va is the authoritative slot and the expectation + * (and the automatic recovery write) follows it. + */ + u64 pool_va; u64 pa = 0; int ap = SVC_APERTURE_SYS; u32 cur = 0; u32 want; - if (svc_lookup_trusted(vgpu, pool_va, &pa, &ap)) + /* + * Only CE channels own a UVM tracking-semaphore pool slot; a GR + * channel's completion signal is its per-channel counter. Deriving a + * pool VA for a GR channel and resolving it here used to start a + * targeted RAM scan per GR channel per pass, starving the pump. + */ + if (!ch->engine_type) + return; + + pool_va = ch->last_rel_va ? ch->last_rel_va : + 0x121000000ULL + 4ULL * ch->ce_index; + + /* + * The expectation is a hint, never a gate. A known exact slot may go + * through the verified pool resolver (injected pin -> validated cache + * -> walk -> targeted scan); a rank-derived guess uses the learned + * map only, so seeding an unknown slot can never launch a multi- + * hundred-millisecond RAM scan from the per-pass hot path. + */ + if (ch->last_rel_va) { + if (svc_pool_pa_resolve(vgpu, ch->h_object, pool_va, &pa, &ap, + NULL) && + svc_lookup_trusted(vgpu, pool_va, &pa, &ap)) + return; /* pool page not mapped yet */ + } else if (svc_lookup_trusted(vgpu, pool_va, &pa, &ap)) { return; /* pool page not mapped yet */ + } if (svc_read_ap(vgpu, pa, ap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS, &cur, sizeof(cur))) @@ -5518,10 +6564,23 @@ static void svc_counter_publish(struct nv_vgpu_device *vgpu, /* * Acknowledge a CE tracking-semaphore release without reading the push that - * carried it: write the expected payload into the semaphore the channel last - * released to (learned from a readable push; the pool index is allocated by - * UVM and does not match the channel rank). Returns false when there is - * nothing to fall back on. Caller owns expected_sem_payload bookkeeping. + * carried it: write the expected payload into the exact semaphore slot the + * channel last released to. That slot is push/CE-side evidence: the decoded + * release VA itself (uvm_channel_tracking_semaphore_get_gpu_va() = + * pool-page GPU VA + semaphore->index * 4; gsp_sema_service.c pool-wr lines + * show one slot per CE channel). The pool index is allocated by UVM in + * channel creation order and does NOT generally match the channel rank, so + * the rank-derived VA (`0x121000000 + 4*ce_index`) is a guess: it is only + * used when the operator explicitly asked for it via gsp_sema_fake_rel / + * gsp_sema_rel_recover. The automatic recovery path (no module parameter) + * requires the decoded VA and never writes into an unproven slot. + * + * Returns false only when there is no exact slot to fall back on at all + * (no decoded release VA and no operator-selected rank guess). A known + * slot is always acknowledged: when its page is not translatable the + * expected write is queued for the pending-release retry pass, the + * expectation still advances and the caller consumes the GPFIFO entry. + * A missing translation must never hold/starve the channel. */ static bool svc_fake_release(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { @@ -5529,31 +6588,82 @@ static bool svc_fake_release(struct nv_vgpu_device *vgpu, struct svc_channel *ch u64 spa = 0; int sap = SVC_APERTURE_SYS; u32 scur = 0; + u32 deliver; + bool have_pa = true; + bool delivered = false; if (!ch->engine_type || !ch->expected_sem_payload) return false; - pool_va = ch->last_rel_va ? ch->last_rel_va : - 0x121000000ULL + 4ULL * ch->ce_index; + pool_va = ch->last_rel_va; + if (!pool_va) { + if (!gsp_sema_fake_rel && !gsp_sema_rel_recover) + return false; + pool_va = 0x121000000ULL + 4ULL * ch->ce_index; + } + deliver = ch->expected_sem_payload; /* * The release can only be delivered when the pool page translates. - * A missing translation is transient (the client's pool page is pinned - * a moment after the channel appears), so report failure and let the - * caller keep the GPFIFO entry pending: consuming it without writing - * the semaphore leaves the client waiting forever. + * svc_pool_pa_resolve() is the verified resolver (injected pin -> + * read-back-validated cache -> walk -> targeted rm_mem record with + * read-back); a translated page is still read at the exact release + * offset before the write. Failure here is not fatal: the exact slot + * is known, so the write is queued and the entry is released. */ - if (svc_lookup_trusted(vgpu, pool_va, &spa, &sap)) - return false; - if (svc_read_ap(vgpu, spa, sap, &scur, sizeof(scur))) - return false; - - if (scur < ch->expected_sem_payload) { - u32 nv = ch->expected_sem_payload; + if (svc_pool_sem_va(pool_va)) { + if (svc_pool_pa_resolve(vgpu, ch->h_object, pool_va, &spa, + &sap, NULL) && + svc_lookup_trusted(vgpu, pool_va, &spa, &sap)) + have_pa = false; + } else if (svc_lookup_trusted(vgpu, pool_va, &spa, &sap)) { + have_pa = false; + } + if (have_pa && + !svc_read_ap(vgpu, spa, svc_ap_of(sap), &scur, sizeof(scur))) { + if (scur >= ch->expected_sem_payload) { + /* Another path already delivered this payload. */ + delivered = true; + } else { + u32 nv = ch->expected_sem_payload; + u32 back = 0; - if (svc_write_ap(vgpu, spa, sap, &nv, sizeof(nv))) - return false; - svc.releases_applied++; + /* + * UVM's update_completed_value() asserts on a jump + * beyond UVM_GPU_SEMAPHORE_MAX_JUMP (a stale + * expectation must never slam the whole page): if the + * expectation is far ahead, deliver exactly one step + * and re-seed it. + */ + if (nv - scur > 0x1000) { + nv = scur + 1; + ch->expected_sem_payload = nv + 1; + } + deliver = nv; + if (!svc_write_ap(vgpu, spa, svc_ap_of(sap), &nv, + sizeof(nv)) && + !svc_read_ap(vgpu, spa, svc_ap_of(sap), &back, + sizeof(back)) && + back == nv) { + delivered = true; + if (svc_pool_sem_va(pool_va)) + svc_pool_pa_store(pool_va, spa, + svc_ap_of(sap)); + svc.releases_applied++; + if (!gsp_sema_fake_rel) + svc.releases_recovered++; + svc.post_hint_until = jiffies + 10 * HZ; + } + } + } + if (!delivered) { + /* + * The exact target is known but not translatable right now. + * Never hold the channel: queue the exact-slot write for the + * retry pass so it still lands when the mapping appears. + */ + svc_pending_add(pool_va, deliver, 0); + svc.releases_forwarded++; } svc.releases_faked++; svc_rel_count++; @@ -5969,6 +7079,22 @@ static void svc_fwd_rel_apply_all(struct nv_vgpu_device *vgpu, svc_fwd_rel_apply_all_ex(vgpu, ch, rel, n, SVC_FWD_REL_SKIP_NONE); } +/* + * Proof line for a held record set resolved without a successful forwarded + * copy (failure/timeout/abort): the releases below are applied directly, so + * none is lost to a hardware copy that never ran. Called with svc_fwd_hold + * populated, before svc_fwd_hold_clear(). + */ +static void svc_fwd_hold_direct_note(void) +{ + u8 i; + + for (i = 0; i < svc_fwd_hold.nrel; i++) + svc_fwd_rel_direct_note(svc_fwd_hold.ch_handle, + svc_fwd_hold.rel[i].va, + svc_fwd_hold.rel[i].payload); +} + static bool svc_fwd_hold_expired(void) { return time_after(jiffies, svc_fwd_hold.since + @@ -6194,6 +7320,8 @@ static void svc_fwd_hold_expire_check(struct nv_vgpu_device *vgpu, svc_fwd_rel_apply_all(hv, &shim, svc_fwd_hold.rel, svc_fwd_hold.nrel); } + if (!(done && svc_fwd_req.hw_rc == 0)) + svc_fwd_hold_direct_note(); if (done && svc_fwd_req.hw_rc == 0) { svc.fwd_rel_applied++; } else { @@ -6295,6 +7423,7 @@ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu, svc.fwd_relay_fallback++; svc_fwd_rel_apply_all(hv, ch, svc_fwd_hold.rel, svc_fwd_hold.nrel); + svc_fwd_hold_direct_note(); svc.fwd_rel_fallback++; if (!done) svc.fwd_hold_timeouts++; @@ -6320,58 +7449,52 @@ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu, } /* - * Channel vanished while a hold was active. A successfully completed work - * item or an abort (quiesce / channel delete seen before the snapshot lost - * the channel) applies the recorded releases through a shim channel - holds - * exist only for CE channels and the vanished channel object is not required; - * only a copy that did not complete successfully (timeout/failure/cancel) - * drops without applying. The stored engage-time device is preferred over the - * caller's so the releases never land in a different device's address space. + * Channel vanished while a hold was active. Any resolved hold (completed, + * aborted, failed or expired) applies the recorded releases through a shim + * channel - holds exist only for CE channels and the vanished channel object + * is not required. A copy that did not complete successfully is no longer + * dropped: the releases land through the same bounded, forward-only apply, so + * a decoded release is never lost to a hardware copy that did not run. The + * stored engage-time device is preferred over the caller's so the releases + * never land in a different device's address space. */ static void svc_fwd_hold_gone_finish(struct nv_vgpu_device *vgpu) { struct nv_vgpu_device *hv = svc_fwd_hold.vgpu ? svc_fwd_hold.vgpu : vgpu; + struct svc_channel shim; bool done = smp_load_acquire(&svc_fwd_req.done); bool aborted = READ_ONCE(svc_fwd_hold.aborted); if (!done && !aborted && !svc_fwd_hold_expired()) return; - if ((done && svc_fwd_req.hw_rc == 0) || aborted) { - struct svc_channel shim; - - /* - * P6b wave 2: an abort (or a vanished channel after a - * failure/timeout) applies its releases through the shim; - * author the skipped data once first unless the hardware copy - * itself completed successfully. - */ - if (!(done && svc_fwd_req.hw_rc == 0)) - svc_fwd_hold_soft_copy(hv); - if (hv) { - memset(&shim, 0, sizeof(shim)); - shim.valid = true; - shim.h_object = svc_fwd_hold.ch_handle; - shim.engine_type = 1; /* holds are CE-only */ - svc_fwd_rel_apply_all(hv, &shim, svc_fwd_hold.rel, - svc_fwd_hold.nrel); - } - if (done && svc_fwd_req.hw_rc == 0) - svc.fwd_rel_applied++; - else { - svc.fwd_rel_fallback++; - if (!done) - svc.fwd_hold_timeouts++; - } + /* + * P6b wave 2: author the skipped data once first unless the hardware + * copy itself completed successfully. + */ + if (!(done && svc_fwd_req.hw_rc == 0)) + svc_fwd_hold_soft_copy(hv); + if (hv) { + memset(&shim, 0, sizeof(shim)); + shim.valid = true; + shim.h_object = svc_fwd_hold.ch_handle; + shim.engine_type = 1; /* holds are CE-only */ + svc_fwd_rel_apply_all(hv, &shim, svc_fwd_hold.rel, + svc_fwd_hold.nrel); + } + if (done && svc_fwd_req.hw_rc == 0) { + svc.fwd_rel_applied++; } else { /* - * Canceled (expired without a successful copy): wave 1.5 - * drops the releases, but the skipped data is still authored - * once before the hold is released so the destination is - * correct if anything else observes it. + * No successful hardware copy ever engaged (failed, + * aborted or the bounded window expired): the recorded + * releases were applied directly above, exactly once. */ - svc_fwd_hold_soft_copy(hv); + svc_fwd_hold_direct_note(); + svc.fwd_rel_fallback++; svc.fwd_hold_canceled++; + if (!done) + svc.fwd_hold_timeouts++; } /* * P6c: a vanished channel cannot run the relay verification edge; a @@ -6409,10 +7532,12 @@ static void svc_fwd_hold_stop_flush(struct nv_vgpu_device *vgpu) svc_fwd_rel_apply_all(hv, &shim, svc_fwd_hold.rel, svc_fwd_hold.nrel); } - if (done && svc_fwd_req.hw_rc == 0) + if (done && svc_fwd_req.hw_rc == 0) { svc.fwd_rel_applied++; - else + } else { + svc_fwd_hold_direct_note(); svc.fwd_rel_fallback++; + } if (svc_fwd_hold.relay_armed) svc.fwd_relay_fallback++; svc_fwd_hold_clear(); @@ -6976,6 +8101,22 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) bool held = false; /* + * Safety net: a GR channel's completion-counter slot must be + * registered - and any release decoded before the slot was + * known flushed - even when this pass cannot read the ring or + * USERD and one of the early-outs below skips the publish at + * the end of the pass. Without it a release recorded while + * push decode stayed broken never lands and the client's + * userspace tracker polls forever. Idempotent and cheap: once + * the slot is registered only the pending-payload check + * remains. + */ + if (!ch->engine_type) { + svc_counter_autoreg(vgpu, ch, n_gr); + svc_counter_flush_pending(vgpu, ch); + } + + /* * P6c: while a relay hold owns this channel, the hold's * verified success edge is the single completion post; skip * the per-pass notifier so it cannot fire before the guest @@ -7146,8 +8287,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (pb) { int rescue_rc; bool unreadable = - svc_read_va_trusted(vgpu, va, - pb, len) != 0; + svc_read_va_push(vgpu, ch, va, + pb, len) != 0; bool injected = false; bool allow_ops = (ch->engine_type != 0); @@ -7157,14 +8298,16 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * (content-scanned map; observed firmware * and kernel data pages). Handle it like * an unreadable push so the content rescue - * can find the real page. Exception: a - * host-injected translation is verified - * truth, and real CE release pushes start - * with an INC_METHOD whose address is not - * zero (observed 0x200381c0, addr 0x700): - * decode those release-only instead of - * discarding them, so the client's - * tracking-semaphore releases still apply. + * can find the real page. An injected pin + * is treated as verified truth by default, + * but content is the arbiter: a pin whose + * page does not parse is re-read from the + * remaining candidates (validated pin / + * window / trusted) and only a plausible + * read replaces it; if that also fails the + * old release-only decode of the injected + * page stands, so the client's tracking- + * semaphore releases still apply. */ if (!unreadable) { u64 ipa; @@ -7174,8 +8317,34 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) &iap) == 0); if (!svc_method_stream_plausible(pb, len)) { svc.false_trans++; - if (!injected) + if (injected) { + u8 *fb = kmalloc(len, + GFP_KERNEL); + + if (fb) { + if (!svc_read_va_push_ex(vgpu, ch, va, + fb, len, true) && + svc_method_stream_plausible(fb, len)) { + memcpy(pb, fb, len); + injected = false; + svc.pushpg_fallback++; + if (gsp_sema_dbg && + svc_pushpg_fb_log < 16) { + u64 wpa = 0; + bool win = !svc_pb_win_lookup(ch, + va & ~(u64)(SEMA_PAGE_SIZE - 1), &wpa); + + svc_pushpg_fb_log++; + pr_info("sema-pushpg: h=0x%x va=0x%llx fallback pin=0 win=%d\n", + ch->h_object, va, + win ? 1 : 0); + } + } + kfree(fb); + } + } else { unreadable = true; + } } } @@ -7306,6 +8475,16 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc_fwd_rel_apply_all(vgpu, ch, svc_fwd_rel, svc_fwd_rel_n); + { + u8 ri; + + for (ri = 0; + ri < svc_fwd_rel_n; + ri++) + svc_fwd_rel_direct_note(ch->h_object, + svc_fwd_rel[ri].va, + svc_fwd_rel[ri].payload); + } /* * P6c: a * selected @@ -7339,7 +8518,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) u32 rel_n2 = svc_rel_count; if (!svc_rescue_push_page(vgpu, ch, va, len) && - !svc_read_va_trusted(vgpu, va, pb, len)) + !svc_read_va_push(vgpu, ch, va, pb, len)) svc_decode_stream(vgpu, pb, len, 0, false, ch, idx, va, true); @@ -7383,6 +8562,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (time_before(jiffies, ch->defer_since + msecs_to_jiffies(gsp_sema_ce_grace_ms))) { + svc.gpfifo_held++; svc.gpfifo_nomap++; kfree(pb); break; @@ -7390,16 +8570,15 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) pr_info("sema-chan: skipping stale entry h=0x%x va=0x%llx len=%u after %u ms grace\n", ch->h_object, va, len, gsp_sema_ce_grace_ms); - if ((gsp_sema_fake_rel || - gsp_sema_rel_recover) && - !svc_fake_release(vgpu, ch)) { - if (!ch->defer_since) - ch->defer_since = - jiffies; - svc.gpfifo_nomap++; - kfree(pb); - break; - } + /* + * Grace expired: deliver through the + * exact-slot fallback and consume the + * entry regardless. A missing + * translation is queued for the retry + * pass; it must never hold the channel. + */ + if (!svc_fake_release(vgpu, ch)) + svc.gpfifo_starved++; } else { ch->expected_sem_payload++; } @@ -7423,28 +8602,35 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * lets the client read unstaged * memory (observed: cuMemcpyDtoH * returning zeros after its HtoD push - * was skipped unreadable). The push - * page's translation usually lands - * within one service pass (content - * discovery or the host watcher), so - * keep the entry pending for the grace - * window first and only then fall back - * to acknowledging the release. + * was skipped unreadable). With a + * decoded release VA (last_rel_va) + * the exact slot is already known: + * deliver or queue it and consume the + * entry at once - never hold the + * channel for a translation that may + * never arrive. Only a channel with no + * exact slot gets one bounded grace so + * a late translation can teach it the + * slot; after that it is consumed + * either way (the release content + * sweep is the safety net). */ - if (gsp_sema_fake_rel && ch->engine_type) { + if (ch->engine_type && + !ch->last_rel_va && + !gsp_sema_fake_rel) { if (!ch->defer_since) ch->defer_since = jiffies; if (time_before(jiffies, ch->defer_since + msecs_to_jiffies(gsp_sema_ce_grace_ms))) { + svc.gpfifo_held++; svc.gpfifo_nomap++; kfree(pb); break; } } -if (gsp_sema_fake_rel && - svc_fake_release(vgpu, ch)) { + if (svc_fake_release(vgpu, ch)) { ch->defer_since = 0; svc_replay_note(ch, idx, put); ch->last_get = (idx + 1) % @@ -7454,29 +8640,6 @@ if (gsp_sema_fake_rel && kfree(pb); continue; } - if (gsp_sema_fake_rel && - ch->engine_type) { - /* - * CE channel and the - * pool page is not - * translated yet: the - * pin lands a moment - * later, so keep the - * entry pending - * instead of burning a - * multi-second RAM - * rescue. Consuming it - * now would drop the - * release the client - * waits on. - */ - if (!ch->defer_since) - ch->defer_since = - jiffies; - svc.gpfifo_nomap++; - kfree(pb); - break; - } if (svc.false_trans && svc.false_trans < 8) pr_info("sema-svc: false translation for push va=0x%llx (h=0x%x len=%u) - rescuing\n", va, ch->h_object, len); @@ -7485,8 +8648,8 @@ if (gsp_sema_fake_rel && if (!rescue_rc) { ch->rescue_fail = 0; /* Rescue succeeded, retry the read. */ - if (!svc_read_va_trusted(vgpu, va, pb, - len)) { + if (!svc_read_va_push(vgpu, ch, va, pb, + len)) { /* * Never run work decoded from a * rescued page: the rescue matches @@ -7574,6 +8737,7 @@ if (gsp_sema_fake_rel && if (time_before(jiffies, ch->defer_since + msecs_to_jiffies(gsp_sema_ce_grace_ms))) { + svc.gpfifo_held++; svc.gpfifo_nomap++; kfree(pb); break; @@ -7581,37 +8745,26 @@ if (gsp_sema_fake_rel && pr_info("sema-chan: skipping stale entry h=0x%x va=0x%llx len=%u after %u ms grace\n", ch->h_object, va, len, gsp_sema_ce_grace_ms); - if (gsp_sema_fake_rel || - gsp_sema_rel_recover) { + { u64 faked_before = svc.releases_faked; - if (!svc_fake_release(vgpu, ch)) { - /* - * The release could not be - * delivered (pool page not - * translated yet): keep the - * entry pending so a pin that - * lands shortly still applies - * it instead of silently - * dropping the client's wait. - */ - if (!ch->defer_since) - ch->defer_since = - jiffies; - svc.gpfifo_nomap++; - kfree(pb); - break; - } + /* + * Deliver through the + * exact-slot fallback and + * consume regardless; a + * missing translation is + * queued for the retry pass. + */ + if (!svc_fake_release(vgpu, ch)) + svc.gpfifo_starved++; if (!gsp_sema_fake_rel && svc.releases_faked != - faked_before) { - svc.releases_recovered++; + faked_before) pr_info("sema-chan: recovery release h=0x%x rank=%u expected=%u (push page unreadable after rescue)\n", ch->h_object, ch->ce_index, ch->expected_sem_payload); - } } ch->rescue_fail = 0; ch->defer_since = 0; @@ -7657,24 +8810,19 @@ if (gsp_sema_fake_rel && pr_info("sema-chan: skipping stale entry h=0x%x va=0x%llx len=%u after %u failed rescues\n", ch->h_object, va, len, ch->rescue_fail); - if (gsp_sema_fake_rel && - !svc_fake_release(vgpu, ch)) { - /* - * The release could not be - * delivered (pool page not - * translated yet): keep the - * entry pending so a pin that - * lands shortly still applies - * it instead of silently - * dropping the client's wait. - */ - if (!ch->defer_since) - ch->defer_since = - jiffies; - svc.gpfifo_nomap++; - kfree(pb); - break; - } + /* + * Never hold on a missing + * exact slot: consume the + * entry and let the content + * sweep / counter publish carry + * the client's wait. GR + * channels have no exact slot + * (svc_fake_release returns + * false) and are counted as + * consumed-without-delivery. + */ + if (!svc_fake_release(vgpu, ch)) + svc.gpfifo_starved++; ch->rescue_fail = 0; ch->defer_since = 0; if (gsp_sema_fake_rel || @@ -7738,11 +8886,15 @@ if (gsp_sema_fake_rel && held = svc_fwd_hold.active && svc_fwd_hold.vgpu == vgpu && svc_fwd_hold.ch_handle == ch->h_object; - /* Advance the client-visible completion counter. */ - if (!held) { - svc_counter_autoreg(vgpu, ch, n_gr); + /* + * Advance the client-visible completion counter. The slot was + * already registered (and any pending release flushed) by the + * safety net at the top of the pass, so this must not + * re-register: once per pass is enough and the derivation is + * not free. + */ + if (!held) svc_counter_publish(vgpu, ch, put); - } spin_lock_irqsave(&svc.chan_lock, flags); for (k = 0; k < SVC_MAX_CHANNELS; k++) { @@ -8365,6 +9517,12 @@ static int svc_thread(void *arg) } /* + * Host CUDA bridge: marshal one queued launch into a daemon + * job and resolve completions. Inert until a launch is seen. + */ + gsp_cuda_bridge_service(vgpu); + + /* * Releases whose target VA was not translatable when the scan * saw them: retry every pass, so a pin or a learned pair applies * them immediately instead of on the next full sweep. @@ -8511,6 +9669,8 @@ int gsp_sema_service_start(void) svc.task = NULL; return ret; } + if (gsp_cuda_bridge_init()) + pr_warn("sema-svc: cuda bridge debugfs unavailable\n"); return 0; } @@ -8544,4 +9704,97 @@ void gsp_sema_service_stop(void) destroy_workqueue(svc_fwd_wq); svc_fwd_wq = NULL; } + /* Thread is gone: release the bridge's pending jobs and debugfs files. */ + gsp_cuda_bridge_fini(); +} + +/* ------------------------------------------------------------------ */ +/* host CUDA bridge exports (gsp_cuda_bridge.c) */ +/* ------------------------------------------------------------------ */ + +/* + * Page-wise GPU-VA read for the bridge: resolve each page through the same + * trusted sources every operand read uses (pins, learned cache, discovered + * map, published page directories, targeted rm_mem resolve) and read through + * the aperture the VA maps to (vidmem shadow vs guest RAM via vfio_dma_rw). + */ +int gsp_sema_service_va_read(struct nv_vgpu_device *vgpu, u64 va, void *buf, + size_t len) +{ + u8 *p = buf; + + while (len) { + size_t chunk = min_t(size_t, len, + SEMA_PAGE_SIZE - + (va & (SEMA_PAGE_SIZE - 1))); + u64 addr = 0; + int ap = SVC_APERTURE_SYS; + + if (svc_lookup(vgpu, va, &addr, &ap)) + return -ENOENT; + if (svc_read_ap(vgpu, addr, svc_ap_of(ap), p, chunk)) + return -EIO; + p += chunk; + va += chunk; + len -= chunk; + } + return 0; +} + +int gsp_sema_service_va_write(struct nv_vgpu_device *vgpu, u64 va, + const void *buf, size_t len) +{ + const u8 *p = buf; + + while (len) { + size_t chunk = min_t(size_t, len, + SEMA_PAGE_SIZE - + (va & (SEMA_PAGE_SIZE - 1))); + u64 addr = 0; + int ap = SVC_APERTURE_SYS; + + if (svc_lookup(vgpu, va, &addr, &ap)) + return -ENOENT; + if (svc_write_ap(vgpu, addr, svc_ap_of(ap), (void *)p, chunk)) + return -EIO; + p += chunk; + va += chunk; + len -= chunk; + } + return 0; +} + +int gsp_sema_service_va_resolve(struct nv_vgpu_device *vgpu, u64 va, u64 *pa, + u32 *ap) +{ + u64 addr = 0; + int iap = SVC_APERTURE_SYS; + + if (svc_lookup(vgpu, va, &addr, &iap)) + return -ENOENT; + if (pa) + *pa = addr; + if (ap) + *ap = svc_ap_of(iap); + return 0; +} + +int gsp_sema_service_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val) +{ + return svc_sem_read(vgpu, va, val); +} + +int gsp_sema_service_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, + u32 val) +{ + int rc = svc_sem_write(vgpu, h, va, val); + + /* + * A bridge-completed job is a work completion like any pump release: + * keep the completion hint window open so a guest that starts waiting + * after the write still receives the completion event. + */ + if (!rc) + svc.post_hint_until = jiffies + 10 * HZ; + return rc; } diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h index d5a62e9..530663e 100644 --- a/kernel/gsp_sema_service.h +++ b/kernel/gsp_sema_service.h @@ -155,6 +155,16 @@ void gsp_sema_service_fwd_fault_stats(u64 *matched, u64 *unmatched, extern bool gsp_sema_gr_rings; /* + * In-kernel GR completion-counter slot derivation (svc_counter_derive): on an + * injected-pin miss, resolve a counter slot from the structural translations + * (the driver-published page directories, the self-discovered directory and + * the targeted rm_mem record) and author the release there instead of waiting + * for the external pin watcher. Default on; set 0 to restore the + * injected-pin-only behaviour for an A/B without a rebuild. + */ +extern bool gsp_sema_counter_derive; + +/* * Acknowledge tracking-semaphore releases from CE pushes the pump could not * read: write the expected payload into the channel's pool semaphore so the * driver's wait clears immediately instead of after the content sweep. diff --git a/kernel/main.c b/kernel/main.c index 39c71c3..7001b3d 100644 --- a/kernel/main.c +++ b/kernel/main.c @@ -2551,6 +2551,16 @@ module_param_named(sema_gr_rings, gsp_sema_gr_rings, bool, 0644); MODULE_PARM_DESC(sema_gr_rings, "Auto-discover+pin GR channel rings (default 1)"); /* + * In-kernel GR completion-counter slot derivation from the structural + * translations (published page directories, self-discovered directory, + * targeted rm_mem record), used when a release's counter page has no injected + * pin yet. Default on; set 0 to restore the injected-pin-only behaviour for + * an A/B without a rebuild. + */ +module_param_named(sema_counter_derive, gsp_sema_counter_derive, bool, 0644); +MODULE_PARM_DESC(sema_counter_derive, "Derive GR completion-counter slots in-kernel from structural translations (default 1)"); + +/* * Acknowledge releases from unreadable CE pushes directly (tracking-semaphore * fallback). The push's data writes are lost either way when the content * rescue cannot find the page; writing the expected payload keeps the |