diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_cuda_bridge.c | 92 | ||||
| -rw-r--r-- | kernel/gsp_phys_booter.c | 4801 | ||||
| -rw-r--r-- | kernel/gsp_phys_booter.h | 53 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.c | 810 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.h | 89 | ||||
| -rw-r--r-- | kernel/gsp_rm_emulation.c | 990 | ||||
| -rw-r--r-- | kernel/gsp_rm_emulation.h | 24 | ||||
| -rw-r--r-- | kernel/gsp_rm_tu104.h | 13 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 11102 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.h | 211 | ||||
| -rw-r--r-- | kernel/main.c | 176 |
11 files changed, 17448 insertions, 913 deletions
diff --git a/kernel/gsp_cuda_bridge.c b/kernel/gsp_cuda_bridge.c index 2fac2b2..7adc8e3 100644 --- a/kernel/gsp_cuda_bridge.c +++ b/kernel/gsp_cuda_bridge.c @@ -141,6 +141,9 @@ MODULE_PARM_DESC(cuda_bridge, /* ------------------------------------------------------------------ */ struct cubb_req { + struct nv_vgpu_device *vgpu; /* owning mdev (guest session); the + * queue is FIFO, so only the owner's + * service pass may marshal this job */ u32 h_object; u64 qmd_dev_va; u64 push_va; @@ -178,6 +181,9 @@ static u64 cubb_dedup_hits; struct cubb_pend { bool valid; + struct nv_vgpu_device *vgpu; /* owning mdev: the completion write-back + * must run through this guest's + * address space, never another's */ u64 seq; u32 h_object; u32 opcode; @@ -576,12 +582,15 @@ void gsp_cuda_bridge_launch(struct nv_vgpu_device *vgpu, u32 h_object, if (!gsp_cuda_bridge_enabled()) return; - (void)vgpu; for (i = 0; i < 64; i++) { hash ^= qmd[i]; hash *= 1099511628211ULL; } + /* The owner is part of the identity: two guests' identical launches + * must never dedup against each other. */ + hash ^= (u64)(unsigned long)vgpu; + hash *= 1099511628211ULL; hash ^= (u64)h_object << 32; hash *= 1099511628211ULL; hash ^= qmd_dev_va; @@ -612,6 +621,7 @@ void gsp_cuda_bridge_launch(struct nv_vgpu_device *vgpu, u32 h_object, struct cubb_req *r = &cubb_req[cubb_req_head % CUBB2_REQ_MAX]; + r->vgpu = vgpu; r->h_object = h_object; r->qmd_dev_va = qmd_dev_va; r->push_va = push_va; @@ -972,6 +982,7 @@ static int cubb_marshal_one(struct nv_vgpu_device *vgpu) } job.seq = ++cubb_seq; cubb_pend[slot].valid = true; + cubb_pend[slot].vgpu = r.vgpu; cubb_pend[slot].seq = job.seq; cubb_pend[slot].h_object = r.h_object; cubb_pend[slot].opcode = opcode; @@ -1303,12 +1314,48 @@ static bool cubb_finish_try(struct nv_vgpu_device *vgpu, struct cubb_pend *p, } /* + * Is this mdev still on the driver list with an open VFIO device? Guards the + * bridge's per-request/per-pend owner pointers (dual-guest goal 1.0): a + * removed mdev must never be dereferenced and its queued work can never be + * observed by a live guest. Pointer comparisons only - safe to call with a + * possibly-stale pointer. + */ +static bool cubb_vgpu_open(const struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_device *d; + bool open = false; + + if (!vgpu) + return false; + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(d, &nv_vgpu_driver.devices, list) { + if (d == vgpu && atomic_read(&d->vfio_dev_open_count) > 0) { + open = true; + break; + } + } + mutex_unlock(&nv_vgpu_driver.lock); + return open; +} + +/* * Service-thread hook: marshal at most one queued launch into a daemon-visible * job, stage any returned done records, then drive staged completions. Called * every service pass; a no-op when no launch has been seen. + * + * Dual-guest (goal 1.0): request and pend records carry their owning mdev + * (the bridge falls back here when forwarding was refused under the + * single-inflight slot). Only the owning guest's pass marshals its job (the + * marshal resolves VAs and reads guest memory through that guest) and only + * the owning pass writes a completion back; a head/record whose owner is + * gone is dropped instead of ever being dereferenced. */ void gsp_cuda_bridge_service(struct nv_vgpu_device *vgpu) { + struct nv_vgpu_device *head_owner = NULL; + unsigned long flags; + u32 head_h = 0; + bool have_head = false; struct cubb2_done d; int guard = 0; int i; @@ -1316,7 +1363,29 @@ void gsp_cuda_bridge_service(struct nv_vgpu_device *vgpu) if (!gsp_cuda_bridge_enabled()) return; - (void)cubb_marshal_one(vgpu); + spin_lock_irqsave(&cubb_req_lock, flags); + if (cubb_req_head != cubb_req_tail) { + struct cubb_req *head = + &cubb_req[cubb_req_tail % CUBB2_REQ_MAX]; + + head_owner = head->vgpu; + head_h = head->h_object; + have_head = true; + } + spin_unlock_irqrestore(&cubb_req_lock, flags); + if (have_head) { + if (head_owner == vgpu) { + (void)cubb_marshal_one(vgpu); + } else if (!cubb_vgpu_open(head_owner)) { + /* The owner's guest is gone: nobody can observe it. */ + cubb_req_drop_head(); + cubb_dropped++; + pr_info_ratelimited("cuda-bridge: dropped a queued launch whose mdev is gone (h=0x%x)\n", + head_h); + } + /* else: another live guest's head; its own pass marshals it. */ + } + while (guard++ < CUBB2_DONE_RING && cubb_done_pop(&d)) { if (d.magic != CUBB2_DONE_MAGIC || d.version != CUBB2_VERSION) { @@ -1330,8 +1399,23 @@ void gsp_cuda_bridge_service(struct nv_vgpu_device *vgpu) mutex_lock(&cubb_lock); for (i = 0; i < CUBB2_PEND_MAX; i++) { - if (cubb_pend[i].valid && cubb_pend[i].have_done && - cubb_finish_try(vgpu, &cubb_pend[i], i)) + struct cubb_pend *p = &cubb_pend[i]; + + if (!p->valid || !p->have_done) + continue; + if (p->vgpu != vgpu) { + /* + * Another guest's staged completion: its own pass + * drives the write-back. A removed owner's record is + * released here so the slot frees. + */ + if (!cubb_vgpu_open(p->vgpu)) { + cubb_pend_release(p, i); + i = -1; /* compaction shifted the entries */ + } + continue; + } + if (cubb_finish_try(vgpu, p, i)) i = -1; /* compaction shifted the entries; rescan */ } mutex_unlock(&cubb_lock); diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 72cda10..7ba74ce 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -39,6 +39,7 @@ #include <linux/seq_file.h> #include <linux/uaccess.h> #include <linux/jiffies.h> +#include <linux/ktime.h> #include <linux/timekeeping.h> #include <linux/byteorder/generic.h> #include <linux/pci_regs.h> @@ -73,6 +74,150 @@ module_param_named(gsp_phys_nocat, gsp_phys_nocat_param, bool, 0644); MODULE_PARM_DESC(gsp_phys_nocat, "Log every GSP-RM NOCAT journal record (status-queue function 0x1020) individually during the stage-3e RM-init poll (default 0; the unique-record summary is always accumulated and dumped when the poll ends)"); +/* + * GRFIX FIX3 (relocation coverage, default 0): the QMD/bank rewrite scan + * counts 8-byte words that pass the rewrite's own device-VA shape test + * (>= 0x10000000, < 2^47, 256-byte aligned) but match NO tracked data page + * (req->data[x].va). Such a word is left as an unrelocated guest VA for the + * engine to walk, which is the multi-pointer QMD failure signature. With the + * parameter clear the scan only logs the per-page count (RELOC-GAP, + * rate-limited); with it set a launch carrying any such word is held + * fail-closed before publish (same backend-refusal path as the copy-in hold; + * the sema hold applies the software releases - bounded, never a hang). + * The hold is opt-in because the shape heuristic can false-positive on + * legitimate non-pointer data words; the RELOC-GAP count measures that rate. + */ +static bool gsp_phys_gr_strict_reloc_param; +module_param_named(gsp_phys_gr_strict_reloc, gsp_phys_gr_strict_reloc_param, + bool, 0644); +MODULE_PARM_DESC(gsp_phys_gr_strict_reloc, + "Hold a GR launch fail-closed when the QMD rewrite finds device-VA-shaped words that match no tracked data page (default 0: RELOC-GAP log only; 1: hold the launch)"); + +/* + * relfix (agent-A FIX A, default 1): the GR QMD rewrite's device-VA-shaped + * words that match NO tracked data page (the RELOC-GAP class - the raw guest + * pointers the real engine then faults on; r216: MMU FAULT_PDE at a gap VA, + * SM warp exceptions) are re-resolved through the sema service's trusted + * resolver (gsp_sema_gr_reloc_resolve). A word that resolves is authored + * into the bounce window exactly like a late-tracked data page - it and the + * contiguous span the resolver also answers are copied in before the + * doorbell and copied back after the retire wait, the same m142 ordering as + * every tracked page - and the word is rewritten to the page's reloc VA plus + * its in-page offset, the exact encoding the matched-word relocation uses. + * Unresolved words keep the historical RELOC-GAP behaviour (counted and left + * as guest VAs; never guessed). 0 = exact pre-relfix behaviour for one-flip + * A/B. The rewrite runs on the single-inflight GR work item with + * gsp_phys_rm_lock held; the resolver is read-only w.r.t. the service tables + * but may sleep in the same vfio walks the fill's resolver uses. + */ +static unsigned int gsp_phys_gr_reloc_resolve_param = 1; +module_param_named(gsp_phys_gr_reloc_resolve, gsp_phys_gr_reloc_resolve_param, + uint, 0644); +MODULE_PARM_DESC(gsp_phys_gr_reloc_resolve, + "Resolve QMD words that match no tracked data page and author them as late data pages (default 1: RELOC-FIXED; 0: RELOC-GAP only)"); + +/* + * COPYWIT (2026-10-18 fail4 discrimination tooling, default 1): witness the + * copy-out of every tracked page the engine changed - device VA, guest HPA + * the results were written back to, and the first/last 8 bytes actually + * shipped. Correlating these lines with the sema-bigcopy D2H resolution lines + * (same VA -> pa) and the client's own check discriminates "engine result + * never produced" from "produced but landed on a different guest page than + * the client's read resolves" - the two classes the all-bad client results + * cannot separate from the aggregate counters alone. Bounded per launch + * (GSP_PHYS_GR_COPYWIT_MAX) and rate-limited; 0 disables both this and the + * per-launch RELOC-CENSUS line. + */ +#define GSP_PHYS_GR_COPYWIT_MAX 12u +static bool gsp_phys_gr_copyout_dump_param = true; +module_param_named(gsp_phys_gr_copyout_dump, gsp_phys_gr_copyout_dump_param, + bool, 0644); +MODULE_PARM_DESC(gsp_phys_gr_copyout_dump, + "Log per-changed-page copy-out witnesses (VA, HPA, first/last 8 bytes) and the per-launch relocation census (default 1)"); + +static inline bool gsp_phys_gr_copywit_enabled(void) +{ + return READ_ONCE(gsp_phys_gr_copyout_dump_param); +} + + +/* + * r405 late-write probes (the r405 "partial write" class: the scale launch + * of streams2 wrote 1 of its 32 expected in-place pages and every other + * launch settled with far fewer changed pages than the kernel should + * produce). Two knobs, both runtime-settable: + * + * - gsp_phys_gr_late_probe (default 0): after the retire wait concludes, + * sleep +500 ms and re-scan the tracked pages. Run-once diagnostic: + * "ENGINE STILL WRITING" proves the copy-out shipped before the stores + * landed (the fix is a longer settle window); "stable" proves the engine + * genuinely stopped, which needs a different treatment. + * - gsp_phys_gr_settle_ms (default 100): the content-stable window the + * partial-changed rule requires before it declares the stores settled. + * Raise it to give a still-writing engine its time (A/B in one cycle). + * + * r552 (2026-10-23): measured A/B on real boots. Every launch with a + * partial changed-page count (read-mostly kernels: 1..64 of 1000+ pages + * written, so the changed>=n_all shortcut can never fire) paid the full + * window plus scan overhead: 683-901 ms per launch at 300 ms. At 100 ms + * the same boots measured 231-247 ms of retire wait and the launch + * latency p50 dropped 1119 ms -> 651 ms (n=22 launches; baseline 684 ms) + * with the battery unchanged (12/15, 0 hangs) and a green 2-round soak + * on the next boot. The content-stability hash still gates the verdict; + * only the minimum stable duration shrank. + */ +static bool gsp_phys_gr_late_probe_param; +module_param_named(gsp_phys_gr_late_probe, gsp_phys_gr_late_probe_param, + bool, 0644); +MODULE_PARM_DESC(gsp_phys_gr_late_probe, + "Diagnostic: after the GR retire wait, re-scan the tracked pages +500 ms later and log whether the engine is still writing (default 0)"); + +static int gsp_phys_gr_settle_ms_param = 100; +module_param_named(gsp_phys_gr_settle_ms, gsp_phys_gr_settle_ms_param, + int, 0644); +MODULE_PARM_DESC(gsp_phys_gr_settle_ms, + "GR retire wait: content-stable window (ms) required before a partial changed-page count counts as settled (default 100; 300 was the pre-r552 value)"); + +static inline u32 gsp_phys_gr_settle_iters(void) +{ + int ms = READ_ONCE(gsp_phys_gr_settle_ms_param); + + if (ms < 60) + ms = 60; + if (ms > 1400) + ms = 1400; + return (u32)(ms / 5); +} + +/* + * COPYWIT fail4 r224 follow-up: the r224 OUT-PAGE-WIT dumps proved the + * engine's outputs are structurally corrupted (row bands >= 16 replicate a + * single tile per buffer) while the copy-out write/read pairs agree + * byte-for-byte. Two witnesses make the next cycle name the source: + * + * - OUT-ALIAS-WIT: during copy-out, every changed page's first 8 bytes are + * matched against the earlier changed pages of the SAME launch; a hit + * logs the (va, bi) pair, i.e. the replicated-page structure and the rec + * partners - the direct evidence for "one tile replicated across the + * band" and against "pass-through of pre-launch content" (the pre/post + * csums in OUT-PAGE-WIT prove the engine changed the page). + * - IN-ALIAS-WIT: during the copy-in fill, consecutive recs with identical + * content (equal per-page csum) are logged - a replicated INPUT buffer + * (e.g. an X or weight region whose H2D/copy-in served one page many + * times) is caught here before any launch runs. + * Bounded boot-wide and per launch; same single-inflight static discipline + * as the other submit scratch. + */ +#define GSP_PHYS_GR_OUTWIT_SLOTS 16u +static u64 gsp_phys_gr_outwit_first8[GSP_PHYS_GR_OUTWIT_SLOTS]; +static u64 gsp_phys_gr_outwit_va[GSP_PHYS_GR_OUTWIT_SLOTS]; +static u32 gsp_phys_gr_outwit_bi[GSP_PHYS_GR_OUTWIT_SLOTS]; +static u32 gsp_phys_gr_outwit_n; +static u32 gsp_phys_gr_alias_log; /* boot-wide alias-line budget */ +static u32 gsp_phys_gr_recdump_budget; /* r233 REC-DUMP boot budget */ +static u32 gsp_phys_gr_rdw_budget; /* r234 RDW arming-line budget */ +static u32 gsp_phys_gr_atrace_budget; /* r234 A-TRACE boot budget */ + bool gsp_phys_boot_enabled(void) { return gsp_phys_boot_param; @@ -999,16 +1144,34 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RMCALL_VA_DST (GSP_PHYS_RMCALL_VA_BASE + 3 * GSP_PHYS_PAGE_SIZE) #define GSP_PHYS_RMCALL_VA_SEM (GSP_PHYS_RMCALL_VA_BASE + 4 * GSP_PHYS_PAGE_SIZE) /* - * m69: window-0 shadow pool - 8 module pages at VAs 0x108000..0x10f000 - * (small-PT slots 0x108..0x10f, the same 128 B PTE fetch line as the - * GPFIFO/PB/QMDWIN slots, planted before the engine's first fetch). The - * Tier-2 copies of the QMD's program/param pages are placed here so the GR - * engine never has to read through the window-1 scratch VA range whose PT - * chain (DMA_UPDATE_PDE_2 pdeIndex=1) has never been proven engine-readable. + * m69: window-0 shadow pool - 16 module pages holding the Tier-2 copies of + * the QMD's program/param pages, so the GR engine never has to read through + * the window-1 scratch VA range whose PT chain (DMA_UPDATE_PDE_2 + * pdeIndex=1) has never been proven engine-readable. The pool is split + * into two 8-page banks; the forwarded launch alternates banks per launch + * (m151) so one launch's copy-in can never clobber the pages the previous + * launch's engine fetches may still be reading (late-fetch safety). + * + * The base MUST stay clear of the m114/m150 GR data bounce window + * (GSP_PHYS_RMCALL_GR_BOUNCE_VA0 .. +PAGES, main-PT slots 0x110..0x1df): + * this pool is re-planted on every gsp_phys_rmcall_pdb_build() while the + * bounce PTEs are a one-shot plant, so any overlap lets the later win0 + * plant steal the bounce slots and the engine writes results into the + * shadow pages, losing them. 0x1e0000 (slots 0x1e0..0x1ef) sits directly + * above the last bounce/mail/marker page (0x1df000) and below the + * mail-injection slot 0x1f0 and the guest scratch window (0x200000). Keep + * it that way (see the static_assert next to the bounce constants). */ #define GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE \ - (GSP_PHYS_RMCALL_VA_BASE + 0x90000ULL) -#define GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS 8u + (GSP_PHYS_RMCALL_VA_BASE + 0xE0000ULL) +#define GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS 16u +/* + * m151: one bank = one launch's worth of QMD shadow pages. A forwarded + * launch consumes at most eight pages (param via qmd_word 32, program 48, + * bank 0/1/2/3/5/7 at qmd_word 34..46 even); the per-launch bounds check + * uses this so a launch can never spill into the other bank. + */ +#define GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS 8u /* * m99: engine-tree branches for identity-mapping the forwarded kernel's * guest data pages. The walker uses our root[0] -> PD2 -> PD1 -> PD0 dual -> @@ -1043,6 +1206,16 @@ bool gsp_phys_boot_requested(void) * engine's first fetch of the line - and never at launch time. */ #define GSP_PHYS_RMCALL_VA_GR_QMDWIN (GSP_PHYS_RMCALL_VA_BASE + 0x7000) +/* + * m156 (E3): double-buffer the inline-QMD destination. The page is + * prewritten every launch; when a single VA is reused the engine's QMD-unit + * look-ahead reads of the previous launch may still be in flight against the + * rewrite (the rationale the m151 win0 banks give). Slot 0x108 sits in the + * same already-proven 128 B PTE fetch line as 0x107 and is otherwise unused; + * launches alternate 0x107000/0x108000 by submission epoch. Both PTEs are + * planted once at stage 3 (no invalidate path exists). + */ +#define GSP_PHYS_RMCALL_GR_QMDWIN_SLOTS 2u /* * P6a guest-memory windows (gsp_phys_guest_mem=1). Two more slots in the same @@ -1100,6 +1273,33 @@ bool gsp_phys_boot_requested(void) /* Full 128 B fetch lines per 4 KB small-PT page: (4096/8)/16 = 32. */ #define GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE \ ((GSP_PHYS_PAGE_SIZE / 8u) / GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES) +/* + * m152: extended scratch-PT pool. The original 16 x 2 MB PD0 windows + * (512 never-reused 128 B fetch lines per boot) exhausted under a normal + * launch mix (each distinct guest page consumed a line per launch) and the + * exhaustion was permanent for the boot: every later launch that needed a + * new mapping was refused ("launch refused ... no Tier-1 mapping and no + * captured inline data"), which is the deterministic once-broken-forever + * failure the longevity runs kept hitting. Two changes fix the class: + * (a) GR alias cache (below): a consumed fetch line is keyed by the HPA + * it maps, so a repeat launch of the same buffers (or the same chunk + * page / semaphore page) reuses the identical never-rewritten line and + * consumes nothing; + * (b) a 48-window extension pool right above the ladder's free gap + * (window VA 0x04000000.., PD0 indices 32..79) backed by one 48-page + * PT chunk, raising capacity to 64 windows x 32 lines = 2048. + * lines/boot. Both are pure capacity mechanics: no PTE and no mapping any + * engine may have fetched is ever rewritten. + */ +#define GSP_PHYS_RMCALL_VA_GUEST_PT_EXT_BASE 0x04000000ULL +#define GSP_PHYS_RMCALL_GUEST_PT_EXT_PAGES 48u +#define GSP_PHYS_RMCALL_GUEST_PT_TOTAL_PAGES \ + (GSP_PHYS_RMCALL_GUEST_PT_PAGES + GSP_PHYS_RMCALL_GUEST_PT_EXT_PAGES) +#define GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES \ + (GSP_PHYS_RMCALL_GUEST_PT_TOTAL_PAGES * \ + GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE) +/* HPA -> shadow-VA reuse cache; hit = no new fetch line is consumed. */ +#define GSP_PHYS_RMCALL_GR_ALIAS_CACHE 768u /* * GR forwarding (gsp_phys_gr_forward=1; default off) scratch resources, @@ -1253,8 +1453,21 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RMCALL_CHANNEL_FLAG_USER_LANE \ (u32)(GSP_PHYS_RMCALL_CHANNEL_FLAG_PRIVILEGED | (4u << 8) | \ (1u << 21)) -/* Captured user-lane GPFIFO depth (#825: 1024 entries). */ -#define GSP_PHYS_RMCALL_USER_LANE_GPFIFO_ENTRIES 1024u +/* + * User-lane GPFIFO depth. MUST equal the host ring's push modulus + * (GSP_PHYS_RMCALL_GPFIFO_ENTRIES, 8): the host publishes into the + * one-page ladder ring modulo 8, so if the channel declares a different + * count the engine's GET/PUT wrap desyncs at the host's first wrap + * (GP_PUT 7 -> 0 while the engine's GET keeps counting to 8) and every + * later launch on the boot is never fetched (r138: launches 1..7 green, + * the 8th and all after it silent with `qmd_rel=5 ch_rel=0`). The captured + * #825 value (1024) describes the real client's own much larger ring, not + * this emulated one; the campaign's 2026-10-06 "fidelity" change to 1024 + * was a wrap-only difference that broke exactly here (M1's two-launch + * acceptance never wrapped). Keep the two ends equal. + */ +#define GSP_PHYS_RMCALL_USER_LANE_GPFIFO_ENTRIES \ + GSP_PHYS_RMCALL_GPFIFO_ENTRIES #define GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_USER_LANE 0x1du #define GSP_PHYS_RMCALL_TSG_FAULT_BUF_SIZE 0x2000u #define GSP_PHYS_RMCALL_USER_LANE_PM_RAW_SIZE 0xd100u @@ -1379,6 +1592,58 @@ enum gsp_phys_rmcall_buf { GSP_PHYS_RMCALL_BUF_GUEST_PT13, GSP_PHYS_RMCALL_BUF_GUEST_PT14, GSP_PHYS_RMCALL_BUF_GUEST_PT15, + /* m152: extension pool PT pages: 48 separate single-page chunks + * (GUEST_PT_EXT0..47). The server only accepts single-page memdescs + * for client buffers (a 48-page chunk was refused with rpc_result + * 0x1f + a firmware ASSERT in stage 2). */ + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT0, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT1, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT2, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT3, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT4, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT5, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT6, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT7, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT8, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT9, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT10, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT11, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT12, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT13, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT14, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT15, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT16, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT17, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT18, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT19, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT20, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT21, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT22, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT23, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT24, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT25, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT26, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT27, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT28, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT29, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT30, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT31, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT32, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT33, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT34, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT35, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT36, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT37, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT38, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT39, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT40, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT41, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT42, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT43, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT44, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT45, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT46, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT47, /* * Golden9g: the three client page-table levels the real golden VAS * hands to the server with VASPACE_COPY_SERVER_RESERVED_PDES @@ -1409,6 +1674,27 @@ enum gsp_phys_rmcall_buf { * generous and progress is logged every ~500 ms. */ #define GSP_PHYS_RMCALL_GR_POLL_MS 5000u +/* + * Bounded retry of a forwarded GR launch whose QMD did not run. A green + * launch completes in ~170-195 ms; when the engine drops it (cold context + * save/unload, or an entry the PBDMA never fetched) `qmd_rel` never reaches + * the wanted value and the client would fall back to the software release + * and read stale/zero results. Because the timeout path never copies the + * result out to the guest, a launch that timed out did not reach the + * client: re-publishing a FRESH ring entry (same PB/QMD) and re-issuing the + * schedule+doorbell gives the front-end another chance and the guest still + * sees only the success path. Bounded so the service thread is never held + * past the existing hold budget. + */ +#define GSP_PHYS_RMCALL_GR_RETRY_MS 2000u +/* + * F2: the short re-poll after the first-timeout fast re-arm (extended + * re-arm + fresh ring entry). The engine ack for a recovered launch is + * observed within tens of ms (r198); 1500 ms covers the ctxsw path without + * burning another full window before the historical ladder continues. + */ +#define GSP_PHYS_RMCALL_GR_FAST_RETRY_MS 1500u +#define GSP_PHYS_RMCALL_GR_ATTEMPTS 3u /* P6a guest-memory CE copies: same bounded poll cap as the P5 shadow path. */ #define GSP_PHYS_RMCALL_GUEST_POLL_MS 200u @@ -2136,17 +2422,31 @@ static struct gsp_phys_boot_stage { bool rmcall_gr_exec_ok; /* a real GR submission proved out */ u32 rmcall_gr_submit_epoch; /* per-submission mapper epoch */ u32 rmcall_gr_pt_used; /* Tier-1 window pool slots used */ - u32 rmcall_guest_pt_win_mask; /* Tier-2 scratch windows installed */ + u64 rmcall_guest_pt_win_mask; /* Tier-2 scratch windows installed */ + /* + * m152: shadow alias reuse cache: hpa -> never-rewritten shadow VA. + * Populated by gsp_phys_rmcall_gr_map_shadow_locked; a hit returns the + * existing line so repeat launches of the same physical pages consume + * no new fetch line. Fill-order/LRU-free by design: entries are dead + * weight once a client exits, but a boot has 2048 lines to give and + * the cache only ever returns identical PTE content. + */ + u64 rmcall_gr_alias_hpa[GSP_PHYS_RMCALL_GR_ALIAS_CACHE]; + u64 rmcall_gr_alias_va[GSP_PHYS_RMCALL_GR_ALIAS_CACHE]; + u32 rmcall_gr_alias_count; struct gsp_phys_boot_chunk rmcall_gr_pt[GSP_PHYS_RMCALL_GR_PT_PAGES]; struct gsp_phys_boot_chunk rmcall_gr_shadow; /* Tier-2 pages */ /* * m69: window-0 shadow pool (gsp_phys_rmcall_gr_shadow_win0=1). - * Eight module pages whose PTEs are planted in the proven window-0 - * PT line at slots 0x108..0x10f (VAs 0x108000..0x10f000) by - * gsp_phys_rmcall_pdb_build, before the engine's first fetch (same - * pattern as the qmdwin slot). The launch copies the QMD's - * program/param page content into them and rewrites the QMD pointers - * there, so the GR engine never reads the window-1 scratch range. + * Sixteen module pages (two 8-page banks; m151 alternates the bank + * per launch so a launch's copy-in cannot clobber pages the previous + * launch's engine fetches may still be reading) whose PTEs are + * planted in the proven window-0 PT (slots derive from + * GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE) by gsp_phys_rmcall_pdb_build, + * before the engine's first fetch (same pattern as the qmdwin slot). + * The launch copies the QMD's program/param page content into them + * and rewrites the QMD pointers there, so the GR engine never reads + * the window-1 scratch range. */ struct gsp_phys_boot_chunk rmcall_gr_shadow_win0; u32 rmcall_gr_shadow_win0_used; @@ -2161,19 +2461,114 @@ static struct gsp_phys_boot_stage { /* m104: module page holding the injected mail_kernel message. */ struct gsp_phys_boot_chunk rmcall_gr_mail; /* - * m114: pre-commit bounce window. 112 pages planted into the main - * small-PT at slots 0x110..0x17f during stage 3 (the proven win0 - * pattern: the SM walker resolves the commit-time tree only). The - * launch copies the kernel's data pages in, rewrites the pointers to - * these VAs (reloc base 0x110000), and copies the results back after - * the release; page 100 (VA 0x174000) carries the mail message. + * m114: pre-commit bounce window, planted into the main small-PT at + * slots 0x110..0x1df during stage 3 (the proven win0 pattern: the SM + * walker resolves the commit-time tree only). The launch copies the + * kernel's data pages in, rewrites the pointers to these VAs (reloc + * base 0x110000), and copies the results back after the release. + * B-2: with the grown data region on (gsp_phys_gr_data_region=1, the + * default) this window no longer carries operand data - only the mail + * slot and marker - but it stays allocated so the mail injection and + * the copy diagnostics are unchanged, and it remains the whole data + * region with the parameter off. + * + * The window is 208 pages so the data region below the mail page can + * carry the operand pages of kernels whose buffers exceed the old + * 128 KB (victory_matmul's 256 KB buffers need 64 pages each; three + * of them plus argument/bank candidates exceed the old 100-page / + * 33-pages-per-pointer caps and the engine then ran on truncated + * windows). The mail page is the LAST data-free page (slot 206) and + * the m132 marker sits at 207; the main PT page deliberately covers + * the whole 0x110000..0x1df000 range with nothing else planted there. */ struct gsp_phys_boot_chunk rmcall_gr_bounce; u32 rmcall_gr_bounce_used; -#define GSP_PHYS_RMCALL_GR_BOUNCE_PAGES 112u + /* + * B-2: grown boot-fixed data-bounce region (gsp_phys_gr_data_region=1, + * the default; see the constants in gsp_phys_booter.h). One 16384-page + * (64 MiB) data chunk plus GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PT_PAGES 4 KB + * PT pages (one per 2 MB window), all planted once at stage 3 before + * any engine fetch. The old 208-page window above keeps its mail slot + * 206 and marker 207 exactly as-is (the mail injection and copy + * diagnostics rely on them); when the grown region is on it no longer + * carries operand data. + */ + struct gsp_phys_boot_chunk rmcall_gr_data_bounce; + struct gsp_phys_boot_chunk rmcall_gr_data_bounce_pt; +#define GSP_PHYS_RMCALL_GR_BOUNCE_PAGES 208u #define GSP_PHYS_RMCALL_GR_BOUNCE_SLOT0 0x110u #define GSP_PHYS_RMCALL_GR_BOUNCE_VA0 0x110000ULL -#define GSP_PHYS_RMCALL_GR_MAIL_SLOT 100u +#define GSP_PHYS_RMCALL_GR_MAIL_SLOT 206u +/* + * m150b: the win0 QMD shadow pool must never overlap the bounce window. + * The win0 plant runs on every pdb_build() (unconditional) and the bounce + * plant is one-shot, so an overlap silently hijacks bounce PTEs and the + * engine's results land in the shadow pages (victory_matmul lost C rows + * 0..31 exactly this way before 0x1e0000 moved the pool clear). + */ +static_assert(GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE + + (u64)GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS * GSP_PHYS_PAGE_SIZE <= + GSP_PHYS_RMCALL_GR_BOUNCE_VA0 || + GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE >= + GSP_PHYS_RMCALL_GR_BOUNCE_VA0 + + (u64)GSP_PHYS_RMCALL_GR_BOUNCE_PAGES * GSP_PHYS_PAGE_SIZE, + "win0 shadow pool must not overlap the GR data bounce window"); +/* + * m151: and the two-bank pool must end below the reserved mail-injection + * slot 0x1f0 (VA 0x1f0000) so the extra slots never steal that PTE. + */ +static_assert(GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE + + (u64)GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS * GSP_PHYS_PAGE_SIZE <= + GSP_PHYS_RMCALL_VA_BASE + 0xF0000ULL, + "win0 shadow pool must end below the reserved mail-injection slot"); +static_assert(GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS == + 2u * GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS, + "win0 shadow pool must be exactly two banks"); +/* + * B-2: the grown data-bounce region is a hard VA-layout contract. One fresh + * PT page per 2 MB window; PD0 slots immediately above the m152 extension + * pool's 32..79 under the same root/PD2/PD1 chain as the ladder; the whole + * region below the non-shared VAS vaSize (a non-shared VAS addresses PD0 + * entries 0..127 = 256 MB, so the region's 64 MB is a 32 MB-slack fit); and + * disjoint from every other planted window in the PDB (old bounce 0x110000.. + * 0x1dffff, win0 pool 0x1e0000.., guest scratch windows 0x200000..). Any + * future move of VA0/SLOT0/size fails at compile time. + */ +static_assert(GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES == + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PT_PAGES * 512u, + "the grown data-bounce region must be exactly one PT page per 2 MB window"); +static_assert(GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 == + (u64)GSP_PHYS_RMCALL_GR_DATA_BOUNCE_SLOT0 * (2ULL << 20), + "the grown data-bounce VA0 must equal its PD0 slot base"); +static_assert(((GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 >> 47) & 0x3) == 0 && + ((GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 >> 38) & 0x1ff) == 0 && + ((GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 >> 29) & 0x1ff) == 0 && + ((GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 >> 21) & 0xff) == + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_SLOT0 && + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_SLOT0 + + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PT_PAGES <= 128, + "the grown data-bounce region must use PD0 entries 80..111 under the ladder's root/PD2/PD1 chain"); +static_assert(GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 + + (u64)GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES * + GSP_PHYS_PAGE_SIZE <= + GSP_PHYS_RMCALL_VAS_SIZE, + "the grown data-bounce region must stay below the VAS vaSize limit"); +static_assert(GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 >= + GSP_PHYS_RMCALL_VA_GUEST_PT_EXT_BASE + + (u64)GSP_PHYS_RMCALL_GUEST_PT_EXT_PAGES * (2ULL << 20), + "the grown data-bounce region must start above the m152 extension scratch pool"); +static_assert(GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 >= + GSP_PHYS_RMCALL_GR_BOUNCE_VA0 + + (u64)GSP_PHYS_RMCALL_GR_BOUNCE_PAGES * + GSP_PHYS_PAGE_SIZE && + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 >= + GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE + + (u64)GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS * + GSP_PHYS_PAGE_SIZE && + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 >= + GSP_PHYS_RMCALL_VA_GUEST_PT_BASE + + (u64)GSP_PHYS_RMCALL_GUEST_PT_PAGES * (2ULL << 20), + "the grown data-bounce region must be disjoint from the old bounce window, the win0 pool and the guest scratch window"); /* m99: guest data identity-map table branches (see the PD*_MAX block). */ struct gsp_phys_boot_chunk rmcall_gr_data_pd1[GSP_PHYS_RMCALL_GR_DATA_PD1_MAX]; @@ -2813,6 +3208,7 @@ static void gsp_phys_rmcall_reset(void) boot_stage.rmcall_gr_submit_epoch = 0; boot_stage.rmcall_gr_pt_used = 0; boot_stage.rmcall_guest_pt_win_mask = 0; + boot_stage.rmcall_gr_alias_count = 0; memset(boot_stage.rmcall_gr_pt, 0, sizeof(boot_stage.rmcall_gr_pt)); memset(&boot_stage.rmcall_gr_shadow, 0, sizeof(boot_stage.rmcall_gr_shadow)); @@ -6442,6 +6838,35 @@ int gsp_phys_backend_static_wait(struct gsp_phys_backend_static_config *out, EXPORT_SYMBOL_GPL(gsp_phys_backend_static_wait); /* + * Mode-35 KGR static-data snapshots. The user-lane query set + * (gsp_phys_rmcall_user_lane_kgr_queries) is answered by the LIVE GSP with + * the device's true GR static data; the replies are kept here so the + * guest-side emulation can serve the identical bytes instead of fabricating + * GR values. The concrete consumer is the X driver's 3D-engine selection: + * nv3dAllocDevice maps GR_GET_INFO index SM_VERSION (0x0c) through a table + * and fails with "Failed to select a 3D engine" when it reads back as + * SM_VERSION_NONE (0) - the fabricated fill never set it, while the live + * GSP's KGR_GET_INFO reply carries the real value (0x75 for TU104) along + * with every other GR static field (warps, caps, topology, ctx buffers). + * + * One entry per captured cmd; the generation key ties the cache to the + * backend session exactly like gsp_phys_backend_static above, and the + * getter refuses entries once the session has moved on. + */ +#define GSP_PHYS_KGR_SNAP_ENTRIES 10 +#define GSP_PHYS_KGR_SNAP_MAX_SIZE 3712u /* KGR_GET_INFO: 8 engines x 58 x 8 */ + +struct gsp_phys_kgr_snap { + u32 cmd; + u32 size; + bool valid; + u32 generation; + u8 data[GSP_PHYS_KGR_SNAP_MAX_SIZE]; +}; + +static struct gsp_phys_kgr_snap gsp_phys_kgr_snaps[GSP_PHYS_KGR_SNAP_ENTRIES]; + +/* * Drop the cached static-config snapshot. Called when the backend session * leaves READY (or a new claim starts): the cache generation key is bumped * past the current backend generation so a concurrent getter cannot serve the @@ -6450,9 +6875,13 @@ EXPORT_SYMBOL_GPL(gsp_phys_backend_static_wait); */ void gsp_phys_backend_static_invalidate(void) { + unsigned int i; + mutex_lock(&gsp_phys_rm_lock); gsp_phys_backend_static.valid = false; gsp_phys_backend_static.generation = gsp_phys_backend_generation() + 1; + for (i = 0; i < ARRAY_SIZE(gsp_phys_kgr_snaps); i++) + gsp_phys_kgr_snaps[i].valid = false; mutex_unlock(&gsp_phys_rm_lock); } @@ -6506,6 +6935,71 @@ int gsp_phys_backend_static_get(struct gsp_phys_backend_static_config *out) } EXPORT_SYMBOL_GPL(gsp_phys_backend_static_get); +/* + * Copy the captured reply for cmd into dst. Returns 0 on success (with + * *out_len set), -ENODATA when nothing was captured for cmd (or the entry's + * generation no longer matches the backend session), -ENOSPC when the + * snapshot does not fit dst_len, -EAGAIN while the backend is not READY. + */ +int gsp_phys_kgr_snap_get(u32 cmd, void *dst, u32 dst_len, u32 *out_len) +{ + unsigned int i; + + if (!dst || !out_len) + return -EINVAL; + + if (gsp_phys_backend_state() != GSP_PHYS_BACKEND_READY) + return -EAGAIN; + + for (i = 0; i < ARRAY_SIZE(gsp_phys_kgr_snaps); i++) { + struct gsp_phys_kgr_snap *s = &gsp_phys_kgr_snaps[i]; + + if (s->cmd != cmd) + continue; + if (!s->valid || + s->generation != gsp_phys_backend_generation()) + return -ENODATA; + if (s->size > dst_len) + return -ENOSPC; + memcpy(dst, s->data, s->size); + *out_len = s->size; + return 0; + } + return -ENODATA; +} +EXPORT_SYMBOL_GPL(gsp_phys_kgr_snap_get); + +/* + * Record one mode-35 KGR query reply. Called under the boot thread only + * (same thread as the query itself), so no lock is needed beyond the + * generation key. + */ +static void gsp_phys_kgr_snap_store(u32 cmd, const void *data, u32 size) +{ + unsigned int i; + struct gsp_phys_kgr_snap *slot = NULL; + + if (size > GSP_PHYS_KGR_SNAP_MAX_SIZE) + return; + + for (i = 0; i < ARRAY_SIZE(gsp_phys_kgr_snaps); i++) { + if (gsp_phys_kgr_snaps[i].cmd == cmd) { + slot = &gsp_phys_kgr_snaps[i]; + break; + } + if (!slot && !gsp_phys_kgr_snaps[i].valid) + slot = &gsp_phys_kgr_snaps[i]; + } + if (!slot) + return; + + slot->cmd = cmd; + slot->size = size; + slot->generation = gsp_phys_backend_generation(); + memcpy(slot->data, data, size); + slot->valid = true; +} + /* ============================================================ * Host RM client ladder (R1): gsp-phys-rm-call * ============================================================ */ @@ -7215,6 +7709,65 @@ static void gsp_phys_rmcall_ctx_diag_dump(const char *what, static void gsp_phys_rmcall_ctx_diag_decode(const char *tag); /* + * m152: scratch-PT pool geometry helpers. Pool local page P (0-based across + * the original 16-window region then the 48-window extension) maps to a PD0 + * window VA and to a page of either the per-window single-page PT chunks + * (P < 16) or the multi-page extension chunk (P >= 16). Keep every + * consumer on these two helpers so the two regions can never drift. + */ +static u64 gsp_phys_rmcall_guest_pt_win_va(u32 page) +{ + if (page < GSP_PHYS_RMCALL_GUEST_PT_PAGES) + return boot_stage.rmcall_va_guest_pt + ((u64)page << 21); + return GSP_PHYS_RMCALL_VA_GUEST_PT_EXT_BASE + + ((u64)(page - GSP_PHYS_RMCALL_GUEST_PT_PAGES) << 21); +} + +static u8 *gsp_phys_rmcall_guest_pt_page_va(u32 page) +{ + if (page < GSP_PHYS_RMCALL_GUEST_PT_PAGES) + return gsp_phys_chunk_page_va( + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GUEST_PT0 + + page], 0); + if (page < GSP_PHYS_RMCALL_GUEST_PT_TOTAL_PAGES) + return gsp_phys_chunk_page_va( + &boot_stage.rmcall_chunks[ + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT0 + + (page - GSP_PHYS_RMCALL_GUEST_PT_PAGES)], 0); + return NULL; +} + +static u64 gsp_phys_rmcall_guest_pt_page_dma(u32 page) +{ + if (page < GSP_PHYS_RMCALL_GUEST_PT_PAGES) + return (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GUEST_PT0 + + page], 0); + return (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_chunks[ + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT0 + + (page - GSP_PHYS_RMCALL_GUEST_PT_PAGES)], 0); +} + +/* + * PD0 index -> pool local page, or 0xffffffffu when the index is not a + * scratch-pool window. Consults both pool regions. + */ +static u32 gsp_phys_rmcall_guest_pt_page_of_pde(u32 pde_index) +{ + u32 base = (u32)(boot_stage.rmcall_va_guest_pt >> 21); + + if (pde_index >= base && + pde_index < base + GSP_PHYS_RMCALL_GUEST_PT_PAGES) + return pde_index - base; + base = (u32)(GSP_PHYS_RMCALL_VA_GUEST_PT_EXT_BASE >> 21); + if (pde_index >= base && + pde_index < base + GSP_PHYS_RMCALL_GUEST_PT_EXT_PAGES) + return pde_index - base + GSP_PHYS_RMCALL_GUEST_PT_PAGES; + return 0xffffffffu; +} + +/* * Shared-management VAS: install the ladder's page-table window through the * server's own MMU walker. NV0080_CTRL_CMD_DMA_UPDATE_PDE_2 (0x80180f, * gpu_vaspace.c gvaspaceUpdatePde2_IMPL) takes the system-physical base of a @@ -9182,14 +9735,68 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0) * (swref/published/pascal/gp100/dev_mmu.h:97-119,157; * uvm_pascal_mmu.c:64-141,235-303). */ -static u64 gsp_phys_rmcall_pde(u64 pa) +/* + * r431/r436 COHERENCY ROOT-CAUSE CANDIDATE: the leaf-PTE aperture for every + * host page this backend maps (the ladder windows, the win0 shadow pool, the + * bounce/data regions, the release/scratch shadows) was hardcoded to + * aperture 3 = SYSTEM_NON_COHERENT_MEMORY. The reference RM derives the + * pair from the allocation's CPU cache attribute (fabric_vaspace.c:1123-1134 + * for the aperture, mem_desc.c memdescGetVolatility for VOL: NV_MEMORY_CACHED + * -> SYSTEM_COHERENT (2) + volatile FALSE, NV_MEMORY_UNCACHED -> NON_COHERENT + * (3) + volatile TRUE). The r431 experiment used aperture 2 with VOL still 1 + * (0xd) - a pair the reference RM never produces - and it made the engine's + * truncated writes visible sooner but worsened the wedge on a fresh boot. + * r436 turns the knob into the full 2x2 matrix so each consistent encoding + * can be A/B'd: + * 0 = aperture 3 + VOL 1 (0xf) - historical, NV_MEMORY_UNCACHED + * 1 = aperture 2 + VOL 1 (0xd) - r431 experiment (inconsistent pair) + * 2 = aperture 2 + VOL 0 (0x5) - NV_MEMORY_CACHED (coherent, non-volatile) + * 3 = aperture 3 + VOL 0 (0x7) - non-coherent, non-volatile + * The PTEs are planted before the engine's first fetch, so the flag is + * boot-time only. + */ +static int gsp_phys_rmcall_pte_coh_param; +module_param_named(gsp_phys_rmcall_pte_coh, gsp_phys_rmcall_pte_coh_param, + int, 0444); +MODULE_PARM_DESC(gsp_phys_rmcall_pte_coh, + "r436: sysmem leaf-PTE aperture/VOL mode: 0=0xf (aperture3+VOL, historical), 1=0xd (aperture2+VOL, r431), 2=0x5 (aperture2, non-volatile), 3=0x7 (aperture3, non-volatile)"); + +static u64 gsp_phys_rmcall_sysmem_base_low(void) { - return 0xeull | ((pa >> 12) << 8); + switch (gsp_phys_rmcall_pte_coh_param) { + case 1: + return 0xdull; /* aperture 2, VOL 1 */ + case 2: + return 0x5ull; /* aperture 2, VOL 0 */ + case 3: + return 0x7ull; /* aperture 3, VOL 0 */ + default: + return 0xfull; /* aperture 3, VOL 1 (historical) */ + } } static u64 gsp_phys_rmcall_pte(u64 pa) { - return 0xfull | ((pa >> 12) << 8); + /* + * Low nibble: VALID (1) | APERTURE<<1 | VOL (8). Aperture 2 = + * SYSTEM_COHERENT_MEMORY -> 1|4|8 = 0xd; aperture 3 = NON_COHERENT + * -> 1|6|8 = 0xf. r436: see gsp_phys_rmcall_pte_coh_param. + */ + u64 base = gsp_phys_rmcall_sysmem_base_low(); + + return base | ((pa >> 12) << 8); +} + +/* + * r436: the single PDE uses the same aperture/VOL pair as the leaf PTEs but + * with IS_PDE=0 in bit 0 (the low bit of the PTE base is VALID there), so the + * PDE base is the PTE base with bit 0 cleared: 0xe/0xc/0x4/0x6. + */ +static u64 gsp_phys_rmcall_pde(u64 pa) +{ + u64 base = gsp_phys_rmcall_sysmem_base_low() & ~1ull; + + return base | ((pa >> 12) << 8); } /* @@ -9336,35 +9943,39 @@ static void gsp_phys_rmcall_pdb_build(void) &boot_stage.rmcall_gr_qmdwin; u32 qw_idx = (u32)((GSP_PHYS_RMCALL_VA_GR_QMDWIN >> 12) & 0x1ffu); + u32 qw_s; if (!qw->pages) { if (!boot_stage.pdev) { pr_warn("vgsp-boot: rm-call: qmdwin plant skipped: no claimed pci_dev; launches keep the guest QMD destination\n"); - } else if (!gsp_phys_chunk_alloc(boot_stage.pdev, qw, 1u)) { + } else if (!gsp_phys_chunk_alloc(boot_stage.pdev, qw, + GSP_PHYS_RMCALL_GR_QMDWIN_SLOTS)) { gsp_phys_chunk_zero(qw); } else { pr_warn("vgsp-boot: rm-call: qmdwin page allocation failed; launches keep the guest QMD destination\n"); } } - if (qw->pages) { - put_unaligned_le64(gsp_phys_rmcall_pte( - (u64)gsp_phys_chunk_page_dma(qw, 0)), - (u8 *)gsp_phys_chunk_page_va(pt, 0) + - (size_t)qw_idx * 8); + if (qw->pages >= GSP_PHYS_RMCALL_GR_QMDWIN_SLOTS) { + for (qw_s = 0; qw_s < GSP_PHYS_RMCALL_GR_QMDWIN_SLOTS; + qw_s++) + put_unaligned_le64(gsp_phys_rmcall_pte( + (u64)gsp_phys_chunk_page_dma(qw, + qw_s)), + (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)(qw_idx + qw_s) * 8); if (!boot_stage.rmcall_gr_qmdwin_va) - pr_info("vgsp-boot: rm-call: qmdwin module page planted VA=0x%llx ptSlot=0x%x pte=0x%016llx dma=0x%llx (before the engine's first fetch)\n", - GSP_PHYS_RMCALL_VA_GR_QMDWIN, qw_idx, - get_unaligned_le64( - (u8 *)gsp_phys_chunk_page_va( - pt, 0) + - (size_t)qw_idx * 8), - (u64)gsp_phys_chunk_page_dma(qw, 0)); + pr_info("vgsp-boot: rm-call: qmdwin module pages planted VA=0x%llx..0x%llx ptSlots=0x%x/0x%x (double-buffer, before the engine's first fetch)\n", + GSP_PHYS_RMCALL_VA_GR_QMDWIN, + GSP_PHYS_RMCALL_VA_GR_QMDWIN + + (GSP_PHYS_RMCALL_GR_QMDWIN_SLOTS - 1u) * + GSP_PHYS_PAGE_SIZE, + qw_idx, qw_idx + 1u); boot_stage.rmcall_gr_qmdwin_va = GSP_PHYS_RMCALL_VA_GR_QMDWIN; } } /* - * m69: plant the window-0 shadow pool PTEs (slots 0x108..0x10f) in the + * m69: plant the window-0 shadow pool PTEs (current win0 slot range) in the * same proven small-PT line, before the engine's first fetch. The * launch copies the QMD's program/param page content into these pages * and rewrites the QMD pointers to these VAs, so the GR engine never @@ -9410,12 +10021,18 @@ static void gsp_phys_rmcall_pdb_build(void) (size_t)idx * 8); } dma_wmb(); - pr_info("vgsp-boot: rm-call: shadow-win0 pool planted %u pages at VA 0x%llx..0x%llx (small-PT slots 0x108..0x10f, before first fetch)\n", + pr_info("vgsp-boot: rm-call: shadow-win0 pool planted %u pages at VA 0x%llx..0x%llx (small-PT slots 0x%x..0x%x, before first fetch)\n", GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS, GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE, GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE + (GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS - 1) * - GSP_PHYS_PAGE_SIZE); + GSP_PHYS_PAGE_SIZE, + (u32)((GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE >> 12) & + 0x1ffu), + (u32)(((GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE + + (GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS - 1) * + GSP_PHYS_PAGE_SIZE) >> 12) & + 0x1ffu)); } } /* @@ -9468,6 +10085,77 @@ static void gsp_phys_rmcall_pdb_build(void) pr_err("vgsp-boot: rm-call: bounce window allocation failed\n"); } } + /* + * B-2: grown data-bounce region (gsp_phys_gr_data_region=1, default). + * One 16384-page (64 MiB) data chunk plus one fresh 4 KB PT page per + * 2 MB window; PD0 dual entries 80..111 in the client PDB (immediately + * above the m152 extension pool's 32..79), every leaf PTE pre-written + * with the chunk page's DMA address. The whole map is final at stage 3 + * - before the engine's first fetch - so the 128 B PTE fetch-line + * granule is fully populated at plant time (the GMMU has no invalidate + * path; a runtime PTE write would be invisible). One-shot like the + * bounce plant above: no later pdb_build() call re-zeroes the PD0 page, + * and the region's other writers (ext pool, ctx, win0) target disjoint + * slots. Planted only when the parameter is on; with 0 the SVC and the + * copy path stay on the legacy 205-page window. + */ + if (gsp_phys_gr_data_region_value() && + (!boot_stage.rmcall_gr_data_bounce.pages || + !boot_stage.rmcall_gr_data_bounce_pt.pages)) { + struct gsp_phys_boot_chunk *ptd = + &boot_stage.rmcall_gr_data_bounce_pt; + u32 i; + + if (!gsp_phys_chunk_alloc(boot_stage.pdev, + &boot_stage.rmcall_gr_data_bounce, + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES)) + gsp_phys_chunk_zero(&boot_stage.rmcall_gr_data_bounce); + if (!gsp_phys_chunk_alloc(boot_stage.pdev, ptd, + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PT_PAGES)) + gsp_phys_chunk_zero(ptd); + if (!boot_stage.rmcall_gr_data_bounce.pages || !ptd->pages) { + pr_err("vgsp-boot: rm-call: data-bounce region allocation failed (data=%u pt=%u pages); launches fail closed on the grown region\n", + boot_stage.rmcall_gr_data_bounce.pages, + ptd->pages); + } else { + for (i = 0; i < GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PT_PAGES; i++) { + u32 pd0_idx = + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_SLOT0 + i; + u64 pde = gsp_phys_rmcall_pde( + (u64)gsp_phys_chunk_page_dma(ptd, i)); + + put_unaligned_le64(0, + (u8 *)gsp_phys_chunk_page_va(pd0, 0) + + (size_t)pd0_idx * 16); + put_unaligned_le64(pde, + (u8 *)gsp_phys_chunk_page_va(pd0, 0) + + (size_t)pd0_idx * 16 + 8); + } + for (i = 0; i < GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES; i++) { + u32 win = i / 512u; + u32 slot = i % 512u; + + put_unaligned_le64(gsp_phys_rmcall_pte( + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_gr_data_bounce, + i)), + (u8 *)gsp_phys_chunk_page_va(ptd, win) + + (size_t)slot * 8); + } + dma_wmb(); + pr_info("vgsp-boot: rm-call: data-bounce region planted: %u pages at VA 0x%llx..0x%llx (PD0 slots %u..%u, %u PT pages, before first fetch)\n", + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES, + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0, + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 + + (GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES - + 1) * GSP_PHYS_PAGE_SIZE, + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_SLOT0, + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_SLOT0 + + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PT_PAGES - + 1, + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PT_PAGES); + } + } /* * P6a guest windows: start from explicit zero (invalid PTE) for both @@ -9491,8 +10179,16 @@ static void gsp_phys_rmcall_pdb_build(void) u64 guest_pte; int gpa_ret; - gpa_ret = gsp_phys_gpa_to_hpa(boot_stage.rmcall_guest_gpa, - &guest_hpa); + /* + * The pinned GPA is a single-guest test aid (the pin write is + * refused while more than one mdev is open; see + * gsp_phys_guest_gpa_write): resolve through the first open + * mdev exactly as before the per-mdev records, via the NULL + * selector (a per-mdev pin is not modelled). + */ + gpa_ret = gsp_phys_gpa_to_hpa_vgpu(NULL, + boot_stage.rmcall_guest_gpa, + &guest_hpa); if (!gpa_ret) { guest_pte = gsp_phys_rmcall_pte(guest_hpa); put_unaligned_le64(guest_pte, @@ -9532,6 +10228,30 @@ static void gsp_phys_rmcall_pdb_build(void) (size_t)(1 + i) * 16 + 8); } /* + * m152: extension scratch pool - PD0 dual entries 32..79 back the + * GSP_PHYS_RMCALL_GUEST_PT_EXT_PAGES pages of the multi-page extension + * PT chunk at GSP_PHYS_RMCALL_VA_GUEST_PT_EXT_BASE (0x04000000; free + * space above the GR-context windows, which end well below 0x03000000). + * Same shape and same once-before-any-engine-access rule as the 16-window + * loop above. + */ + for (i = 0; i < GSP_PHYS_RMCALL_GUEST_PT_EXT_PAGES; i++) { + u64 guest_pt_pde = gsp_phys_rmcall_pde( + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_chunks[ + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT0 + i], + 0)); + u32 pd0_idx = (u32)(GSP_PHYS_RMCALL_VA_GUEST_PT_EXT_BASE >> 21) + + i; + + put_unaligned_le64(0, + (u8 *)gsp_phys_chunk_page_va(pd0, 0) + + (size_t)pd0_idx * 16); + put_unaligned_le64(guest_pt_pde, + (u8 *)gsp_phys_chunk_page_va(pd0, 0) + + (size_t)pd0_idx * 16 + 8); + } + /* * H1 GR context window: when the GR ladder prepared the promoted MAIN * context buffer (gsp_phys_rmcall_gr_ctx_prepare ran before this * builder), append its page tables as fresh PD0 dual entries 17.. (the @@ -9698,11 +10418,12 @@ static void gsp_phys_rmcall_pdb_build(void) src_idx, pt_src, (u64)gsp_phys_chunk_page_dma(src, 0), dst_idx, pt_dst, (u64)gsp_phys_chunk_page_dma(dst, 0), sem_idx, pt_sem, (u64)gsp_phys_chunk_page_dma(sem, 0)); - pr_info("vgsp-boot: rm-call: guest scratch PT built: pages=%u vaBase=0x%llx capacity=%u\n", + pr_info("vgsp-boot: rm-call: guest scratch PT built: pages=%u (+%u ext) vaBase=0x%llx extBase=0x%llx capacity=%u\n", GSP_PHYS_RMCALL_GUEST_PT_PAGES, + GSP_PHYS_RMCALL_GUEST_PT_EXT_PAGES, GSP_PHYS_RMCALL_VA_GUEST_PT_BASE, - GSP_PHYS_RMCALL_GUEST_PT_PAGES * - GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE); + GSP_PHYS_RMCALL_VA_GUEST_PT_EXT_BASE, + GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES); } /* @@ -12993,6 +13714,12 @@ static int gsp_phys_rmcall_user_lane_userd(void __iomem *bar0) * (CONTEXT_BUFFERS_INFO in particular) may be prepared by the query. All are * zero-in best-effort getters at the capture's payload sizes; a failure * warns and the lane continues. + * + * Every successful reply is retained via gsp_phys_kgr_snap_store(): the + * guest-side emulation serves these exact bytes for the same commands, so + * the guest's GL/3D stack reads the live GSP's true GR static values + * (SM_VERSION in particular - see the snapshot comment near + * gsp_phys_backend_static_get). */ static void gsp_phys_rmcall_user_lane_kgr_queries(void __iomem *bar0) { @@ -13031,6 +13758,8 @@ static void gsp_phys_rmcall_user_lane_kgr_queries(void __iomem *bar0) q[i].name); pr_info("vgsp-boot: rm-call: mode 35 KGR query %s: size=%u status=0x%x rpc_result=0x%x ret=%d\n", q[i].name, q[i].size, status, rpc_result, ret); + if (ret == 0 && rpc_result == 0 && status == 0) + gsp_phys_kgr_snap_store(q[i].cmd, buf, q[i].size); kvfree(buf); } } @@ -14536,6 +15265,39 @@ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, * semaphore. On timeout the existing status-queue drain runs and the helper * returns -ETIMEDOUT; it never retries. */ +static int gsp_phys_rmcall_gr_resched_locked(void __iomem *bar0); +static int gsp_phys_rmcall_gr_sem_push_locked(void __iomem *bar0); + +/* + * r445 wedge hard re-arm: the outcome-retry's warm semaphore probe proves the + * engine refused the channel's proven GR block (the sticky post-dispatch + * no-store wedge). One bounded recovery attempt instead of an immediate + * break: force a context save/unload with a bWait TSG preempt, re-run the + * mode-35 lane-enable tail + schedules (the m155 extended re-arm), then + * repeat the warm probe. Caller holds gsp_phys_rm_lock. Returns 0 when the + * engine executed the probe after the re-arm. + */ +static int gsp_phys_gr_wedge_hard_rearm_locked(void __iomem *bar0) +{ + u32 pre[2] = { 1u, 0u }; /* bWait=1, bManualTimeout=0 */ + u32 rpc_result = 0, status = 0; + int ret; + + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_tsg, + GSP_PHYS_RMCALL_CTRL_TSG_PREEMPT, + pre, sizeof(pre), sizeof(pre), NULL, 0, + &rpc_result, &status, + "TSG PREEMPT bWait=1 (r445 wedge hard re-arm)"); + pr_warn("vgsp-gr: WEDGE-REARM preempt status=0x%x rpc=0x%x ret=%d\n", + status, rpc_result, ret); + ret = gsp_phys_rmcall_gr_resched_locked(bar0); + pr_warn("vgsp-gr: WEDGE-REARM resched ret=%d\n", ret); + ret = gsp_phys_rmcall_gr_sem_push_locked(bar0); + pr_warn("vgsp-gr: WEDGE-REARM probe rc=%d (engine %s after re-arm)\n", + ret, ret ? "still wedged" : "ALIVE"); + return ret; +} + static int gsp_phys_rmcall_gr_sem_push_locked(void __iomem *bar0) { struct gsp_phys_boot_chunk *userd_c = @@ -15162,7 +15924,6 @@ static int gsp_phys_rmcall_guest_map_locked(u64 src_hpa, u64 dst_hpa, u64 sem_hpa, u64 *src_va, u64 *dst_va, u64 *sem_va) { - struct gsp_phys_boot_chunk *guest_pt_chunk; u32 line = boot_stage.rmcall_guest_pt_next_line; u32 page, line_in_page; u64 va; @@ -15170,19 +15931,14 @@ static int gsp_phys_rmcall_guest_map_locked(u64 src_hpa, u64 dst_hpa, if (!boot_stage.rmcall_va_guest_pt) return -EIO; - if (line >= GSP_PHYS_RMCALL_GUEST_PT_PAGES * - GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE) + if (line >= GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES) return -ENOSPC; page = line / GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; line_in_page = line % GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; - guest_pt_chunk = &boot_stage.rmcall_chunks[ - GSP_PHYS_RMCALL_BUF_GUEST_PT0 + page]; - if (!guest_pt_chunk->pages) - return -EIO; /* 2 MB PD0 slot stride; a full line is 16 PTEs = 0x10000 of VA. */ - va = boot_stage.rmcall_va_guest_pt + ((u64)page << 21) + + va = gsp_phys_rmcall_guest_pt_win_va(page) + ((u64)line_in_page << 16); - pt = gsp_phys_chunk_page_va(guest_pt_chunk, 0); + pt = gsp_phys_rmcall_guest_pt_page_va(page); if (!pt) return -EIO; /* @@ -15403,15 +16159,13 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa, if ((gsp_phys_relay_pool_exhausted & 63) == 1) pr_err("vgsp-work: GUEST CE RELAY FAILED: scratch-PT pool exhausted (no line reuse) pool_exhausted=%llu capacity=%u; falling back to the software release\n", gsp_phys_relay_pool_exhausted, - GSP_PHYS_RMCALL_GUEST_PT_PAGES * - GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE); + GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES); } else { gsp_phys_guest_pool_exhausted++; if ((gsp_phys_guest_pool_exhausted & 63) == 1) pr_err("vgsp-work: GUEST CE COPY FAILED: scratch-PT pool exhausted (no line reuse) pool_exhausted=%llu capacity=%u; falling back to the ladder path\n", gsp_phys_guest_pool_exhausted, - GSP_PHYS_RMCALL_GUEST_PT_PAGES * - GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE); + GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES); } } else { if (relay) @@ -15730,6 +16484,19 @@ EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_ce_copy); #define GSP_PHYS_RMCALL_GR_CWD_REFCOUNT 1536u #define GSP_PHYS_RMCALL_GR_CWD_LEVELS 64u #define GSP_PHYS_RMCALL_GR_MAX_SM_COUNT 24u +/* + * r434 CWD-slot-count path: the falcon PRI write (CUDA's + * turingFalconPRIWriteMasked: flag/data/mask through SET_MME_SHADOW_SCRATCH + * 0/1/2, then SET_FALCON04 = register offset) and the gated method. Used only + * when gsp_phys_rmcall_gr_cwd_slot >= 0. + */ +#define GSP_PHYS_RMCALL_GR_MME_SCRATCH0 0x3400u +#define GSP_PHYS_RMCALL_GR_MME_SCRATCH1 0x3404u +#define GSP_PHYS_RMCALL_GR_MME_SCRATCH2 0x3408u +#define GSP_PHYS_RMCALL_GR_SET_FALCON04 0x0510u +#define GSP_PHYS_RMCALL_GR_SET_CWD_SLOT_COUNT 0x02b0u +#define GSP_PHYS_RMCALL_PRI_SKED_DEBUG_1 0x00407040u +#define GSP_PHYS_RMCALL_PRI_SKED_DEBUG_1_EN_CWD_SLOT_COUNT 0x800u /* Method block bound: init(152) + inline run(67) + PCAS(4) + REPORT_SEM(8) + * the optional QMD-virt pair(2). */ #define GSP_PHYS_RMCALL_GR_METHOD_WORDS 512u @@ -15738,7 +16505,109 @@ static u64 gsp_phys_gr_submitted; static u64 gsp_phys_gr_ok; static u64 gsp_phys_gr_refused; static u64 gsp_phys_gr_timed_out; +static u64 gsp_phys_gr_retries; /* fresh-entry launch retries published */ static u64 gsp_phys_gr_pool_exhausted; +/* m156 (E5): zero-write retries where the warm sem push proved the engine + * wedged (the persistent-lottery class); further retries are skipped. */ +static u64 gsp_phys_gr_sem_wedged; +/* + * STALLFREE: bounded-completion telemetry. stall_wait_max counts launches + * that crossed gsp_phys_gr_wait_max_ms and failed out of the submit ladder + * (the sema service then authors the QMD release in software); stall_retry + * counts launches whose retirement retry loop stopped early on the same + * budget (the launch proceeds with its current outcome). Both are zero on a + * green run. + */ +static u64 gsp_phys_gr_stall_wait_max; +static u64 gsp_phys_gr_stall_retry; +/* + * r410 wedge quiet window state. nw_streak counts consecutive accepted + * zero-write launches with tracked data (reset by any launch that wrote); + * the sticky wedge is the state where this grows without bound while the + * lane is kept busy. quiet counts the quiet holds issued; heals counts the + * quiet retries after which the engine wrote. Single-inflight submit path, + * so plain statics. + */ +static u32 gsp_phys_gr_nw_streak; +static u64 gsp_phys_gr_wedge_quiet; +static u64 gsp_phys_gr_wedge_heals; +/* + * r411: once a quiet heal has succeeded on this boot, the lane is in a + * wedge-prone episode. r418 showed the wedge can alternate one-no-write / + * one-heal, and the healed launch resets nw_streak, so a streak-only trigger + * quieted only every second wedged launch and the other was accepted. The + * heal count makes the episode sticky for the boot: any later zero-write + * launch gets the quiet path on its first retry, so every wedged launch + * either heals (engine writes, client data correct) or is loudly classified. + */ +static bool gsp_phys_gr_wedge_episode; +/* + * r420: true exactly while a WEDGE-QUIET hold sleeps with NO submission + * outstanding. The hold is a deliberate, bounded wait for the sticky wedge to + * clear (r410/r411); during it the in-flight forwarded request is NOT done. + * The sema service's completion deadlines - the 12 s GR release hold + * (SVC_FWD_GR_HOLD_TIMEOUT_MS) and the 3 s per-launch consumption bound + * (gsp_phys_gr_hold_max_ms) - are shorter than this hold, so without the + * publish they fire UNDER it and author the launch's release word before the + * post-hold copy-out: the client then verifies stale data (r419 soak: + * `victory_matmul bad=65536` while its launch was held; the authoring paths + * are GR-STALL-fwd-hold / the busy-defer inline software release). The + * accessor lets both deadlines pause instead; the flag is bounded by the + * msleep itself and by the pause caps on the sema side. Single-inflight + * submit path, so a plain static. + */ +static bool gsp_phys_gr_quiet_hold_on; + +bool gsp_phys_gr_quiet_hold_active(void) +{ + return READ_ONCE(gsp_phys_gr_quiet_hold_on); +} +EXPORT_SYMBOL_GPL(gsp_phys_gr_quiet_hold_active); + +/* + * r425 experiment (the second-launch wedge): the stock guest driver emits the + * per-channel compute init block ONCE at channel creation (turing_init.c), + * long before any launch, while this backend re-emits it as the preamble of + * every forwarded launch. The wedge strikes the SECOND launch of a channel, so + * test whether the re-emission is the trigger: with gsp_phys_rmcall_gr_init_once + * set, the init preamble is emitted only on the first forwarded launch after + * the module load (the engine's compute-class/CWD state persists in the + * context; every launch still carries its QMD + report semaphore). 0 keeps the + * historical per-launch emission (default; runtime-writable A/B). + */ +static bool gsp_phys_rmcall_gr_init_once_param; +module_param_named(gsp_phys_rmcall_gr_init_once, gsp_phys_rmcall_gr_init_once_param, + bool, 0644); +MODULE_PARM_DESC(gsp_phys_rmcall_gr_init_once, + "r425: emit the GR compute init preamble only on the first forwarded launch (0 = every launch, default)"); +static bool gsp_phys_rmcall_gr_init_emitted; + +/* + * r425 stale-QMD witness: the first engine-changed page of the LAST completed + * launch (bounce index + post-copy-out checksum). At a zero-write accept the + * page is re-scanned; a change proves a late store (a stuck/queued grid or a + * stale-QMD execution) landed after its launch was shipped. + */ +static u32 gsp_phys_gr_prev_out_bi; +static u64 gsp_phys_gr_prev_out_csum; +static bool gsp_phys_gr_prev_out_valid; + +/* r425: SKED/FECS/CWD register witness (one line; the healthy-vs-wedged + * comparison baseline lives in the same log). */ +static void gsp_phys_gr_sked_wit(void __iomem *bar0, const char *tag) +{ + if (!bar0) + return; + pr_info("vgsp-gr: SKED-WIT %s cwd0=%08x cwd1=%08x cwd2=%08x cwd3=%08x info1=%08x act=%08x grfifo=%08x fecs=%08x new=%08x ctxsw1=%08x qmdvirt0=%08x tpcm0=%08x\n", + tag, + readl(bar0 + 0x00407060u), readl(bar0 + 0x00407064u), + readl(bar0 + 0x00407068u), readl(bar0 + 0x0040706cu), + readl(bar0 + 0x00407028u), readl(bar0 + 0x00407054u), + readl(bar0 + 0x00400504u), readl(bar0 + 0x00409b00u), + readl(bar0 + 0x00409b04u), readl(bar0 + 0x00409400u), + readl(bar0 + 0x0040a500u), readl(bar0 + 0x0040a680u)); +} + /* m86: program-page override blob (see gsp-phys-gr-override). */ static u8 gsp_phys_gr_override[GSP_PHYS_PAGE_SIZE]; static u32 gsp_phys_gr_override_len; @@ -15848,6 +16717,29 @@ static u32 *gsp_phys_rmcall_build_gr_init(u32 *p) GSP_PHYS_RMCALL_GR_MAX_SM_COUNT); p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_INIT_SM_DISP_CTRL, 0u); + /* + * r434 CWD-slot-count path (gsp_phys_rmcall_gr_cwd_slot >= 0): masked + * PRI write of SKED_DEBUG_1.ENABLE_CWD_SLOT_COUNT (bit 11) through the + * falcon scratch registers, then the gated SET_CWD_SLOT_COUNT method - + * the CUDA driver's own turingLaunchSetCwdSlotCount sequence. -1 (the + * default) emits nothing, so the historical preamble is unchanged. + */ + if (gsp_phys_rmcall_gr_cwd_slot_value() >= 0) { + p = gsp_phys_rmcall_push_gr( + p, GSP_PHYS_RMCALL_GR_MME_SCRATCH0, 0u); + p = gsp_phys_rmcall_push_gr( + p, GSP_PHYS_RMCALL_GR_MME_SCRATCH1, + GSP_PHYS_RMCALL_PRI_SKED_DEBUG_1_EN_CWD_SLOT_COUNT); + p = gsp_phys_rmcall_push_gr( + p, GSP_PHYS_RMCALL_GR_MME_SCRATCH2, + GSP_PHYS_RMCALL_PRI_SKED_DEBUG_1_EN_CWD_SLOT_COUNT); + p = gsp_phys_rmcall_push_gr( + p, GSP_PHYS_RMCALL_GR_SET_FALCON04, + GSP_PHYS_RMCALL_PRI_SKED_DEBUG_1); + p = gsp_phys_rmcall_push_gr( + p, GSP_PHYS_RMCALL_GR_SET_CWD_SLOT_COUNT, + (u32)gsp_phys_rmcall_gr_cwd_slot_value()); + } return p; } @@ -15868,7 +16760,17 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, int qmdvirt = gsp_phys_rmcall_gr_qmdvirt_value(); int two_step = gsp_phys_rmcall_gr_pcas_two_step_value(); - p = gsp_phys_rmcall_build_gr_init(p); + /* + * r425 A/B: with gsp_phys_rmcall_gr_init_once the compute init preamble + * is emitted only on the first forwarded launch (the stock driver's + * per-channel shape); subsequent launches start at the inline-QMD + * window. + */ + if (!READ_ONCE(gsp_phys_rmcall_gr_init_once_param) || + !READ_ONCE(gsp_phys_rmcall_gr_init_emitted)) { + p = gsp_phys_rmcall_build_gr_init(p); + WRITE_ONCE(gsp_phys_rmcall_gr_init_emitted, true); + } if (qmdvirt) p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_QMD_VIRT_CONTROL, @@ -16077,26 +16979,26 @@ static void __iomem *gsp_phys_gr_scratch_bar0; static int gsp_phys_rmcall_gr_scratch_win_install_locked(u32 page) { struct gsp_phys_rmcall_update_pde2_params p; - struct gsp_phys_boot_chunk *pt; u32 rpc_result = 0, status = 0, pde_index; + u64 pt_dma; int ret; - if (page >= GSP_PHYS_RMCALL_GUEST_PT_PAGES) + if (page >= GSP_PHYS_RMCALL_GUEST_PT_TOTAL_PAGES) return -EINVAL; if (!gsp_phys_rmcall_vas_shared_value()) return 0; - if (boot_stage.rmcall_guest_pt_win_mask & BIT(page)) + if (boot_stage.rmcall_guest_pt_win_mask & BIT_ULL(page)) return 0; - pt = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GUEST_PT0 + page]; - if (!pt->pages) + pt_dma = gsp_phys_rmcall_guest_pt_page_dma(page); + if (!pt_dma) return -EIO; if (!boot_stage.rmcall_va_guest_pt) return -EIO; - pde_index = (u32)((boot_stage.rmcall_va_guest_pt >> 21) + page); + pde_index = (u32)(gsp_phys_rmcall_guest_pt_win_va(page) >> 21); memset(&p, 0, sizeof(p)); p.pde_index = cpu_to_le32(pde_index); p.flags = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_FLAG_FLUSH); - p.pt[0].phys_addr = cpu_to_le64((u64)gsp_phys_chunk_page_dma(pt, 0)); + p.pt[0].phys_addr = cpu_to_le64(pt_dma); p.pt[0].aperture = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_APERTURE_SYS_COH); p.pt[1].aperture = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_APERTURE_INVALID); p.h_va_space = cpu_to_le32(boot_stage.rmcall_vas); @@ -16113,10 +17015,9 @@ static int gsp_phys_rmcall_gr_scratch_win_install_locked(u32 page) ret); return ret; } - boot_stage.rmcall_guest_pt_win_mask |= BIT(page); + boot_stage.rmcall_guest_pt_win_mask |= BIT_ULL(page); pr_info("vgsp-gr: scratch window installed pdeIndex=%u (VA 0x%llx) ptPa=0x%llx status=0x%x (Tier-2 shadow legs now reachable through the server walker)\n", - pde_index, (u64)pde_index << 21, - (u64)gsp_phys_chunk_page_dma(pt, 0), status); + pde_index, (u64)pde_index << 21, pt_dma, status); return 0; } @@ -16129,13 +17030,11 @@ static int gsp_phys_rmcall_gr_scratch_win_install_locked(u32 page) static bool gsp_phys_rmcall_gr_win_is_installed_locked(u64 va) { u32 pde_index = (u32)(va >> 21); - u32 base = (u32)(boot_stage.rmcall_va_guest_pt >> 21); + u32 page = gsp_phys_rmcall_guest_pt_page_of_pde(pde_index); u32 i; - if (pde_index >= base && - pde_index < base + GSP_PHYS_RMCALL_GUEST_PT_PAGES) - return !!(boot_stage.rmcall_guest_pt_win_mask & - BIT(pde_index - base)); + if (page != 0xffffffffu) + return !!(boot_stage.rmcall_guest_pt_win_mask & BIT_ULL(page)); for (i = 0; i < boot_stage.rmcall_gr_pt_used; i++) if (boot_stage.rmcall_gr_win[i].used && boot_stage.rmcall_gr_win[i].pde_index == pde_index) @@ -16154,13 +17053,11 @@ static bool gsp_phys_rmcall_gr_win_is_installed_locked(u64 va) static int gsp_phys_rmcall_gr_win_ensure_locked(void __iomem *bar0, u64 va) { u32 pde_index = (u32)(va >> 21); - u32 base = (u32)(boot_stage.rmcall_va_guest_pt >> 21); + u32 page = gsp_phys_rmcall_guest_pt_page_of_pde(pde_index); u32 slot; - if (pde_index >= base && - pde_index < base + GSP_PHYS_RMCALL_GUEST_PT_PAGES) - return gsp_phys_rmcall_gr_scratch_win_install_locked( - pde_index - base); + if (page != 0xffffffffu) + return gsp_phys_rmcall_gr_scratch_win_install_locked(page); return gsp_phys_rmcall_gr_win_slot_locked(bar0, pde_index, &slot); } @@ -16171,16 +17068,35 @@ static int gsp_phys_rmcall_gr_win_ensure_locked(void __iomem *bar0, u64 va) * supplies the backing page. Called with gsp_phys_rm_lock held. */ static int gsp_phys_rmcall_gr_map_shadow_locked(u32 slot, u64 hpa, - u64 *va_out) + u64 *va_out, u32 *line_out) { struct gsp_phys_boot_chunk *sh = &boot_stage.rmcall_gr_shadow; - struct gsp_phys_boot_chunk *pt; u32 line = boot_stage.rmcall_guest_pt_next_line; - u32 page, line_in_page; + u32 page, line_in_page, i; u64 va; - u8 *pt_va; + u8 *pt; int ret; + /* + * m152: reuse before consume. A shadow VA is a never-rewritten PTE: + * once hpa X maps at VA V, every later request for X can be served by + * V with byte-identical engine-visible content, so the pool only + * needs one line per distinct HPA per boot instead of one per launch. + * This is safe against the GMMU's no-invalidate fetch-line cache by + * construction: the PTE value never differs from what the engine may + * have already fetched. A hit reports line_out = ~0 so a caller that + * records contiguity bookkeeping never contig-extends an old line + * (a new PTE in a line the engine may have already fetched would be + * invisible). + */ + for (i = 0; i < boot_stage.rmcall_gr_alias_count; i++) { + if (boot_stage.rmcall_gr_alias_hpa[i] == hpa) { + *va_out = boot_stage.rmcall_gr_alias_va[i]; + if (line_out) + *line_out = 0xffffffffu; + return 0; + } + } if (!sh->pages) { ret = gsp_phys_chunk_alloc(boot_stage.pdev, sh, GSP_PHYS_RMCALL_GR_SHADOW_PAGES); @@ -16192,28 +17108,34 @@ static int gsp_phys_rmcall_gr_map_shadow_locked(u32 slot, u64 hpa, return -ENOSPC; if (!boot_stage.rmcall_va_guest_pt) return -EIO; - if (line >= GSP_PHYS_RMCALL_GUEST_PT_PAGES * - GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE) + if (line >= GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES) return -ENOSPC; page = line / GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; line_in_page = line % GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; ret = gsp_phys_rmcall_gr_scratch_win_install_locked(page); if (ret) return ret; - pt = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GUEST_PT0 + page]; - if (!pt->pages) - return -EIO; - pt_va = gsp_phys_chunk_page_va(pt, 0); - if (!pt_va) + pt = gsp_phys_rmcall_guest_pt_page_va(page); + if (!pt) return -EIO; put_unaligned_le64(gsp_phys_rmcall_pte(hpa), - pt_va + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * + pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * line_in_page) * 8); dma_wmb(); boot_stage.rmcall_guest_pt_next_line = line + 1; - va = boot_stage.rmcall_va_guest_pt + ((u64)page << 21) + + va = gsp_phys_rmcall_guest_pt_win_va(page) + ((u64)line_in_page << 16); *va_out = va; + if (line_out) + *line_out = line; + if (boot_stage.rmcall_gr_alias_count < + GSP_PHYS_RMCALL_GR_ALIAS_CACHE) { + u32 ci = boot_stage.rmcall_gr_alias_count; + + boot_stage.rmcall_gr_alias_hpa[ci] = hpa; + boot_stage.rmcall_gr_alias_va[ci] = va; + boot_stage.rmcall_gr_alias_count = ci + 1u; + } return 0; } @@ -16227,32 +17149,27 @@ static int gsp_phys_rmcall_gr_map_shadow_locked(u32 slot, u64 hpa, static int gsp_phys_rmcall_gr_map_shadow_contig_locked(u32 line, u32 pte, u64 hpa, u64 *va_out) { - struct gsp_phys_boot_chunk *pt; u32 page, line_in_page; - u8 *pt_va; + u8 *pt; int ret; if (pte >= GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES) return -EINVAL; - if (line >= GSP_PHYS_RMCALL_GUEST_PT_PAGES * - GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE) + if (line >= GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES) return -ENOSPC; page = line / GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; line_in_page = line % GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; ret = gsp_phys_rmcall_gr_scratch_win_install_locked(page); if (ret) return ret; - pt = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GUEST_PT0 + page]; - if (!pt->pages) - return -EIO; - pt_va = gsp_phys_chunk_page_va(pt, 0); - if (!pt_va) + pt = gsp_phys_rmcall_guest_pt_page_va(page); + if (!pt) return -EIO; put_unaligned_le64(gsp_phys_rmcall_pte(hpa), - pt_va + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * + pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * line_in_page + pte) * 8); dma_wmb(); - *va_out = boot_stage.rmcall_va_guest_pt + ((u64)page << 21) + + *va_out = gsp_phys_rmcall_guest_pt_win_va(page) + ((u64)line_in_page << 16) + (u64)pte * GSP_PHYS_PAGE_SIZE; return 0; } @@ -16315,14 +17232,23 @@ static void gsp_phys_rmcall_gr_gpc_mmu_init(void __iomem *bar0) * the affine-relocated server-managed VA (the engine cannot walk the * client-managed guest VAs: UPDATE_PDE_2 refuses them with 0x1f). */ -static void gsp_phys_gr_rewrite_data_vars(u32 *w, u32 n_words, - const struct gsp_phys_gr_submit *req, - u32 bank, u32 *mirror) +/* + * relfix (agent-A FIX A): resolve-and-author one unmatched device-VA word. + * Defined after gsp_phys_gr_data_bi (the bounce-index helpers it needs); + * returns true when the word was rewritten to a reloc VA (and the pages + * queued for copy-in/out), false to leave it to the RELOC-GAP accounting. + */ +static bool gsp_phys_gr_reloc_fix_word(const struct gsp_phys_gr_submit *req, + u32 *w, u32 k, u32 *mirror); +static u32 gsp_phys_gr_rewrite_data_vars(u32 *w, u32 n_words, + const struct gsp_phys_gr_submit *req, + u32 bank, u32 *mirror, u64 *gap_first, + bool fix_enabled) { - u32 rew = 0, k; + u32 rew = 0, skipped = 0, gap = 0, k; if (!req->n_data) - return; + return 0; /* * m117: 8-byte stride with a 256-byte alignment guard. The m103 * 4-byte stride mis-matched arbitrary adjacent 4-byte words (the @@ -16335,19 +17261,6 @@ static void gsp_phys_gr_rewrite_data_vars(u32 *w, u32 n_words, u64 wb; u32 x; - /* - * m135: never relocate below offset 0x100. The param-page - * header (grid/block dims and driver bookkeeping at 0x0..0xff) - * can contain server-region VAs (e.g. 0x100000100) that are - * NOT kernel parameters; relocating them corrupted the - * vadd launch's blockDim word (read as 0x170100 instead of - * 0x100, so every CTA past 0 computed out-of-range indices - * and exited: VADD bad=32512 exactly one block correct). - * Kernel parameters live at >= 0x140 (JIT sm_75 layout: - * a/b/c at 0x160/0x168/0x170, n at 0x178). - */ - if (k * 4u < 0x100u) - continue; if (v < 0x10000000ULL || v >= (1ULL << 47) || (v & 0xff)) continue; wb = v & ~0xfffULL; @@ -16355,6 +17268,34 @@ static void gsp_phys_gr_rewrite_data_vars(u32 *w, u32 n_words, if (req->data[x].va != wb || !req->data[x].reloc_va) continue; + /* + * m135: never relocate below offset 0x100. The + * param-page header (grid/block dims and driver + * bookkeeping at 0x0..0xff) can contain + * server-region VAs (e.g. 0x100000100) that are NOT + * kernel parameters; relocating them corrupted the + * vadd launch's blockDim word (read as 0x170100 + * instead of 0x100, so every CTA past 0 computed + * out-of-range indices and exited: VADD bad=32512 + * exactly one block correct). Kernel parameters live + * at >= 0x140 (JIT sm_75 layout: a/b/c at + * 0x160/0x168/0x170, n at 0x178). + * + * m181 diagnostic: a below-0x100 value whose page is + * an operand VA of this launch is still logged (the + * observed class is the driver control block at + * bank offset 0, present in green clients too); the + * guard itself is unchanged until a dereferenced + * pointer in that range is proven. + */ + if (k * 4u < 0x100u) { + if (skipped < 16) { + skipped++; + pr_info("vgsp-gr: bank %u RELOC-SKIP @0x%x: 0x%llx is operand page 0x%llx (below 0x100, left guest-VA)\n", + bank, k * 4, v, wb); + } + break; + } /* m120: log the hit site so coverage is provable. */ if (rew < 16) pr_info("vgsp-gr: bank %u hit @0x%x: 0x%llx -> 0x%llx\n", @@ -16388,10 +17329,30 @@ static void gsp_phys_gr_rewrite_data_vars(u32 *w, u32 n_words, rew++; break; } + /* + * GRFIX FIX3: the inner scan ran out (x reached n_data) + * without matching req->data[x].va - the word passes the + * exact shape heuristic the match test above uses, so it + * will be left as an unrelocated guest VA in this page. + * relfix: first give the sema service's trusted resolver a + * chance to translate it; a word that resolves is authored + * as a late data page and rewritten to its reloc VA (the + * exact matched-word encoding). Only an unresolved word is + * counted as a gap for the caller's RELOC-GAP log/hold. + */ + if (x == req->n_data) { + if (fix_enabled && gsp_phys_gr_reloc_resolve_param && + gsp_phys_gr_reloc_fix_word(req, w, k, mirror)) + continue; + gap++; + if (gap_first && !*gap_first) + *gap_first = v; + } } if (rew) pr_info("vgsp-gr: bank %u: %u data pointers relocated\n", bank, rew); + return gap; } /* @@ -16471,6 +17432,24 @@ static int gsp_phys_rmcall_gr_resched_locked(void __iomem *bar0) u32 runlist_id, ch_id, doorbell; int ret; + /* + * m155 (HF-3 extended re-arm): after the channel has been idle long + * enough for the FECS to save/unload the context, re-issuing only the + * two GPFIFO_SCHEDULE verbs does not revive dispatch - r154: matmul's + * post-smiley launches acked (qmd_rel/ch_rel) but SKED stayed idle + * (cwd0=0x2086/fecs_cur=0) for every attempt including outcome + * retries, while the same channel executed mail/smiley earlier in the + * boot. Replay the mode-35 lane-enable tail (CILP preempt mode, + * timeslice, device sched) that stage 4 ran when the lane first went + * live - it is proven accepted and is the design's extended re-arm. + * The poison variants (explicit PREEMPT/tail, BIND, PROMOTE) stay out. + */ + ret = gsp_phys_rmcall_user_lane_lane_enable(bar0); + if (ret) + pr_info("vgsp-gr: rearm lane-enable rc=%d (continuing to schedules)\n", + ret); + rpc_result = 0; + status = 0; ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_channel, GSP_PHYS_RMCALL_CTRL_GPFIFO_SCHEDULE, sched_params, sizeof(sched_params), 16, @@ -16495,6 +17474,1588 @@ static int gsp_phys_rmcall_gr_resched_locked(void __iomem *bar0) return 0; } +/* + * m151: FNV-1a 64-bit over one page. Used to compare the win0 param page + * and the guest param page before and after the poll, so a launch that + * wrote (or clobbered) the page the engine read is provable from the log. + */ +static inline u64 gsp_phys_gr_fnv1a64(const u8 *p, size_t len) +{ + u64 h = 0xcbf29ce484222325ULL; + size_t i; + + for (i = 0; i < len; i++) { + h ^= p[i]; + h *= 0x100000001b3ULL; + } + return h; +} + +/* + * B-2: the data chunk the active operand region lives in. With + * gsp_phys_gr_data_region=1 (default) that is the grown 16384-page region; + * with 0 it is the legacy 208-page bounce window (whose data pages sit below + * the mail slot). The copy path always asks this instead of naming a chunk so + * the two paths share one index space per mode. + */ +static struct gsp_phys_boot_chunk *gsp_phys_gr_data_chunk(void) +{ + if (gsp_phys_gr_data_region_value()) + return &boot_stage.rmcall_gr_data_bounce; + return &boot_stage.rmcall_gr_bounce; +} + +/* + * r452: scan the whole active GR data-bounce chunk for an 8-byte pattern + * (write "0x<pattern>" to /sys/module/vgsp/parameters/gsp_phys_gr_scan). + * Verifies by content where a lost engine store actually landed: the chunk + * is the only bulk memory the operand window PTEs map, so a store that + * executed but is absent from the writer's tracked pages must be findable + * here (a stale-PTE landing) or nowhere (a dropped store). + */ +static int gsp_phys_gr_scan_set(const char *val, + const struct kernel_param *kp) +{ + struct gsp_phys_boot_chunk *c = gsp_phys_gr_data_chunk(); + u64 pattern = 0; + u32 i, hits = 0; + + if (kstrtoull(val, 0, &pattern)) + return -EINVAL; + if (!c->pages) + return -ENODEV; + for (i = 0; i < c->pages; i++) { + const u8 *p = gsp_phys_chunk_page_va(c, i); + u32 off; + + if (!p) + continue; + for (off = 0; off + 8 <= GSP_PHYS_PAGE_SIZE; off += 8) { + if (get_unaligned_le64(p + off) == pattern) { + if (hits < 32) + pr_info("vgsp-gr: CHUNK-SCAN found 0x%llx at bi=%u off=0x%x (va=0x%llx)\n", + pattern, i, off, + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 + + ((u64)i << 12) + off); + hits++; + } + } + } + pr_info("vgsp-gr: CHUNK-SCAN 0x%llx done: %u hit(s) over %u pages\n", + pattern, hits, c->pages); + return 0; +} + +static const struct kernel_param_ops gsp_phys_gr_scan_ops = { + .set = gsp_phys_gr_scan_set, +}; +module_param_cb(gsp_phys_gr_scan, &gsp_phys_gr_scan_ops, NULL, 0200); +MODULE_PARM_DESC(gsp_phys_gr_scan, + "Scan the GR data-bounce chunk for an 8-byte pattern (hex)"); + +/* + * B-2: page index of a relocated operand VA inside the engine-visible data + * region, replacing the five inlined (reloc_va - 0x110000) >> 12 derivations + * and their MAIL_SLOT bounds. Grown region: VAs from + * GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0, bound + * GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES. Legacy window: VAs from + * GSP_PHYS_RMCALL_GR_BOUNCE_VA0, bound GSP_PHYS_RMCALL_GR_MAIL_SLOT so the + * mail (206) and marker (207) pages are never treated as data. Returns + * GSP_PHYS_GR_DATA_BI_NONE when the VA is outside the active region (the copy + * sites skip such records exactly as they skipped the old mail-slot bound). + */ +#define GSP_PHYS_GR_DATA_BI_NONE 0xffffffffu +static u32 gsp_phys_gr_data_bi(u64 reloc_va) +{ + u32 bi; + + if (gsp_phys_gr_data_region_value()) { + if (reloc_va < GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0) + return GSP_PHYS_GR_DATA_BI_NONE; + bi = (u32)((reloc_va - GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0) >> + 12); + if (bi >= GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES) + return GSP_PHYS_GR_DATA_BI_NONE; + return bi; + } + if (reloc_va < GSP_PHYS_RMCALL_GR_BOUNCE_VA0) + return GSP_PHYS_GR_DATA_BI_NONE; + bi = (u32)((reloc_va - GSP_PHYS_RMCALL_GR_BOUNCE_VA0) >> 12); + if (bi >= GSP_PHYS_RMCALL_GR_MAIL_SLOT) + return GSP_PHYS_GR_DATA_BI_NONE; + return bi; +} + +/* + * relfix: late-resolved data pages. A QMD word that matches no tracked data + * page yet resolves at rewrite time becomes a real operand page of this + * launch: it gets the next bounce-region index after the fill's tracked set, + * is copied in before the doorbell and copied back after the retire wait + * (the same m142 ordering as every tracked page), and the word is rewritten + * to its reloc VA - so the engine stops walking an untranslatable guest VA + * (r216 MMU FAULT_PDE class). Bounds are per-launch and fail-open: at the + * caps, or when no source answers, the word simply stays a RELOC-GAP (the + * pre-relfix behaviour). Static scratch: the single-inflight ordered + * gsp-phys-work item is the only caller, reset per submission exactly like + * bounce_csum_in. + */ +/* + * r564: minimum data records for a forwardable launch. Healthy vadd-class + * launches resolve 968-969 records (the client's whole arena); a launch + * with only a handful after the fill's drops has an unresolved argument + * set and must not be handed to the engine (see the submit guard). + */ +#define GSP_PHYS_GR_MIN_DATA_RECS 16u +#define GSP_PHYS_GR_GAP_MAX 4096u /* total late pages per launch */ +/* + * relfix PART 3 (r217 span containment): the late path authors ONLY the + * pages actually needed by the fixed word. r217 showed two fixed words + * (the launch's own param/program descriptor fields, which the fill skips + * as launch-internal) walking ~1242 fully-mapped neighbour pages each - + * 2484 late pages, ~10 MB copied in and out per launch, for zero engine + * writes - and the clients converted from fast-fail (~60 s verdict) to + * killed at 250 s. The span is now a small fetch line, not a region walk: + * the multi-page operand class is owned by the fill's own span pipeline + * (which the PART 3 candidate superset now feeds), and this path is a + * last-resort proxy for words the fill could not own. + */ +#define GSP_PHYS_GR_GAP_SPAN_MAX 16u /* contiguous pages per word max */ +#define GSP_PHYS_GR_GAP_WORD_MAX 32u /* distinct fixed words per launch */ +#define GSP_PHYS_GR_GAP_LOG_MAX 16u /* distinct gap VAs stage-logged */ +struct gsp_phys_gr_gap_page { + u64 va; /* device VA of the page (unique-proxy key) */ + u64 hpa; /* guest page copied in (results copied back to) */ + u64 reloc_va; /* engine-visible bounce-region VA of that page */ + u32 bi; /* bounce page index (n_data + late index) */ +}; +static struct gsp_phys_gr_gap_page + gsp_phys_gr_gap[GSP_PHYS_GR_GAP_MAX]; +static u32 gsp_phys_gr_gap_n; +struct gsp_phys_gr_gap_word { + u64 va; /* device VA of the fixed word (page base) */ + u64 reloc_va; /* reloc VA of its first page */ + u64 pa; /* resolved guest PA (logging) */ +}; +static struct gsp_phys_gr_gap_word + gsp_phys_gr_gap_word[GSP_PHYS_GR_GAP_WORD_MAX]; +static u32 gsp_phys_gr_gap_word_n; +static u64 gsp_phys_gr_reloc_fixed; /* words rewritten this launch */ +static u64 gsp_phys_gr_reloc_fixed_first; /* first fixed device VA */ +static u64 gsp_phys_gr_gap_logged[GSP_PHYS_GR_GAP_LOG_MAX]; +static u32 gsp_phys_gr_gap_logged_n; + +/* + * relfix: reloc VA of the i-th bounce-region index, region mode exactly as + * gsp_phys_gr_data_bi; the legacy window's small bound fails open. + */ +static bool gsp_phys_gr_data_reloc_va(u32 idx, u64 *va_out) +{ + if (gsp_phys_gr_data_region_value()) { + if (idx >= GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES) + return false; + *va_out = GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 + + ((u64)idx << 12); + return true; + } + if (idx >= GSP_PHYS_RMCALL_GR_MAIL_SLOT) + return false; + *va_out = GSP_PHYS_RMCALL_GR_BOUNCE_VA0 + + ((u64)idx << 12); + return true; +} + +/* + * relfix: append one late page at the next free bounce index (n_data + + * gap_n, the slot right after the fill's tracked set). Fail-open on the + * caps or an absent bounce page. + */ +static bool gsp_phys_gr_gap_add(u32 n_data, u64 va, u64 hpa, u64 *reloc_out) +{ + u32 gi = gsp_phys_gr_gap_n; + u32 idx = n_data + gi; + struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); + u64 reloc_va; + + if (gi >= GSP_PHYS_GR_GAP_MAX) + return false; + if (!gsp_phys_gr_data_reloc_va(idx, &reloc_va)) + return false; + if (!dchunk->pages || !gsp_phys_chunk_page_va(dchunk, idx)) + return false; + gsp_phys_gr_gap[gi].va = va; + gsp_phys_gr_gap[gi].hpa = hpa; + gsp_phys_gr_gap[gi].reloc_va = reloc_va; + gsp_phys_gr_gap[gi].bi = idx; + gsp_phys_gr_gap_n = gi + 1u; + *reloc_out = reloc_va; + return true; +} + +/* + * relfix: is this device page already covered by a proxy this launch? + * Either the fill's tracked set (req->data, each page exactly once - the + * fill dedupes) or a late page queued by this fix. A second proxy for one + * guest page is unsound: the engine may author results through either + * slot, and the two copy-outs would race to write back to the same guest + * page, so one would clobber the other's results. The fix therefore never + * creates a second proxy (see the span rollback in + * gsp_phys_gr_reloc_fix_word). + */ +static bool gsp_phys_gr_page_tracked(const struct gsp_phys_gr_submit *req, + u64 va) +{ + u32 i; + + for (i = 0; i < req->n_data; i++) + if (req->data[i].va == va) + return true; + for (i = 0; i < gsp_phys_gr_gap_n; i++) + if (gsp_phys_gr_gap[i].va == va) + return true; + return false; +} + +/* + * relfix PART 3: one rate-limited log per distinct gap VA per launch + * (the dedup helper both gap diagnostics below share). + */ +static bool gsp_phys_gr_gap_log_once(u64 va) +{ + u32 i; + + for (i = 0; i < gsp_phys_gr_gap_logged_n; i++) + if (gsp_phys_gr_gap_logged[i] == va) + return false; + if (gsp_phys_gr_gap_logged_n < ARRAY_SIZE(gsp_phys_gr_gap_logged)) + gsp_phys_gr_gap_logged[gsp_phys_gr_gap_logged_n++] = va; + return true; +} + +/* + * relfix PART 3/4: one rate-limited RELOC-GAP-STAGE line per distinct gap VA + * per launch. `stage` is the resolver source chain (r.stage, e.g. + * "ipin:-2,pin:-2,lat:-2,map:-2,vas:-2,disc:-2,ledger:-2," - the resolving + * source ends in ":0,") for a word no source answered, or a short authoring + * refusal token ("span-overlap", "region-full", "wordmax-full", + * "base-tracked") for a word that resolved but was deliberately not + * authored. A FULL chain of -2 is classified as RELOC-GAP-UNMAPPED instead + * (see gsp_phys_gr_gap_unmapped_log): the sources are exhausted, so the word + * cannot be authored this launch; note the map stage's population is + * range-gated to [SEMA_VA_LO 4 GiB, SEMA_VA_HI 128 GiB) so its -2 is + * uninformative for VAs above that window - the vas/disc WALKS (49-bit + * full-range) are the authoritative absence signal. + */ +static void gsp_phys_gr_gap_stage_log(u64 va, const char *stage, int rc) +{ + if (!gsp_phys_gr_gap_log_once(va)) + return; + pr_info_ratelimited("vgsp-gr: RELOC-GAP-STAGE va=0x%llx stage=%s rc=%d\n", + va, stage, rc); +} + +/* + * relfix PART 4: definitive classification for a shape-passing word that NO + * resolver source answers (chain fully -2, r218: 0x760275000000, + * 0x760273000000, 0x3000000000, 0x17400000000, 0x17c00000000 - every stage + * -ENOENT). The walks cover the full 49-bit range, so a walk miss + no + * pin/cache/ledger entry means no mapping we can see exists anywhere; + * r218 additionally shows these words produce NO engine MMU fault (unlike + * the dereferenced argument words the late path now fixes), i.e. the engine + * does not consume them: they are descriptor/metadata values that are merely + * VA-shaped ("junk" as pointers). Any remaining engine fault is therefore + * NOT from this class - it redirects diagnosis to the code-pointer/entry + * word and page-content paths. + */ +static void gsp_phys_gr_gap_unmapped_log(u64 va) +{ + if (!gsp_phys_gr_gap_log_once(va)) + return; + pr_info_ratelimited("vgsp-gr: RELOC-GAP-UNMAPPED va=0x%llx (all stages -2; VA-shape junk)\n", + va); +} + +/* + * relfix PART 4: bounded content bound for a late-resolved word whose page is + * one of the launch's OWN pages (req->pages[], the only class with ground + * truth). The late copy-in itself is a direct copy of the resolved source + * page (gsp_phys_gr_fill_gap_bounce: src = __va(r.hpa)), so a dst-vs-src + * compare is vacuous; the meaningful question is whether the RESOLVER named + * the same page the launch pipeline uses. Compare the resolved source page + * against req->pages[i].hpa content and log the mission's + * LATE-CONTENT-MISMATCH line once per launch when they differ: that is the + * only way the engine could have been handed wrong bytes through a late + * proxy. Words whose page is not launch-internal have no ground truth and + * are skipped (their resolution is validated by the same walk/pin sources as + * every tracked data page). + */ +static void gsp_phys_gr_late_content_check(const struct gsp_phys_gr_submit *req, + u64 wb, u64 resolved_hpa, u32 off) +{ + u32 pi; + + for (pi = 0; pi < req->n_pages; pi++) { + const u8 *gt, *src; + + if ((req->pages[pi].guest_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) != wb) + continue; + if (!req->pages[pi].hpa) + break; + gt = (const u8 *)__va(req->pages[pi].hpa); + src = (const u8 *)__va(resolved_hpa); + if (!gt || !src) + break; + if (memcmp(gt, src, GSP_PHYS_PAGE_SIZE) && + gsp_phys_gr_gap_log_once(wb)) + pr_info_ratelimited("vgsp-gr: LATE-CONTENT-MISMATCH word_va=0x%llx page=0x%llx off=%u src=0x%llx\n", + wb, (u64)req->pages[pi].hpa, off, + resolved_hpa); + break; + } +} + +/* + * relfix: resolve and author one unmatched word (see gsp_phys_gr_gap_page). + * The base VA resolves through the sema service's trusted resolver; on a hit + * the page is queued and the word rewritten to <reloc VA> | offset - the + * exact encoding the matched-word relocation uses - and up to + * GSP_PHYS_GR_GAP_SPAN_MAX contiguous pages the resolver also answers are + * queued behind it so a small operand's base+offset reads stay contiguous. + * PART 3 (r217): that span is a small fetch line (16 pages), NOT a region + * walk - r217's 1242-page spans turned fast fails into 250 s kills; the + * multi-page operand class belongs to the fill's own span pipeline, which + * the PART 3 candidate superset now feeds. A repeated word (same base VA + * authored in another QMD page) reuses its recorded reloc VA. NEVER guesses: + * only a successful resolve rewrites, and a page that already has a proxy + * (the fill's or an earlier fix's) stops the fix - the queued pages of THIS + * word are rolled back and the word stays a RELOC-GAP - so one guest page + * can never gain a second copy-out writer (the clobber class + * gsp_phys_gr_page_tracked describes). Every word that stays a gap is + * stage-logged once (RELOC-GAP-STAGE) with the resolver chain or the + * authoring refusal token. + */ +static bool gsp_phys_gr_reloc_fix_word(const struct gsp_phys_gr_submit *req, + u32 *w, u32 k, u32 *mirror) +{ + u64 v = (u64)w[k] | ((u64)w[k + 1] << 32); + u64 wb = v & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + struct gsp_sema_gr_reloc r; + struct gsp_phys_gr_gap_word *gw = NULL; + u32 gap_start = gsp_phys_gr_gap_n; + u64 reloc_va = 0; + u32 wi, i; + + for (wi = 0; wi < gsp_phys_gr_gap_word_n; wi++) { + if (gsp_phys_gr_gap_word[wi].va == wb) { + gw = &gsp_phys_gr_gap_word[wi]; + break; + } + } + if (!gw) { + bool overlap = false; + u64 base_pa; + + if (gsp_phys_gr_gap_word_n >= GSP_PHYS_GR_GAP_WORD_MAX) { + gsp_phys_gr_gap_stage_log(wb, "wordmax-full", + -ENOSPC); + return false; + } + if (gsp_phys_gr_page_tracked(req, wb)) { + gsp_phys_gr_gap_stage_log(wb, "base-tracked", -EEXIST); + return false; + } + if (gsp_sema_gr_reloc_resolve(wb, &r)) { + /* + * A FULL chain of misses is the definitive + * no-mapping-anywhere class (PART 4); anything else + * (hpa conversion failure) keeps the stage chain. + */ + if (r.stage_rc == -ENOENT) + gsp_phys_gr_gap_unmapped_log(wb); + else + gsp_phys_gr_gap_stage_log(wb, r.stage, + r.stage_rc); + return false; + } + base_pa = r.pa; + /* + * PART 4: the copy-in is a direct copy of this source page; + * for a launch-internal word, bound it against the launch's + * own page content (the LATE-CONTENT-MISMATCH witness). + */ + gsp_phys_gr_late_content_check(req, wb, r.hpa, + (u32)(v & (u64)(GSP_PHYS_PAGE_SIZE - 1))); + if (!gsp_phys_gr_gap_add(req->n_data, wb, r.hpa, &reloc_va)) { + gsp_phys_gr_gap_stage_log(wb, "region-full", -ENOSPC); + return false; + } + { + u32 span_done = 1; /* pages authored this word */ + bool span_capped = false; + + for (i = 1; i < GSP_PHYS_GR_GAP_SPAN_MAX; i++) { + u64 va = wb + ((u64)i << 12); + u64 ignored = 0; + + /* + * A page with an existing proxy cannot get a + * second one (two copy-out writers for one + * guest page would clobber each other); stop + * the span before it. A merely unresolved + * page (the common end-of-buffer case) + * truncates the span like the fill's own span + * walk does. + */ + if (gsp_phys_gr_page_tracked(req, va)) { + overlap = true; + break; + } + if (gsp_sema_gr_reloc_resolve(va, &r)) + break; + if (!gsp_phys_gr_gap_add(req->n_data, va, r.hpa, + &ignored)) { + /* capacity cut: tail cannot be + * authored though more mapping may + * exist - reported below. */ + span_capped = true; + break; + } + span_done++; + } + /* + * COPYAUDIT completeness probe (2026-10-18): the + * span cap is intentional (r217 containment), but a + * word cut at the cap with a still-resolvable next + * page means the engine's base+offset reads past + * the authored run into later operands' bounce + * pages - silent partial data. One probe page past + * the cut proves it and makes the truncation loud. + */ + if (!overlap && span_done == GSP_PHYS_GR_GAP_SPAN_MAX) { + struct gsp_sema_gr_reloc r2; + u64 next_va = wb + + ((u64)GSP_PHYS_GR_GAP_SPAN_MAX << 12); + + if (!gsp_sema_gr_reloc_resolve(next_va, &r2)) + span_capped = true; + } + if (span_capped) + pr_info_ratelimited("vgsp: COPY-TRUNC va=0x%llx want>=0x%llx done=0x%llx path=reloc-fix-span\n", + wb, + ((u64)span_done + 1) << 12, + (u64)span_done << 12); + } + if (overlap) { + /* + * Roll the whole word back (base included) and + * leave it a RELOC-GAP: a partial run would make + * the engine's next base+offset read land on a + * later word's page. + */ + gsp_phys_gr_gap_stage_log(wb, "span-overlap", -EEXIST); + gsp_phys_gr_gap_n = gap_start; + return false; + } + gw = &gsp_phys_gr_gap_word[gsp_phys_gr_gap_word_n]; + gw->va = wb; + gw->reloc_va = gsp_phys_gr_gap[gap_start].reloc_va; + gw->pa = base_pa; + gsp_phys_gr_gap_word_n++; + } + + v = gw->reloc_va | (v & (u64)(GSP_PHYS_PAGE_SIZE - 1)); + w[k] = (u32)v; + w[k + 1] = (u32)(v >> 32); + if (mirror) { + mirror[k] = (u32)v; + mirror[k + 1] = (u32)(v >> 32); + } + gsp_phys_gr_reloc_fixed++; + if (!gsp_phys_gr_reloc_fixed_first) + gsp_phys_gr_reloc_fixed_first = wb; + pr_info_ratelimited("vgsp-gr: RELOC-FIXED va=0x%llx pa=0x%llx off=%u\n", + wb, gw->pa, (u32)(v & (u64)(GSP_PHYS_PAGE_SIZE - 1))); + return true; +} + +/* + * m151: output retirement wait. qmd_rel is a schedule/fetch ack (the front + * end latched and fetched the QMD), not a data-path retirement: the SM's + * stores into the bounce pages can still be in flight when the ack lands. + * Copying out - and the next launch's copy-in - before those stores land + * loses them: the guest observes zeros for a completed launch, or + * stale/mixed content when the next copy-in races the previous engine's + * write tail. + * + * The wait covers EVERY resolved data page (footprint-by-inference was + * tried twice and is unreliable: the fill's candidate list ends with the + * launch's own bank/param pages, so the max-reloc run is 3-4 pages, not + * the output operand). Adaptive exits make a whole-window count cheap: + * - all resolved pages changed -> done (normal case, one poll) + * - changed>0 and stable for 3 polls -> stores settled (masked or + * repeated-identical kernels) + * - no writes at all for 600 ms -> no output for this launch + * - hard cap 1.5 s -> genuinely late remainder + * noinline so this rarely-taken wait does not grow the submit frame, which + * is already near the warning bound. + */ +/* + * m180: scan the request's unique tracked bounce pages and return how many + * differ from the copy-in checksum baseline. `hash_out` (optional) receives + * an FNV-1a fold of the per-page checksums - the content fingerprint the + * settle rule needs so a store ramp with a constant changed-page COUNT can + * never read as settled again. Static scratch: the only caller is the + * single-inflight ordered gsp-phys-work item (same discipline as the other + * submit-scope statics). + */ +static noinline u32 +gsp_phys_gr_scan_pages(const struct gsp_phys_gr_submit *req, + const u64 *csum_in, u32 *n_tracked, u64 *hash_out) +{ + static u32 checked[GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES]; + struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); + u32 n_all = 0, di, j, changed = 0; + u64 hash = 1469598103934665603ULL; + + for (di = 0; di < req->n_data; di++) { + u32 bi = gsp_phys_gr_data_bi(req->data[di].reloc_va); + u32 k; + bool dup = false; + + if (bi == GSP_PHYS_GR_DATA_BI_NONE) + continue; + for (k = 0; k < n_all; k++) + if (checked[k] == bi) { + dup = true; + break; + } + if (!dup) + checked[n_all++] = bi; + } + /* + * relfix: late-resolved pages are tracked exactly like fill-tracked + * data pages, so the retire wait, the late-check and the NOWRITE + * classification see stores the engine authors into them. + */ + for (di = 0; di < gsp_phys_gr_gap_n; di++) { + u32 bi = gsp_phys_gr_gap[di].bi; + u32 k; + bool dup = false; + + if (bi >= GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES) + continue; + for (k = 0; k < n_all; k++) + if (checked[k] == bi) { + dup = true; + break; + } + if (!dup) + checked[n_all++] = bi; + } + if (n_tracked) + *n_tracked = n_all; + for (j = 0; j < n_all; j++) { + const u8 *src = gsp_phys_chunk_page_va(dchunk, checked[j]); + u64 fp; + + if (!src) + continue; + /* + * r452: FNV-1a per page, not a 32-bit additive sum. The + * add-sum was PROVABLY blind to a power-of-two in-place + * scale: doubling every word of a 4 KB page shifts each + * exponent by +1, i.e. +2^23 per word, and 1024 words give + * 2^33 == 0 (mod 2^32) - so c[i] *= 2.0f left the checksum + * bit-identical, the page classified "unchanged", and the + * client read the pre-scale values (the deterministic + * streams2/scale class). FNV-1a has no such structure. + */ + fp = gsp_phys_gr_fnv1a64(src, GSP_PHYS_PAGE_SIZE); + if (fp != csum_in[checked[j]]) + changed++; + hash ^= fp; + hash *= 1099511628211ULL; + } + if (hash_out) + *hash_out = hash; + return changed; +} + +/* + * r434: post-launch deposit dump (hardware-only words). The r430 QMD64 dump + * runs at submission, before the engine touches the deposit, so the + * hardware-only fields (w50 INNER_GET, w51 INNER_PUT, w52 span-list head, + * w53 next-QMD) are always zero there. Reading the same page AFTER the + * retire wait shows what the SKED actually consumed: w50/w51 are the CTA + * (INNER) get/put counters, so a wedged launch whose grid "completed" with + * zero stores is distinguishable from a genuinely dispatched grid by + * whether INNER_PUT advanced to the CTA count. One line per launch, gated + * by the skedrec param. + */ +static void gsp_phys_gr_qmd64_post(const u8 *deposit, const char *tag) +{ + char qb[64 * 9 + 1]; + int qo = 0; + u32 i; + + if (!deposit || !gsp_phys_rmcall_gr_skedrec_value()) + return; + for (i = 0; i < GSP_PHYS_GR_QMD_WORDS && qo < (int)sizeof(qb) - 10; + i++) + qo += scnprintf(qb + qo, sizeof(qb) - qo, "%08x ", + get_unaligned_le32(deposit + i * 4)); + pr_info("vgsp-gr: QMD64-POST %s %s\n", tag, qb); +} + +static noinline u32 +gsp_phys_gr_output_retire_wait(const struct gsp_phys_gr_submit *req, + const u64 *csum_in, u32 *n_tracked) +{ + u32 n_all = 0, iter; + u32 changed = 0; + u64 prev_hash = 0; + const char *why = " (TIMEOUT)"; + unsigned long wait_start = jiffies; + + /* + * m179: report how many pages were actually trackable. "wrote == 0" + * from a launch with n_tracked == 0 (no data entry resolves to a + * bounce page) is not a no-execution signature and must not trigger + * the no-write retry/accept path. + */ + if (n_tracked) + *n_tracked = 0; + if (!req->n_data) + return 0; + /* + * r343: scan_pages RETURNS the changed count and reports the tracked + * total through the out-param. The old call put the return (changed) + * into n_all while the tracker total went to *n_tracked, so the + * loop's `changed >= n_all` was `changed >= changed` - always true + * after the first scan (observed: "1/1 pages changed after 4 ms"), + * and the `!n_all` early-return also skipped the no-write settle + * when zero pages changed. Net effect: the copy-out shipped the + * bounce mid-ramp, so a cold engine (slowest store ramp) produced + * all-bad first runs (the matmul cold flake) and late stores could + * be dropped. Fix: tracked total into n_all; the content-stable + * rule below becomes the real settle bound. + */ + (void)gsp_phys_gr_scan_pages(req, csum_in, &n_all, NULL); + /* + * r405: the r343 fix moved the tracked total from the out-param to + * the local n_all but FORGOT to write it back, so *n_tracked stayed + * 0 for every launch and the whole no-write classification chain in + * the caller (`wrote == 0 && n_tracked`: first-attempt witness, + * outcome retries, NOWRITE accept + release restore, late-store + * check) has been dead since ed91540a/r345 - the r404 forensics + * show "RETIRE-WIT n_data=968 n_tracked=0" while the same wait + * reports n_all=1032 in the "0/1032 pages changed (no writes)" + * line. Propagate the tracked total so the classifier can fire. + */ + if (n_tracked) + *n_tracked = n_all; + /* + * RETIRE-WIT (r227): the r227 capture shows n_tracked far below the + * copy-in/copy-out rec counts for the same launch (4 tracked vs 1482 + * written back) - either the rec array genuinely aliases reloc VAs + * mid-launch or the observed kernel differs from this source. One + * line per launch proves which: n_data, the unique-bi count, and the + * first/last recs' va/reloc_va/bi must be sequential and in-region + * for a healthy launch. + */ + if (gsp_phys_gr_copywit_enabled()) + pr_info_ratelimited("vgsp-gr: RETIRE-WIT n_data=%u n_tracked=%u first_va=0x%llx first_rel=0x%llx first_bi=%u last_va=0x%llx last_rel=0x%llx last_bi=%u\n", + req->n_data, n_tracked ? *n_tracked : 0, + req->data[0].va, req->data[0].reloc_va, + gsp_phys_gr_data_bi(req->data[0].reloc_va), + req->data[req->n_data - 1].va, + req->data[req->n_data - 1].reloc_va, + gsp_phys_gr_data_bi(req->data[req->n_data - 1].reloc_va)); + if (!n_all) + return 0; + for (iter = 0; iter < 300u; iter++) { + u64 hash = 0; + + changed = gsp_phys_gr_scan_pages(req, csum_in, NULL, &hash); + if (changed >= n_all) { + why = ""; + break; + } + /* + * m180 (H3): a partial count must be CONTENT-stable for + * ~300 ms before it counts as settled. The m162 count-only + * rule accepted a continuing store ramp whose page count + * happened to be constant (the always-failing class's + * "17/32 settled" copy-outs), and accepted an + * un-dispatched grid as settled. A genuinely masked/short + * kernel costs at most 300 ms extra; a ramping engine gets + * the time its stores need. + */ + if (changed > 0 && hash == prev_hash && + iter >= gsp_phys_gr_settle_iters()) { + why = " (settled)"; + break; + } + if (changed == 0 && iter >= 120u) { + why = " (no writes)"; + break; + } + prev_hash = hash; + usleep_range(4000, 6000); + } + pr_info("vgsp-gr: output retirement wait: %u/%u pages changed after %u ms%s\n", + changed, n_all, jiffies_to_msecs(jiffies - wait_start), why); + /* + * r405: run-once late-write probe (see the param comment above). + */ + if (READ_ONCE(gsp_phys_gr_late_probe_param) && changed < n_all) { + u32 late_n = 0, late; + unsigned long lp_start = jiffies; + + msleep(500); + late = gsp_phys_gr_scan_pages(req, csum_in, &late_n, NULL); + pr_info("vgsp-gr: retire LATE-PROBE %u -> %u of %u after +%u ms%s\n", + changed, late, late_n, + jiffies_to_msecs(jiffies - lp_start), + late != changed ? + " (ENGINE STILL WRITING - stores landed late)" : + " (stable - writes are final)"); + } + return changed; +} + +/* + * m161: hardware fetch witness. The host writes GSP_PHYS_GR_QMD_SENTINEL + * into the inline-QMD page before every deposit; the engine's QMD fetch is + * the only actor that overwrites it (the post-mortem has used this as its + * ran/not-run marker since m110). Computing it as a standalone check lets + * the outcome-retry path classify a zero-write result beyond the release + * word, which the sema service's pending-release sweep can satisfy in + * software. + */ +static bool gsp_phys_gr_qmd_fetched(const u8 *qmd_mem) +{ + if (!qmd_mem) + return false; + return get_unaligned_le32(qmd_mem + + GSP_PHYS_GR_QMD_SENTINEL_WORD * 4) != + GSP_PHYS_GR_QMD_SENTINEL; +} + +/* + * r427 idempotent fast-accept (CONT.86 §6.1). A zero-write launch is either + * the no-execute wedge OR a legitimate idempotent re-launch (the same QMD + * content with the same staged inputs, whose result is already in place) - + * the host cannot tell them apart, and both burn the 40 s + 80 s quiet holds. + * The launch identity below (QMD minus the per-launch release0 address and + * payload, folded with the staged content checksums of every tracked page) is + * recorded for every launch that WROTE; a zero-write launch whose identity + * matches an entry accepts immediately, with no retries and no quiet window. + * Safe by construction: identical code + identical staged content + a prior + * execution that produced this launch's own footprint means the correct + * result is already in the buffers; a side-effecting kernel's re-launch is + * equally wrong under the current accept (the client sees the stale result + * either way) - this only removes the stall. + */ +#define GSP_PHYS_GR_IDEM_SLOTS 64u +static u64 gsp_phys_gr_idem_fp[GSP_PHYS_GR_IDEM_SLOTS]; +static u32 gsp_phys_gr_idem_n; + +static u64 gsp_phys_gr_launch_fp(const u32 *qmd, + const struct gsp_phys_gr_submit *req, + const u64 *csum_in) +{ + u64 h = 1469598103934665603ULL; + u32 i; + + for (i = 0; i < GSP_PHYS_GR_QMD_WORDS; i++) { + /* Skip release0's address (w23/w24) and payload (w25): the + * driver allocates a fresh release slot per launch, so they + * differ even for an identical re-launch. */ + u32 w; + + if (i == 23u || i == 24u || i == 25u) + continue; + w = qmd[i]; + h ^= w; + h *= 1099511628211ULL; + } + for (i = 0; i < req->n_data; i++) { + u32 bi = gsp_phys_gr_data_bi(req->data[i].reloc_va); + + if (bi == GSP_PHYS_GR_DATA_BI_NONE) + continue; + h ^= csum_in[bi]; + h *= 1099511628211ULL; + } + return h; +} + +static bool gsp_phys_gr_idem_match(u64 fp) +{ + u32 i; + + for (i = 0; i < gsp_phys_gr_idem_n; i++) + if (gsp_phys_gr_idem_fp[i] == fp) + return true; + return false; +} + +static void gsp_phys_gr_idem_record(u64 fp) +{ + u32 i; + + for (i = 0; i < gsp_phys_gr_idem_n; i++) + if (gsp_phys_gr_idem_fp[i] == fp) + return; + if (gsp_phys_gr_idem_n < GSP_PHYS_GR_IDEM_SLOTS) { + gsp_phys_gr_idem_fp[gsp_phys_gr_idem_n++] = fp; + } else { + /* ring: drop the oldest */ + memmove(&gsp_phys_gr_idem_fp[0], &gsp_phys_gr_idem_fp[1], + (GSP_PHYS_GR_IDEM_SLOTS - 1u) * sizeof(u64)); + gsp_phys_gr_idem_fp[GSP_PHYS_GR_IDEM_SLOTS - 1u] = fp; + } +} + +/* + * COPYAUDIT (2026-10-18 copy-completeness audit): per-copy sighting of the + * GR operand staging copies. Every staged page is spot-checked (16 bytes + * at the start and end of the page, source vs destination, right after the + * memcpy) and any difference logs COPY-MISMATCH; the callers log the + * per-launch COPYAUDIT aggregate when the staged set exceeds 32 KB, and + * COPY-TRUNC whenever fewer pages were staged than the launch declared + * (want>done). Purely diagnostic: no control flow, locking or timing + * change; the memcmp reads memory the memcpy just touched. + */ +#define GSP_PHYS_GR_AUDIT_MIN (32u << 10) /* >32 KB staged sets */ + +static void gsp_phys_gr_page_spot_check(const u8 *dst, const u8 *src, + u64 va, const char *path) +{ + if (memcmp(dst, src, 16) || + memcmp(dst + GSP_PHYS_PAGE_SIZE - 16, + src + GSP_PHYS_PAGE_SIZE - 16, 16)) + pr_info_ratelimited("vgsp: COPY-MISMATCH va=0x%llx off=0x%llx path=%s\n", + va, 0ull, path); +} + +/* + * r452: the legacy 32-bit additive page sum, kept ONLY for the all-zero + * filter. It is never a change-detection fingerprint any more: doubling every + * word of a 4 KB page leaves the sum bit-identical (1024 x 2^23 == 2^33 == 0 + * mod 2^32), which silently classified an in-place x2 scale as "no engine + * store" and shipped stale values (the deterministic streams2 class). + */ +static inline u32 gsp_phys_gr_page_sum32(const u8 *p) +{ + u32 ci, csum = 0; + + for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) + csum += get_unaligned_le32(p + ci); + return csum; +} + +/* + * COPYWIT r226 zero filter: an all-zero page is never a meaningful alias + * partner. A fresh client's fill stages long runs of zero pages + * (uninitialized cudaMalloc buffers), and consecutive zero pages produced + * the IN-ALIAS-WIT false positives of the r226a green run (csum=0x00000000 + * sequences). A page counts as zero only when its csum is zero AND every + * word is zero (csum==0 alone can hide cancelling contents). + */ +static inline bool gsp_phys_gr_page_all_zero(u32 csum, const u8 *p) +{ + u32 i; + + if (csum) + return false; + for (i = 0; i < GSP_PHYS_PAGE_SIZE; i += 4) + if (get_unaligned_le32(p + i)) + return false; + return true; +} + +/* + * COPYWIT r22x INPUT-ALIAS-WIT: per-launch census of the copy-in recs (the + * exact VA -> HPA -> content triples the engine will read for this launch, + * inputs included). Each new non-zero rec is compared against the last + * GSP_PHYS_GR_IN_CENSUS recs: + * - distinct VA resolving to the SAME host page (hpa collision) is the + * resolver-aliasing smoking gun: several guest pages served by one host + * page would replicate those rows through every consumer (the observed + * band-replication signature); + * - distinct HPA with identical content is the upstream-write-clone class + * (an H2D that landed on one page, or guest pages that genuinely hold + * the same bytes). + * Zero pages are excluded on both sides (fresh-client zero runs are not + * aliasing evidence). Bounded boot-wide; same single-inflight discipline. + */ +#define GSP_PHYS_GR_IN_CENSUS 64u +static u32 gsp_phys_gr_in_census_log; + +/* + * m153: fill the bounce window from the request's guest pages and capture a + * per-page checksum baseline. Used before the first publish and again before + * every outcome retry (a mis-executing first attempt can leave stale or + * wrong-window content behind, so a retry must start from the guest inputs + * again). Returns the number of pages staged. + */ +static u32 gsp_phys_gr_fill_bounce(const struct gsp_phys_gr_submit *req, + u64 *csum_in, u64 *staged_hpa) +{ + struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); + struct { + u64 va; + u64 hpa; + u32 csum; + } census[GSP_PHYS_GR_IN_CENSUS]; + u32 n_census = 0; + u32 di, copied = 0; + + for (di = 0; di < req->n_data; di++) { + u32 bi = gsp_phys_gr_data_bi(req->data[di].reloc_va); + const u8 *src; + u8 *dst; + u32 ci, csum = 0; + bool zero; + + if (bi == GSP_PHYS_GR_DATA_BI_NONE || !req->data[di].hpa) + continue; + src = (const u8 *)__va(req->data[di].hpa); + dst = gsp_phys_chunk_page_va(dchunk, bi); + if (!dst) + continue; + memcpy(dst, src, GSP_PHYS_PAGE_SIZE); + /* + * r378: record which guest page's content now occupies this + * bounce slot. The copy-out may only write the slot back to + * that page - a rec whose reloc VA aliases another rec's slot + * would otherwise ship the other page's content into this + * guest page (the multi-writer clobber class). + */ + staged_hpa[bi] = req->data[di].hpa; + gsp_phys_gr_page_spot_check(dst, src, req->data[di].va, + "gr-bounce-copy-in"); + for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) + csum += get_unaligned_le32(dst + ci); + /* r452: the change baseline is the FNV-1a fingerprint (the + * add-sum above survives only as the all-zero test). */ + csum_in[bi] = gsp_phys_gr_fnv1a64(dst, GSP_PHYS_PAGE_SIZE); + zero = gsp_phys_gr_page_all_zero(csum, dst); + if (!zero && gsp_phys_gr_copywit_enabled()) { + u32 j; + + for (j = 0; j < n_census; j++) { + if (census[j].hpa == req->data[di].hpa && + census[j].va != req->data[di].va) { + if (gsp_phys_gr_in_census_log < 24) { + gsp_phys_gr_in_census_log++; + pr_info("vgsp-gr: INPUT-ALIAS-WIT kind=hpa-collision va=0x%llx hpa=0x%llx prev_va=0x%llx (two guest pages served by one host page)\n", + req->data[di].va, + req->data[di].hpa, + census[j].va); + } + } else if (census[j].csum == csum && + census[j].hpa != req->data[di].hpa) { + if (gsp_phys_gr_in_census_log < 24) { + gsp_phys_gr_in_census_log++; + pr_info("vgsp-gr: INPUT-ALIAS-WIT kind=content-clone va=0x%llx hpa=0x%llx prev_va=0x%llx prev_hpa=0x%llx csum=0x%08x\n", + req->data[di].va, + req->data[di].hpa, + census[j].va, + census[j].hpa, + csum); + } + } + } + if (n_census < GSP_PHYS_GR_IN_CENSUS) { + census[n_census].va = req->data[di].va; + census[n_census].hpa = req->data[di].hpa; + census[n_census].csum = csum; + n_census++; + } + } + copied++; + } + dma_wmb(); + return copied; +} + +/* + * relfix: copy the late-resolved pages' guest content into their bounce + * pages and capture the copy-in checksum baseline, exactly as + * gsp_phys_gr_fill_bounce does for the fill's tracked data pages. Called + * before the doorbell (initial copy-in) and again before every outcome + * retry, so a retried launch starts from the guest inputs in both sets. + * Returns the number of pages staged. + */ +static u32 gsp_phys_gr_fill_gap_bounce(u64 *csum_in, u64 *staged_hpa) +{ + struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); + u32 gi, copied = 0; + + for (gi = 0; gi < gsp_phys_gr_gap_n; gi++) { + const u8 *src = (const u8 *)__va(gsp_phys_gr_gap[gi].hpa); + u8 *dst = gsp_phys_chunk_page_va(dchunk, gsp_phys_gr_gap[gi].bi); + + if (!src || !dst) + continue; + memcpy(dst, src, GSP_PHYS_PAGE_SIZE); + /* r378: the slot now holds THIS page's content (see the + * tracked-page fill above). */ + staged_hpa[gsp_phys_gr_gap[gi].bi] = gsp_phys_gr_gap[gi].hpa; + gsp_phys_gr_page_spot_check(dst, src, gsp_phys_gr_gap[gi].va, + "gr-bounce-copy-in"); + /* r452: FNV-1a baseline (see gsp_phys_gr_scan_pages). */ + csum_in[gsp_phys_gr_gap[gi].bi] = + gsp_phys_gr_fnv1a64(dst, GSP_PHYS_PAGE_SIZE); + copied++; + } + dma_wmb(); + return copied; +} + +/* + * F3 retry hygiene: an outcome retry re-publishes the same QMD. If the QMD's + * pointer pairs name the win0 shadow bank the previous attempt used, move the + * already-prepared param/program/bank pages into the alternate bank and + * rewrite the pairs (and the program prefetch) to the fresh VAs. The copies + * come from the module shadow pages - after relocation, inline replay and + * pointer fix-ups - so the engine-visible content is byte-identical to the + * first attempt and only the VA changes; the engine therefore cannot serve + * the retry from a cached fetch line of the previous attempt's bank. With + * gsp_phys_rmcall_gr_ptr_guest the pairs hold guest VAs, match nothing here, + * and the engine keeps reading the guest VAs exactly as before. Returns true + * when at least one pair moved (the caller must rebuild the method block). + */ +static bool gsp_phys_gr_win0_retry_rearm(u32 *qmd, + const struct gsp_phys_gr_submit *req, + u32 old_bank_base, u32 new_bank_base, + u32 new_bank_limit) +{ + struct gsp_phys_boot_chunk *pool = &boot_stage.rmcall_gr_shadow_win0; + u64 old_lo = GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE + + (u64)old_bank_base * GSP_PHYS_PAGE_SIZE; + u64 old_hi = old_lo + + (u64)GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS * + GSP_PHYS_PAGE_SIZE; + u32 next_slot = new_bank_base; + u32 i; + bool moved = false; + + if (!pool->pages) + return false; + for (i = 0; i < req->n_pages; i++) { + const struct gsp_phys_gr_page *pg = &req->pages[i]; + u32 w = pg->qmd_word; + u64 pair_va, new_va; + u32 off, old_slot; + u8 *src, *dst; + + if (w != 32u && w != 48u && + !(w >= 34u && w <= 46u && !(w & 1u))) + continue; + pair_va = ((u64)(qmd[w + 1] & 0x1ffffu) << 32) | qmd[w]; + if (pair_va < old_lo || pair_va >= old_hi) + continue; + if (next_slot >= new_bank_limit) + break; + off = (u32)(pair_va & (GSP_PHYS_PAGE_SIZE - 1)); + old_slot = (u32)((pair_va - + GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE) / + GSP_PHYS_PAGE_SIZE); + src = gsp_phys_chunk_page_va(pool, old_slot); + dst = gsp_phys_chunk_page_va(pool, next_slot); + if (!src || !dst) + break; + memcpy(dst, src, GSP_PHYS_PAGE_SIZE); + new_va = GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE + + (u64)next_slot * GSP_PHYS_PAGE_SIZE + off; + qmd[w] = (u32)new_va; + qmd[w + 1] = (qmd[w + 1] & ~0x1ffffu) | + ((u32)(new_va >> 32) & 0x1ffffu); + if (w == 48u) { + u64 pref = ((u64)(qmd[31] & 0x1ffu) << 40) | + ((u64)qmd[8] << 8); + + if ((pref & ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) == + (pair_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1))) { + u64 np = new_va | + (pref & (GSP_PHYS_PAGE_SIZE - 1)); + + qmd[8] = (u32)(np >> 8); + qmd[31] = (qmd[31] & ~0x1ffu) | + ((u32)(np >> 40) & 0x1ffu); + } + } + next_slot++; + moved = true; + } + if (moved) + dma_wmb(); + return moved; +} + +/* + * F4-a: the first poll break, rendered from a noinline helper so its + * addresses/counter scratch never grow the (already warning-bound) submit + * frame. The USERD GP_GET/GP_PUT at the break discriminate an engine ack + * (GP_GET advanced, break latency long enough for a real retirement) from a + * host-authored word (break within a few ms of the doorbell, GP_GET frozen). + */ +static noinline void +gsp_phys_gr_poll_break_log(u32 w0, u32 want_qmd, u32 w1, u32 since_us, + const struct gsp_phys_rmcall_ce_submit *sub) +{ + struct gsp_phys_boot_chunk *ud_c = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD]; + u8 *ud = ud_c->pages ? gsp_phys_chunk_page_va(ud_c, 0) : NULL; + u32 usr_gpget = ud ? + get_unaligned_le32(ud + gsp_phys_rmcall_userd_ctl_off() + + GSP_PHYS_RMCALL_USERD_GPGET_OFF) : 0; + + pr_info("vgsp-gr: poll break w0=0x%x want=0x%x ch=0x%x after %u us usr_gpget=0x%x gpget=0x%x gpput=0x%x\n", + w0, want_qmd, w1, + (u32)ktime_to_us(ktime_get()) - since_us, + usr_gpget, sub->gpget, sub->gpput); + /* r425: the healthy-launch SKED/FECS baseline for the wedge compare. */ + gsp_phys_gr_sked_wit(gsp_phys_gpu_bar0(), "post-poll"); +} + +/* + * F4-b: one-shot deposit readback at the first timeout (noinline for the same + * frame reason as above). Reads the qmdwin/shadow destination (or the guest + * QMD page) with the booter's existing unaligned host readback; w63 is the + * engine fetch sentinel. + */ +static noinline void +gsp_phys_gr_deposit_rb_log(const struct gsp_phys_gr_submit *req, + u8 *qmd_dst_mem) +{ + u8 *qp = qmd_dst_mem ? qmd_dst_mem : + (req->qmd_hpa ? (u8 *)__va(req->qmd_hpa) : NULL); + + if (!qp) + return; + pr_info("vgsp-gr: deposit readback w0=0x%08x w4=0x%08x w12=0x%08x w13=0x%08x w18=0x%08x w19=0x%08x w23=0x%08x w32=0x%08x w33=0x%08x w48=0x%08x w49=0x%08x w63=0x%08x\n", + get_unaligned_le32(qp), + get_unaligned_le32(qp + 16), + get_unaligned_le32(qp + 48), + get_unaligned_le32(qp + 52), + get_unaligned_le32(qp + 72), + get_unaligned_le32(qp + 76), + get_unaligned_le32(qp + 92), + get_unaligned_le32(qp + 128), + get_unaligned_le32(qp + 132), + get_unaligned_le32(qp + 192), + get_unaligned_le32(qp + 196), + get_unaligned_le32(qp + + GSP_PHYS_GR_QMD_SENTINEL_WORD * 4)); +} + +/* + * CODEHASH (2026-10-18): code-fetch content verification. At author time, + * sample the host-readable bytes the engine is about to fetch for this + * launch's program (the QMD word 48/49 pair): the first 16 B at the entry + * offset plus 16 B at PA+0 / PA+mid / PA+end of the entry page, and the + * same three windows on up to three further consecutive tracked pages of + * the code run. One bounded line per launch, emitted when the content is + * suspicious (all-zero samples, or the first four entry words identical - + * a page that never landed or a repeated filler word) or for the first few + * healthy launches per boot (so the healthy signature is visible without + * flooding). A missing host-readable landing (hpa 0 on the entry page) + * logs CODEHASH-UNREADABLE. sh0 is the engine-visible shadow page's entry + * word when the Tier-2 copy exists (NULL on the Tier-1/guest path). + * noinline: keeps the (warning-bound) submit frame unchanged. + */ +static bool gsp_phys_gr_codehash_done; +static noinline void +gsp_phys_gr_codehash(const struct gsp_phys_gr_submit *req, + const struct gsp_phys_gr_page *pg, const u8 *sh_page) +{ + static u32 ch_checked, ch_logged, ch_unread; + u64 va, h0 = 0, ent = 0, sh0 = 0; + u32 off, n = 0, w[4] = { 0, 0, 0, 0 }; + bool zero3 = false, suspect = false; + const u8 *p = NULL; + u32 i; + + if (gsp_phys_gr_codehash_done) + return; + gsp_phys_gr_codehash_done = true; + ch_checked++; + off = (u32)(req->qmd[48] & (GSP_PHYS_PAGE_SIZE - 1)); + va = (u64)req->qmd[48] | ((u64)(req->qmd[49] & 0x1ffffu) << 32); + if (!pg->hpa) { + if (ch_unread < 32) { + ch_unread++; + pr_info("vgsp-gr: CODEHASH-UNREADABLE va=0x%llx off=0x%x (entry page has no host-readable landing)\n", + va, off); + } + return; + } + p = (const u8 *)__va(pg->hpa); + n = 1; + ent = get_unaligned_le64(p + off); + h0 = get_unaligned_le64(p); + for (i = 0; i < 4; i++) + w[i] = get_unaligned_le32(p + off + i * 4); + zero3 = !(get_unaligned_le64(p) | get_unaligned_le64(p + 8) | + get_unaligned_le64(p + 0x800) | + get_unaligned_le64(p + 0x808) | + get_unaligned_le64(p + 0xff0) | + get_unaligned_le64(p + 0xff8)); + for (i = 1; i < 4; i++) { + u64 tva = pg->guest_va + ((u64)i << 12); + u32 j; + + for (j = 0; j < req->n_pages; j++) + if (req->pages[j].guest_va == tva && + req->pages[j].hpa) + break; + if (j == req->n_pages) + break; + { + const u8 *q = (const u8 *)__va(req->pages[j].hpa); + + n++; + if (get_unaligned_le64(q) || get_unaligned_le64(q + 8) || + get_unaligned_le64(q + 0x800) || + get_unaligned_le64(q + 0x808) || + get_unaligned_le64(q + 0xff0) || + get_unaligned_le64(q + 0xff8)) + zero3 = false; + } + } + if (sh_page) + sh0 = get_unaligned_le64(sh_page + off); + /* + * Conservative plausibility test: all three sampled windows zero; or + * the first two entry words zero; or the first four entry words + * identical (an all-zero page also lands here). + */ + suspect = zero3 || (!w[0] && !w[1]) || + (w[0] == w[1] && w[1] == w[2] && w[2] == w[3]); + if (!suspect && ch_logged >= 3) + return; + if (ch_logged >= 64) + return; + ch_logged++; + pr_info("vgsp-gr: CODEHASH va=0x%llx pa=0x%llx n=%u h0=0x%016llx ent=0x%016llx off=0x%x w=%08x %08x %08x %08x sh0=0x%016llx %s (first 16B)\n", + va, pg->hpa, n, h0, ent, off, w[0], w[1], w[2], w[3], sh0, + suspect ? "SUSPECT" : "ok"); +} + +/* + * M91 FAULT-CAP (2026-10-18): first-fault capture for the transient SM + * instruction-fetch class ("m91 class": the first attempt of a mid-sequence + * launch faults on SM garbage - "Illegal Instruction Encoding" / "Out Of + * Range Register" - while a retry of the identical QMD and identical + * shadow content succeeds). The submit path calls this once per launch at + * the end of the launch window (poll break or timeout) with the retry + * levers that fired; it emits one vgsp-m91: FAULT-CAP line per faulting + * launch carrying: the fault-epoch delta since the launch started, the + * forwarded QMD's address and first 64 B, the live program pair (w48/49) + * with its resolved engine-visible VA/PA, the first 64 B at that code + * address (the bytes the SM will fetch), the retry decision and outcome, + * and the newest relay-decoded 0x1006 note (exceptType/chid/errString). + * Statically inert on a fault-free launch: one READ_ONCE of the fault + * epoch and one compare, no reads of content or engine state. Full dumps + * are capped at GSP_PHYS_M91_CAP_MAX per module-load cycle, followed by a + * single suppression notice; the existing vgsp-fault lines keep decoding + * every fault either way. Static state follows the other per-launch submit + * scratch: the single-inflight ordered work item is the only caller. + */ +#define GSP_PHYS_M91_CAP_MAX 20u +static u32 gsp_phys_m91_fault_start; +static u32 gsp_phys_m91_cap_used; +static bool gsp_phys_m91_cap_suppressed; +static bool gsp_phys_m91_icache_log; +static bool gsp_phys_r449_ccache_log; +static bool gsp_phys_gr_rel_membar_log; +static noinline void +gsp_phys_gr_m91_fault_cap(const struct gsp_phys_gr_submit *req, + const u32 *qmd, u64 qmd_dst, const u8 *qmd_dst_mem, + u32 attempts, bool rearmed, bool fast, bool ok) +{ + u32 faults = READ_ONCE(gsp_phys_fault_epoch) - gsp_phys_m91_fault_start; + const struct gsp_phys_fault_info *fn; + const u8 *code = NULL; + const char *src = "unresolved"; + u64 pva, ppa = 0; + u32 off = 0, cn = 0, i; + u8 qbuf[64], cbuf[64]; + char qhex[2 * 64 + 1], chex[2 * 64 + 1]; + + /* INERT until a fault: fault-free launches take one compare and return. */ + if (!faults || !req || !qmd) + return; + if (gsp_phys_m91_cap_used >= GSP_PHYS_M91_CAP_MAX) { + if (!gsp_phys_m91_cap_suppressed) { + gsp_phys_m91_cap_suppressed = true; + pr_info("vgsp-m91: FAULT-CAP suppressed: %u dumps already emitted this cycle (the existing vgsp-fault lines still decode every fault)\n", + GSP_PHYS_M91_CAP_MAX); + } + return; + } + gsp_phys_m91_cap_used++; + + /* First 64 B of the forwarded QMD (qmdwin copy preferred). */ + memset(qbuf, 0, sizeof(qbuf)); + if (qmd_dst_mem) { + memcpy(qbuf, qmd_dst_mem, sizeof(qbuf)); + } else if (req->qmd_hpa) { + memcpy(qbuf, (const u8 *)__va(req->qmd_hpa), sizeof(qbuf)); + } + bin2hex(qhex, qbuf, sizeof(qbuf)); + qhex[sizeof(qhex) - 1] = '\0'; + + /* Program pair (QMD w48/49) and its engine-visible resolution. */ + pva = ((u64)(qmd[49] & 0x1ffffu) << 32) | qmd[48]; + off = (u32)(pva & (GSP_PHYS_PAGE_SIZE - 1)); + if (boot_stage.rmcall_gr_shadow_win0.pages && + pva >= GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE && + pva < GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE + + (u64)GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS * GSP_PHYS_PAGE_SIZE) { + u32 slot = (u32)((pva - GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE) / + GSP_PHYS_PAGE_SIZE); + + if (slot < boot_stage.rmcall_gr_shadow_win0.pages) { + code = gsp_phys_chunk_page_va( + &boot_stage.rmcall_gr_shadow_win0, slot); + ppa = (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_gr_shadow_win0, slot); + src = "win0"; + } + } else { + u64 base = pva & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + + for (i = 0; i < boot_stage.rmcall_gr_alias_count; i++) { + if (boot_stage.rmcall_gr_alias_va[i] == base) { + ppa = boot_stage.rmcall_gr_alias_hpa[i]; + code = (const u8 *)__va(ppa); + src = "shadow-line"; + break; + } + } + for (i = 0; req->pages && !code && i < req->n_pages; i++) { + if ((req->pages[i].guest_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) != base || + !req->pages[i].hpa) + continue; + ppa = req->pages[i].hpa; + code = (const u8 *)__va(ppa); + src = "guest-hpa"; + } + } + memset(cbuf, 0, sizeof(cbuf)); + memset(chex, 0, sizeof(chex)); + if (code) { + cn = min_t(u32, sizeof(cbuf), GSP_PHYS_PAGE_SIZE - off); + memcpy(cbuf, code + off, cn); + bin2hex(chex, cbuf, cn); + chex[sizeof(chex) - 1] = '\0'; + } + + /* Newest relay-decoded 0x1006 (valid whenever the epoch advanced). */ + fn = &gsp_phys_fault_notes[(gsp_phys_fault_note_next + + GSP_PHYS_FAULT_NOTE_MAX - 1u) % + GSP_PHYS_FAULT_NOTE_MAX]; + pr_info("vgsp-m91: FAULT-CAP ok=%u faults=%u retry=%s attempts=%u qmd=0x%llx qmd64=%s prog_w48=0x%08x w49=0x%08x prog_va=0x%llx prog_pa=0x%llx code_src=%s off=0x%x code64=%s fault_et=0x%x fault_chid=%u fault_err=\"%.48s\"\n", + ok ? 1u : 0u, faults, + fast ? "fast-rearm" : (rearmed ? "resched/fresh-entry" : "none"), + attempts, qmd_dst, qhex, qmd[48], qmd[49], pva, ppa, src, off, + cn ? chex : "unresolved", + fn->valid ? fn->except_type : 0u, + fn->valid ? fn->chid : 0u, + fn->valid ? fn->err : "no relayed 0x1006 note"); +} + +/* + * CODESRC (2026-10-18, r221 follow-up): conservative refusal of a + * non-plausible Tier-2 program-pair source. The program page (QMD w48/49) + * is copied verbatim into the shadow the engine fetches from (win0 or the + * Tier-2 line); a source that is empty, PTE-shaped or one of the engine + * page-tree's own module pages would execute PTE/filler bytes as SASS. + * Detection is deliberately narrow so it is statically inert on every + * observed launch (greens included): + * (a) empty: the entry 16 B AND all three sampled windows are zero + * (a page that never landed anywhere the engine starts); + * (b) PTE page: >=12 of the 16 8-byte entries at the entry offset are + * identically-flagged valid PTEs (dense table page), or the run is + * >=12 zero entries with only PTE-signed entries (sparse table); + * (c) module page: the source HPA is a page of one of our own page-tree + * chunks (main PT, guest PT pool, Tier-2 shadow pools, qmdwin) - a + * guest page can never legitimately resolve there. + * A refused source is re-resolved through the late resolver + * (gsp_sema_gr_reloc_resolve: pins/map/walks before the ledger, the + * guest-walk-first counterpart of the fill's ledger-first order). Only a + * different, plausibly-code page is adopted (mode ALT; the engine-visible + * copy is taken from it); anything else leaves the QMD pair at the guest + * VA (mode GUEST_VA, the ptr_guest behaviour for this pair). One + * rate-limited CODESRC-SUSPECT line per detection. + */ +#define GSP_PHYS_GR_CODESRC_OK 0 +#define GSP_PHYS_GR_CODESRC_ALT 1 +#define GSP_PHYS_GR_CODESRC_GUEST_VA 2 + +static bool +gsp_phys_gr_codesrc_chunk_has(const struct gsp_phys_boot_chunk *c, u64 hpa) +{ + u32 i, n; + + if (!c || !c->pages) + return false; + n = c->pages < 32u ? c->pages : 32u; + for (i = 0; i < n; i++) { + u64 d = (u64)gsp_phys_chunk_page_dma(c, i); + + if (d && (d & ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) == hpa) + return true; + } + return false; +} + +static bool gsp_phys_gr_codesrc_modpt(u64 hpa) +{ + u32 i; + + if (gsp_phys_gr_codesrc_chunk_has( + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PT], hpa)) + return true; + for (i = 0; i < GSP_PHYS_RMCALL_GR_PT_PAGES; i++) + if (gsp_phys_gr_codesrc_chunk_has(&boot_stage.rmcall_gr_pt[i], + hpa)) + return true; + return gsp_phys_gr_codesrc_chunk_has(&boot_stage.rmcall_gr_shadow, + hpa) || + gsp_phys_gr_codesrc_chunk_has( + &boot_stage.rmcall_gr_shadow_win0, hpa) || + gsp_phys_gr_codesrc_chunk_has(&boot_stage.rmcall_gr_qmdwin, hpa); +} + +/* Entry sampling window: 16 8-byte entries starting at the entry offset. */ +static bool gsp_phys_gr_codesrc_ptelike(const u8 *p, u32 off) +{ + u32 start = off & ~7u, i, hits = 0, zeros = 0, nz = 0, flag = 0; + bool flag_set = false, sig_ok = true; + + if (start + 128u > GSP_PHYS_PAGE_SIZE) + start = GSP_PHYS_PAGE_SIZE - 128u; + for (i = 0; i < 16u; i++) { + u64 v = get_unaligned_le64(p + start + i * 8); + u32 lo = (u32)(v & 0xffu); + /* VALID, aperture not PEER(1): the PTE shape family. */ + bool pte = (v & 1u) && (((v >> 1) & 0x3u) != 1u); + + if (!v) { + zeros++; + continue; + } + nz++; + if (!pte) { + sig_ok = false; + continue; + } + if (!flag_set) { + flag = lo; + flag_set = true; + } + if (lo == flag) + hits++; + } + if (hits >= 12u) /* dense table page: a uniform valid-PTE run */ + return true; + /* sparse table page: mostly zero, only PTE-signed entries */ + return sig_ok && nz >= 1u && nz <= 4u && zeros >= 12u; +} + +static bool gsp_phys_gr_codesrc_empty(const u8 *p, u32 off) +{ + const u8 *e = p + off; + + return !(get_unaligned_le64(e) | get_unaligned_le64(e + 8)) && + !(get_unaligned_le64(p) | get_unaligned_le64(p + 8) | + get_unaligned_le64(p + 0x800) | get_unaligned_le64(p + 0x808) | + get_unaligned_le64(p + 0xff0) | get_unaligned_le64(p + 0xff8)); +} + +static noinline int +gsp_phys_gr_code_src(const struct gsp_phys_gr_submit *req, + const struct gsp_phys_gr_page *pg, u64 *alt_hpa) +{ + u64 page = pg->guest_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + u32 off = (u32)(req->qmd[48] & (GSP_PHYS_PAGE_SIZE - 1)); + const u8 *p = (const u8 *)__va(pg->hpa); + const char *why = NULL, *used; + struct gsp_sema_gr_reloc r; + int mode = GSP_PHYS_GR_CODESRC_GUEST_VA; + + *alt_hpa = 0; + if (off > GSP_PHYS_PAGE_SIZE - 16u) + off = GSP_PHYS_PAGE_SIZE - 16u; /* keep 16 B in-page */ + if (gsp_phys_gr_codesrc_empty(p, off)) + why = "empty-page"; + else if (gsp_phys_gr_codesrc_ptelike(p, off)) + why = "pte-page"; + else if (gsp_phys_gr_codesrc_modpt(pg->hpa)) + why = "module-pt-page"; + if (!why) + return GSP_PHYS_GR_CODESRC_OK; + /* + * Alternative resolution order; only a different page that itself + * passes the content checks is adopted. + */ + if (!gsp_sema_gr_reloc_resolve(page, &r) && r.hpa && + (r.hpa & ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) != + (pg->hpa & ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) && + !gsp_phys_gr_codesrc_empty((const u8 *)__va(r.hpa), off) && + !gsp_phys_gr_codesrc_ptelike((const u8 *)__va(r.hpa), off)) { + *alt_hpa = r.hpa; + mode = GSP_PHYS_GR_CODESRC_ALT; + used = "alt"; + } else { + used = "guest-va"; + } + pr_info_ratelimited("vgsp-gr: CODESRC-SUSPECT va=0x%llx pa=0x%llx alt=%s used=%s why=%s\n", + page, pg->hpa, + mode == GSP_PHYS_GR_CODESRC_ALT ? r.stage : "none", + used, why); + return mode; +} + +/* + * STALLFREE: the absolute per-launch wait budget. Every ladder phase is + * individually bounded, but a launch can chain them (fast re-arm + fresh + * entries + retirement retries), so the sum - and with it the time the + * single-inflight work item holds the client's completion - is not. This + * predicate bounds the whole submit: once it is true, no further re-arm, + * re-publish or re-execution runs. Green launches complete in ~0.2-11 s and + * never approach the default 60 s. + */ +static bool gsp_phys_gr_wait_budget_over(unsigned long start) +{ + unsigned int budget = gsp_phys_gr_wait_max_ms_value(); + + return budget && time_after_eq(jiffies, + start + msecs_to_jiffies(budget)); +} + +/* + * STALLFREE: one distinguishable bounded-completion line per stall episode. + * `action=fail` means the submit gives up (the sema service's failure edge + * authors the QMD completion release in software, so the guest proceeds); + * `action=proceed` means the phase gave up but the launch continues with its + * current outcome. + */ +static void gsp_phys_gr_stall_log(const char *reason, u64 push_va, + unsigned long start, const char *action) +{ + pr_warn("vgsp-gr: GR-STALL-%s va=0x%llx waited_ms=%u action=%s\n", + reason, push_va, jiffies_to_msecs(jiffies - start), action); +} + int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, struct gsp_phys_backend_copy_result *out) { @@ -16504,50 +19065,135 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB]; void __iomem *bar0; struct gsp_phys_rmcall_ce_submit sub; - struct gsp_phys_fault_info fault = { }; /* * Static scratch: the only caller is the single-inflight sema work * item (ordered gsp-phys-work queue), so no concurrent submission can - * touch these. Keeping the 96-dword method block and the 64-word QMD - * copy off the stack keeps the frame under the warning bound. + * touch these. Keeping the 96-dword method block, the 64-word QMD + * copy and the ~370-byte fault record off the stack keeps the frame + * under the warning bound. */ static u32 methods[GSP_PHYS_RMCALL_GR_METHOD_WORDS]; static u32 qmd[GSP_PHYS_GR_QMD_WORDS]; + static struct gsp_phys_fault_info fault; unsigned long start = jiffies; unsigned long deadline; u8 *sem_va; u32 epoch, words, i, n_shadow = 0; + u32 win0_bank_base = 0, shadow_slot_limit = 0; + const u8 *guest_param_pg; u32 want_qmd = 0, want_ch = 0; bool qmd_rel = false, ch_rel = false; + bool qmd_rel_redirected = false; /* r420b: release0 -> module sem */ u64 qmd_rel_hpa = 0, ch_rel_hpa = 0; u64 ch_sem_va = GSP_PHYS_RMCALL_VA_SEM + 4; u64 qmd_dst = GSP_PHYS_RMCALL_VA_SRC; u8 *qmd_dst_mem = NULL; u32 qmd_rel_off = 0, ch_rel_off = 0; u32 poll = 0, w0 = 0, w1 = 0, grpoll_n = 0; + bool sem_gated = false; static u32 sked_prev[11]; static u32 sked_sample[11]; static u32 sked_tick; static bool sked_first; /* - * m142: per-bounce-page checksums captured at copy-in. The copy-out + * m142: per-data-page checksums captured at copy-in. The copy-out * compares them to report how many pages the engine actually modified; * an all-unchanged window after a reported completion is the * "front-end ran, grid never dispatched / read zeros" signature that - * previously masqueraded as a successful launch. Safe as static: the - * only caller is the single-inflight ordered gsp-phys-work item. + * previously masqueraded as a successful launch. B-2: one entry per + * page of the grown region (64 KB, static - must not sit on the kernel + * stack). Safe as static: the only caller is the single-inflight + * ordered gsp-phys-work item. */ - static u32 bounce_csum_in[GSP_PHYS_RMCALL_GR_MAIL_SLOT]; + static u64 bounce_csum_in[GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES]; + /* + * r378: which guest page's content each bounce slot holds this + * launch (set by the copy-in fills, cleared per submission). The + * copy-out writes a slot back only to that exact page, so a rec + * aliasing another rec's slot - or a slot left over from a previous + * launch - can never ship foreign content into a guest page. + * Static like bounce_csum_in: the single-inflight ordered + * gsp-phys-work item is the only caller. + */ + static u64 bounce_staged_hpa[GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES]; sked_tick = 0; sked_first = true; + guest_param_pg = NULL; u32 qmd_win_pde = 0, qmd_win_installed = 0; int qmd_win_ret = 0; bool qmd_win_diag = false; bool qmdwin_on = false; + u32 qw_bank = 0; char qmdwin_note[32] = ""; bool rearmed = false; + u32 attempts = 0; + /* + * F2/F4 state: the fast first-timeout re-arm runs once per launch; the + * deposit readback is one-shot per launch (the first timeout is the + * evidence window - the old r198 post-mortem only ran after attempt 3, + * which recovered launches never reached); poll_ms is the window in + * force for the current poll, so the timeout log reports it accurately. + * Static like the other submit scratch: the single-inflight ordered + * work item is the only caller, and statics keep the frame at its + * warning-bound size. + */ + static bool fast_rearmed; + static bool deposit_rb_done; + static u16 poll_ms; + static u32 poll_start_us; + /* + * GRFIX FIX3: per-launch relocation coverage. Static like the other + * submit scratch: the single-inflight ordered work item is the only + * caller, and statics keep the frame at its warning-bound size. + */ + static u32 reloc_gap; + static u64 reloc_gap_first; int ret = 0; + fast_rearmed = false; + deposit_rb_done = false; + gsp_phys_gr_codehash_done = false; /* CODEHASH: once per launch */ + /* + * M91 FAULT-CAP: snapshot the fault epoch per launch, so the capture + * at the launch's end attributes exactly the faults consumed inside + * this window (and a refusal path that never publishes compares equal + * and stays inert). + */ + gsp_phys_m91_fault_start = READ_ONCE(gsp_phys_fault_epoch); + reloc_gap = 0; + reloc_gap_first = 0; + /* + * relfix: the late-resolved page list is per-launch scratch exactly + * like bounce_csum_in; the single-inflight work item is the only + * caller. + */ + gsp_phys_gr_gap_n = 0; + gsp_phys_gr_gap_word_n = 0; + gsp_phys_gr_reloc_fixed = 0; + gsp_phys_gr_reloc_fixed_first = 0; + gsp_phys_gr_outwit_n = 0; /* COPYWIT: per-launch alias table */ + gsp_phys_gr_gap_logged_n = 0; + /* + * The fault record is static scratch (frame bound): reset it per + * submission so a previous launch's fault can never leak into this + * one's result - the old on-stack `= { }` semantics. + */ + memset(&fault, 0, sizeof(fault)); + /* + * r378: the bounce staging identity is per launch. A slot left over + * from a previous launch (or never staged this one) must not be + * treated as this launch's page even when the HPA matches. + */ + memset(bounce_staged_hpa, 0, sizeof(bounce_staged_hpa)); + /* + * F2: the shortened first window is part of the fast path; with + * gsp_phys_gr_retry_fast=0 the launch keeps today's exact behavior + * (the historical 5 s first window and ladder). + */ + poll_ms = gsp_phys_gr_retry_fast_value() ? + (u16)gsp_phys_gr_first_poll_ms_value() : + (u16)GSP_PHYS_RMCALL_GR_POLL_MS; + if (out) memset(out, 0, sizeof(*out)); if (!req || !req->qmd || @@ -16557,6 +19203,25 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, gsp_phys_gr_refused++; return -EINVAL; } + /* + * r564: refuse a degenerate data set BEFORE the engine ever sees it. + * r565 evidence: the fill dropped 4 argument bases (GRFILL-DROP), + * RDW showed nrecs=2 where healthy launches run 968-969, the launch + * was still submitted and the backend timed out after ~19 s + * ("GR forward failed rc=-110"), after which the tail authored the + * software release and the client kept spinning on a completion for a + * kernel that never ran - the worst of both (20 s of backend + * occupancy plus a false completion). Refusing up front fails fast + * into the same software-completion fallback without touching the + * engine, and cannot wedge it for the next client. + */ + if (req->n_data < GSP_PHYS_GR_MIN_DATA_RECS) { + gsp_phys_gr_refused++; + pr_err_ratelimited("vgsp-gr: GR submit refused: degenerate data set n_data=%u (min %u) launch_va=0x%llx\n", + req->n_data, GSP_PHYS_GR_MIN_DATA_RECS, + req->push_va); + return -ENODATA; + } if (!gsp_phys_backend_gr_ready() || !gsp_phys_guest_mem_enabled() || gsp_phys_dma_domain_identity() != 1 || !gsp_phys_guest_ram_active()) { gsp_phys_gr_refused++; @@ -16588,8 +19253,13 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * the program page (silent corruption; no launch can survive it). * The win0 PTEs are planted once at stage 3; reusing the same VAs * with fresh content is the proven pattern. + * + * m151: the cursor now selects one of two 8-page banks, alternating + * with the submission epoch (set below). The engine's index/data + * fetches of the previous launch's QMD shadow pages can still be in + * flight when this launch runs, so the bank alternation protects + * them from this launch's copy-in (late-fetch safety). */ - boot_stage.rmcall_gr_shadow_win0_used = 0; mutex_lock(&gsp_phys_rm_lock); if (!gsp_phys_backend_gr_ready()) { mutex_unlock(&gsp_phys_rm_lock); @@ -16600,8 +19270,84 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, epoch = ++boot_stage.rmcall_gr_submit_epoch; if (!epoch) epoch = ++boot_stage.rmcall_gr_submit_epoch; + boot_stage.rmcall_gr_shadow_win0_used = + (epoch & 1u) * GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS; + win0_bank_base = boot_stage.rmcall_gr_shadow_win0_used; + shadow_slot_limit = win0_bank_base + + GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS; memcpy(qmd, req->qmd, sizeof(qmd)); /* + * M91 icache coherence (gsp_phys_gr_icache_inv, default 1): the + * window-0 shadow pool reuses code VAs across launches (two 8-page + * banks alternate per submit epoch), so the bytes at a program VA + * change between launches, while every captured guest QMD carries + * the QMDV02_03 cache-control block with INVALIDATE_INSTRUCTION_CACHE + * (word 5 bit 30) clear - the same field the CUDA driver arms only + * under its forceIcacheInvalidate knob. An SM instruction-fetch line + * fetched for launch N can then survive into launch N+2 and execute + * the previous kernel's bytes at the same VA, the observed + * "first attempt faults, identical retry succeeds" class. Set the + * bit only when the QMD carries the driver's standard cache-control + * signature (word 5 bits 26..29) so an unknown QMD revision is never + * touched; 0 restores the exact pre-change QMD for a one-flip A/B. + */ + if (gsp_phys_gr_icache_inv_value() && + (qmd[5] & 0x3c000000u) == 0x3c000000u && + !(qmd[5] & (1u << 30))) { + qmd[5] |= 1u << 30; + if (!gsp_phys_m91_icache_log) { + gsp_phys_m91_icache_log = true; + pr_info("vgsp-gr: M91 icache-invalidate bit set in the forwarded QMD (w5=0x%08x; QMDV02_03 word5 bit30, the guest driver's forceIcacheInvalidate field)\n", + qmd[5]); + } + } + /* + * r449 SHADER-CONSTANT-CACHE invalidate (gsp_phys_gr_ccache_inv; default + * 0 = off): the same cache-control word's bit 31 (the V01_06-layout + * SHADER_CONSTANT_CACHE position) is never set by the captured guest + * QMDs. The forwarded param page is rewritten in place between launches + * (the two-bank win0 pool), so a stale constant-cache line can serve + * the PREVIOUS launch's parameters and the current launch's output + * pages stay untouched - the post-dispatch no-store signature. Mode 1 + * sets bit 31; mode 2 forces the whole invalidate set (bits 24..31). + * Only when the QMD carries the driver's standard cache-control + * signature (bits 26..29) so an unknown QMD revision is never touched. + */ + if (gsp_phys_gr_ccache_inv_value() && + (qmd[5] & 0x3c000000u) == 0x3c000000u) { + u32 mode = gsp_phys_gr_ccache_inv_value(); + + if (mode == 2) + qmd[5] |= 0xff000000u; + else + qmd[5] |= 1u << 31; + if (!gsp_phys_r449_ccache_log) { + gsp_phys_r449_ccache_log = true; + pr_info("vgsp-gr: r449 constant-cache invalidate mode=%u applied (w5=0x%08x)\n", + mode, qmd[5]); + } + } + /* + * r433 RELEASE_MEMBAR_TYPE A/B (gsp_phys_gr_rel_membar): force the + * forwarded QMD's RELEASE_MEMBAR_TYPE to FE_SYSMEMBAR (QMDV00_06 + * word 11 bit 14) so the front-end flushes its pending system-memory + * stores before writing the release semaphore. The captured guest + * QMDs carry FE_NONE, so a grid that completes with its data still + * in the GPU's L2 releases the client while no store is host-visible + * - the post-dispatch store-loss signature. The release ordering then + * guarantees the host's retire scan and copy-out can never observe a + * completion whose data has not landed. 0 (default) keeps the + * guest's captured value. + */ + if (gsp_phys_gr_rel_membar_value() && !(qmd[11] & (1u << 14))) { + qmd[11] |= 1u << 14; + if (!gsp_phys_gr_rel_membar_log) { + gsp_phys_gr_rel_membar_log = true; + pr_info("vgsp-gr: r433 RELEASE_MEMBAR_TYPE forced to FE_SYSMEMBAR in the forwarded QMD (w11=0x%08x)\n", + qmd[11]); + } + } + /* * Two-step PCAS (gsp_phys_rmcall_gr_pcas_two_step): set the QMD's * REQUIRE_SCHEDULING_PCAS (w4 bit12) so the copy PCAS does not * schedule and the trailing schedule-only PCAS does. @@ -16628,7 +19374,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, ret = gsp_phys_rmcall_gr_map_shadow_locked(n_shadow, req->rel_hpa, - &sh_va); + &sh_va, NULL); if (!ret) { n_shadow++; qmd_rel_hpa = req->rel_hpa; @@ -16660,6 +19406,75 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, */ qmd_rel_hpa = 0; qmd_rel_off = 0; + /* + * r420b: the ENGINE will not write the guest's QMD + * release word this launch (rel_mod, or no targetable + * guest release page). Tell the caller so it authors + * the word in software on every outcome - otherwise a + * launch without deferred release records is never + * completed on the guest word (the r414 saxpy hang). + */ + qmd_rel_redirected = true; + } + } + + /* + * GRID-WIT (r226): the engine reads CTA geometry from this verbatim + * QMD copy, and the copy-out witnesses show the corruption of the + * failing clients starts exactly at a CTA-band boundary (rows >= 16 + * replicate one tile). Dump the geometry-relevant words once per + * launch so the next capture proves what raster/thread dimensions the + * engine actually sees and lets them be compared against the client's + * own launch config: w12 = CTA_RASTER_WIDTH (gridDim.x), + * w13.lo = CTA_RASTER_HEIGHT (gridDim.y), w13.hi = DEPTH (z), + * w14/w15 = raster resume W/H, w18.hi = CTA_THREAD_DIMENSION0 + * (blockDim.x), w19.lo/hi = DIM1/DIM2 (blockDim.y/z), w18.lo = + * QMD_VERSION/MAJOR nibbles. w4/w5/w8/w17: PCAS/cache-control, + * prefetch pair low word, shared-memory size. + */ + if (gsp_phys_gr_copywit_enabled()) + pr_info("vgsp-gr: GRID-WIT w4=%08x w5=%08x w8=%08x w12=%08x w13=%08x w14=%08x w15=%08x w17=%08x w18=%08x w19=%08x\n", + qmd[4], qmd[5], qmd[8], qmd[12], qmd[13], qmd[14], + qmd[15], qmd[17], qmd[18], qmd[19]); + + /* + * r234 RDW arming witness: one line per launch with the rec count, + * so a capture missing REC-DUMP lines can never be confused with + * "the witness did not run" - probe-class launches must show + * nrecs>0 here. + */ + if (req->n_data && gsp_phys_gr_rdw_budget < 256u) { + gsp_phys_gr_rdw_budget++; + pr_info("vgsp-gr: RDW nrecs=%u launch_va=0x%llx\n", + req->n_data, req->push_va); + } + + /* + * REC-DUMP (r229): the probe-class launches (small data sets) get + * their full rec table logged - every (va, hpa, reloc_va) the engine + * will read/write - so the next capture can compare the resolved + * HPAs against the buffer the client actually uploaded (distinct + * VAs collapsing onto one HPA, or HPAs in a foreign range, are + * visible directly). Big launches log the first/last 8 recs only. + * r233: plain pr_info with a boot-wide line budget - the earlier + * pr_info_ratelimited version was suppressed wholesale during the + * probe's post-stall logging burst, so the probe-class recs never + * reached the ring. + */ + if (gsp_phys_gr_copywit_enabled() && req->n_data && + gsp_phys_gr_recdump_budget < 512u) { + u32 d; + + for (d = 0; d < req->n_data; d++) { + if (req->n_data > 64u && d >= 8u && + d < req->n_data - 8u) + continue; + if (gsp_phys_gr_recdump_budget >= 512u) + break; + gsp_phys_gr_recdump_budget++; + pr_info("vgsp-gr: REC-DUMP[%u/%u] va=0x%llx hpa=0x%llx reloc=0x%llx\n", + d, req->n_data, req->data[d].va, + req->data[d].hpa, req->data[d].reloc_va); } } @@ -16700,9 +19515,16 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, if (pg->hpa) { ret = gsp_phys_rmcall_gr_map_va_locked(bar0, pg_va, pg->hpa, epoch); - if (!ret) + if (!ret) { mapped = true; - else if (ret == -ENOSPC) + /* + * CODEHASH: on the Tier-1 identity path the + * engine reads the guest page itself; sample + * the entry before the QMD pair is authored. + */ + if (pg->qmd_word == 48u) + gsp_phys_gr_codehash(req, pg, NULL); + } else if (ret == -ENOSPC) gsp_phys_gr_pool_exhausted++; } /* @@ -16719,6 +19541,8 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, for (k = 0; k < n_sh_rec; k++) { u64 va = 0; + if (sh_rec[k].line == 0xffffffffu) + continue; /* cached line: never extended */ if (pg_va != sh_rec[k].guest_va + (u64)sh_rec[k].npte * GSP_PHYS_PAGE_SIZE) @@ -16745,11 +19569,11 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, */ if (!mapped && pg->hpa && pg->qmd_word == GSP_PHYS_GR_QMD_WORD_NONE) { - u32 sh_line = boot_stage.rmcall_guest_pt_next_line; + u32 sh_line = 0xffffffffu; if (n_shadow < GSP_PHYS_RMCALL_GR_SHADOW_PAGES && !gsp_phys_rmcall_gr_map_shadow_locked( - n_shadow, pg->hpa, &sh_va)) { + n_shadow, pg->hpa, &sh_va, &sh_line)) { u32 k, slot = n_shadow; mapped = true; @@ -16801,6 +19625,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, &boot_stage.rmcall_gr_shadow; u8 *sh_page; u32 j, copied = 0; + bool code_keep_guest = false; if (n_shadow >= GSP_PHYS_RMCALL_GR_SHADOW_PAGES) { gsp_phys_gr_pool_exhausted++; @@ -16826,8 +19651,32 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * domain, copy the full guest page into the shadow and * rewrite the QMD pointer pair, so no inline coverage * is needed for pages the engine only reads. + * + * CODESRC: the program page (qmd_word 48) is the page + * the engine's I-fetch reads; screen its source + * before publishing it. A refused source is either + * replaced by the alternative resolver's page or the + * pair is left at the guest VA (no copy, no rewrite, + * no mapping - the page is fully withdrawn from the + * submission). */ - if (pg->hpa) { + if (pg->hpa && pg->qmd_word == 48u) { + u64 alt_hpa = 0; + int cs = gsp_phys_gr_code_src(req, pg, &alt_hpa); + + if (cs == GSP_PHYS_GR_CODESRC_GUEST_VA) { + code_keep_guest = true; + mapped = true; + } else { + const u8 *csrc = __va(cs == + GSP_PHYS_GR_CODESRC_ALT ? + alt_hpa : pg->hpa); + + memcpy(sh_page, csrc, + GSP_PHYS_PAGE_SIZE); + copied++; + } + } else if (pg->hpa) { memcpy(sh_page, (u8 *)__va(pg->hpa), GSP_PHYS_PAGE_SIZE); copied++; @@ -16845,10 +19694,11 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, } if (pg->hpa && pg->qmd_word >= 32 && pg->qmd_word <= 47 && !(pg->qmd_word & 1)) - gsp_phys_gr_rewrite_data_vars((u32 *)sh_page, + (void)gsp_phys_gr_rewrite_data_vars( + (u32 *)sh_page, GSP_PHYS_PAGE_SIZE / 4, req, (pg->qmd_word - 32) / 2, - (u32 *)__va(pg->hpa)); + (u32 *)__va(pg->hpa), NULL, false); if (pg->hpa && pg->qmd_word == 40u && gsp_phys_rmcall_gr_mail_patch_value()) { u32 *w = (u32 *)sh_page; @@ -16884,7 +19734,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * data-access instruction and the guest VA arguments it * fetches can be read from the log. */ - if (pg->hpa && pg->qmd_word == 48u) { + if (pg->hpa && pg->qmd_word == 48u && !code_keep_guest) { u32 po = (qmd[48] & (GSP_PHYS_PAGE_SIZE - 1)) / 4; const u32 *w = (const u32 *)sh_page + po; @@ -16899,6 +19749,12 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, w[27], w[28], w[29], w[30], w[31], w[32], w[33], w[34], w[35], w[36], w[37], w[38], w[39]); + /* + * CODEHASH: the engine-visible code copy (sh_page) + * is authored here - sample source + shadow + * before the QMD pair points at this page. + */ + gsp_phys_gr_codehash(req, pg, sh_page); } if (pg->hpa && pg->qmd_word == 32u) { const u32 *w = (const u32 *)sh_page; @@ -16959,11 +19815,31 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * relocated values no longer match a guest data VA. */ if (pg->hpa && pg->qmd_word >= 32 && pg->qmd_word <= 47 && - !(pg->qmd_word & 1)) - gsp_phys_gr_rewrite_data_vars((u32 *)sh_page, + !(pg->qmd_word & 1)) { + u64 gf = 0; + u32 gg; + + /* + * GRFIX FIX3: this is the final authoring + * of the engine-visible page (after the + * inline replay), so its gap count is the + * launch's relocation-coverage measure. + * Log it (rate-limited) and accumulate it + * for the strict hold below. + */ + gg = gsp_phys_gr_rewrite_data_vars( + (u32 *)sh_page, GSP_PHYS_PAGE_SIZE / 4, req, (pg->qmd_word - 32) / 2, - (u32 *)__va(pg->hpa)); + (u32 *)__va(pg->hpa), &gf, true); + if (gg) { + reloc_gap += gg; + if (!reloc_gap_first) + reloc_gap_first = gf; + pr_info_ratelimited("vgsp-gr: RELOC-GAP words=%u first=0x%llx\n", + gg, gf); + } + } if (copied) { u32 sh_line = boot_stage.rmcall_guest_pt_next_line; @@ -16975,7 +19851,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, gsp_phys_rmcall_gr_shadow_win0_value() && boot_stage.rmcall_gr_shadow_win0.pages && boot_stage.rmcall_gr_shadow_win0_used < - GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS; + shadow_slot_limit; /* * m86: SASS override. When gsp-phys-gr-override @@ -17036,7 +19912,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, n_shadow, (u64)gsp_phys_chunk_page_dma(sh, n_shadow), - &sh_va); + &sh_va, NULL); } if (!ret) { bool keep_guest = @@ -17147,6 +20023,40 @@ gr_page_unmapped: } /* + * GRFIX FIX3 strict hold: with gsp_phys_gr_strict_reloc=1, a launch + * whose rewritten pages carry device-VA-shaped words that match no + * tracked data page is refused before publish - the word stays what + * it was (an unrelocated guest VA), but the engine never sees the + * QMD, so it can never walk an untranslatable pointer. The refusal + * rides the existing backend-failure path (caller bridge fallback + + * the sema service's recorded-release hold; bounded, never a hang). + * The hold is opt-in because the shape heuristic can false-positive + * on non-pointer data words; the RELOC-GAP line (emitted with the + * parameter off as well) is the false-positive evidence. + */ + if (reloc_gap && gsp_phys_gr_strict_reloc_param) { + mutex_unlock(&gsp_phys_rm_lock); + gsp_phys_gr_refused++; + pr_err("vgsp-gr: RELOC-GAP-STRICT words=%u first=0x%llx; launch held fail-closed (unrelocated device-VA words never reach the engine)\n", + reloc_gap, reloc_gap_first); + ret = -EAGAIN; + goto out; + } + /* + * COPYWIT fail4: one per-launch census of the rewrite outcome so the + * resolved/unresolved split of every bank pass is provable from a + * single line (gaps left as guest VAs vs late-resolved words authored + * via RELOC-FIXED). Rate-limited like the per-word lines. + */ + if (gsp_phys_gr_copywit_enabled() && + (reloc_gap || gsp_phys_gr_reloc_fixed)) + pr_info_ratelimited("vgsp-gr: RELOC-CENSUS gaps=%u first_gap=0x%llx fixed=%llu first_fixed=0x%llx late_pages=%u\n", + reloc_gap, reloc_gap_first, + gsp_phys_gr_reloc_fixed, + gsp_phys_gr_reloc_fixed_first, + gsp_phys_gr_gap_n); + + /* * Rank-2 qmdwin (gsp_phys_rmcall_gr_qmdwin=1): relocate the inline-QMD * destination to the dedicated module page planted in the ladder's * window-0 small-PT line (VA 0x107000). The PTE was written by @@ -17163,11 +20073,16 @@ gr_page_unmapped: NULL; if (boot_stage.rmcall_gr_qmdwin_va && qw) { + qw_bank = (epoch & 1u) % + GSP_PHYS_RMCALL_GR_QMDWIN_SLOTS; + qmdwin_on = true; - qmd_dst = boot_stage.rmcall_gr_qmdwin_va; - qmd_dst_mem = qw; - pr_info("vgsp-gr: qmdwin: relocating inline-QMD dest 0x%llx -> 0x%llx (module ring chunk page)\n", - req->qmd_va, qmd_dst); + qmd_dst = boot_stage.rmcall_gr_qmdwin_va + + (u64)qw_bank * GSP_PHYS_PAGE_SIZE; + qmd_dst_mem = gsp_phys_chunk_page_va( + &boot_stage.rmcall_gr_qmdwin, qw_bank); + pr_info("vgsp-gr: qmdwin: relocating inline-QMD dest 0x%llx -> 0x%llx (module ring chunk page, bank %u)\n", + req->qmd_va, qmd_dst, qw_bank); } else { pr_warn("vgsp-gr: qmdwin=1 but no planted module page (stage-3 plant skipped or failed); using the guest QMD destination\n"); } @@ -17197,7 +20112,7 @@ gr_page_unmapped: if (n_shadow < GSP_PHYS_RMCALL_GR_SHADOW_PAGES && !gsp_phys_rmcall_gr_map_shadow_locked( - n_shadow, req->qmd_hpa, &sh_va)) { + n_shadow, req->qmd_hpa, &sh_va, NULL)) { qmd_dst_mem = (u8 *)gsp_phys_chunk_page_va( &boot_stage.rmcall_gr_shadow, n_shadow) + qmd_off; @@ -17240,7 +20155,8 @@ gr_page_unmapped: u64 sh_va = 0; ret = gsp_phys_rmcall_gr_map_shadow_locked(n_shadow, - req->sem_hpa, &sh_va); + req->sem_hpa, &sh_va, + NULL); if (!ret) { n_shadow++; ch_rel_hpa = req->sem_hpa; @@ -17258,6 +20174,27 @@ gr_page_unmapped: /* A stale release from a previous submission must not be observed. */ WRITE_ONCE(*(u32 *)sem_va, 0); WRITE_ONCE(*(u32 *)(sem_va + 4), 0); + /* + * r441: zero the GUEST-VISIBLE release words for the first attempt + * too, not only for outcome retries. The r439 evidence shows a first + * attempt whose poll broke on a STALE value (qmd_rel=0x00000005 from + * the previous launch's engine write, want=6) while the engine never + * fetched this launch's deposit (sentinel intact, "engine fetch + * NONE"): the host then shipped a completion for a launch that did + * not run. Zeroing forces the poll to wait for THIS launch's own + * engine write; when the engine does not write it, the timeout ladder + * and the software-authored completion take over (the honest path), + * and the wedge classification stops being satisfiable by a stale + * value. The write guard re-validates the target like the retry path. + */ + if (qmd_rel_hpa && qmd_rel_off < GSP_PHYS_PAGE_SIZE && + gsp_sema_gr_write_guard(req->rel_va, qmd_rel_hpa, + "gr-rel-zero-first")) + WRITE_ONCE(*(u32 *)((u8 *)__va(qmd_rel_hpa) + qmd_rel_off), 0); + if (ch_rel_hpa && ch_rel_off < GSP_PHYS_PAGE_SIZE && + gsp_sema_gr_write_guard(req->rel_va, ch_rel_hpa, + "gr-chrel-zero-first")) + WRITE_ONCE(*(u32 *)((u8 *)__va(ch_rel_hpa) + ch_rel_off), 0); wmb(); /* @@ -17284,6 +20221,20 @@ gr_page_unmapped: pr_info("vgsp-gr: qmd w0..3=%08x %08x %08x %08x w4..7=%08x %08x %08x %08x w23..25=%08x %08x %08x w32/33=%08x %08x w48/49=%08x %08x\n", qmd[0], qmd[1], qmd[2], qmd[3], qmd[4], qmd[5], qmd[6], qmd[7], qmd[23], qmd[24], qmd[25], qmd[32], qmd[33], qmd[48], qmd[49]); + /* + * r430: full 64-word deposit dump for the wedge's healthy-vs-wedged + * QMD diff (hardware-only fields w50/w51 INNER_GET/PUT, w52 span-list, + * w53 next-QMD pointer are not in the partial line above). Gated by + * the skedrec param (default on); one long line per launch. + */ + if (gsp_phys_rmcall_gr_skedrec_value()) { + char qb[64 * 9 + 1]; + int qo = 0; + + for (i = 0; i < GSP_PHYS_GR_QMD_WORDS && qo < (int)sizeof(qb) - 10; i++) + qo += scnprintf(qb + qo, sizeof(qb) - qo, "%08x ", qmd[i]); + pr_info("vgsp-gr: QMD64 %s\n", qb); + } /* m120: dump the main PT page slots 0x100..0x11f so the PTE content * the walker would read (win0 + bounce window) is visible in the log. */ { @@ -17332,38 +20283,125 @@ gr_page_unmapped: * removes the race entirely; per-page checksums are kept so the * copy-out can prove whether the engine modified the window at all. */ - if (req->n_data && boot_stage.rmcall_gr_bounce.pages) { + if (req->n_data && gsp_phys_gr_data_chunk()->pages) { + struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); u32 dcopied = 0, dfailed = 0, di; + /* + * GRFIX FIX2 (copy-in fail-closed): classify every data page + * BEFORE the bounce copy. An entry with no resolved HPA, no + * bounce index or no backing bounce page cannot be freshly + * filled for this launch; publishing it would hand the engine + * a QMD data pointer into a bounce page still holding a + * previous launch's bytes (the multi-page stale-output + * class). Hold on the existing backend-refusal path: the + * caller's bridge fallback and the sema service's recorded- + * release hold apply the software releases - bounded, never + * a hang - and the engine never sees the QMD. + */ for (di = 0; di < req->n_data; di++) { - u32 bi = (u32)((req->data[di].reloc_va - - GSP_PHYS_RMCALL_GR_BOUNCE_VA0) >> - 12); - const u8 *src; - u8 *dst; - u32 ci = 0, csum = 0; + u32 bi = gsp_phys_gr_data_bi(req->data[di].reloc_va); - if (bi >= GSP_PHYS_RMCALL_GR_MAIL_SLOT || - !req->data[di].hpa) { + if (bi == GSP_PHYS_GR_DATA_BI_NONE || + !req->data[di].hpa || + !gsp_phys_chunk_page_va(dchunk, bi)) dfailed++; - continue; - } - src = (const u8 *)__va(req->data[di].hpa); - dst = gsp_phys_chunk_page_va( - &boot_stage.rmcall_gr_bounce, bi); - memcpy(dst, src, GSP_PHYS_PAGE_SIZE); - for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) - csum += get_unaligned_le32(dst + ci); - bounce_csum_in[bi] = csum; - dcopied++; } - dma_wmb(); + if (dfailed) { + mutex_unlock(&gsp_phys_rm_lock); + gsp_phys_gr_refused++; + pr_err("vgsp-gr: copy-in hold: %u/%u data pages unresolved at copy-in (no fresh bounce content); launch held fail-closed, no publish\n", + dfailed, req->n_data); + ret = -EAGAIN; + goto out; + } + dcopied = gsp_phys_gr_fill_bounce(req, bounce_csum_in, + bounce_staged_hpa); pr_info("vgsp-gr: bounce copy-in: %u pages, %u skipped (pre-publish)\n", dcopied, dfailed); - for (di = 0; di < req->n_data && di < 3; di++) - pr_info("vgsp-gr: data[%u] va=0x%llx hpa=0x%llx reloc=0x%llx\n", + /* + * r234 A-TRACE pt=P3 (engine-read point): small launches get + * the source-page content of their first recs - the exact + * bytes the copy-in stages and the engine will read. The probe + * A buffer is 16 pages of ints 0..16383 (w0=0 w1=1 w2=2 ...); + * any point in the pipeline where that pattern breaks names + * the bug site. + */ + if (req->n_data <= 1024u && gsp_phys_gr_atrace_budget < 128u) { + u32 d; + + for (d = 0; d < req->n_data && d < 16u; d++) { + const u32 *pp = + (const u32 *)__va(req->data[d].hpa); + + if (!pp || gsp_phys_gr_atrace_budget >= 128u) + break; + gsp_phys_gr_atrace_budget++; + pr_info("vgsp: A-TRACE pt=P3 va=0x%llx hpa=0x%llx w0..15=%08x %08x %08x %08x %08x %08x %08x %08x | %08x %08x %08x %08x %08x %08x %08x %08x\n", + req->data[d].va, req->data[d].hpa, + pp[0], pp[1], pp[2], pp[3], pp[4], + pp[5], pp[6], pp[7], pp[8], pp[9], + pp[10], pp[11], pp[12], pp[13], + pp[14], pp[15]); + } + } + if ((u64)req->n_data << 12 >= GSP_PHYS_GR_AUDIT_MIN) + pr_info("vgsp: COPYAUDIT path=gr-bounce-copy-in va=0x%llx len=0x%llx done=0x%llx\n", + req->data[0].va, (u64)req->n_data << 12, + (u64)dcopied << 12); + if (dcopied < req->n_data) + pr_err_ratelimited("vgsp: COPY-TRUNC va=0x%llx want=0x%llx done=0x%llx path=gr-bounce-copy-in\n", + req->data[0].va, + (u64)req->n_data << 12, + (u64)dcopied << 12); + for (di = 0; di < req->n_data && di < 3; di++) { + u32 w0 = 0, w1 = 0; + + /* + * fail4 definitive witness: the bytes the engine will + * read for the first recs (the rec hpa's content) + * beside the rec identity - names a +2 MB rec-page + * phase directly (client a[0..3] = 00000000 + * 0000003f 0000003f... would appear as exp[2M] + * values if the hpa points 512 pages late). + */ + if (req->data[di].hpa) { + const u32 *p = (const u32 *)__va( + req->data[di].hpa); + + w0 = p[0]; + w1 = p[1]; + } + pr_info("vgsp-gr: data[%u] va=0x%llx hpa=0x%llx reloc=0x%llx first8=%08x %08x\n", di, req->data[di].va, req->data[di].hpa, - req->data[di].reloc_va); + req->data[di].reloc_va, w0, w1); + } + } + + /* + * relfix: the late-resolved pages copy in exactly like the tracked + * data pages (same m142 pre-doorbell ordering). The word rewrites + * already point the engine at these bounce pages; filling them here + * is what makes those pointers real. Still under gsp_phys_rm_lock, + * like the block above. + */ + if (gsp_phys_gr_gap_n && gsp_phys_gr_data_chunk()->pages) { + u32 gcopied = gsp_phys_gr_fill_gap_bounce(bounce_csum_in, + bounce_staged_hpa); + + pr_info("vgsp-gr: gap copy-in: %u late-resolved pages, %llu word(s) fixed first=0x%llx\n", + gsp_phys_gr_gap_n, gsp_phys_gr_reloc_fixed, + gsp_phys_gr_reloc_fixed_first); + if ((u64)gsp_phys_gr_gap_n << 12 >= GSP_PHYS_GR_AUDIT_MIN) + pr_info("vgsp: COPYAUDIT path=gr-gap-copy-in va=0x%llx len=0x%llx done=0x%llx\n", + gsp_phys_gr_gap[0].va, + (u64)gsp_phys_gr_gap_n << 12, + (u64)gcopied << 12); + if (gcopied < gsp_phys_gr_gap_n) + pr_err_ratelimited("vgsp: COPY-TRUNC va=0x%llx want=0x%llx done=0x%llx path=gr-gap-copy-in\n", + gsp_phys_gr_gap[0].va, + (u64)gsp_phys_gr_gap_n << 12, + (u64)gcopied << 12); } ret = gsp_phys_rmcall_publish_locked(bar0, boot_stage.rmcall_va_pb, @@ -17445,24 +20483,37 @@ gr_page_unmapped: gsp_phys_rmcall_gr_gpc_mmu_init(bar0); /* * m143 DIAGNOSTIC (second-client launch-state divergence): dump the - * exact param page the FE/SM will read (win0 slot 0, the QMD bank-0 - * target) and the guest param page at the three JIT parameter slots, - * right before the doorbell. The JIT reads a@0x160/b@0x168/c@0x170 - * and n@0x178 (verified with ptxas -arch=sm_75). If the win0 copy is - * correct here but the engine still stores to the wrong relocated - * window, the divergence is engine-side (a stale mapping), not the - * relaunched page content. + * exact param page the FE/SM will read (this launch's win0 bank base + * slot, the QMD bank-0 target) and the guest param page at the three + * JIT parameter slots, right before the doorbell. The JIT reads + * a@0x160/b@0x168/c@0x170 and n@0x178 (verified with ptxas + * -arch=sm_75). If the win0 copy is correct here but the engine still + * stores to the wrong relocated window, the divergence is engine-side + * (a stale mapping), not the relaunched page content. */ if (boot_stage.rmcall_gr_shadow_win0.pages) { const u8 *wp = gsp_phys_chunk_page_va( - &boot_stage.rmcall_gr_shadow_win0, 0); + &boot_stage.rmcall_gr_shadow_win0, win0_bank_base); - if (wp) - pr_info("vgsp-gr: DIAG win0[0] 160=%llx 168=%llx 170=%llx 178=%llx\n", + if (wp) { + u32 off; + + pr_info("vgsp-gr: DIAG win0[%u] 160=%llx 168=%llx 170=%llx 178=%llx\n", + win0_bank_base, (u64)get_unaligned_le64(wp + 0x160), (u64)get_unaligned_le64(wp + 0x168), (u64)get_unaligned_le64(wp + 0x170), (u64)get_unaligned_le64(wp + 0x178)); + pr_info("vgsp-gr: win0 param hash pre=0x%016llx\n", + gsp_phys_gr_fnv1a64(wp, GSP_PHYS_PAGE_SIZE)); + for (off = 0x100u; off < 0x200u; off += 0x20u) + pr_info("vgsp-gr: win0 page +0x%03x: %016llx %016llx %016llx %016llx\n", + off, + (u64)get_unaligned_le64(wp + off), + (u64)get_unaligned_le64(wp + off + 8), + (u64)get_unaligned_le64(wp + off + 16), + (u64)get_unaligned_le64(wp + off + 24)); + } } { u32 pi; @@ -17474,12 +20525,15 @@ gr_page_unmapped: !req->pages[pi].hpa) continue; gp = (const u8 *)__va(req->pages[pi].hpa); + guest_param_pg = gp; pr_info("vgsp-gr: DIAG guest-param pa=0x%llx 160=%llx 168=%llx 170=%llx 178=%llx\n", req->pages[pi].hpa, (u64)get_unaligned_le64(gp + 0x160), (u64)get_unaligned_le64(gp + 0x168), (u64)get_unaligned_le64(gp + 0x170), (u64)get_unaligned_le64(gp + 0x178)); + pr_info("vgsp-gr: guest-param hash pre=0x%016llx\n", + gsp_phys_gr_fnv1a64(gp, GSP_PHYS_PAGE_SIZE)); break; } } @@ -17489,13 +20543,15 @@ gr_page_unmapped: gsp_phys_rmcall_gr_prelude_value(), gsp_phys_rmcall_gr_qmdvirt_value(), boot_stage.rmcall_token, - qmd_rel, want_ch, GSP_PHYS_RMCALL_GR_POLL_MS, qmdwin_note); + qmd_rel, want_ch, poll_ms, qmdwin_note); if (qmd_win_diag) - pr_info("vgsp-gr: qmd dest va=0x%llx pdeIndex=%u installed=%u win_mask=0x%x ensure_ret=%d\n", + pr_info("vgsp-gr: qmd dest va=0x%llx pdeIndex=%u installed=%u win_mask=0x%llx ensure_ret=%d\n", qmd_dst, qmd_win_pde, qmd_win_installed, - boot_stage.rmcall_guest_pt_win_mask, qmd_win_ret); + (unsigned long long)boot_stage.rmcall_guest_pt_win_mask, + qmd_win_ret); - deadline = jiffies + msecs_to_jiffies(GSP_PHYS_RMCALL_GR_POLL_MS); + poll_start_us = (u32)ktime_to_us(ktime_get()); + deadline = jiffies + msecs_to_jiffies(poll_ms); for (;;) { /* * m72: SKED transition recorder - sample the CWD/HWW registers @@ -17503,8 +20559,14 @@ gr_page_unmapped: * cancel or dispatch sequence of the launch window is on the * record even when the poll times out. */ - if (gsp_phys_rmcall_gr_skedrec_value() && bar0 && - (++sked_tick & 7u) == 0u) { + if (gsp_phys_rmcall_gr_skedrec_value() && bar0) { + /* + * r430: sample EVERY poll iteration (not every 8th): + * the wedge's release word lands ~7 ms after the + * doorbell, so the old 8-iteration gate saw at most one + * sample of the dispatch window. Changes only. + */ + sked_tick++; sked_sample[0] = ioread32(bar0 + 0x00407060u); sked_sample[1] = ioread32(bar0 + 0x00407064u); sked_sample[2] = ioread32(bar0 + 0x00407068u); @@ -17520,7 +20582,7 @@ gr_page_unmapped: memcmp(sked_sample, sked_prev, sizeof(sked_sample))) { pr_info("vgsp-gr: sked rec +%ums cwd0=%08x cwd1=%08x cwd2=%08x info1=%08x info2=%08x act=%08x esr=%08x grfifo=%08x fecs_cur=%08x fecs_new=%08x ctxsw1=%08x\n", - (sked_tick / 8u) * + sked_tick * GSP_PHYS_RMCALL_POLL_MS, sked_sample[0], sked_sample[1], sked_sample[2], sked_sample[3], @@ -17545,17 +20607,99 @@ gr_page_unmapped: ch_rel_off)); else w1 = READ_ONCE(*(u32 *)(sem_va + 4)); - if ((!qmd_rel || w0 == want_qmd) && w1 == want_ch) + if ((!qmd_rel || w0 == want_qmd) && w1 == want_ch) { + /* F4-a: first poll break + USERD witnesses. */ + gsp_phys_gr_poll_break_log(w0, want_qmd, w1, + poll_start_us, &sub); break; + } + /* + * STALLFREE: the whole-launch budget. Checked after the release + * acks (a completed launch always wins) and before any further + * ladder work; on expiry the submit fails out now and the sema + * service's failure edge authors the QMD completion release in + * software, so the guest client is never left waiting on a + * launch this host will not finish. + */ + if (gsp_phys_gr_wait_budget_over(start)) { + gsp_phys_gr_stall_wait_max++; + gsp_phys_gr_stall_log("wait-max", req->push_va, start, + "fail"); + pr_err("vgsp-gr: GR LAUNCH FAILED: wait budget crossed (qmd_rel=0x%08x want=0x%08x ch_rel=0x%08x want=0x%08x) h=0x%x push=0x%llx; ladder stopped\n", + w0, want_qmd, w1, want_ch, req->h_object, + req->push_va); + ret = -ETIMEDOUT; + goto out; + } if ((++grpoll_n % 100) == 0) pr_info("vgsp-gr: poll +%ums qmd_rel=0x%08x ch_rel=0x%08x\n", grpoll_n * GSP_PHYS_RMCALL_POLL_MS, w0, w1); if (time_after_eq(jiffies, deadline)) { gsp_phys_gr_timed_out++; pr_err("vgsp-gr: GR LAUNCH FAILED: timeout after %u ms (qmd_rel=0x%08x want=0x%08x ch_rel=0x%08x want=0x%08x) h=0x%x push=0x%llx\n", - GSP_PHYS_RMCALL_GR_POLL_MS, w0, want_qmd, w1, + poll_ms, w0, want_qmd, w1, want_ch, req->h_object, req->push_va); /* + * F4-b: one-shot deposit readback at the FIRST timeout + * (the old post-mortem only ran after attempt 3, which + * a recovered launch never reached). Reading the + * qmdwin/shadow destination here proves whether THIS + * attempt's inline-QMD copy landed and what the engine + * would fetch (w63 is the fetch sentinel). Bound: one + * line per launch. + */ + if (!deposit_rb_done) { + deposit_rb_done = true; + gsp_phys_gr_deposit_rb_log(req, qmd_dst_mem); + } + /* + * F2: fast re-arm at the first timeout. The m155 + * extended re-arm (lane-enable replay + GPFIFO/TSG + * schedule + doorbell) plus a FRESH ring entry recovers + * the r198 class in ~11 s only because it runs after two + * full 5 s windows. Do it now and re-poll briefly; if it + * still fails, fall through to the historical ladder + * (the fast entry counts against the same attempt + * budget, so the total number of host publishes and the + * ring wrap exposure stay as before). + */ + if (gsp_phys_gr_retry_fast_value() && !fast_rearmed) { + int fr; + + fast_rearmed = true; + attempts++; + mutex_lock(&gsp_phys_rm_lock); + fr = gsp_phys_rmcall_publish_locked( + bar0, boot_stage.rmcall_va_pb, words, + &sub); + if (!fr) + fr = gsp_phys_rmcall_gr_resched_locked( + bar0); + mutex_unlock(&gsp_phys_rm_lock); + if (!fr) { + gsp_phys_gr_retries++; + poll_ms = + GSP_PHYS_RMCALL_GR_FAST_RETRY_MS; + pr_info("vgsp-gr: fast re-arm after first timeout: extended re-arm + fresh ring entry slot=%u published; re-polling %u ms (attempts %u/%u)\n", + sub.slot, poll_ms, attempts, + GSP_PHYS_RMCALL_GR_ATTEMPTS); + deadline = jiffies + + msecs_to_jiffies(poll_ms); + /* + * The immediate m142 re-arm would + * only duplicate the resched that just + * ran; the ladder below still owns the + * warm-sem gate and the remaining + * fresh entries. + */ + rearmed = true; + grpoll_n = 0; + continue; + } + pr_info("vgsp-gr: fast re-arm failed rc=%d; falling through to the timeout ladder\n", + fr); + } + /* * m142: one bounded re-arm before declaring the launch * dead. Re-issue the stage-4 schedule controls and re-ring * the doorbell, then poll again; if the channel was merely @@ -17570,17 +20714,81 @@ gr_page_unmapped: rr = gsp_phys_rmcall_gr_resched_locked(bar0); mutex_unlock(&gsp_phys_rm_lock); if (!rr) { + poll_ms = GSP_PHYS_RMCALL_GR_POLL_MS; pr_info("vgsp-gr: rearm after timeout: schedule re-issued + doorbell re-rung; re-polling %u ms\n", - GSP_PHYS_RMCALL_GR_POLL_MS); + poll_ms); deadline = jiffies + - msecs_to_jiffies( - GSP_PHYS_RMCALL_GR_POLL_MS); + msecs_to_jiffies(poll_ms); grpoll_n = 0; continue; } pr_info("vgsp-gr: rearm after timeout failed rc=%d\n", rr); } + /* + * m166: liveness gate before spending the fresh-entry + * retries. The m156 warm sem push (the stage-4 GR sem + * block) is the only proven "engine alive" probe; if + * it times out, the engine is wedged for the boot and + * every further retry is futile - fail fast instead of + * burning ~20 s with the single-inflight slot held + * (r167 memset2d: 8 internal ladder timeouts over 31 s + * for an entry the front-end never fetched). The probe + * itself is bounded by the sem push's own poll. + */ + if (!sem_gated) { + int sp_ret; + + sem_gated = true; + mutex_lock(&gsp_phys_rm_lock); + sp_ret = gsp_phys_rmcall_gr_sem_push_locked(bar0); + mutex_unlock(&gsp_phys_rm_lock); + if (sp_ret) { + gsp_phys_gr_sem_wedged++; + pr_info("vgsp-gr: timeout ladder: warm sem WEDGED (rc=%d) - failing fast instead of fresh-entry retries\n", + sp_ret); + break; + } + pr_info("vgsp-gr: timeout ladder: warm sem ALIVE - continuing fresh-entry retries\n"); + } + /* + * The re-arm above only replays the schedule verbs; if the + * entry was already fetched or the front-end never latched + * it, that cannot make the QMD run. Re-publish a FRESH ring + * entry (same PB content and QMD) with a re-issued schedule + * + doorbell, bounded. The timeout path never copied the + * result out to the guest, so a timed-out attempt did not + * reach the client; the guest therefore still sees only the + * successful attempt. + */ + if (attempts < GSP_PHYS_RMCALL_GR_ATTEMPTS) { + int pr_ret; + + attempts++; + mutex_lock(&gsp_phys_rm_lock); + pr_ret = gsp_phys_rmcall_publish_locked( + bar0, boot_stage.rmcall_va_pb, words, + &sub); + if (!pr_ret) + pr_ret = gsp_phys_rmcall_gr_resched_locked( + bar0); + mutex_unlock(&gsp_phys_rm_lock); + if (!pr_ret) { + gsp_phys_gr_retries++; + poll_ms = GSP_PHYS_RMCALL_GR_RETRY_MS; + pr_info("vgsp-gr: retry %u/%u: fresh ring entry slot=%u published + rescheduled; re-polling %u ms\n", + attempts, + GSP_PHYS_RMCALL_GR_ATTEMPTS, + sub.slot, poll_ms); + deadline = jiffies + + msecs_to_jiffies(poll_ms); + rearmed = false; + grpoll_n = 0; + continue; + } + pr_info("vgsp-gr: retry %u publish failed rc=%d\n", + attempts, pr_ret); + } if (gsp_phys_gr_fault_active()) { struct gsp_phys_fault_info fi; u32 fep = 0; @@ -17871,39 +21079,764 @@ gr_page_unmapped: gsp_phys_gr_ok++; /* + * m151: retire the output footprint before the copy-out (and before + * any later launch's copy-in). qmd_rel above is a schedule/fetch ack, + * not a data-path retirement, so the engine's stores into the bounce + * pages can land after it and be lost by an early copy-out; the next + * launch's copy-in would also race that write tail. Success path + * only - the timeout/fault paths never copied out. See + * gsp_phys_gr_output_retire_wait(). + * + * m153: outcome-verifying retry. The intermittent per-boot + * no-execution class (qmd_rel/ch_rel satisfied but the engine wrote + * nothing at all) cannot be caught by the completion poll; the + * retirement wait is the only execution witness. On zero writes, + * refill the bounce window from the guest inputs, re-publish the + * same method block on a fresh ring entry, re-issue the schedule and + * re-poll, then re-wait. + * + * m179: the budget is now DEDICATED (gsp_phys_gr_nowrite_retries, + * default 3) instead of sharing the timeout ladder's attempt counter - + * a launch that had already spent attempts on the ladder used to reach + * the accept path with no re-execution at all (the r200-r202 + * memcpy_roundtrip / multikernel all-wrong class). The trigger is + * specific: only a launch with n_tracked > 0 tracked data pages can be + * a no-execution, so a launch whose data entries resolve nothing + * legitimately reports zero writes without entering this path. After + * the budget the launch proceeds to copy-out with whatever the engine + * produced, and the accept is logged loudly (pr_warn) and classified + * by the QMD fetch sentinel (CONFIRMED lost vs AMBIGUOUS): a + * genuinely read-only kernel is indistinguishable from a + * no-execution here and simply pays the bounded extra polls (safe to + * re-execute by construction - a kernel that writes nothing has no + * side effects to duplicate). A + * wrong-place write detector was tried and removed: the highest-reloc + * data entry is often a UVM-internal buffer, not the output, so it + * false-fired on correct launches and would re-execute + * side-effecting kernels (atomics) up to 3x. + */ + if (req->n_data && gsp_phys_gr_data_chunk()->pages) { + u32 n_tracked = 0; + u32 nw_attempts = 0; + bool idem = false; + u32 wrote = gsp_phys_gr_output_retire_wait(req, bounce_csum_in, + &n_tracked); + + /* + * r434: post-launch hardware-only QMD dump (w50/w51 INNER + * get/put = the SKED's CTA dispatch counters) so the + * healthy-vs-wedged diff has the fields the submission-time + * dump cannot show. + */ + gsp_phys_gr_qmd64_post(qmd_dst_mem, "retire"); + + /* + * m179: the zero-write test is specific to launches with a + * non-empty TRACKED data set. A launch whose data entries do + * not resolve to any bounce page legitimately reports zero + * writes; treating that as a lost launch would re-execute a + * kernel for no reason (a false positive). Only + * n_tracked > 0 makes "wrote == 0" a possible no-execution + * signature. + */ + if (wrote == 0 && n_tracked) { + bool ff = gsp_phys_gr_qmd_fetched(qmd_dst_mem); + + pr_info("vgsp-gr: first-attempt witness: engine fetch %s (qmd sentinel %s) tracked=%u\n", + ff ? "OK" : "NONE", + ff ? "overwritten" : "intact", n_tracked); + } + /* + * r427 idempotent fast-accept: a zero-write launch whose identity + * (QMD minus release0 + staged content checksums) matches an + * earlier WRITING launch of this boot is a legitimate idempotent + * re-launch - the result is already in place. Accept immediately: + * no retries, no quiet holds. (See gsp_phys_gr_launch_fp.) + */ + if (wrote == 0 && n_tracked) { + u64 fp = gsp_phys_gr_launch_fp(qmd, req, bounce_csum_in); + + if (gsp_phys_gr_idem_match(fp)) { + pr_warn("vgsp-gr: IDEMPOTENT-ACCEPT h=0x%x push=0x%llx fp=0x%llx tracked=%u (same QMD+content as an earlier writing launch; no retries, no hold)\n", + req->h_object, req->push_va, fp, + n_tracked); + idem = true; + } + } + if (wrote == 0 && n_tracked && !idem && + gsp_phys_rmcall_gr_nowrite_retry_value()) { + u32 nw_retries = + (u32)gsp_phys_gr_nowrite_retries_value(); + /* + * r410: the sticky per-lane wedge (previous launch + * also no-wrote while the lane stays busy) is only + * cleared by idleness (r409 ~37 s, r416 ~52 s), so a + * back-to-back retry is futile and keeps it alive. + * When the streak says the lane is wedged, hold a + * quiet window with NO submissions before the retry + * (base on the first retry, doubled on the second), + * then let the normal fresh-bank + resched retry run. + * The quiet path relaxes the whole-launch budget + * (its own hold is the bound: 2 holds max) - the + * alternative is accepting every launch silently. + */ + u32 quiet_used = 0; + + while (wrote == 0 && nw_attempts < nw_retries) { + u32 quiet_ms = 0; + + if ((READ_ONCE(gsp_phys_gr_nw_streak) > 0 || + READ_ONCE(gsp_phys_gr_wedge_episode)) && + nw_attempts <= 1) { + quiet_ms = + gsp_phys_gr_wedge_quiet_ms_value(); + if (quiet_ms && nw_attempts) + quiet_ms <<= 1; + if (quiet_ms > 240000u) + quiet_ms = 240000u; + } + /* + * STALLFREE: the same whole-launch budget caps + * the retirement retry chain. When it is crossed + * the loop stops and the launch PROCEEDS with + * its current outcome (the zero-write accept + * below restores the release words and logs + * loudly), so a chain of slow dud retries can + * never hold the client past the budget. The + * r410 quiet path is exempt: it is entered only + * for a launch whose previous launch also + * no-wrote, and its own holds are bounded. + */ + if (!quiet_ms && + gsp_phys_gr_wait_budget_over(start)) { + gsp_phys_gr_stall_retry++; + gsp_phys_gr_stall_log("retry", + req->push_va, start, + "proceed"); + break; + } + nw_attempts++; + pr_info("vgsp-gr: outcome retry %u/%u (wrote=%u tracked=%u): refill + re-publish\n", + nw_attempts, nw_retries, + wrote, n_tracked); + /* + * r445 wedge hard re-arm (gsp_phys_gr_wedge_rearm, + * default off): the r445 evidence shows every + * post-dispatch no-store episode leaves the warm + * ladder sem probe ALIVE (the ladder block executes) + * while the client launch keeps writing nothing, so + * the plain refill+re-publish retries are futile. + * On the FINAL retry attempt, force one context + * save/unload (TSG PREEMPT bWait=1) and re-run the + * mode-35 lane-enable tail + schedules before the + * re-publish, so the retried launch meets a freshly + * restored context. Bounded: one preempt per launch + * (2 s RPC timeout), only on the last retry. + */ + if (nw_attempts == nw_retries && + gsp_phys_gr_wedge_rearm_value()) { + mutex_lock(&gsp_phys_rm_lock); + gsp_phys_gr_wedge_hard_rearm_locked(bar0); + mutex_unlock(&gsp_phys_rm_lock); + } + if (quiet_ms) { + gsp_phys_gr_wedge_quiet++; + pr_warn("vgsp-gr: WEDGE-QUIET hold %u ms before outcome retry %u (push=0x%llx; %u consecutive zero-write launch(es); no submissions in between)\n", + quiet_ms, nw_attempts, + req->push_va, + READ_ONCE(gsp_phys_gr_nw_streak)); + /* + * r420: publish the deliberate hold so + * the sema service's completion + * deadlines (the 12 s GR release hold + * and the 3 s per-launch consumption + * bound) pause instead of authoring + * this launch's release word under it - + * the client must not be released + * before the post-hold copy-out + * (gsp_phys_gr_quiet_hold_active). + */ + WRITE_ONCE(gsp_phys_gr_quiet_hold_on, + true); + msleep(quiet_ms); + WRITE_ONCE(gsp_phys_gr_quiet_hold_on, + false); + quiet_used = quiet_ms; + pr_warn("vgsp-gr: WEDGE-QUIET hold done (push=0x%llx); lanes re-armed by the retry below\n", + req->push_va); + } + /* + * m156 (E5): on the first zero-write retry, push the + * proven stage-4 GR semaphore block before re-publishing. + * It is the discriminator the lottery report asked for: + * if the channel can still dispatch the ladder's own + * 10-dword sem block, the engine is alive and the failed + * launch is a launch-form race (retry can recover); if + * the sem push times out, the engine/channel is wedged + * for the boot and every further retry is futile - break + * immediately instead of burning ~20 s of futile polls + * (r155: 3 outcome retries + rearm all dead after the + * wedge). Both outcomes are logged distinctly. + */ + if (nw_attempts == 1) { + int sp_ret; + + mutex_lock(&gsp_phys_rm_lock); + sp_ret = gsp_phys_rmcall_gr_sem_push_locked( + bar0); + mutex_unlock(&gsp_phys_rm_lock); + pr_info("vgsp-gr: outcome retry %u: warm sem push rc=%d (engine %s)\n", + nw_attempts, sp_ret, + sp_ret ? "WEDGED - skipping further retries" : + "ALIVE"); + if (sp_ret) { + gsp_phys_gr_sem_wedged++; + break; + } + } + /* + * m160 (fresh-ack + fresh-deposit retry): the m153 + * retry re-published the same PB and then polled + * the release words WITHOUT clearing them - the + * previous attempt's ack (the identical payload) + * satisfied the poll instantly, so every retry + * burned only the retirement wait and concluded + * "engine wrote 0" without ever having waited for + * the engine to consume the re-published entry + * (r162: 3 dud retries, ~870 ms each, then + * accepted). Clear the guest release words (and + * the module semaphore when the QMD won't reach a + * guest page) so the poll below waits for a FRESH + * ack, and move the inline-QMD deposit to the + * other qmdwin bank (fresh epoch) and rebuild the + * method block for it, so the engine cannot have + * a cached fetch line or a stale deposit for the + * retry's entry. The mechanical sequence is + * exactly the one r162's ok=7 saga used when it + * recovered the no-execute launch (fresh ring + * entry + resched + fresh deposit). + */ + /* + * r378: re-validate the guest release page + * before zeroing it - the HPA was resolved at + * submission and the retry runs seconds later. + */ + if (qmd_rel_hpa && + gsp_sema_gr_write_guard(req->rel_va, + qmd_rel_hpa, + "gr-rel-zero")) + WRITE_ONCE(*(u32 *)((u8 *)__va(qmd_rel_hpa) + + qmd_rel_off), 0); + if (ch_rel_hpa) + WRITE_ONCE(*(u32 *)((u8 *)__va(ch_rel_hpa) + + ch_rel_off), 0); + WRITE_ONCE(*(u32 *)sem_va, 0); + WRITE_ONCE(*(u32 *)(sem_va + 4), 0); + wmb(); + /* + * F3 retry hygiene: recompute the win0 bank + * and slot limit for THIS attempt (they were + * fixed at the first submission's epoch) and + * re-copy the prepared param/program/bank + * shadow pages into the alternate bank before + * the re-publish. The qmdwin bank swap (below) + * is unchanged; when both are active they share + * the same fresh epoch. + */ + { + bool win0_on = + gsp_phys_rmcall_gr_shadow_win0_value() && + boot_stage.rmcall_gr_shadow_win0.pages; + bool win0_moved = false; + bool qmdwin_swap = qmdwin_on && + boot_stage.rmcall_gr_qmdwin.pages; + + if (qmdwin_swap || win0_on) { + epoch = ++boot_stage.rmcall_gr_submit_epoch; + if (!epoch) + epoch = ++boot_stage.rmcall_gr_submit_epoch; + boot_stage.rmcall_gr_shadow_win0_used = + (epoch & 1u) * + GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS; + if (win0_on) + win0_moved = + gsp_phys_gr_win0_retry_rearm( + qmd, req, + win0_bank_base, + boot_stage.rmcall_gr_shadow_win0_used, + boot_stage.rmcall_gr_shadow_win0_used + + GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS); + win0_bank_base = + boot_stage.rmcall_gr_shadow_win0_used; + shadow_slot_limit = + win0_bank_base + + GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS; + } + if (qmdwin_swap) { + qw_bank = (epoch & 1u) % + GSP_PHYS_RMCALL_GR_QMDWIN_SLOTS; + qmd_dst = + boot_stage.rmcall_gr_qmdwin_va + + (u64)qw_bank * + GSP_PHYS_PAGE_SIZE; + qmd_dst_mem = + gsp_phys_chunk_page_va( + &boot_stage.rmcall_gr_qmdwin, + qw_bank); + if (qmd_dst_mem) { + memcpy(qmd_dst_mem, qmd, + GSP_PHYS_GR_QMD_WORDS * 4); + put_unaligned_le32( + GSP_PHYS_GR_QMD_SENTINEL, + qmd_dst_mem + + GSP_PHYS_GR_QMD_SENTINEL_WORD * 4); + /* + * m179: order the fresh + * deposit before the + * re-publish/doorbell + * below (the first-attempt + * path has the same + * barrier; the publish tail + * also carries dma_wmb + + * wmb, this keeps the + * deposit->kick order + * explicit on the retry + * path). + */ + wmb(); + } + } + if (qmdwin_swap || win0_moved) { + words = gsp_phys_rmcall_build_gr_launch( + methods, req, qmd, + qmd_dst, ch_sem_va, + want_ch); + memcpy(gsp_phys_chunk_page_va(pb, 0), + methods, words * 4); + } + if (qmdwin_swap) + pr_info("vgsp-gr: outcome retry %u: release words zeroed + fresh qmdwin bank %u dest 0x%llx + win0 bank %u%s + PB rebuilt\n", + nw_attempts, qw_bank, + qmd_dst, win0_bank_base, + win0_moved ? + " (shadow pages re-copied)" : + ""); + else if (win0_moved) + pr_info("vgsp-gr: outcome retry %u: release words zeroed + fresh win0 bank %u (shadow pages re-copied) + PB rebuilt\n", + nw_attempts, win0_bank_base); + else + pr_info("vgsp-gr: outcome retry %u: release words zeroed (win0/qmdwin off)\n", + nw_attempts); + } + gsp_phys_gr_fill_bounce(req, bounce_csum_in, + bounce_staged_hpa); + gsp_phys_gr_fill_gap_bounce(bounce_csum_in, + bounce_staged_hpa); + mutex_lock(&gsp_phys_rm_lock); + ret = gsp_phys_rmcall_publish_locked( + bar0, boot_stage.rmcall_va_pb, + words, &sub); + if (!ret) + ret = gsp_phys_rmcall_gr_resched_locked( + bar0); + mutex_unlock(&gsp_phys_rm_lock); + if (ret) { + pr_info("vgsp-gr: outcome retry %u publish/resched failed rc=%d (accepting current outcome)\n", + nw_attempts, ret); + ret = 0; + break; + } + { + unsigned long dl = jiffies + + msecs_to_jiffies( + GSP_PHYS_RMCALL_GR_POLL_MS); + unsigned long ack_start = jiffies; + u32 rw0 = 0, rw1 = 0; + bool acked = false; + + for (;;) { + if (qmd_rel_hpa) + rw0 = READ_ONCE( + *(u32 *)((u8 *)__va( + qmd_rel_hpa) + + qmd_rel_off)); + else if (qmd_rel) + rw0 = READ_ONCE( + *(u32 *)sem_va); + if (ch_rel_hpa) + rw1 = READ_ONCE( + *(u32 *)((u8 *)__va( + ch_rel_hpa) + + ch_rel_off)); + else + rw1 = READ_ONCE( + *(u32 *)(sem_va + 4)); + if ((!qmd_rel || + rw0 == want_qmd) && + rw1 == want_ch) { + acked = true; + break; + } + if (time_after_eq(jiffies, dl)) + break; + msleep(GSP_PHYS_RMCALL_POLL_MS); + } + pr_info("vgsp-gr: outcome retry %u: fresh ack %s (qmd_rel=0x%08x want=0x%08x ch_rel=0x%08x after %u ms)\n", + nw_attempts, + acked ? "OK" : "TIMEOUT", + rw0, want_qmd, rw1, + jiffies_to_msecs(jiffies - + ack_start)); + } + /* + * m161: hardware fetch witness. The fresh-ack + * poll cannot distinguish the engine's release + * from the sema service's software release (its + * pending-release sweep re-applies the payload + * as soon as the word reads below it). The QMD + * page's sentinel word is written only by the + * engine's QMD fetch, so it is the one side + * effect the sweep cannot fake: intact after the + * wait = this deposit never reached the engine. + */ + if (qmd_dst_mem) { + unsigned long wdl = jiffies + + msecs_to_jiffies(1500); + bool fetched = false; + + while (time_before(jiffies, wdl)) { + if (gsp_phys_gr_qmd_fetched( + qmd_dst_mem)) { + fetched = true; + break; + } + msleep(5); + } + pr_info("vgsp-gr: outcome retry %u: engine fetch witness %s (qmd sentinel %s)\n", + nw_attempts, + fetched ? "OK" : "NONE", + fetched ? "overwritten" : "intact"); + } + wrote = gsp_phys_gr_output_retire_wait( + req, bounce_csum_in, &n_tracked); + gsp_phys_gr_qmd64_post(qmd_dst_mem, + "retry-retire"); + pr_info("vgsp-gr: outcome retry %u done: engine wrote %u page(s)\n", + nw_attempts, wrote); + /* r410: the quiet-window verdict. */ + if (quiet_used) { + if (wrote) { + gsp_phys_gr_wedge_heals++; + WRITE_ONCE(gsp_phys_gr_wedge_episode, + true); + pr_warn("vgsp-gr: WEDGE-QUIET healed: engine wrote %u page(s) after the %u ms hold (streak was %u)\n", + wrote, quiet_used, + READ_ONCE( + gsp_phys_gr_nw_streak)); + } else { + pr_warn("vgsp-gr: WEDGE-QUIET no heal: %u ms quiet and retry %u still wrote 0 page(s)\n", + quiet_used, nw_attempts); + } + quiet_used = 0; + } + } + } + /* + * m160/m179: if every fresh-ack retry still produced no engine + * writes, the release words may be left zeroed from the last + * retry. Restore the expected payloads so the guest client + * observes its completion and proceeds to its own result check + * (a genuinely lost launch then surfaces as the client's own + * FAIL, never as a hang). This accept runs even when the retry + * loop is disabled (gsp_phys_rmcall_gr_nowrite_retry=0) so a + * zero-write outcome is never silent. + */ + if (wrote == 0 && n_tracked && !idem) { + bool restored = false; + bool fetched = gsp_phys_gr_qmd_fetched(qmd_dst_mem); + + /* + * m179: the no-write accept is the last resort after + * the dedicated re-execution budget + * (gsp_phys_gr_nowrite_retries). It stays + * unconditional - the alternative is stalling the + * client forever - but it is now LOUD and classified + * by the hardware fetch witness: an intact sentinel + * means the deposit never reached the engine (a + * CONFIRMED lost launch, the sweep/hold-faked-ack + * class); an overwritten sentinel means the engine + * consumed the deposit, so the zero tracked writes + * may also be a legitimately read-only kernel + * (AMBIGUOUS). + */ + pr_warn("vgsp-gr: NOWRITE-%s h=0x%x push=0x%llx n_data=%u tracked=%u wrote=0 retries=%u qmd_rel=0x%x (engine fetch %s) consec=%u quiet=%llu heals=%llu\n", + fetched ? "AMBIGUOUS" : "CONFIRMED", + req->h_object, req->push_va, req->n_data, + n_tracked, nw_attempts, want_qmd, + fetched ? "OK" : "NONE", + READ_ONCE(gsp_phys_gr_nw_streak) + 1u, + gsp_phys_gr_wedge_quiet, + gsp_phys_gr_wedge_heals); + + /* + * m180 (lottery discriminator, agent-order H1/H2). + * Read BOTH qmdwin banks' sentinel words. Facts only: + * the other bank is normally the previous launch's + * deposit (legitimately consumed), so only THIS + * launch's sentinel classifies the event: intact = + * this deposit never reached the engine; overwritten + * = the engine consumed it (or something zeroed the + * word), so the zero stores are an engine-side + * non-execution of a fetched deposit, not a lost + * deposit. + */ + if (qmdwin_on && boot_stage.rmcall_gr_qmdwin.pages && + qmd_dst_mem) { + u8 *om = gsp_phys_chunk_page_va( + &boot_stage.rmcall_gr_qmdwin, + qw_bank ^ 1u); + + pr_warn("vgsp-gr: zero-write deposit stamp: bank=%u sentinel=0x%08x (%s) other-bank=%u %s\n", + qw_bank, + get_unaligned_le32( + qmd_dst_mem + + GSP_PHYS_GR_QMD_SENTINEL_WORD * 4), + fetched ? "this deposit fetched" : + "this deposit NOT fetched", + qw_bank ^ 1u, + om ? (gsp_phys_gr_qmd_fetched(om) ? + "sentinel overwritten (previous deposit)" : + "sentinel intact") : "n/a"); + } + /* + * (b) Deposit readback at the accept (m181): the same + * QMD-word dump the first-timeout path logs, so a + * zero-write launch's deposit content (w0/w4/w12/.../ + * w63) shows whether the engine consumed the header + * words or the page is untouched, without needing a + * timeout to have happened first. + */ + gsp_phys_gr_deposit_rb_log(req, qmd_dst_mem); + /* + * GRFIX FIX4 (agent-A decisive experiment): bounded + * real-GR register dump at the zero-write accept, one + * line per classified launch, using the same readl + * accessor and the same offsets as the read-only + * debugfs probe (gsp_phys_reg_probe_show): FE_HWW_ESR + * 0x404000, SKED_HWW_ESR 0x407020, PGRAPH_TRAPPED_ADDR + * 0x400704 + TRAPPED_DATA_LO/HI 0x400708/70c, + * FECS_IRQSTAT 0x409008, FECS_CUR/NEW_CTX 0x409b00/04. + * No new locks or sleeps: the submit context already + * holds gsp_phys_rm_lock exactly as the surrounding + * post-mortem reads do. Zero is a legitimate value for + * several of these (no context resident), so the line + * is classified with the sentinel/faultbuf evidence, + * not on its own. + */ + if (bar0) + pr_warn("vgsp-gr: NOWRITE-REGS fe_esr=0x%08x sked_esr=0x%08x trap=0x%08x/%08x/%08x fecs_irq=0x%08x ctx=0x%08x->0x%08x\n", + readl(bar0 + 0x00404000u), + readl(bar0 + 0x00407020u), + readl(bar0 + 0x00400704u), + readl(bar0 + 0x00400708u), + readl(bar0 + 0x0040070cu), + readl(bar0 + 0x00409008u), + readl(bar0 + 0x00409b00u), + readl(bar0 + 0x00409b04u)); + /* r425: the wedged-launch SKED/FECS baseline. */ + gsp_phys_gr_sked_wit(bar0, "accept"); + /* + * r425 stale-store witness: re-scan the last completed + * launch's first engine-changed page. A change since its + * copy-out proves a late store landed after the launch + * was shipped (a stuck/queued grid or a stale-QMD + * execution), the mechanism behind the "1 page" heals. + */ + if (gsp_phys_gr_prev_out_valid && + bounce_staged_hpa[gsp_phys_gr_prev_out_bi] == 0 && + gsp_phys_gr_data_chunk()->pages) { + const u8 *pp = gsp_phys_chunk_page_va( + gsp_phys_gr_data_chunk(), + gsp_phys_gr_prev_out_bi); + u64 pc; + + if (pp) { + pc = gsp_phys_gr_fnv1a64(pp, GSP_PHYS_PAGE_SIZE); + if (pc != gsp_phys_gr_prev_out_csum) + pr_warn("vgsp-gr: PREV-OUT-CHANGED bi=%u fp %016llx -> %016llx (late store after its launch shipped)\n", + gsp_phys_gr_prev_out_bi, + gsp_phys_gr_prev_out_csum, + pc); + } + } + /* + * (c) Late-store check: a store ramp that retires after + * the host's window is indistinguishable from a lost + * dispatch at the accept moment. Re-scan once at +1 s + * and report whether the tracked pages changed in the + * meantime; the copy-out below picks up any stores + * that arrived late, so this both classifies and heals + * the late sub-case. + */ + { + u32 late_tracked = 0; + u32 late; + + msleep(1000); + late = gsp_phys_gr_scan_pages(req, bounce_csum_in, + &late_tracked, NULL); + pr_warn("vgsp-gr: zero-write late-check: %u/%u pages changed after +1000 ms (%s)\n", + late, late_tracked, + late ? "LATE-STORES - copy-out will ship them" : + "NO-STORES - dispatch lost or read-only kernel"); + } + + if (qmd_rel_hpa) { + u8 *rp = (u8 *)__va(qmd_rel_hpa) + qmd_rel_off; + + /* + * r378: same target re-validation as the + * retry zero above. + */ + if (READ_ONCE(*(u32 *)rp) != want_qmd && + gsp_sema_gr_write_guard(req->rel_va, + qmd_rel_hpa, + "gr-rel-restore")) { + WRITE_ONCE(*(u32 *)rp, want_qmd); + restored = true; + } + } + if (ch_rel_hpa) { + u8 *rp = (u8 *)__va(ch_rel_hpa) + ch_rel_off; + + if (READ_ONCE(*(u32 *)rp) != want_ch) { + WRITE_ONCE(*(u32 *)rp, want_ch); + restored = true; + } + } + if (restored) + pr_warn("vgsp-gr: no-write accept: release words restored (qmd=0x%08x ch=0x%08x); client's own check decides\n", + want_qmd, want_ch); + } + /* + * r427: record the launch identity of every launch that WROTE, + * so a later idempotent re-launch (zero writes, same identity) + * can accept immediately (see gsp_phys_gr_launch_fp). + */ + if (wrote && n_tracked) + gsp_phys_gr_idem_record( + gsp_phys_gr_launch_fp(qmd, req, bounce_csum_in)); + /* + * r410: streak bookkeeping for the quiet-window trigger. A + * launch with tracked data that wrote nothing extends the + * zero-write streak; any launch that wrote resets it. Runs + * even when the retry loop is disabled so the streak always + * reflects the accepted-outcome history. + */ + if (n_tracked) { + WRITE_ONCE(gsp_phys_gr_nw_streak, + wrote ? 0u : + READ_ONCE(gsp_phys_gr_nw_streak) + 1u); + /* r411: arm the wedge episode on any accepted + * zero-write launch, so the next zero-write launch + * quiets even after a heal reset the streak. */ + if (!wrote) + WRITE_ONCE(gsp_phys_gr_wedge_episode, true); + } + } + /* * m114: the kernel ran against the pre-commit bounce window; copy the * results back to the guest's real pages (identity HPAs in the * request). All mapped pages are written back; inputs are unchanged * by the kernels, so copying them back is a no-op. */ - if (req->n_data && boot_stage.rmcall_gr_bounce.pages) { - u32 wbi, wrote = 0, changed = 0; + if (req->n_data && gsp_phys_gr_data_chunk()->pages) { + struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); + u32 wbi, wrote = 0, changed = 0, eligible = 0; + u32 aliased = 0, unchanged = 0, skipped = 0; u32 chg[8] = { }; + u32 outwit = 0; /* COPYWIT: bounded per-launch witness lines */ for (wbi = 0; wbi < req->n_data; wbi++) { - u32 bi = (u32)((req->data[wbi].reloc_va - - GSP_PHYS_RMCALL_GR_BOUNCE_VA0) >> - 12); + u32 bi = gsp_phys_gr_data_bi(req->data[wbi].reloc_va); const u8 *src; - u32 ci, csum = 0; + u64 fp; - if (bi >= GSP_PHYS_RMCALL_GR_MAIL_SLOT || + if (bi == GSP_PHYS_GR_DATA_BI_NONE || !req->data[wbi].hpa) continue; - src = gsp_phys_chunk_page_va( - &boot_stage.rmcall_gr_bounce, bi); - for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) - csum += get_unaligned_le32(src + ci); - if (csum != bounce_csum_in[bi]) { + src = gsp_phys_chunk_page_va(dchunk, bi); + if (!src) + continue; + eligible++; + /* + * r378: only the page whose content was staged into + * this slot this launch may be written back. A rec + * aliasing another rec's reloc slot would otherwise + * ship the other page's content into this guest page + * (the multi-writer clobber class), and a slot left + * over from a previous launch likewise. + */ + if (bounce_staged_hpa[bi] != req->data[wbi].hpa) { + aliased++; + continue; + } + fp = gsp_phys_gr_fnv1a64(src, GSP_PHYS_PAGE_SIZE); + if (fp == bounce_csum_in[bi]) { + /* + * r378: the engine did not touch this page, + * so the guest page still holds exactly this + * content (the slot was copied in from it). + * Writing it back is a no-op at best and + * stale-content corruption at worst - skip. + */ + unchanged++; + /* + * r452: witness the UNCHANGED page's chunk + * content (first 4 words) and the guest + * page's, bounded per launch. This is the + * only place the two can be compared for a + * page the csum rule classified as "engine + * did not touch" - it separates "the store + * never landed in the chunk" from "the + * store landed but the csum rule missed + * it" (or "the reader resolves a different + * page than the writer wrote"). + */ + if (gsp_phys_gr_copywit_enabled() && + outwit < GSP_PHYS_GR_COPYWIT_MAX) { + const u8 *gp = req->data[wbi].hpa ? + (const u8 *)__va(req->data[wbi].hpa) : + NULL; + + outwit++; + pr_info("vgsp-gr: OUT-UNCH bi=%u va=0x%llx fp=%016llx pre=%016llx chunk=%08x %08x %08x %08x guest=%08x %08x %08x %08x\n", + bi, req->data[wbi].va, fp, + bounce_csum_in[bi], + get_unaligned_le32(src), + get_unaligned_le32(src + 4), + get_unaligned_le32(src + 8), + get_unaligned_le32(src + 12), + gp ? get_unaligned_le32(gp) : 0, + gp ? get_unaligned_le32(gp + 4) : 0, + gp ? get_unaligned_le32(gp + 8) : 0, + gp ? get_unaligned_le32(gp + 12) : 0); + } + continue; + } + { + u64 f8 = (u64)get_unaligned_le64(src); + /* - * m142: which pages the engine touched. The - * indexes name the relocated VA (bi page of - * 0x110000); a c-range write is bi >= 0x40. - * Also dump the first words of the first two - * changed pages: float sums prove the kernel - * wrote them (the c-pointer landed on this - * buffer); pointers/QMD words prove something - * else wrote. + * m142: which pages the engine touched. bi is + * the page index inside the active data region + * (grown: VA 0x0a000000 + bi*4096; legacy: + * 0x110000 + bi*4096). Also dump the first + * words of the first two changed pages: float + * sums prove the kernel wrote them (the + * c-pointer landed on this buffer); + * pointers/QMD words prove something else + * wrote. */ if (changed < 2) pr_info("vgsp-gr: changed page bi=%u w0..3=%08x %08x %08x %08x\n", @@ -17912,34 +21845,173 @@ gr_page_unmapped: get_unaligned_le32(src + 4), get_unaligned_le32(src + 8), get_unaligned_le32(src + 12)); + /* + * COPYWIT fail4: per-changed-page copy-out + * witness - the exact (device VA -> guest + * HPA) pair the results were written back to + * plus the first/last 8 bytes shipped and the + * pre/post csum pair (proof the engine changed + * the page vs the copy-in baseline). The D2H + * side of the same VA is named by the + * sema-bigcopy resolution lines. + */ + if (gsp_phys_gr_copywit_enabled() && + outwit < GSP_PHYS_GR_COPYWIT_MAX) { + /* + * r391 diagnostic (sum=0 class): the + * landing-table resolution of this VA + * (the page the D2H reader will use) + * and its first 8 bytes beside the + * write target - a launch/readback + * disagreement in one line. + */ + u64 rpa = 0, rval = 0; + u32 rap = 0; + bool rhave = gsp_sema_gr_va_snap( + req->data[wbi].va, &rpa, + &rap, &rval); + + outwit++; + pr_info("vgsp-gr: OUT-PAGE-WIT va=0x%llx hpa=0x%llx bi=%u first8=%016llx last8=%016llx pre=%016llx post=%016llx res=%s pa=0x%llx ap=%u val8=%016llx vbase=%px\n", + req->data[wbi].va, + req->data[wbi].hpa, bi, f8, + (u64)get_unaligned_le64( + src + GSP_PHYS_PAGE_SIZE - + 8), + bounce_csum_in[bi], fp, + rhave ? "ledger" : "none", + rpa, rap, rval, + (void *)gsp_sema_gr_shadow_base()); + } + /* + * OUT-ALIAS-WIT: identical first 8 bytes as + * an earlier changed page of this launch = + * the replicated-tile structure, with both + * recs named. r226: all-zero pages are + * excluded on both sides (the fresh-client + * fill stages zero runs; they are not alias + * evidence). + */ + if (gsp_phys_gr_copywit_enabled() && + !gsp_phys_gr_page_all_zero( + gsp_phys_gr_page_sum32(src), src)) { + u32 wi; + + for (wi = 0; wi < gsp_phys_gr_outwit_n; + wi++) { + if (gsp_phys_gr_outwit_first8[wi] != + f8) + continue; + if (gsp_phys_gr_alias_log < 12) { + gsp_phys_gr_alias_log++; + pr_info("vgsp-gr: OUT-ALIAS-WIT va=0x%llx bi=%u partner_va=0x%llx partner_bi=%u first8=%016llx\n", + req->data[wbi].va, + bi, + gsp_phys_gr_outwit_va[wi], + gsp_phys_gr_outwit_bi[wi], + f8); + } + break; + } + if (gsp_phys_gr_outwit_n < + GSP_PHYS_GR_OUTWIT_SLOTS) { + gsp_phys_gr_outwit_first8[gsp_phys_gr_outwit_n] = f8; + gsp_phys_gr_outwit_va[gsp_phys_gr_outwit_n] = req->data[wbi].va; + gsp_phys_gr_outwit_bi[gsp_phys_gr_outwit_n] = bi; + gsp_phys_gr_outwit_n++; + } + } if (changed < 8) chg[changed] = bi; + /* + * r425: remember the first engine-changed page + * (post-copy-out content) for the next zero-write + * accept's late-store witness. + */ + if (changed == 0) { + gsp_phys_gr_prev_out_bi = bi; + gsp_phys_gr_prev_out_csum = fp; + gsp_phys_gr_prev_out_valid = true; + } changed++; } + /* + * r378: the target HPA was resolved at submission; + * the poll/retire/retry window is seconds wide and a + * guest buffer freed and re-homed in it would receive + * this full page through the stale target. Re-resolve + * the VA and skip (bounded log) on a positive + * disagreement. + */ + if (!gsp_sema_gr_write_guard(req->data[wbi].va, + req->data[wbi].hpa, + "gr-bounce-copy-out")) { + skipped++; + continue; + } memcpy((u8 *)__va(req->data[wbi].hpa), src, GSP_PHYS_PAGE_SIZE); + gsp_phys_gr_page_spot_check( + (const u8 *)__va(req->data[wbi].hpa), src, + req->data[wbi].va, "gr-bounce-copy-out"); + /* + * r391d post-write witness (sum=0 class): read the VA's + * landing page right after the write through the same + * path the D2H reader uses. If val8 is not first8 the + * write target and the reader's page are different + * (aperture/staleness disagreement), not a lost write. + */ + if (outwit < GSP_PHYS_GR_COPYWIT_MAX) { + u64 rpa2 = 0, rval2 = 0; + u32 rap2 = 0; + + if (gsp_sema_gr_va_snap(req->data[wbi].va, + &rpa2, &rap2, &rval2)) { + outwit++; + pr_info("vgsp-gr: OUT-PAGE-POST va=0x%llx hpa=0x%llx res_pa=0x%llx ap=%u val8=%016llx vbase=%px\n", + req->data[wbi].va, + req->data[wbi].hpa, rpa2, + rap2, rval2, + (void *)gsp_sema_gr_shadow_base()); + } + } wrote++; } + if ((u64)req->n_data << 12 >= GSP_PHYS_GR_AUDIT_MIN) + pr_info("vgsp: COPYAUDIT path=gr-bounce-copy-out va=0x%llx len=0x%llx done=0x%llx\n", + req->data[0].va, (u64)req->n_data << 12, + (u64)wrote << 12); + /* + * r378: the copy-out is now selective (staged + changed + + * re-validated pages only), so completeness is an accounting + * check over the eligible recs: every one is either written, + * skipped as unchanged, skipped as aliased, or refused by the + * write guard (logged as WGUARD-SKIP). This can only fire if + * a rec vanished between the filters. + */ + if (wrote + unchanged + aliased + skipped < eligible) + pr_err_ratelimited("vgsp: COPY-TRUNC va=0x%llx want=0x%llx done=0x%llx path=gr-bounce-copy-out\n", + req->data[0].va, + (u64)eligible << 12, + (u64)(wrote + unchanged + aliased + + skipped) << 12); /* m143b: unconditional first words of the first page of each - * 32-page relocation window (a=0x110000, b=0x130000, c=0x150000) - * after the run, so the engine's read/write footprint per window - * is provable without relying on the changed-page counter. */ + * 32-page relocation window after the run, so the engine's + * read/write footprint per window is provable without relying + * on the changed-page counter. */ { u32 wi; for (wi = 0; wi + 1 < req->n_data; wi += 32) { - u32 bi = (u32)((req->data[wi].reloc_va - - GSP_PHYS_RMCALL_GR_BOUNCE_VA0) >> - 12); + u32 bi = gsp_phys_gr_data_bi( + req->data[wi].reloc_va); const u8 *wp; u32 wc = 0, k; - if (bi >= GSP_PHYS_RMCALL_GR_MAIL_SLOT) + if (bi == GSP_PHYS_GR_DATA_BI_NONE) continue; - wp = gsp_phys_chunk_page_va( - &boot_stage.rmcall_gr_bounce, bi); - for (k = 0; k < GSP_PHYS_PAGE_SIZE; k += 4) - wc += get_unaligned_le32(wp + k); + wp = gsp_phys_chunk_page_va(dchunk, bi); + (void)k; (void)wc; pr_info("vgsp-gr: window[%u] reloc=0x%llx first=%08x %08x %08x %08x changed=%u\n", bi, req->data[wi].reloc_va, @@ -17947,13 +22019,84 @@ gr_page_unmapped: get_unaligned_le32(wp + 4), get_unaligned_le32(wp + 8), get_unaligned_le32(wp + 12), - wc != bounce_csum_in[bi]); + gsp_phys_gr_fnv1a64( + wp, GSP_PHYS_PAGE_SIZE) != + bounce_csum_in[bi]); } } - pr_info("vgsp-gr: bounce copy-out: %u pages written back, %u changed by engine (first bi: %u %u %u %u %u %u %u %u)\n", - wrote, changed, chg[0], chg[1], chg[2], chg[3], chg[4], + pr_info("vgsp-gr: bounce copy-out: %u pages written back, %u changed by engine, %u unchanged-skipped, %u alias-skipped, %u stale-skipped (first bi: %u %u %u %u %u %u %u %u)\n", + wrote, changed, unchanged, aliased, skipped, + chg[0], chg[1], chg[2], chg[3], chg[4], chg[5], chg[6], chg[7]); } + /* + * relfix: copy the late-resolved pages back to the guest pages the + * resolver named - the same identity the word rewrite handed the + * engine - after the retire wait, exactly like the tracked data + * pages above. + */ + if (gsp_phys_gr_gap_n && gsp_phys_gr_data_chunk()->pages) { + struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); + u32 gi, gwrote = 0, gchanged = 0; + u32 gunchanged = 0, gskipped = 0; + u32 gchg[4] = { }; + + for (gi = 0; gi < gsp_phys_gr_gap_n; gi++) { + u32 bi = gsp_phys_gr_gap[gi].bi; + const u8 *src = gsp_phys_chunk_page_va(dchunk, bi); + u64 fp; + + if (!src) + continue; + /* + * r378: same staging-identity rule as the tracked + * copy-out above - only the page staged into this + * slot may receive the slot's content. + */ + if (bounce_staged_hpa[bi] != gsp_phys_gr_gap[gi].hpa) { + gskipped++; + continue; + } + fp = gsp_phys_gr_fnv1a64(src, GSP_PHYS_PAGE_SIZE); + if (fp == bounce_csum_in[bi]) { + /* Engine did not touch it: nothing to ship. */ + gunchanged++; + continue; + } + if (gchanged < 4) + gchg[gchanged] = bi; + gchanged++; + if (!gsp_sema_gr_write_guard(gsp_phys_gr_gap[gi].va, + gsp_phys_gr_gap[gi].hpa, + "gr-gap-copy-out")) { + gskipped++; + continue; + } + memcpy((u8 *)__va(gsp_phys_gr_gap[gi].hpa), src, + GSP_PHYS_PAGE_SIZE); + gsp_phys_gr_page_spot_check( + (const u8 *)__va(gsp_phys_gr_gap[gi].hpa), src, + gsp_phys_gr_gap[gi].va, "gr-gap-copy-out"); + gwrote++; + } + pr_info("vgsp-gr: gap copy-out: %u pages written back, %u changed by engine, %u unchanged-skipped, %u stale-skipped (first bi: %u %u %u %u)\n", + gwrote, gchanged, gunchanged, gskipped, + gchg[0], gchg[1], gchg[2], gchg[3]); + if ((u64)gsp_phys_gr_gap_n << 12 >= GSP_PHYS_GR_AUDIT_MIN) + pr_info("vgsp: COPYAUDIT path=gr-gap-copy-out va=0x%llx len=0x%llx done=0x%llx\n", + gsp_phys_gr_gap[0].va, + (u64)gsp_phys_gr_gap_n << 12, + (u64)gwrote << 12); + /* + * r378: same accounting rule as the tracked copy-out above - + * the intentional unchanged/stale skips are not truncation. + */ + if (gwrote + gunchanged + gskipped < gsp_phys_gr_gap_n) + pr_err_ratelimited("vgsp: COPY-TRUNC va=0x%llx want=0x%llx done=0x%llx path=gr-gap-copy-out\n", + gsp_phys_gr_gap[0].va, + (u64)gsp_phys_gr_gap_n << 12, + (u64)gwrote << 12); + } poll = w1; if (bar0) pr_info("vgsp-gr: SKED readback (ok): HWW_ESR=0x%08x INFO2=0x%08x CWD_T0=0x%08x CWD_T1=0x%08x CWD_T2=0x%08x\n", @@ -17962,13 +22105,42 @@ gr_page_unmapped: ioread32(bar0 + 0x00407060u), ioread32(bar0 + 0x00407064u), ioread32(bar0 + 0x00407068u)); + /* m151: post-poll page hashes; compare with the pre-doorbell ones. */ + { + const u8 *wp = boot_stage.rmcall_gr_shadow_win0.pages ? + gsp_phys_chunk_page_va( + &boot_stage.rmcall_gr_shadow_win0, + win0_bank_base) : NULL; + + if (wp || guest_param_pg) + pr_info("vgsp-gr: win0 param hash post=0x%016llx guest-param hash post=0x%016llx\n", + wp ? gsp_phys_gr_fnv1a64(wp, + GSP_PHYS_PAGE_SIZE) : + 0ULL, + guest_param_pg ? + gsp_phys_gr_fnv1a64(guest_param_pg, + GSP_PHYS_PAGE_SIZE) : + 0ULL); + } pr_info("vgsp-gr: GUEST GR LAUNCH OK h=0x%x push=0x%llx qmd=0x%llx prog/pages=%u qmd_rel=0x%08x ch_rel=0x%08x elapsed_ms=%u ok=%llu\n", req->h_object, req->push_va, req->qmd_va, req->n_pages, w0, w1, jiffies_to_msecs(jiffies - start), gsp_phys_gr_ok); /* m128: post-success qmdwin words so a probe that stores a value into * the qmdwin page (e.g. p6's constant-read dump at w48) is readable - * from the success log (the post-mortem only prints on timeout). */ - if (boot_stage.rmcall_gr_qmdwin.pages) { + * from the success log (the post-mortem only prints on timeout). + * r434: read the launch's OWN bank (qmd_dst_mem) when available - + * the old bank-0-only read showed a stale page for odd epochs. */ + if (qmd_dst_mem) { + const u32 *qw = (const u32 *)qmd_dst_mem; + + pr_info("vgsp-gr: post-OK qmdwin w48..59=%08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n", + get_unaligned_le32(qw + 48), get_unaligned_le32(qw + 49), + get_unaligned_le32(qw + 50), get_unaligned_le32(qw + 51), + get_unaligned_le32(qw + 52), get_unaligned_le32(qw + 53), + get_unaligned_le32(qw + 54), get_unaligned_le32(qw + 55), + get_unaligned_le32(qw + 56), get_unaligned_le32(qw + 57), + get_unaligned_le32(qw + 58), get_unaligned_le32(qw + 59)); + } else if (boot_stage.rmcall_gr_qmdwin.pages) { const u32 *qw = (const u32 *)gsp_phys_chunk_page_va( &boot_stage.rmcall_gr_qmdwin, 0); @@ -17980,11 +22152,19 @@ gr_page_unmapped: ret = 0; out: + /* + * M91 FAULT-CAP: one line per faulting launch, emitted at the end of + * the launch window (fault epoch compared against the snapshot taken + * at submission; inert - one compare - when no fault was seen). + */ + gsp_phys_gr_m91_fault_cap(req, qmd, qmd_dst, qmd_dst_mem, attempts, + rearmed, fast_rearmed, ret == 0); if (out) { out->sem_value = poll; out->elapsed_ms = jiffies_to_msecs(jiffies - start); out->rc = ret; out->fault = fault; + out->qmd_rel_redirected = qmd_rel_redirected; } return ret; } @@ -18511,7 +22691,11 @@ static int gsp_phys_rmcall_guest_g2h(void __iomem *bar0) return -EIO; } - ret = gsp_phys_gpa_to_hpa(gpa, &hpa); + /* + * Single-guest pin (see gsp_phys_guest_gpa_write): NULL selects the + * first open mdev, exactly as before the per-mdev records. + */ + ret = gsp_phys_gpa_to_hpa_vgpu(NULL, gpa, &hpa); if (ret) { pr_err("vgsp-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: GPA 0x%llx -> HPA translation failed (%d: no active mdev/VFIO domain, GPA unmapped, or not a RAM PFN)\n", hpa, pte, gpa, ret); @@ -18594,7 +22778,11 @@ static int gsp_phys_rmcall_guest_w2g(void __iomem *bar0) return -EIO; } - ret = gsp_phys_gpa_to_hpa(gpa, &hpa); + /* + * Single-guest pin (see gsp_phys_guest_gpa_write): NULL selects the + * first open mdev, exactly as before the per-mdev records. + */ + ret = gsp_phys_gpa_to_hpa_vgpu(NULL, gpa, &hpa); if (ret) { pr_err("vgsp-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: GPA 0x%llx -> HPA translation failed (%d: no active mdev/VFIO domain, GPA unmapped, or not a RAM PFN)\n", hpa, pte, gpa, ret); @@ -18821,6 +23009,14 @@ static void gsp_phys_rmcall_fini(struct pci_dev *pdev) gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_pt[i]); gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_shadow); gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_qmdwin); + /* + * B-2: the old 208-page bounce window (previously leaked here) and the + * grown data-bounce region chunks. The region's PT chunk and data chunk + * are freed only when allocated. + */ + gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_bounce); + gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_data_bounce); + gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_data_bounce_pt); gsp_phys_chunk_free(pdev, &boot_stage.rmcall_patch); gsp_phys_chunk_free(pdev, &boot_stage.rmcall_pm); gsp_phys_chunk_free(pdev, &boot_stage.rmcall_fecs); @@ -18992,8 +23188,7 @@ static int gsp_phys_state_show(struct seq_file *m, void *v) fwd_cand_none, fwd_cand_sys, fwd_cand_mixed, fwd_cand_vid, fwd_resolve_fail, boot_stage.rmcall_guest_pt_next_line, - GSP_PHYS_RMCALL_GUEST_PT_PAGES * - GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE); + GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES); } { /* @@ -19073,11 +23268,18 @@ static int gsp_phys_state_show(struct seq_file *m, void *v) gsp_phys_backend_gr_counters(&b_sub, &b_ok, &b_refused, &b_timeout, &b_pool); seq_printf(m, - "work_gr: req=%llu fallback=%llu ok=%llu refused=%llu timeout=%llu fault=%llu submitted=%llu backend_ok=%llu backend_refused=%llu backend_timeout=%llu pool_exhausted=%llu forward_param=%u exec_ok=%u\n", + "work_gr: req=%llu fallback=%llu ok=%llu refused=%llu timeout=%llu fault=%llu submitted=%llu backend_ok=%llu backend_refused=%llu backend_timeout=%llu pool_exhausted=%llu forward_param=%u exec_ok=%u sem_wedged=%llu stall_wait_max=%llu stall_retry=%llu wait_max_ms=%u nw_streak=%u wedge_quiet=%llu wedge_heals=%llu\n", gr_req, gr_fallback, gr_ok, gr_refused, gr_timeout, gr_fault, b_sub, b_ok, b_refused, b_timeout, b_pool, gsp_phys_gr_forward_enabled(), - boot_stage.rmcall_gr_exec_ok); + boot_stage.rmcall_gr_exec_ok, + READ_ONCE(gsp_phys_gr_sem_wedged), + READ_ONCE(gsp_phys_gr_stall_wait_max), + READ_ONCE(gsp_phys_gr_stall_retry), + gsp_phys_gr_wait_max_ms_value(), + READ_ONCE(gsp_phys_gr_nw_streak), + READ_ONCE(gsp_phys_gr_wedge_quiet), + READ_ONCE(gsp_phys_gr_wedge_heals)); } { /* @@ -19806,6 +24008,18 @@ static ssize_t gsp_phys_guest_gpa_write(struct file *file, gpa); return -EINVAL; } + /* + * Dual-guest guard (goal 1.0): the pin is a single-guest test aid. The + * triggers translate it through the first open mdev (see + * gsp_phys_gpa_to_hpa / boot_stage.rmcall_guest_gpa), so with more than + * one open mdev the pin cannot name its owner and is refused instead of + * silently resolving through whichever guest opened first. + */ + if (gsp_phys_open_mdev_count() > 1) { + pr_err("vgsp-boot: gsp-phys-guest-gpa: refused: %d mdevs are open; the guest-GPA pin is a single-guest test aid (translates through the first open mdev). Close all but one mdev, or use the per-mdev guest/GPA paths\n", + gsp_phys_open_mdev_count()); + return -EBUSY; + } if (boot_stage.rmcall_guest_gpa_valid && boot_stage.rmcall_guest_gpa != gpa) { pr_err("vgsp-boot: gsp-phys-guest-gpa: refused: this boot already pinned GPA 0x%llx; one GPA per boot (stale GMMU TLB safety). Reuse that GPA, or release/reboot before changing it\n", @@ -19828,18 +24042,26 @@ static const struct file_operations gsp_phys_guest_gpa_fops = { }; /* - * gsp-phys-guest-ram (write-only, mode 0200): configure the shared guest RAM - * backing file that gsp_phys_gpa_to_hpa() translates through (QEMU + * gsp-phys-guest-ram (write-only, mode 0200): configure a guest RAM backing + * file that gsp_phys_gpa_to_hpa_vgpu() translates through (QEMU * memory-backend-file,...,share=yes). One write is * * "<path> <gpa_base_hex> <file_offset_hex> <size_hex>" * * (path without spaces; all three numbers hex with 0x, e.g. - * "/dev/shm/vgpu-ram 0x100000000 0x0 0x40000000"). The first write opens the - * file and fixes the path for this module cycle; later writes may add ranges - * (up to four) or replace a range with the same gpa_base. The ranges are - * only consulted by gsp_phys_gpa_to_hpa(), and only while a live mdev - * session exists. A single "-" clears the configuration and closes the file. + * "/dev/shm/vgpu-ram 0x100000000 0x0 0x40000000"). This historical form + * configures the legacy record and serves the FIRST/ONLY open mdev exactly + * as before the per-mdev records existed. With two live guests each writes + * its own record by naming its mdev UUID first: + * + * "<mdev-uuid> <path> <gpa_base_hex> <file_offset_hex> <size_hex>" + * + * Paths and UUIDs contain no spaces, so the token count (4 vs 5) decides. + * The first write for a record opens the file and fixes the path for that + * record; later writes may add ranges (up to four) or replace a range with + * the same gpa_base. The ranges are only consulted by the translators, and + * only while a live mdev session exists. A single "-" clears ALL records + * (legacy and per-mdev) and closes the files. */ static char *gsp_phys_guest_ram_token(char **cursor) { @@ -19854,9 +24076,11 @@ static ssize_t gsp_phys_guest_ram_write(struct file *file, const char __user *ubuf, size_t len, loff_t *ppos) { - char buf[384]; - char *p, *tok, *path; + char buf[448]; + char *t[5]; + char *path; u64 gpa_base, file_offset, size; + int nt = 0, off; int ret; if (len == 0 || len >= sizeof(buf)) @@ -19865,30 +24089,63 @@ static ssize_t gsp_phys_guest_ram_write(struct file *file, return -EFAULT; buf[len] = '\0'; - p = strim(buf); - if (!strcmp(p, "-")) { + if (!strcmp(strim(buf), "-")) { gsp_phys_guest_ram_clear(); pr_info("vgsp-boot: gsp-phys-guest-ram: cleared the guest RAM file configuration\n"); return len; } - path = gsp_phys_guest_ram_token(&p); - tok = gsp_phys_guest_ram_token(&p); - if (!path || !tok || kstrtoull(tok, 0, &gpa_base)) { + { + char *p = buf; + + while (nt < 5 && (t[nt] = gsp_phys_guest_ram_token(&p))) + nt++; + } + if (nt != 4 && nt != 5) { + pr_info("vgsp-boot: gsp-phys-guest-ram: write ignored: expected \"<path> <gpa_base_hex> <file_offset_hex> <size_hex>\", \"<mdev-uuid> <path> <gpa_base_hex> <file_offset_hex> <size_hex>\" or \"-\"\n"); + return -EINVAL; + } + off = nt - 4; /* 0 = legacy form, 1 = per-mdev form */ + path = t[off]; + if (!path || kstrtoull(t[off + 1], 0, &gpa_base)) { pr_info("vgsp-boot: gsp-phys-guest-ram: write ignored: expected \"<path> <gpa_base_hex> <file_offset_hex> <size_hex>\" or \"-\"\n"); return -EINVAL; } - tok = gsp_phys_guest_ram_token(&p); - if (!tok || kstrtoull(tok, 0, &file_offset)) { + if (kstrtoull(t[off + 2], 0, &file_offset)) { pr_info("vgsp-boot: gsp-phys-guest-ram: write ignored: expected a hex file offset after the GPA\n"); return -EINVAL; } - tok = gsp_phys_guest_ram_token(&p); - if (!tok || kstrtoull(tok, 0, &size)) { + if (kstrtoull(t[off + 3], 0, &size)) { pr_info("vgsp-boot: gsp-phys-guest-ram: write ignored: expected a hex size after the file offset\n"); return -EINVAL; } + if (off) { + /* + * Per-mdev form: the selector must name an OPEN mdev (the mdev + * UUID is its device name). Refusing an unknown selector keeps + * a typo from silently configuring the legacy record. + */ + struct nv_vgpu_device *mdev = + gsp_phys_open_mdev_by_name(t[0]); + + if (!mdev) { + pr_err("vgsp-boot: gsp-phys-guest-ram: no open mdev named %s; write refused\n", + t[0]); + return -ENODEV; + } + ret = gsp_phys_guest_ram_configure_vgpu(mdev, path, gpa_base, + file_offset, size); + if (ret) { + pr_err("vgsp-boot: gsp-phys-guest-ram: configure failed: mdev=%s path=%s gpa=0x%llx off=0x%llx size=0x%llx ret=%d\n", + t[0], path, gpa_base, file_offset, size, ret); + return ret; + } + pr_info("vgsp-boot: gsp-phys-guest-ram: configured mdev=%s path=%s gpa=0x%llx off=0x%llx size=0x%llx\n", + t[0], path, gpa_base, file_offset, size); + return len; + } + ret = gsp_phys_guest_ram_configure(path, gpa_base, file_offset, size); if (ret) { pr_err("vgsp-boot: gsp-phys-guest-ram: configure failed: path=%s gpa=0x%llx off=0x%llx size=0x%llx ret=%d\n", @@ -20001,6 +24258,30 @@ static int gsp_phys_reg_probe_show(struct seq_file *m, void *unused) sked_cwd1 = gsp_phys_reg_probe_line(m, bar0, "SKED_DEBUG_CWD_TABLE_1", 0x00407064u); + /* r425: the wedge diagnostics - the rest of the SKED CWD table, the + * per-8-group GR TPC masks (a zero compute mask would dispatch no + * TPCs), and the FE QMD-virtualization control for VEID 0. */ + (void)gsp_phys_reg_probe_line(m, bar0, "SKED_DEBUG_CWD_TABLE_2", + 0x00407068u); + (void)gsp_phys_reg_probe_line(m, bar0, "SKED_DEBUG_CWD_TABLE_3", + 0x0040706cu); + (void)gsp_phys_reg_probe_line(m, bar0, "SKED_DEBUG_INFO_1", + 0x00407028u); + { + u32 gi; + + for (gi = 0; gi < 8u; gi++) { + gsp_phys_reg_probe_line(m, bar0, + "FE_TPCM_GR_TPC_MASK", + 0x0040a680u + gi * 4u); + gsp_phys_reg_probe_line(m, bar0, + "FE_LAST_GR_TPC_MASK", + 0x0040a6a0u + gi * 4u); + gsp_phys_reg_probe_line(m, bar0, + "FE_CWD_OWN_TPC_MASK", + 0x0040a6c0u + gi * 4u); + } + } fecs_irq = gsp_phys_reg_probe_line(m, bar0, "FECS_IRQSTAT", 0x00409008u); fecs_cpuctl = gsp_phys_reg_probe_line(m, bar0, "FECS_CPUCTL", @@ -20625,8 +24906,8 @@ static_assert(offsetof(struct gsp_phys_rmcall_channel_params, process_id) == 296 "NV_CHANNEL_ALLOC_PARAMS.ProcessID must be at offset 296"); static_assert(offsetof(struct gsp_phys_rmcall_channel_params, tpc_config_id) == 360, "NV_CHANNEL_ALLOC_PARAMS.tpcConfigID must be at offset 360"); -static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 33, - "the rm-call ladder registers exactly thirty-three DMA buffers (14 ladder + 16 single-page guest scratch PTs + 3 golden9g VAS levels)"); +static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 81, + "the rm-call ladder registers exactly eighty-one DMA buffers (14 ladder + 16 single-page guest scratch PTs + 48 single-page extension scratch PTs + 3 golden9g VAS levels)"); static_assert((GSP_PHYS_RMCALL_HANDLE_MEM_BASE + (GSP_PHYS_RMCALL_BUF_COUNT - 1) * GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE) < 0xcaf80000u, diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h index 896a436..af74f7c 100644 --- a/kernel/gsp_phys_booter.h +++ b/kernel/gsp_phys_booter.h @@ -164,6 +164,16 @@ int gsp_phys_backend_static_wait(struct gsp_phys_backend_static_config *out, void gsp_phys_backend_static_invalidate(void); /* + * Mode-35 KGR static-data snapshots (see gsp_phys_booter.c): the live GSP's + * replies to the real client's KGR query set, captured at backend boot and + * served byte-identically by the guest-side emulation. Returns 0 with + * *out_len set, -ENODATA when cmd was not captured for the current backend + * generation, -ENOSPC when the snapshot exceeds dst_len, -EAGAIN when the + * backend is not READY. + */ +int gsp_phys_kgr_snap_get(u32 cmd, void *dst, u32 dst_len, u32 *out_len); + +/* * Run one fn1+fn65 probe on a READY backend to fill the static snapshot * cache. Takes gsp_phys_rm_lock inside (via gsp_phys_rm_probe), so it must * run on a work item, never inline under a caller-held rm lock or on the mdev @@ -275,6 +285,17 @@ struct gsp_phys_backend_copy_result { u32 elapsed_ms; /* return-to-caller elapsed milliseconds */ int rc; /* duplicated function return code */ struct gsp_phys_fault_info fault; /* decoded engine fault, if any */ + /* + * r420b: the forwarded launch's QMD release0 was redirected to the + * module semaphore (gsp_phys_rmcall_gr_rel_mod=1, or an unresolved + * guest release page), so the ENGINE will never write the guest's own + * QMD release word. The caller must author it in software on every + * outcome (the same write the hold's success edge performs); without + * this a rel_mod=1 launch with no deferred release records is never + * completed on the guest word and the client hangs (the r414 saxpy + * f0 cur=5 stall). + */ + bool qmd_rel_redirected; }; int gsp_phys_backend_guest_ce_copy_ex(u64 src_hpa, u32 src_off, u64 dst_hpa, @@ -349,6 +370,25 @@ void gsp_phys_backend_relay_counters(u64 *requested, u64 *ok, u64 *verify_fail, #define GSP_PHYS_GR_QMD_WORD_NONE 0xffffffffu #define GSP_PHYS_GR_RAW_WORDS 256u +/* + * B-2 (large-operand data path): the grown, boot-fixed GR data-bounce region. + * One PD0 dual entry per 2 MB window (PD0 slots 80..111, immediately above + * the m152 extension scratch pool's 32..79) backed by one fresh 4 KB small-PT + * page each, every leaf PTE pointing at one of the 16384 pages (64 MiB) of a + * single data chunk. The whole map is planted once at stage 3, before any + * engine fetch, so the GMMU's no-invalidate 128 B fetch-line rule is + * satisfied by construction (no PTE is ever rewritten at runtime). The SVC + * assigns operand reloc VAs from VA0 upward; gsp_sema_service.c derives its + * reloc base and page cap from these constants so the two sides cannot drift + * (the pre-B-2 205-page window below the mail page stays allocated for the + * mail injection and diagnostics, it just no longer carries operand data when + * gsp_phys_gr_data_region=1, the default). + */ +#define GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 0x0A000000ULL +#define GSP_PHYS_RMCALL_GR_DATA_BOUNCE_SLOT0 80u +#define GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PT_PAGES 32u +#define GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES (32u * 512u) /* 16384 */ + struct gsp_phys_gr_page { u64 guest_va; /* guest VA (page aligned) */ u64 hpa; /* host PA (page aligned; 0 = unresolved) */ @@ -402,6 +442,9 @@ struct gsp_phys_gr_submit { u64 rel_hpa; /* guest-visible QMD release0 page, or 0 */ u32 rel_off; u32 rel_payload; + u64 rel_va; /* r378: device VA rel_hpa was resolved from, + * for write-target re-validation before the + * release word is zeroed/restored */ u64 sem_hpa; /* guest-visible channel release page, or 0 */ u32 sem_off; u32 sem_payload; @@ -414,6 +457,16 @@ struct gsp_phys_gr_submit { * whose boot submission proved out (rmcall_gr_exec_ok). */ bool gsp_phys_backend_gr_ready(void); +/* + * r420: true exactly while the GR submit path sleeps in a WEDGE-QUIET hold + * (a deliberate, bounded wait for the sticky wedge to clear). The in-flight + * forwarded request is NOT done during it, so the sema service's completion + * deadlines (the 12 s GR release hold and the 3 s per-launch consumption + * bound) pause while this returns true instead of authoring the launch's + * release word under the hold (the r419 mid-boot stale-data escape). + */ +bool gsp_phys_gr_quiet_hold_active(void); + int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, struct gsp_phys_backend_copy_result *out); diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c index c5419a5..74062dd 100644 --- a/kernel/gsp_phys_gpu.c +++ b/kernel/gsp_phys_gpu.c @@ -638,13 +638,20 @@ int gsp_phys_rmcall_gr_qmdvirt_value(void) EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_qmdvirt_value); /* - * Explicit-PCAS gate (default 1). The rebuilt launch carries the vendor's - * turingPushPcas(CU_PCAS_TYPE_COPY) pair (SEND_PCAS_A = QMD>>8 followed by - * SEND_SIGNALING_PCAS_B = INVALIDATE|SCHEDULE). The stock guest push does - * not emit these (it relies on the IQ2M window's implied copy PCAS); setting - * this to 0 emits the pure vendor window-only form for the A/B test. - */ -static int gsp_phys_rmcall_gr_pcas_param = 1; + * Explicit-PCAS gate (default 0 since r343 - the double-execution fix). + * The rebuilt launch used to carry the vendor's turingPushPcas + * (CU_PCAS_TYPE_COPY) pair (SEND_PCAS_A = QMD>>8 followed by + * SEND_SIGNALING_PCAS_B = INVALIDATE|SCHEDULE) ADDED to the inline-QMD run + * that the stock guest push already carries. The QMD's + * REQUIRE_SCHEDULING_PCAS bit is clear, so the inline run schedules by + * itself - the explicit pair was a SECOND scheduling edge and the same QMD + * was dispatched twice (launch_count: one atomicAdd +100 landed as +200; + * victory_reduce exactly 2x; scale/saxpy/memset2d doubled). The stock guest + * push has NO PCAS pair (captured push dumps); default 0 restores that + * single-edge form. 1 = re-add the pair (A/B debug only - with it on, the + * grid executes twice). + */ +static int gsp_phys_rmcall_gr_pcas_param; module_param_named(gsp_phys_rmcall_gr_pcas, gsp_phys_rmcall_gr_pcas_param, int, 0644); MODULE_PARM_DESC(gsp_phys_rmcall_gr_pcas, @@ -657,6 +664,29 @@ int gsp_phys_rmcall_gr_pcas_value(void) EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_pcas_value); /* + * r434 CWD-slot-count path (gsp_phys_rmcall_gr_cwd_slot; default -1 = off). + * On Volta+ the SET_CWD_SLOT_COUNT method is gated by the falcon PRI bit + * NV_PGRAPH_PRI_SKED_DEBUG_1.ENABLE_CWD_SLOT_COUNT (the CUDA driver's + * turingLaunchSetCwdSlotCount: a masked PRI write via SET_MME_SHADOW_SCRATCH + * 0/1/2 + SET_FALCON04, then the method itself). When >= 0 the init preamble + * emits that PRI write (ENABLE_CWD_SLOT_COUNT=1) followed by + * SET_CWD_SLOT_COUNT with the given value, so the wedge's "grid truncated to + * ~4 CTAs" signature can be A/B'd against the CWD slot-count watermark. + * Read per launch; -1 restores the historical preamble byte-for-byte. + */ +static int gsp_phys_rmcall_gr_cwd_slot_param = -1; +module_param_named(gsp_phys_rmcall_gr_cwd_slot, + gsp_phys_rmcall_gr_cwd_slot_param, int, 0644); +MODULE_PARM_DESC(gsp_phys_rmcall_gr_cwd_slot, + "r434: enable the CWD slot count PRI (SKED_DEBUG_1 bit11) and emit SET_CWD_SLOT_COUNT with this value in the GR init preamble (-1 = off, default)"); + +int gsp_phys_rmcall_gr_cwd_slot_value(void) +{ + return gsp_phys_rmcall_gr_cwd_slot_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_cwd_slot_value); + +/* * Two-step PCAS gate (default 0). With the explicit PCAS pair on, setting * this marks the QMD's REQUIRE_SCHEDULING_PCAS (w4 bit12) and emits the copy * PCAS (INVALIDATE only) followed by a schedule-only PCAS (SCHEDULE), the @@ -704,20 +734,30 @@ int gsp_phys_rmcall_gr_qmdwin_value(void) EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_qmdwin_value); /* - * m64 forced module-semaphore release (gsp_phys_rmcall_gr_rel_mod; default 0). - * When set, a forwarded GR launch ignores the resolved guest release page and - * points QMD release0 at the module window-0 semaphore VA - * (GSP_PHYS_RMCALL_VA_SEM), which the host polls at the SEM chunk's host VA. - * Discriminant for "engine completed but the shadow release write was lost" - * vs "engine never completed": with the module release, a completed QMD is - * always observable. Also makes the existing module-release fallback - * observable (it previously set no poll source, so it could never complete). - */ -static int gsp_phys_rmcall_gr_rel_mod_param; + * m64 forced module-semaphore release (gsp_phys_rmcall_gr_rel_mod). + * + * DEFAULT 1 since r453. When set, a forwarded GR launch ignores the resolved + * guest release page and points QMD release0 at the module window-0 semaphore + * VA (GSP_PHYS_RMCALL_VA_SEM), which the host polls at the SEM chunk's host + * VA; the GUEST-VISIBLE release word is authored by the host only on the + * completion edge - after the retire wait and after the bounce copy-out has + * shipped the engine's stores to the guest pages. Evidence (r453, the FNV + * change-detection fix on top): + * - with 0: the guest's release could be satisfied by the engine's own + * early write while the host copy-out was still pending, so a client's + * stream-ordered readback (streams2's c2, memcpy_roundtrip's memset + * phase, streams_events' chain) read PRE-launch content: 11/15 battery. + * - with 1: 13/15 battery (streams_events and memcpy_roundtrip flipped + * green), no stalls across the full battery, mail+saxpy clean. + * The r414 stall hole (a launch that never reached the hold edge left the + * guest word unauthored) is closed by the r420b per-outcome authoring path; + * one-line rollback: echo 0 > .../parameters/gsp_phys_rmcall_gr_rel_mod. + */ +static int gsp_phys_rmcall_gr_rel_mod_param = 1; module_param_named(gsp_phys_rmcall_gr_rel_mod, gsp_phys_rmcall_gr_rel_mod_param, int, 0644); MODULE_PARM_DESC(gsp_phys_rmcall_gr_rel_mod, - "Force the forwarded GR QMD release0 to the module semaphore (default 0; 1 = m64 engine-completion discriminant)"); + "Force the forwarded GR QMD release0 to the module semaphore; guest release authored only on the post-copy-out completion edge (default 1 since r453; 0 = historical engine-authored guest release)"); int gsp_phys_rmcall_gr_rel_mod_value(void) { @@ -949,6 +989,286 @@ int gsp_phys_rmcall_gr_skedrec_value(void) EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_skedrec_value); /* + * m153 no-write/wrong-place launch retry (gsp_phys_rmcall_gr_nowrite_retry; + * default 1). After a forwarded launch reports completion, the output + * retirement wait measures how many resolved data pages the engine actually + * changed. Two mis-execution signatures are known to be intermittent per + * boot: the engine writes nothing at all, or it writes into a lower operand + * window while the highest (output) operand stays untouched (a stale + * constant-source state). Both re-execute correctly on a fresh ring entry + * often enough to be worth a bounded retry: refill the bounce window from + * the guest inputs, re-publish the same method block, re-issue the schedule + * and re-poll, then re-wait. Set to 0 to disable the retry and accept the + * first outcome unconditionally. + */ +static int gsp_phys_rmcall_gr_nowrite_retry_param = 1; +module_param_named(gsp_phys_rmcall_gr_nowrite_retry, + gsp_phys_rmcall_gr_nowrite_retry_param, int, 0644); +MODULE_PARM_DESC(gsp_phys_rmcall_gr_nowrite_retry, + "Re-execute a forwarded GR launch whose retirement wait saw zero/wrong-place engine writes (default 1)"); + +int gsp_phys_rmcall_gr_nowrite_retry_value(void) +{ + return gsp_phys_rmcall_gr_nowrite_retry_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_nowrite_retry_value); + +/* + * F2 fast re-arm (gsp_phys_gr_retry_fast; default 1). r198: the first + * forwarded-launch timeout costs 2x5 s before the fresh-entry ladder runs, + * yet the m155 extended re-arm (lane-enable replay + GPFIFO/TSG schedule + + * doorbell) plus one fresh ring entry recovers most of these launches. With + * this on, the first timeout immediately runs that re-arm + a fresh entry + * and re-polls with a short window; only if that still fails does the + * historical timeout ladder continue. Set to 0 for today's exact behavior. + */ +static int gsp_phys_gr_retry_fast_param = 1; +module_param_named(gsp_phys_gr_retry_fast, + gsp_phys_gr_retry_fast_param, int, 0644); +MODULE_PARM_DESC(gsp_phys_gr_retry_fast, + "On the first GR launch timeout, run the extended re-arm + a fresh ring entry immediately and re-poll briefly (default 1; 0 = legacy ladder timing)"); + +int gsp_phys_gr_retry_fast_value(void) +{ + return gsp_phys_gr_retry_fast_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_gr_retry_fast_value); + +/* + * F2 first-poll window (gsp_phys_gr_first_poll_ms; default 2500, clamped to + * [1000, 30000]). With gsp_phys_gr_retry_fast=1 the FIRST poll window of a + * forwarded GR launch (before any re-arm) uses this instead of the fixed 5 s, + * so the fast re-arm path above starts earlier; with retry_fast=0 the launch + * keeps today's exact behavior and this knob is not read. The later ladder + * windows keep their own constants. + */ +static unsigned int gsp_phys_gr_first_poll_ms_param = 2500u; +module_param_named(gsp_phys_gr_first_poll_ms, + gsp_phys_gr_first_poll_ms_param, uint, 0644); +MODULE_PARM_DESC(gsp_phys_gr_first_poll_ms, + "First GR launch poll window in ms, clamped [1000,30000], read when gsp_phys_gr_retry_fast=1 (default 2500)"); + +unsigned int gsp_phys_gr_first_poll_ms_value(void) +{ + unsigned int v = gsp_phys_gr_first_poll_ms_param; + + if (v < 1000u) + v = 1000u; + if (v > 30000u) + v = 30000u; + return v; +} +EXPORT_SYMBOL_GPL(gsp_phys_gr_first_poll_ms_value); + +/* + * M91 icache coherence (gsp_phys_gr_icache_inv; default 1). The m91 class is + * the transient SM instruction-fetch lottery: the first attempt of a + * mid-sequence launch faults on SM garbage ("Illegal Instruction Encoding" / + * "Out Of Range Register") while a retry of the identical QMD and identical + * shadow content succeeds. The window-0 shadow pool reuses code VAs across + * launches (two 8-page banks alternate by submit epoch), so the bytes at a + * program VA change between launches, and every captured guest QMD carries + * the QMDV02_03 cache-control block with INVALIDATE_INSTRUCTION_CACHE (word 5 + * bit 30) CLEAR - the field the CUDA driver arms only under its + * forceIcacheInvalidate knob. An SM fetch line cached for launch N can then + * execute the previous kernel's bytes at the same VA in launch N+2. With this + * on, the forwarded QMD sets that bit (only when the QMD carries the driver's + * standard cache-control signature, bits 26..29 of word 5, so an unknown QMD + * revision is never touched), forcing a refetch at dispatch. 0 restores the + * exact pre-change QMD for a one-flip A/B. + */ +static int gsp_phys_gr_icache_inv_param = 1; +module_param_named(gsp_phys_gr_icache_inv, gsp_phys_gr_icache_inv_param, + int, 0644); +MODULE_PARM_DESC(gsp_phys_gr_icache_inv, + "Set QMDV02_03 INVALIDATE_INSTRUCTION_CACHE (word5 bit30) on forwarded GR launches whose QMD has the driver cache-control signature (default 1; 0 = pre-m91 QMD words)"); + +int gsp_phys_gr_icache_inv_value(void) +{ + return gsp_phys_gr_icache_inv_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_gr_icache_inv_value); + +/* + * r433 RELEASE_MEMBAR_TYPE A/B (gsp_phys_gr_rel_membar; default 0). The + * captured guest QMDs carry RELEASE_MEMBAR_TYPE=FE_NONE (QMDV00_06 word 11 + * bit 14 clear, w11=0x44010000), so when the front-end completes a grid it + * writes the QMD release semaphore without first flushing its pending + * system-memory stores. The post-dispatch store-loss signature (the engine + * processes the QMD, writes the release, the grid completes and ZERO data + * pages are host-visible) is exactly what an unflushed write-back L2 looks + * like from the host: the release is host-visible while the data sits in the + * GPU's L2. Forcing RELEASE_MEMBAR_TYPE=FE_SYSMEMBAR makes the release + * ordering guarantee the stores are in memory before the semaphore write, so + * the host's retire scan (and the copy-out) can never observe a completion + * whose data has not landed. 0 keeps the guest's captured value (the + * historical behaviour) for a one-flip A/B; read per launch. + */ +static int gsp_phys_gr_rel_membar_param; +module_param_named(gsp_phys_gr_rel_membar, gsp_phys_gr_rel_membar_param, + int, 0644); +MODULE_PARM_DESC(gsp_phys_gr_rel_membar, + "r433: force QMD RELEASE_MEMBAR_TYPE=FE_SYSMEMBAR (word11 bit14) on forwarded GR launches (default 0 = guest's captured FE_NONE)"); + +int gsp_phys_gr_rel_membar_value(void) +{ + return gsp_phys_gr_rel_membar_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_gr_rel_membar_value); + +/* + * m179 no-write retry budget (gsp_phys_gr_nowrite_retries; default 3, clamped + * to [0, 4]). The m153 outcome retry loop re-executes a data-bearing launch + * whose retirement wait saw zero engine writes; it shared the timeout ladder's + * attempt counter, so a launch that had already spent attempts on the ladder + * (fast re-arm + fresh ring entries) reached the accept path with NO + * re-execution at all - the r200/r201/r202 memcpy_roundtrip / multikernel + * class (a lost launch silently accepted, client output all wrong). This + * budget is dedicated to the no-write loop and independent of the ladder; it + * is read once per submission. 0 keeps the accept-without-retry behavior + * (still logged loudly). The clamp keeps the worst-case host publishes per + * launch (initial + fast re-arm + 2 fresh entries + budget) within the + * 8-entry GPFIFO ring. + */ +static int gsp_phys_gr_nowrite_retries_param = 3; +module_param_named(gsp_phys_gr_nowrite_retries, + gsp_phys_gr_nowrite_retries_param, int, 0644); +MODULE_PARM_DESC(gsp_phys_gr_nowrite_retries, + "Dedicated re-execution budget for a data-bearing GR launch that wrote nothing tracked, clamped [0,4] (default 3)"); + +/* + * STALLFREE: absolute per-launch wall-clock budget (gsp_phys_gr_wait_max_ms; + * default 60000, clamped [10000, 600000]; 0 disables the extra cap and the + * built-in ladder bounds stand alone). The GR submit ladder and its retirement + * (no-write) retries are individually bounded, but their SUM grows with every + * re-arm/re-publish: a launch whose engine ack never arrives could hold the + * single-inflight work item for tens of seconds while the guest client waits + * on its QMD completion word. When this budget is crossed the submit stops its + * ladder immediately (no further re-arm/publish/retry), logs + * 'vgsp-gr: GR-STALL-wait-max ... action=fail' and returns -ETIMEDOUT; the + * sema service's failure edge then authors the QMD completion release in + * software, so the client always completes. A green launch (~0.2-11 s end to + * end, including the fast-re-arm recovery) never approaches the default, so + * the default does not change green behavior. + */ +static unsigned int gsp_phys_gr_wait_max_ms_param = 60000u; +module_param_named(gsp_phys_gr_wait_max_ms, + gsp_phys_gr_wait_max_ms_param, uint, 0644); +MODULE_PARM_DESC(gsp_phys_gr_wait_max_ms, + "Absolute per-launch GR submit budget in ms, clamped [10000,600000]; 0 = only the built-in per-ladder bounds (default 60000)"); + +unsigned int gsp_phys_gr_wait_max_ms_value(void) +{ + unsigned int v = gsp_phys_gr_wait_max_ms_param; + + if (!v) + return 0; + if (v < 10000u) + v = 10000u; + if (v > 600000u) + v = 600000u; + return v; +} +EXPORT_SYMBOL_GPL(gsp_phys_gr_wait_max_ms_value); + +int gsp_phys_gr_nowrite_retries_value(void) +{ + int v = gsp_phys_gr_nowrite_retries_param; + + if (v < 0) + v = 0; + if (v > 4) + v = 4; + return v; +} +EXPORT_SYMBOL_GPL(gsp_phys_gr_nowrite_retries_value); + +/* + * r445 wedge hard re-arm (gsp_phys_gr_wedge_rearm; default 0 = off). When the + * outcome-retry's warm GR semaphore probe proves the engine wedged for the + * context (the proven-sticky post-dispatch no-store state), this knob makes + * the submit attempt ONE bounded recovery instead of breaking immediately: + * a TSG PREEMPT(bWait=1) forces the FECS to save/unload the channel's + * context, then the mode-35 lane-enable tail + GPFIFO schedules are re-issued + * (the m155 extended re-arm) and the semaphore probe is repeated. A pass + * means the lane revived for this launch and the normal retry ladder + * continues; a fail leaves the historical WEDGED break. Read once per wedge + * episode. Off by default for A/B. + */ +static int gsp_phys_gr_wedge_rearm_param; +module_param_named(gsp_phys_gr_wedge_rearm, gsp_phys_gr_wedge_rearm_param, + int, 0644); +MODULE_PARM_DESC(gsp_phys_gr_wedge_rearm, + "r445: on the proven engine-wedged sem probe, attempt a bounded preempt+resched hard re-arm before giving up (default 0=off)"); + +int gsp_phys_gr_wedge_rearm_value(void) +{ + return gsp_phys_gr_wedge_rearm_param ? 1 : 0; +} +EXPORT_SYMBOL_GPL(gsp_phys_gr_wedge_rearm_value); + +/* + * r449 SHADER-CONSTANT-CACHE invalidate (gsp_phys_gr_ccache_inv; default 0). + * The forwarded QMD's cache-control block (QMDV02_03 word 5) carries the + * per-cache invalidate bits; the captured guest QMDs always set bits 26..29 + * (texture header/sampler/data + shader DATA) and the M91 fix adds bit 30 + * (INSTRUCTION), but bit 31 (SHADER CONSTANT - where a kernel's launch + * parameters live) is never set. The forwarded launch's param page is + * rewritten in place between launches (the two-bank win0 pool), so a stale + * constant-cache line can serve the PREVIOUS launch's parameters - the + * kernel then computes with the previous launch's pointers/sizes and the + * current launch's tracked output pages stay untouched ("no stores"). + * Mode: 0 = off, 1 = set bit 31 only, 2 = force the full 0xff000000 + * invalidate set. Read at submission time, so it can be A/B'd at runtime. + */ +static int gsp_phys_gr_ccache_inv_param; +module_param_named(gsp_phys_gr_ccache_inv, gsp_phys_gr_ccache_inv_param, + int, 0644); +MODULE_PARM_DESC(gsp_phys_gr_ccache_inv, + "r449: forwarded-QMD shader-constant-cache invalidate: 0=off, 1=bit31, 2=all invalidate bits (default 0)"); + +unsigned int gsp_phys_gr_ccache_inv_value(void) +{ + if (gsp_phys_gr_ccache_inv_param < 0) + return 0; + if (gsp_phys_gr_ccache_inv_param > 2) + return 2; + return (unsigned int)gsp_phys_gr_ccache_inv_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_gr_ccache_inv_value); + +/* + * r410 wedge quiet window (gsp_phys_gr_wedge_quiet_ms; default 40000, clamped + * [0,120000]; 0 disables). The sticky per-lane no-execute wedge (100% of + * launches no-write while kept busy) is proven to CLEAR when the lane stays + * quiet for tens of seconds (r409: a ~37 s client gap healed a wedged run; + * r416: 58 consecutive no-write launches during a 126 s client run, first + * launches green again after the lane sat ~52 s idle; r417: a 20 s host hold + * did not heal, a 40 s hold did - "WEDGE-QUIET healed: engine wrote 1 + * page(s) after the 40000 ms hold"). Immediate retries and + * re-arms do not heal it (r409/r154). So when a zero-write outcome follows a + * previous accepted zero-write launch, the retry loop first holds this many ms + * with NO submissions to the engine, then re-arms (lane enable + schedules) + * and re-publishes once. The value doubles per further quiet retry. The + * quiet window is recovery-only: a launch that wrote anything never enters it. + */ +static unsigned int gsp_phys_gr_wedge_quiet_ms_param = 40000u; +module_param_named(gsp_phys_gr_wedge_quiet_ms, + gsp_phys_gr_wedge_quiet_ms_param, uint, 0644); +MODULE_PARM_DESC(gsp_phys_gr_wedge_quiet_ms, + "Quiet-window ms before retrying a launch on a lane whose previous launch also no-wrote (0=disable, default 40000; r417: a 20 s hold did not heal, 40 s healed)"); + +unsigned int gsp_phys_gr_wedge_quiet_ms_value(void) +{ + unsigned int v = gsp_phys_gr_wedge_quiet_ms_param; + + if (v > 120000u) + v = 120000u; + return v; +} +EXPORT_SYMBOL_GPL(gsp_phys_gr_wedge_quiet_ms_value); + +/* * m69 window-0 shadow pool (gsp_phys_rmcall_gr_shadow_win0; default 0). When * set, the forwarded GR launch copies the QMD's program/param pages into the * window-0 shadow pool (module pages at VAs 0x108000..0x10f000, PTEs planted @@ -1050,6 +1370,28 @@ int gsp_phys_rmcall_gr_data_prewin_value(void) EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_data_prewin_value); /* + * B-2: grown boot-fixed data-bounce region (gsp_phys_gr_data_region; default + * 1). When set, gsp_phys_rmcall_pdb_build plants the 16384-page (64 MiB) + * region at GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 (PD0 80..111) and the SVC + * relocates operand pages into it; the booter's copy-in/copy-out index the + * grown chunk. When 0, everything falls back to the legacy 205-page / + * 68-pages-per-pointer window below the mail page, for one-flip A/B + * trialing. Boot-fixed: it is read by the stage-3 plant, so set it at + * insmod, not at runtime. + */ +static int gsp_phys_gr_data_region_param = 1; +module_param_named(gsp_phys_gr_data_region, + gsp_phys_gr_data_region_param, int, 0644); +MODULE_PARM_DESC(gsp_phys_gr_data_region, + "Use the grown 64 MiB data-bounce region for GR operands (default 1; 0 = legacy 205-page window)"); + +int gsp_phys_gr_data_region_value(void) +{ + return gsp_phys_gr_data_region_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_gr_data_region_value); + +/* * m113: after planting the launch's data mappings, re-register the VAS page * directory (UNSET+SET) to force the server walker to re-snapshot the tree; * all evidence says the SM walker resolves only the commit-time tree, and @@ -1231,16 +1573,28 @@ EXPORT_SYMBOL_GPL(gsp_phys_gpu_fw_crc32); * of a shared file. A GPA is translated by reading the file page that holds * it (read_cache_page() pulls it into the page cache and pins it for the * read) and converting that page to a host PA with page_to_phys(). The - * ranges map guest GPA windows to file offsets; up to four windows of one - * file per module cycle. + * ranges map guest GPA windows to file offsets; up to four windows per file. + * + * Per-mdev records (dual-guest goal 1.0): a guest RAM backing file belongs + * to ONE guest session, so the record moves with the mdev. Slot 0 is the + * legacy record (vgpu == NULL): it is what the historical top-level + * gsp-phys-guest-ram debugfs write configures and it serves the first/only + * open mdev exactly as the pre-dual-guest code did (one guest: byte-identical + * behaviour). A per-mdev record is configured with the mdev UUID as an extra + * first token on the same debugfs file and serves only that mdev; the + * record is released when the mdev is removed (gsp_phys_guest_ram_vgpu_clear) + * or when the physical cycle ends (gsp_phys_guest_ram_clear). * * Locking: guest_ram_lock serialises configure/clear against each other and * against the translators; it is held across read_cache_page() (which may * sleep) so a clear cannot fput the file while a translation is using it. - * One file per module cycle: a second path is refused -EBUSY because a fresh - * cycle reloads the module; more ranges of the same file may be added. + * Within one record the historical rule is unchanged: the first configure + * opens the file, a different path is refused -EBUSY, more ranges of the + * same file may be added. */ #define GSP_PHYS_GUEST_RAM_RANGES 4 +/* Per-mdev guest-RAM records (a handful of guests; 4 is far beyond 1.0). */ +#define GSP_PHYS_GUEST_RAM_MDEVS 4 struct gsp_phys_guest_ram_range { u64 gpa_base; @@ -1248,35 +1602,121 @@ struct gsp_phys_guest_ram_range { u64 size; }; -static struct file *guest_ram_file; -static char guest_ram_path[128]; -static struct gsp_phys_guest_ram_range guest_ram_ranges[GSP_PHYS_GUEST_RAM_RANGES]; -static unsigned int guest_ram_ranges_n; +struct gsp_phys_guest_ram_rec { + struct nv_vgpu_device *vgpu; /* NULL = legacy (first/only open mdev) */ + struct file *file; + char path[128]; + struct gsp_phys_guest_ram_range ranges[GSP_PHYS_GUEST_RAM_RANGES]; + unsigned int ranges_n; +}; + +static struct gsp_phys_guest_ram_rec guest_ram_legacy; +static struct gsp_phys_guest_ram_rec guest_ram_mdevs[GSP_PHYS_GUEST_RAM_MDEVS]; static DEFINE_MUTEX(guest_ram_lock); +/* Caller holds guest_ram_lock. NULL when this vgpu has no bound record. */ +static struct gsp_phys_guest_ram_rec * +gsp_phys_guest_ram_rec_find_locked(const struct nv_vgpu_device *vgpu) +{ + unsigned int i; + + if (!vgpu) + return NULL; + for (i = 0; i < GSP_PHYS_GUEST_RAM_MDEVS; i++) + if (guest_ram_mdevs[i].vgpu == vgpu) + return &guest_ram_mdevs[i]; + return NULL; +} + +/* Caller holds guest_ram_lock. NULL when every per-mdev slot is taken. */ +static struct gsp_phys_guest_ram_rec * +gsp_phys_guest_ram_rec_alloc_locked(struct nv_vgpu_device *vgpu) +{ + unsigned int i; + + for (i = 0; i < GSP_PHYS_GUEST_RAM_MDEVS; i++) + if (!guest_ram_mdevs[i].vgpu) { + guest_ram_mdevs[i].vgpu = vgpu; + return &guest_ram_mdevs[i]; + } + return NULL; +} + /* - * Configure the guest RAM backing file and add (or replace, keyed by - * gpa_base) one GPA window. The first call opens the file read-only and - * fixes the path for this module cycle; later calls may only add ranges of - * that same file. The file must be a regular file and every range must fit - * inside its current size, so a translation can never index a page beyond - * EOF. Returns 0, -EBUSY for a different path, -ENOSPC with all four ranges - * in use, or the file/argument error. + * Number of mdevs whose VFIO device is currently open. Shared by the + * per-mdev debugfs selector (which mdev does this UUID name?) and the + * single-guest guard on the legacy test-only paths. */ -int gsp_phys_guest_ram_configure(const char *path, u64 gpa_base, - u64 file_offset, u64 size) +int gsp_phys_open_mdev_count(void) +{ + struct nv_vgpu_device *d; + int n = 0; + + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(d, &nv_vgpu_driver.devices, list) { + if (atomic_read(&d->vfio_dev_open_count) > 0) + n++; + } + mutex_unlock(&nv_vgpu_driver.lock); + return n; +} +EXPORT_SYMBOL_GPL(gsp_phys_open_mdev_count); + +/* + * Resolve an OPEN mdev by its device name (the mdev UUID, e.g. + * "b6acdc5e-..."). Returns the device or NULL. The caller must use the + * result only while that mdev's VFIO fd is known to be open (the same + * lifetime rule as the first-open resolution below): the driver list is + * walked under nv_vgpu_driver.lock, but the returned pointer is not + * reference-counted. + */ +struct nv_vgpu_device *gsp_phys_open_mdev_by_name(const char *name) +{ + struct nv_vgpu_device *d, *found = NULL; + + if (!name || !*name) + return NULL; + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(d, &nv_vgpu_driver.devices, list) { + if (atomic_read(&d->vfio_dev_open_count) > 0 && d->mdev && + !strcmp(dev_name(&d->mdev->dev), name)) { + found = d; + break; + } + } + mutex_unlock(&nv_vgpu_driver.lock); + return found; +} +EXPORT_SYMBOL_GPL(gsp_phys_open_mdev_by_name); + +/* + * Shared configure body for one record: open the record's file (first call) + * and add (or replace, keyed by gpa_base) one GPA window. The first call + * opens the file read-only and fixes the path for the record's lifetime; + * later calls may only add ranges of that same file. The file must be a + * regular file and every range must fit inside its current size, so a + * translation can never index a page beyond EOF. Returns 0, -EBUSY for a + * different path, -ENOSPC with all four ranges in use, or the + * file/argument error. `who` is "" for the legacy record and + * " for mdev <uuid>" for a per-mdev record; the legacy strings stay + * byte-identical. + */ +static int gsp_phys_guest_ram_rec_configure(struct gsp_phys_guest_ram_rec *rec, + const char *path, u64 gpa_base, + u64 file_offset, u64 size, + const char *who) { struct gsp_phys_guest_ram_range *range; struct file *f = NULL; unsigned int i; int ret = 0; - if (!path || !*path) + if (!rec || !path || !*path) return -EINVAL; mutex_lock(&guest_ram_lock); - if (!guest_ram_file) { + if (!rec->file) { /* * O_LARGEFILE is required: the guest RAM file is multi-GiB and * generic_file_open() (shmem's open) refuses a file beyond @@ -1306,35 +1746,38 @@ int gsp_phys_guest_ram_configure(const char *path, u64 gpa_base, ret = -ERANGE; goto out_fput; } - guest_ram_file = f; - strscpy(guest_ram_path, path, sizeof(guest_ram_path)); - } else if (strcmp(guest_ram_path, path)) { - pr_err("vgsp: guest RAM already configured from %s; one file per module cycle (reload to switch); refusing %s\n", - guest_ram_path, path); + rec->file = f; + strscpy(rec->path, path, sizeof(rec->path)); + } else if (strcmp(rec->path, path)) { + if (who[0]) + pr_err("vgsp: guest RAM already configured from %s; one file per guest session (reload or mdev remove to switch); refusing %s\n", + rec->path, path); + else + pr_err("vgsp: guest RAM already configured from %s; one file per module cycle (reload to switch); refusing %s\n", + rec->path, path); ret = -EBUSY; goto out; } - for (i = 0, range = NULL; i < guest_ram_ranges_n; i++) { - if (guest_ram_ranges[i].gpa_base == gpa_base) { - range = &guest_ram_ranges[i]; + for (i = 0, range = NULL; i < rec->ranges_n; i++) { + if (rec->ranges[i].gpa_base == gpa_base) { + range = &rec->ranges[i]; break; } } if (!range) { - if (guest_ram_ranges_n >= GSP_PHYS_GUEST_RAM_RANGES) { + if (rec->ranges_n >= GSP_PHYS_GUEST_RAM_RANGES) { ret = -ENOSPC; goto out; } - range = &guest_ram_ranges[guest_ram_ranges_n++]; + range = &rec->ranges[rec->ranges_n++]; range->gpa_base = gpa_base; } range->file_offset = file_offset; range->size = size; - pr_info("vgsp: guest RAM configured: path=%s gpa=0x%llx off=0x%llx size=0x%llx (ranges=%u)\n", - guest_ram_path, gpa_base, file_offset, size, - guest_ram_ranges_n); + pr_info("vgsp: guest RAM configured%s: path=%s gpa=0x%llx off=0x%llx size=0x%llx (ranges=%u)\n", + who, rec->path, gpa_base, file_offset, size, rec->ranges_n); out: mutex_unlock(&guest_ram_lock); @@ -1344,35 +1787,148 @@ out_fput: fput(f); goto out; } + +/* + * Configure the legacy guest RAM record (the historical top-level + * gsp-phys-guest-ram write). Serves the first/only open mdev exactly as + * before the per-mdev records existed. + */ +int gsp_phys_guest_ram_configure(const char *path, u64 gpa_base, + u64 file_offset, u64 size) +{ + return gsp_phys_guest_ram_rec_configure(&guest_ram_legacy, path, + gpa_base, file_offset, size, + ""); +} EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_configure); /* - * Close the configured guest RAM file (if any) and drop every range. Called - * from the physical-GPU release path so module unload and a backend release - * end the file reference; a fresh cycle reconfigures from debugfs. + * Configure the guest RAM record bound to one mdev (per-mdev selector on the + * debugfs write). A record for this vgpu is created on first use and closed + * by gsp_phys_guest_ram_vgpu_clear() when the mdev is removed. A failed + * first configure leaves no bound record behind. + */ +int gsp_phys_guest_ram_configure_vgpu(struct nv_vgpu_device *vgpu, + const char *path, u64 gpa_base, + u64 file_offset, u64 size) +{ + struct gsp_phys_guest_ram_rec *rec; + char who[80]; + const char *name = (vgpu && vgpu->mdev) ? + dev_name(&vgpu->mdev->dev) : "?"; + bool fresh = false; + int ret; + + if (!vgpu) + return -EINVAL; + + mutex_lock(&guest_ram_lock); + rec = gsp_phys_guest_ram_rec_find_locked(vgpu); + if (!rec) { + rec = gsp_phys_guest_ram_rec_alloc_locked(vgpu); + if (!rec) { + mutex_unlock(&guest_ram_lock); + pr_err("vgsp: guest RAM: no free per-mdev record slot (max %u); refusing mdev %s\n", + GSP_PHYS_GUEST_RAM_MDEVS, name); + return -ENOSPC; + } + fresh = true; + } + mutex_unlock(&guest_ram_lock); + + scnprintf(who, sizeof(who), " for mdev %s", name); + ret = gsp_phys_guest_ram_rec_configure(rec, path, gpa_base, + file_offset, size, who); + + /* + * A fresh record whose first configure failed must not hold the slot + * (the vgpu pointer would otherwise be reserved with no file and the + * next attempt could not create a clean record). + */ + if (ret && fresh) { + mutex_lock(&guest_ram_lock); + if (rec->file) + ret = 0; /* raced with another configure; keep it */ + else + rec->vgpu = NULL; + mutex_unlock(&guest_ram_lock); + } + return ret; +} +EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_configure_vgpu); + +/* Lock held: close one record's file and drop its ranges. */ +static void gsp_phys_guest_ram_rec_close(struct gsp_phys_guest_ram_rec *rec, + const char *release_log) +{ + if (rec->file) { + pr_info("%s: path=%s ranges=%u\n", release_log, + rec->path, rec->ranges_n); + fput(rec->file); + rec->file = NULL; + } + rec->path[0] = '\0'; + memset(rec->ranges, 0, sizeof(rec->ranges)); + rec->ranges_n = 0; +} + +/* + * Close every configured guest RAM record (legacy and per-mdev) and drop + * every range. Called from the physical-GPU release path so module unload + * and a backend release end the file references; a fresh cycle reconfigures + * from debugfs. The legacy release log line is unchanged. */ void gsp_phys_guest_ram_clear(void) { + unsigned int i; + mutex_lock(&guest_ram_lock); - if (guest_ram_file) { - pr_info("vgsp: guest RAM released: path=%s ranges=%u\n", - guest_ram_path, guest_ram_ranges_n); - fput(guest_ram_file); - guest_ram_file = NULL; - } - guest_ram_path[0] = '\0'; - memset(guest_ram_ranges, 0, sizeof(guest_ram_ranges)); - guest_ram_ranges_n = 0; + gsp_phys_guest_ram_rec_close(&guest_ram_legacy, + "vgsp: guest RAM released"); + for (i = 0; i < GSP_PHYS_GUEST_RAM_MDEVS; i++) { + gsp_phys_guest_ram_rec_close(&guest_ram_mdevs[i], + "vgsp: guest RAM released for mdev"); + guest_ram_mdevs[i].vgpu = NULL; + } mutex_unlock(&guest_ram_lock); } EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_clear); +/* + * Release the guest RAM record bound to one mdev. Called from the mdev + * cleanup path so the file reference and the vgpu pointer never outlive the + * session (a removed mdev's translations must stop, and the record slot must + * free up for a restarted guest). + */ +void gsp_phys_guest_ram_vgpu_clear(struct nv_vgpu_device *vgpu) +{ + unsigned int i; + + if (!vgpu) + return; + mutex_lock(&guest_ram_lock); + for (i = 0; i < GSP_PHYS_GUEST_RAM_MDEVS; i++) { + if (guest_ram_mdevs[i].vgpu == vgpu) { + gsp_phys_guest_ram_rec_close(&guest_ram_mdevs[i], + "vgsp: guest RAM released for mdev"); + guest_ram_mdevs[i].vgpu = NULL; + } + } + mutex_unlock(&guest_ram_lock); +} +EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_vgpu_clear); + bool gsp_phys_guest_ram_active(void) { - bool active; + bool active = false; + unsigned int i; mutex_lock(&guest_ram_lock); - active = guest_ram_file != NULL; + if (guest_ram_legacy.file) + active = true; + for (i = 0; !active && i < GSP_PHYS_GUEST_RAM_MDEVS; i++) + if (guest_ram_mdevs[i].file) + active = true; mutex_unlock(&guest_ram_lock); return active; } @@ -1380,33 +1936,78 @@ EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_active); /* * Lock-safe one-line snapshot for the gsp-phys-state dump: active flag, - * path, range count and every range (gpa/offset/size). The caller supplies - * the buffer; nothing is written when the file is not configured beyond the - * active=0 marker. + * path, range count and every range (gpa/offset/size). With only the legacy + * record configured (the one-mdev campaign) the output is byte-identical to + * the pre-dual-guest format; a per-mdev record adds its mdev UUID so a + * dual-guest operator can tell the records apart. */ +static void gsp_phys_guest_ram_rec_state(char *buf, size_t len, + unsigned int *n, + const struct gsp_phys_guest_ram_rec *rec, + const char *label) +{ + unsigned int i; + + if (*n >= len) + return; + *n += scnprintf(buf + *n, len - *n, "%s path=%s ranges=%u", + label, rec->path, rec->ranges_n); + for (i = 0; i < rec->ranges_n && *n < len; i++) + *n += scnprintf(buf + *n, len - *n, + " gpa=0x%llx off=0x%llx size=0x%llx", + rec->ranges[i].gpa_base, + rec->ranges[i].file_offset, + rec->ranges[i].size); +} + void gsp_phys_guest_ram_state(char *buf, size_t len) { unsigned int i, n = 0; + bool any_mdev = false; if (!buf || !len) return; buf[0] = '\0'; mutex_lock(&guest_ram_lock); - if (!guest_ram_file) { - scnprintf(buf, len, "active=0"); + for (i = 0; i < GSP_PHYS_GUEST_RAM_MDEVS; i++) + if (guest_ram_mdevs[i].file) { + any_mdev = true; + break; + } + + if (!any_mdev) { + /* + * Legacy-only (or nothing): keep the historical shape - the + * campaign's parser (if any) sees the same text as before. + */ + if (!guest_ram_legacy.file) { + scnprintf(buf, len, "active=0"); + mutex_unlock(&guest_ram_lock); + return; + } + n += scnprintf(buf + n, len - n, "active=1"); + gsp_phys_guest_ram_rec_state(buf, len, &n, + &guest_ram_legacy, ""); mutex_unlock(&guest_ram_lock); return; } - n += scnprintf(buf + n, len - n, "active=1 path=%s ranges=%u", - guest_ram_path, guest_ram_ranges_n); - for (i = 0; i < guest_ram_ranges_n && n < len; i++) - n += scnprintf(buf + n, len - n, - " gpa=0x%llx off=0x%llx size=0x%llx", - guest_ram_ranges[i].gpa_base, - guest_ram_ranges[i].file_offset, - guest_ram_ranges[i].size); + n += scnprintf(buf + n, len - n, "active=1"); + if (guest_ram_legacy.file) + gsp_phys_guest_ram_rec_state(buf, len, &n, &guest_ram_legacy, + " legacy"); + for (i = 0; i < GSP_PHYS_GUEST_RAM_MDEVS && n < len; i++) { + char label[64]; + + if (!guest_ram_mdevs[i].file) + continue; + scnprintf(label, sizeof(label), " mdev=%s", + (guest_ram_mdevs[i].vgpu && guest_ram_mdevs[i].vgpu->mdev) ? + dev_name(&guest_ram_mdevs[i].vgpu->mdev->dev) : "?"); + gsp_phys_guest_ram_rec_state(buf, len, &n, + &guest_ram_mdevs[i], label); + } mutex_unlock(&guest_ram_lock); } EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_state); @@ -1427,12 +2028,20 @@ EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_state); * an unmapped hole or an MMIO frame can never be handed to the engine as * guest RAM. * - * Guards kept from the original path: the active-mdev walk + * Guards kept from the original path: the mdev walk * (nv_vgpu_driver.lock for the bounded list scan) gates every translation - - * the engine path must only run with a live guest session; no such session - * returns -ENODEV. The IOMMU-domain query remains as a documented fallback - * for a setup whose mdev has a real domain (it is inert with the - * emulated-IOMMU mdev used here). + * the engine path must only run with a live guest session; a vgpu given by + * the caller must itself be open, and no open session at all returns + * -ENODEV. The IOMMU-domain query remains as a documented fallback for a + * setup whose mdev has a real domain (it is inert with the emulated-IOMMU + * mdev used here). + * + * Per-mdev resolution (dual-guest goal 1.0): the caller names the owning + * vgpu (a captured forwarding request, a held release, ...). The vgpu's own + * record resolves the translation; the legacy record (configured by the + * historical top-level debugfs write) still serves the FIRST open mdev so + * single-guest scripts are unchanged. vgpu == NULL keeps the historical + * resolution: the first open mdev. * * Locking: guest_ram_lock is held across the range walk and the * read_cache_page() so a concurrent configure/clear cannot change the file @@ -1442,13 +2051,15 @@ EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_state); * own lifetime guarantees for an open fd. No guest or rm lock may be held by * the caller. */ -int gsp_phys_gpa_to_hpa(u64 gpa, u64 *hpa_out) +int gsp_phys_gpa_to_hpa_vgpu(struct nv_vgpu_device *vgpu, u64 gpa, u64 *hpa_out) { - struct nv_vgpu_device *vgpu = NULL; struct nv_vgpu_device *d; + struct nv_vgpu_device *first = NULL; + struct gsp_phys_guest_ram_rec *rec = NULL; struct iommu_domain *dom; phys_addr_t p; unsigned int i; + bool vgpu_gone = false; if (!hpa_out) return -EINVAL; @@ -1456,19 +2067,28 @@ int gsp_phys_gpa_to_hpa(u64 gpa, u64 *hpa_out) mutex_lock(&nv_vgpu_driver.lock); list_for_each_entry(d, &nv_vgpu_driver.devices, list) { if (atomic_read(&d->vfio_dev_open_count) > 0) { - vgpu = d; + first = d; break; } } + if (vgpu && atomic_read(&vgpu->vfio_dev_open_count) <= 0) + vgpu_gone = true; /* named session is gone: never retarget */ mutex_unlock(&nv_vgpu_driver.lock); + if (vgpu_gone) + return -ENODEV; + if (!vgpu) + vgpu = first; /* historical resolution: first open mdev */ if (!vgpu) return -ENODEV; mutex_lock(&guest_ram_lock); - if (guest_ram_file) { - for (i = 0; i < guest_ram_ranges_n; i++) { + rec = gsp_phys_guest_ram_rec_find_locked(vgpu); + if (!rec && vgpu == first) + rec = &guest_ram_legacy; /* legacy file: first/only mdev */ + if (rec && rec->file) { + for (i = 0; i < rec->ranges_n; i++) { const struct gsp_phys_guest_ram_range *r = - &guest_ram_ranges[i]; + &rec->ranges[i]; struct page *page; u64 file_off; @@ -1477,7 +2097,7 @@ int gsp_phys_gpa_to_hpa(u64 gpa, u64 *hpa_out) continue; file_off = r->file_offset + (gpa - r->gpa_base); - page = read_cache_page(guest_ram_file->f_mapping, + page = read_cache_page(rec->file->f_mapping, file_off >> PAGE_SHIFT, NULL, NULL); mutex_unlock(&guest_ram_lock); @@ -1513,6 +2133,18 @@ int gsp_phys_gpa_to_hpa(u64 gpa, u64 *hpa_out) *hpa_out = (u64)p; return 0; } +EXPORT_SYMBOL_GPL(gsp_phys_gpa_to_hpa_vgpu); + +/* + * Historical entry point: resolve through the first open mdev (and that + * mdev's own record, else the legacy record). Used by the single-guest + * boot-ladder guest-GPA pin (test-only; see gsp_phys_guest_gpa_write) and + * kept as the compatibility wrapper for callers without a request context. + */ +int gsp_phys_gpa_to_hpa(u64 gpa, u64 *hpa_out) +{ + return gsp_phys_gpa_to_hpa_vgpu(NULL, gpa, hpa_out); +} EXPORT_SYMBOL_GPL(gsp_phys_gpa_to_hpa); /* diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h index f93603c..36c2220 100644 --- a/kernel/gsp_phys_gpu.h +++ b/kernel/gsp_phys_gpu.h @@ -3,6 +3,8 @@ #include <linux/types.h> +struct nv_vgpu_device; + /* * Physical GPU ownership (real-GSP execution backend, bring-up rung 1). * Claim is disabled unless the gsp_phys_gpu module parameter is set. @@ -352,6 +354,10 @@ int gsp_phys_rmcall_gr_mail_patch_value(void); /* m106: pre-install the data PT window pool in stage 3. */ int gsp_phys_rmcall_gr_data_prewin_value(void); +/* B-2: use the grown 64 MiB boot-fixed data-bounce region (default 1; 0 = + * legacy 205-page window). Read by the stage-3 plant and the SVC fill. */ +int gsp_phys_gr_data_region_value(void); + /* m113: re-register the VAS page directory after planting data. */ int gsp_phys_rmcall_gr_recommit_value(void); @@ -365,6 +371,56 @@ unsigned int gsp_phys_rmcall_gr_lmem_tpc_size_value(void); /* m72: log SKED CWD/HWW register transitions during the launch poll. */ int gsp_phys_rmcall_gr_skedrec_value(void); +/* m153: re-execute a forwarded launch whose retirement wait saw zero or + * wrong-place engine writes (default 1 = enabled). */ +int gsp_phys_rmcall_gr_nowrite_retry_value(void); + +/* F2: on the first GR launch timeout, run the extended re-arm + a fresh ring + * entry immediately and re-poll briefly (default 1; 0 = legacy ladder). */ +int gsp_phys_gr_retry_fast_value(void); + +/* F2: first GR launch poll window in ms, clamped [1000,30000], read while + * gsp_phys_gr_retry_fast=1 (default 2500). */ +unsigned int gsp_phys_gr_first_poll_ms_value(void); + +/* M91: set QMDV02_03 INVALIDATE_INSTRUCTION_CACHE (word5 bit30) on forwarded + * GR launches whose QMD carries the driver cache-control signature + * (default 1; 0 = pre-m91 QMD words). */ +int gsp_phys_gr_icache_inv_value(void); + +/* r433: force QMD RELEASE_MEMBAR_TYPE=FE_SYSMEMBAR (QMDV00_06 word11 bit14) + * on forwarded GR launches so the engine flushes pending sysmem stores + * before writing the release semaphore (default 0 = guest's FE_NONE). */ +int gsp_phys_gr_rel_membar_value(void); + +/* r434: enable the CWD slot count PRI (SKED_DEBUG_1 bit11) and emit + * SET_CWD_SLOT_COUNT with this value in the GR init preamble + * (-1 = off, default). */ +int gsp_phys_rmcall_gr_cwd_slot_value(void); + +/* m179: dedicated re-execution budget for a data-bearing GR launch whose + * retirement wait saw zero engine writes, independent of the timeout ladder's + * attempt counter (default 3, clamped [0,4]). */ +int gsp_phys_gr_nowrite_retries_value(void); +int gsp_phys_gr_wedge_rearm_value(void); +unsigned int gsp_phys_gr_ccache_inv_value(void); + +/* STALLFREE: absolute per-launch GR submit wall-clock budget in ms, clamped + * [10000,600000]; 0 = only the built-in per-ladder bounds (default 60000). + * On expiry the submit fails out immediately with a GR-STALL-wait-max line + * and the sema service authors the QMD completion release in software. */ +unsigned int gsp_phys_gr_wait_max_ms_value(void); + +/* r410 wedge quiet window: when a launch's zero-write outcome follows a + * previous accepted zero-write launch (the sticky per-lane no-execute wedge), + * hold the retry for this many ms with NO submissions to the engine before + * re-publishing. The wedge is proven to clear when the lane stays quiet for + * tens of seconds (r409 ~37 s, r416 ~52 s); immediate retries keep it alive. + * Base value for the first quiet retry, doubled per further attempt, clamped + * [0,120000]; 0 disables the quiet path (retries run back-to-back as before). + * Default 20000. */ +unsigned int gsp_phys_gr_wedge_quiet_ms_value(void); + /* m73: use host-visible sysmem chunks for V7 instance/RAMFC (m62 shape). */ int gsp_phys_rmcall_gr_inst_sys_value(void); @@ -425,21 +481,50 @@ bool gsp_phys_rmcall_vas_shared_value(void); int gsp_phys_gpa_to_hpa(u64 gpa, u64 *hpa_out); /* + * Per-mdev variant: translate the GPA for the named owning vgpu (a captured + * forwarding request, a held release). The vgpu's own guest-RAM record is + * authoritative; the legacy record still serves the FIRST open mdev so + * single-guest callers are unchanged. NULL keeps the historical resolution + * (first open mdev). A named vgpu whose session is closed returns -ENODEV + * rather than being retargeted at another guest. + */ +int gsp_phys_gpa_to_hpa_vgpu(struct nv_vgpu_device *vgpu, u64 gpa, + u64 *hpa_out); + +/* * Guest RAM backing-file configuration (debugfs gsp-phys-guest-ram). One - * shared file per module cycle, up to four GPA->file-offset ranges keyed by + * record per guest session, up to four GPA->file-offset ranges keyed by * gpa_base. The first configure opens the file; the same path adds/replaces * ranges, a different path is refused -EBUSY, and the ranges are dropped and - * the file closed by gsp_phys_guest_ram_clear() on physical-GPU release. The + * the file closed by gsp_phys_guest_ram_clear() on physical-GPU release (or + * gsp_phys_guest_ram_vgpu_clear() when one mdev is removed). The + * gsp_phys_guest_ram_configure() entry point configures the legacy record + * (the historical top-level debugfs write: serves the first/only open mdev); + * gsp_phys_guest_ram_configure_vgpu() binds the record to one mdev. The * state accessor is lock-safe and writes "active=0" or the active path, * range count and per-range gpa/off/size into the caller's buffer. */ int gsp_phys_guest_ram_configure(const char *path, u64 gpa_base, u64 file_offset, u64 size); +int gsp_phys_guest_ram_configure_vgpu(struct nv_vgpu_device *vgpu, + const char *path, u64 gpa_base, + u64 file_offset, u64 size); void gsp_phys_guest_ram_clear(void); +void gsp_phys_guest_ram_vgpu_clear(struct nv_vgpu_device *vgpu); bool gsp_phys_guest_ram_active(void); void gsp_phys_guest_ram_state(char *buf, size_t len); /* + * Open-mdev helpers for the per-mdev debugfs selector and the single-guest + * guards: count the mdevs whose VFIO device is open, or resolve one open + * mdev by its device name (the mdev UUID). The returned device is not + * reference-counted; the caller must only use it while that mdev's fd is + * known to be open. + */ +int gsp_phys_open_mdev_count(void); +struct nv_vgpu_device *gsp_phys_open_mdev_by_name(const char *name); + +/* * Domain shape of the physical (claimed) device, as the P6a engine path * needs it: the module programs raw host PAs into the CE page tables, so the * engine may only run when the physical device's DMA addresses ARE host PAs. diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c index 8318f56..2a957b3 100644 --- a/kernel/gsp_rm_emulation.c +++ b/kernel/gsp_rm_emulation.c @@ -452,8 +452,26 @@ static u32 gsp_rm_build_record(struct gsp_rm_emu *rm, u8 *buf, u32 buf_size, /* * Write a status queue record (1+ elements) and advance the writePtr. */ -static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function, - u32 sequence, u32 result, const void *data, u32 data_len) +/* + * m171: the event burst budget is a rate over a short time window, not a + * monotonic session counter. Without the time reset a flood left the counter + * pinned at the cap and silently dropped the completion post of a client + * that only started waiting later - a minimal cudaEventSynchronize repro sat + * forever while its registrations were valid and 'completion hint rc=-22' + * repeated. Channel-completion posts get headroom over hint re-posts: a hint + * can be retried a moment later, a completion wake-up cannot. + */ +#define GSPRM_EVENT_BURST_CRIT_MAX 48 +#define GSPRM_EVENT_BURST_WINDOW (HZ) +/* Strictly shorter than the service's 0.5 s hint cadence: an equal window + * races the boundary and every later hint skips forever (observed r181: + * 'completion hint rc=-2' forever after one success and the stream client + * still stalled - the late-waiter wake-up never fired again). */ +#define GSPRM_EVENT_HINT_DEDUP (HZ / 4) + +static int gsp_rm_send_flags(struct gsp_emu_context *ctx, u32 function, + u32 sequence, u32 result, const void *data, + u32 data_len, bool critical) { struct gsp_rm_emu *rm = ctx->rm_emu; u8 *buf; @@ -489,13 +507,22 @@ static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function, * newer record and the client aborts with "Bad sequence number". */ if (function == GSPRM_RPC_EVENT_POST_EVENT) { - if (rm->event_burst >= GSPRM_EVENT_BURST_MAX) { + u32 cap = critical ? GSPRM_EVENT_BURST_CRIT_MAX + : GSPRM_EVENT_BURST_MAX; + + /* m171: expire the budget by wall time so a long-ago + * flood can never starve a fresh completion post. */ + if (time_after(jiffies, rm->event_burst_at + + GSPRM_EVENT_BURST_WINDOW)) + rm->event_burst = 0; + rm->event_burst_at = jiffies; + if (rm->event_burst >= cap) { rm->events_dropped++; if (rm->events_dropped <= 8 || (rm->events_dropped % 1000) == 0) nv_vgpu_info(ctx->vgpu, - "vgsp-rm: event burst capped (%u), dropped total=%llu", - GSPRM_EVENT_BURST_MAX, + "vgsp-rm: event burst capped (%u/%u), dropped total=%llu", + rm->event_burst, cap, rm->events_dropped); mutex_unlock(&rm->stat_lock); kfree(buf); @@ -504,6 +531,7 @@ static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function, rm->event_burst++; } else { rm->event_burst = 0; + rm->event_burst_at = jiffies; } record_size = gsp_rm_build_record(rm, buf, buf_size, function, @@ -524,6 +552,106 @@ static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function, return -EOVERFLOW; } + /* + * fail4: free-space discipline. The guest's consumption cursor for + * the status queue (msgqRxHeader.readPtr, at stat+rxHdrOff under the + * queue's SWAP_RX layout - same location msgqTxGetFreeSpace polls) + * gates this write. Overwriting unread records mixes entries in the + * guest's RPC stream: under a completion burst the guest then runs + * garbage RM callbacks (rm_run_rc_callback+0x0) and dies. Events are + * wake-ups and drop freely; a reply waits briefly for the ring to + * drain and is dropped (never overwritten) if it stays full. + */ + { + u32 peer_read = 0; + bool have_peer = false; + int w; + + for (w = 0; w < 51; w++) { + u32 used, space, rxoff = 0; + u8 shdr[32]; + + /* + * r332 SWAP_RX cursor fix: the guest's consumption + * cursor for the STATUS queue lives in the COMMAND + * queue's RX header, not in the status queue's own + * rxHdrOff. Under MSGQ_FLAGS_SWAP_RX (both sides set + * it: flags=1 in both headers) the driver's + * msgqRxMarkConsumed writes its status readPtr to + * pOurRxHdr->readPtr, and the driver's pOurRxHdr is + * the COMMAND-queue buffer's RX header (msgq.c + * msgqRxLink swap branch: pReadOutgoing = + * &pOurRxHdr->readPtr with pOurRxHdr from + * msgqTxCreate's pBackingStore = pCommandQueue). The + * status queue's own +rxHdrOff is the mirror we + * publish: OUR command-queue read position + * (written below after every consumed command + * record). Reading that mirror here made the + * free-space math track our own writes: the ring was + * declared "full" with the guest mid-drain, replies + * were written into it anyway, unread records were + * overwritten, and the guest RM read a newer record + * ("Bad sequence number. Expected N got N+63") - the + * first-client cudaMalloc then fails with "device + * busy or unavailable" (the boot lottery) and every + * later RPC fails until a fresh boot. + */ + if (gsp_rm_read_buf(ctx, rm->cmd_off, shdr, sizeof(shdr))) + break; + memcpy(&rxoff, shdr + 24, 4); + if (!rxoff || rxoff >= 0x1000) + break; + if (gsp_rm_read_buf(ctx, rm->cmd_off + rxoff, &peer_read, 4) || + peer_read >= rm->stat_msgcount) + break; /* no usable cursor: blind write */ + have_peer = true; + used = (rm->stat_write + rm->stat_msgcount - + peer_read) % rm->stat_msgcount; + space = rm->stat_msgcount - used - 1u; + if (elem_count <= space) + break; + /* + * Only event wake-ups drop freely; a reply waits + * (bounded) and then proceeds - dropping replies + * breaks the guest's RM init handshake (observed: + * two fn=0x1 replies dropped at boot -> the device + * never comes up). + */ + if (function == GSPRM_RPC_EVENT_POST_EVENT || w == 50) + break; + mutex_unlock(&rm->stat_lock); + msleep(1); + mutex_lock(&rm->stat_lock); + } + if (have_peer) { + u32 used = (rm->stat_write + rm->stat_msgcount - + peer_read) % rm->stat_msgcount; + u32 space = rm->stat_msgcount - used - 1u; + + if (elem_count > space && + function == GSPRM_RPC_EVENT_POST_EVENT) { + rm->events_dropped++; + if (rm->events_dropped <= 8 || + (rm->events_dropped % 1000) == 0) + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: status ring full (free=%u need=%u peer=%u write=%u mc=%u); event dropped, total=%llu", + space, elem_count, + peer_read, rm->stat_write, + rm->stat_msgcount, + rm->events_dropped); + kfree(buf); + mutex_unlock(&rm->stat_lock); + return -EAGAIN; + } + if (elem_count > space) + nv_vgpu_warn(ctx->vgpu, + "vgsp-rm: status ring full for reply fn=0x%x (free=%u need=%u peer=%u write=%u mc=%u); writing anyway after wait", + function, space, elem_count, + peer_read, rm->stat_write, + rm->stat_msgcount); + } + } + /* Write all elements to consecutive queue slots. Page-aware: the * queue buffer is non-contiguous, so translate per page. */ slot = rm->stat_write % rm->stat_msgcount; @@ -589,6 +717,114 @@ static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function, return 0; } +/* Non-event sends: the historical entry point, budget untouched. */ +static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function, + u32 sequence, u32 result, const void *data, u32 data_len) +{ + return gsp_rm_send_flags(ctx, function, sequence, result, data, + data_len, false); +} + +/* + * r576 DIAG: event-post payload witness (gsp_rm_ev_log=N budget per boot). + * Every EVENT_POST_EVENT record actually sent is logged with its full + * payload and send result: when a client parks waiting for a completion + * wake-up that never arrives, this shows whether the post was generated, + * with which hClient/hEvent, and whether the status-queue write accepted it + * (rc=0) or dropped it (burst cap / ring full => rc<0). The companion + * sentinel fires only when post_all_events found valid events but could not + * post any of them while none were dedup-skipped - the "client waits, host + * thinks it posted" anomaly. 0 (default) = off. + */ +unsigned gsp_rm_ev_log_max; +static unsigned gsp_rm_ev_log; + +static void gsp_rm_evlog_post(struct nv_vgpu_device *vgpu, + const struct gsp_rm_emu *rm, u32 idx, + const char *src, int rc) +{ + if (!gsp_rm_ev_log_max || gsp_rm_ev_log >= gsp_rm_ev_log_max) + return; + gsp_rm_ev_log++; + nv_vgpu_info(vgpu, + "vgsp-rm: EVPOST %s ev[%u] cl=0x%x ev=0x%x nidx=0x%x data=0x%x rc=%d", + src, idx, rm->events[idx].h_client, + rm->events[idx].h_event, rm->events[idx].notify_index, + rm->events[idx].data, rc); +} + +/* + * r577 experiment knobs (runtime-writable; default 0 = legacy behaviour). + * The r576 live experiment showed the guest consumes our POST_EVENT records + * (rmstate cursors match) yet never enters nv_post_event (clean single-probe + * window, forced rmpost rounds -> entries-written 0): the guest RM drops + * them somewhere between the status-queue read and osNotifyEvent. These + * knobs let one boot A/B the record contents without rebuilding: + * gsp_rm_ev_seq - when nonzero, event records carry the status-queue + * sequence as the RPC-level sequence (legacy: 0) + * gsp_rm_ev_nlist - when nonzero, bNotifyList (payload byte 28) is set, + * selecting the driver's notifier-list dispatch path + * gsp_rm_ev_cl - when nonzero, overrides the payload hClient (e.g. to + * the kernel RM client) for the CliGetEventInfo lookup + */ +unsigned gsp_rm_ev_seq; +unsigned gsp_rm_ev_nlist; +unsigned gsp_rm_ev_cl; +/* + * r578: reachability probe for the guest-RM event drop. When nonzero, event + * records carry this RPC function instead of POST_EVENT (0x1003). Posting a + * function with an unmistakable guest-side side effect (e.g. + * NV_VGPU_MSG_EVENT_UCODE_LIBOS_PRINT 0x100C with an unsupported ucodeEngDesc + * trips NV_ASSERT_FAILED in _kgspRpcUcodeLibosPrint, or + * NV_VGPU_MSG_EVENT_OS_ERROR_LOG 0x1006 logs an errString) proves whether + * the guest ever DISPATCHES our status-queue event records: visible side + * effect => the dispatch works and POST_EVENT is dropped inside + * _kgspRpcPostEvent; no side effect => the records are consumed without + * dispatch. + */ +unsigned gsp_rm_ev_func; + +static void gsp_rm_fill_ev_payload(const struct gsp_rm_emu *rm, u32 i, + u8 *payload) +{ + u32 h_client = rm->events[i].h_client; + /* + * r579 THE EVENT-DROP ROOT CAUSE: NV0005_ALLOC_PARAMETERS.notifyIndex + * is OVERLOADED (cl0005.h): INDEX=bits[15:0], SUBDEVICE=bits[23:16], + * FLAGS=bits[31:24]. The registration word (e.g. 0x3c00000c) must not + * be posted verbatim: the guest's _kgspRpcPostEvent hands the payload + * notifyIndex straight to gpuNotifySubDeviceEvent_IMPL, which indexes + * its notifier array by it - posting 0x3c00000c produced a wild + * pointer (base + 0x3c00000c*4 + 0x1508) and a kernel Oops + * ("BUG: unable to handle page fault ... gpuNotifySubDeviceEvent_IMPL + * +0x21f", RAX=0x3c00000c) on every dispatched event, which is why + * nv_post_event never fired and clients waiting on event wakes + * wedged. Post the INDEX field only. + */ + u32 notify_index = rm->events[i].notify_index & 0xffffu; + + memset(payload, 0, 32); + if (gsp_rm_ev_cl) + h_client = gsp_rm_ev_cl; + memcpy(payload + 0, &h_client, 4); + memcpy(payload + 4, &rm->events[i].h_event, 4); + memcpy(payload + 8, ¬ify_index, 4); + memcpy(payload + 12, &rm->events[i].data, 4); + /* info16@16, status@20, eventDataSize@24 = 0 */ + if (gsp_rm_ev_nlist) + payload[28] = 1; /* bNotifyList */ +} + +static u32 gsp_rm_ev_sequence(const struct gsp_rm_emu *rm) +{ + return gsp_rm_ev_seq ? rm->stat_seq : 0; +} + +static u32 gsp_rm_ev_function(void) +{ + return gsp_rm_ev_func ? gsp_rm_ev_func : GSPRM_RPC_EVENT_POST_EVENT; +} + /* * Post the GPU event that tells the guest's CPU-side RM a channel's * submitted work is complete. A real GPU raises an interrupt, the host GSP @@ -621,15 +857,18 @@ int gsp_rm_post_channel_event(struct nv_vgpu_device *vgpu, u32 h_src) if (!rm->events[i].valid || rm->events[i].h_src != h_src) continue; - memset(payload, 0, sizeof(payload)); - memcpy(payload + 0, &rm->events[i].h_client, 4); - memcpy(payload + 4, &rm->events[i].h_event, 4); - memcpy(payload + 8, &rm->events[i].notify_index, 4); - memcpy(payload + 12, &rm->events[i].data, 4); - /* info16@16, status@20, eventDataSize@24, bNotifyList@28 = 0 */ - if (gsp_rm_send(ctx, GSPRM_RPC_EVENT_POST_EVENT, 0, 0, - payload, sizeof(payload))) - continue; + gsp_rm_fill_ev_payload(rm, i, payload); + { + int rc = gsp_rm_send_flags(ctx, gsp_rm_ev_function(), + gsp_rm_ev_sequence(rm), 0, + payload, sizeof(payload), + true); + + gsp_rm_evlog_post(vgpu, rm, i, "chan", rc); + if (rc) + continue; + } + rm->events[i].last_post = jiffies; posted++; } @@ -643,14 +882,34 @@ int gsp_rm_post_channel_event(struct nv_vgpu_device *vgpu, u32 h_src) for (i = 0; i < rm->events_n; i++) { if (!rm->events[i].valid) continue; - memset(payload, 0, sizeof(payload)); - memcpy(payload + 0, &rm->events[i].h_client, 4); - memcpy(payload + 4, &rm->events[i].h_event, 4); - memcpy(payload + 8, &rm->events[i].notify_index, 4); - memcpy(payload + 12, &rm->events[i].data, 4); - if (gsp_rm_send(ctx, GSPRM_RPC_EVENT_POST_EVENT, 0, 0, - payload, sizeof(payload))) + /* + * r580: apply the hint dedup to the per-completion + * shotgun. Every completion posts ALL registered + * events (the ABI gives no source association), so a + * completion burst burns the burst budget 8 records + * per completion and real wake-ups get dropped + * (r579 boot: events_dropped=724). A redundant + * re-post within 250 ms adds no wake-up - the + * client's dispatcher re-reads channel state on any + * event - and the 0.5 s hint re-posts late waiters. + */ + if (time_before(jiffies, rm->events[i].last_post + + GSPRM_EVENT_HINT_DEDUP)) continue; + gsp_rm_fill_ev_payload(rm, i, payload); + { + int rc = gsp_rm_send_flags(ctx, + gsp_rm_ev_function(), + gsp_rm_ev_sequence(rm), + 0, payload, + sizeof(payload), + true); + + gsp_rm_evlog_post(vgpu, rm, i, "chanfb", rc); + if (rc) + continue; + } + rm->events[i].last_post = jiffies; posted++; } } @@ -675,18 +934,18 @@ static int gsp_rm_post_event_idx(struct gsp_emu_context *ctx, u32 idx) { struct gsp_rm_emu *rm = ctx->rm_emu; u8 payload[32]; + int rc; if (idx >= rm->events_n || !rm->events[idx].valid) return -EINVAL; - memset(payload, 0, sizeof(payload)); - memcpy(payload + 0, &rm->events[idx].h_client, 4); - memcpy(payload + 4, &rm->events[idx].h_event, 4); - memcpy(payload + 8, &rm->events[idx].notify_index, 4); - memcpy(payload + 12, &rm->events[idx].data, 4); + gsp_rm_fill_ev_payload(rm, idx, payload); - return gsp_rm_send(ctx, GSPRM_RPC_EVENT_POST_EVENT, 0, 0, - payload, sizeof(payload)); + rc = gsp_rm_send(ctx, gsp_rm_ev_function(), + gsp_rm_ev_sequence(rm), 0, payload, sizeof(payload)); + if (!rc) + rm->events[idx].last_post = jiffies; + return rc; } int gsp_rm_force_post(struct nv_vgpu_device *vgpu, u32 idx) @@ -709,8 +968,14 @@ int gsp_rm_force_post(struct nv_vgpu_device *vgpu, u32 idx) } for (i = 0; i < rm->events_n; i++) { - int r = gsp_rm_post_event_idx(ctx, i); + int r; + /* m171: retired slots are not errors - skipping them + * keeps the hint's rc meaningful (a valid table must not + * report EINVAL because the last allocated slot is dead). */ + if (!rm->events[i].valid) + continue; + r = gsp_rm_post_event_idx(ctx, i); if (r) last = r; else @@ -725,10 +990,68 @@ int gsp_rm_force_post(struct nv_vgpu_device *vgpu, u32 idx) * guest allocates carry no source object, so a per-channel association is * not available in the ABI). Used by the service's completion hint so a * client that starts waiting after the event fired still gets a wake-up. + * + * m171: this is a HINT, and hints repeat every 0.5 s while the window is + * open. Re-posting an event that a real completion post already woke moments + * ago burns the burst budget and floods the status ring without adding a + * wake-up, so skip events posted within GSPRM_EVENT_HINT_DEDUP. A waiter + * that arrives after the edge still gets a re-post within half a second. */ int gsp_rm_post_all_events(struct nv_vgpu_device *vgpu) { - return gsp_rm_force_post(vgpu, 0xffffffffu); + struct gsp_emu_context *ctx = vgpu ? vgpu->gsp_comm_ctx : NULL; + struct gsp_rm_emu *rm; + u8 payload[32]; + u32 i; + int n = 0, skipped = 0; + + if (!ctx) + return -ENODEV; + rm = ctx->rm_emu; + if (!rm) + return -ENODEV; + + for (i = 0; i < rm->events_n; i++) { + if (!rm->events[i].valid) + continue; + if (time_before(jiffies, rm->events[i].last_post + + GSPRM_EVENT_HINT_DEDUP)) { + skipped++; + continue; + } + + gsp_rm_fill_ev_payload(rm, i, payload); + { + int rc = gsp_rm_send(ctx, gsp_rm_ev_function(), + gsp_rm_ev_sequence(rm), 0, + payload, sizeof(payload)); + + gsp_rm_evlog_post(vgpu, rm, i, "all", rc); + if (rc) + continue; + } + rm->events[i].last_post = jiffies; + n++; + } + + /* + * r576 DIAG: the anomaly sentinel - valid events existed, none were + * dedup-skipped, yet not one post could be sent. Every send rc is in + * the EVPOST lines above (budget permitting). + */ + if (!n && !skipped && gsp_rm_ev_log_max && + gsp_rm_ev_log < gsp_rm_ev_log_max) { + gsp_rm_ev_log++; + nv_vgpu_info(vgpu, + "vgsp-rm: EVPOST all: no postable event (events_n=%u dropped=%llu)", + rm->events_n, rm->events_dropped); + } + + if (n) + return 0; + /* -EAGAIN: valid events existed but every one was dedup-skipped + * (normal during an active post burst); -ENOENT: nothing to post. */ + return skipped ? -EAGAIN : -ENOENT; } /* @@ -977,6 +1300,7 @@ static const char *gsp_rm_rpc_name(u32 function) case GSPRM_RPC_FUNC_GSP_RM_CONTROL: return "GSP_RM_CONTROL"; case GSPRM_RPC_FUNC_UNLOADING_GUEST_DRIVER: return "UNLOADING_GUEST_DRIVER"; case GSPRM_RPC_FUNC_UPDATE_BAR_PDE: return "UPDATE_BAR_PDE"; + case GSPRM_RPC_FUNC_DUP_OBJECT: return "DUP_OBJECT"; default: return "UNKNOWN"; } } @@ -2264,6 +2588,7 @@ static void gsp_rm_handle_bus_get_info_v2(struct gsp_emu_context *ctx, #define GSPRM_COPY_PDES_MAX_LEVELS 6 static void gsp_rm_handle_copy_server_pdes(struct gsp_emu_context *ctx, + u32 h_object, const u8 *params, u32 params_size) { struct gsp_rm_emu *rm = ctx->rm_emu; @@ -2314,6 +2639,46 @@ static void gsp_rm_handle_copy_server_pdes(struct gsp_emu_context *ctx, "vgsp-rm: pt level[%u] phys=0x%llx size=0x%llx aperture=%u pageShift=%u", i, phys, size, aperture, page_shift); } + + /* + * r496j: this control IS the kernel RM publishing the page-table + * chain of its internal VAS (the level[0..] instances with their + * phys addresses and page shifts). The chain can name the X/GL + * channel's pushbuffer pages and progress semaphore, for which no + * other translation source exists (the fn-54 SET_PAGE_DIRECTORY + * path is external-VAS/UVM only). It is NOT registered directly: + * a blast registration of every client's directory let a stale + * kernel directory answer another client's VA in the walker's + * first-success order (the r459 stale-directory class; observed + * as a CUDA client hanging after DEVICE with r496g). Instead the + * root is offered as a CANDIDATE; the progress service registers + * it only after it has been VERIFIED against a content-validated + * pair of that very client (the ring found by the RAM content + * scan). + */ + if (num_levels >= 1 && rm->pt.levels[0].phys && + rm->pt.levels[0].size >= 16 && + rm->pt.levels[0].size <= 4096 * 8) { + /* + * The level phys values are guest PAs on this platform + * (observed 4+ GiB, above any FB shadow); aperture 3 in the + * message does not match the walker's FB/SYS encoding, so + * classify by address: values inside the emulated VRAM are + * FB, everything else sysmem. The walker also retries the + * other aperture, so a misclassification only costs a walk. + */ + bool vidmem = rm->pt.levels[0].phys < + (u64)ctx->vgpu->bars[1].size; + + gsp_sema_service_vaspace_candidate_add( + h_object, rm->pt.levels[0].phys, + (u32)(rm->pt.levels[0].size / 8), vidmem); + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: VAS 0x%x page directory offered as candidate pdb=0x%llx entries=%u (%s; registered only after a content-verified ring walk)", + h_object, (u64)rm->pt.levels[0].phys, + (u32)(rm->pt.levels[0].size / 8), + vidmem ? "vidmem" : "sysmem"); + } } static bool gsp_rm_alloc_interesting(u32 class_id); @@ -2357,10 +2722,24 @@ static void gsp_rm_register_channel(struct gsp_emu_context *ctx, u32 h_object, return; memset(&desc, 0, sizeof(desc)); + desc.vgpu = ctx->vgpu; desc.h_object = h_object; memcpy(&desc.gpfifo_va, params + GSPRM_CHAN_GPFIFO_OFFSET, 8); memcpy(&desc.gpfifo_entries, params + GSPRM_CHAN_GPFIFO_ENTRIES, 4); memcpy(&desc.engine_type, params + GSPRM_CHAN_ENGINE_TYPE, 4); + /* + * The wire carries NV2080_ENGINE_TYPE_*: NULL = 0, GR0 (== GRAPHICS) + * = 1, COPY0 = 9, NVDEC0 = 0x13, ... The progress service's + * convention is "engine_type == 0 means the GR channel" (CUDA's + * kernel-RM GR channels arrive with engineType NULL), so an X/GL 3D + * channel - which carries the real NV2080_ENGINE_TYPE_GRAPHICS = 1 - + * was misclassified as a CE and its push never executed ("Failed to + * initialize DMA" in the X driver's pushbuffer NOP test). Normalize + * GR0..GR7 (1..8) to 0: the emulated device has a single GR and every + * other engine id is >= COPY0 (9), so this cannot capture a CE. + */ + if (desc.engine_type >= 1 && desc.engine_type <= 8) + desc.engine_type = 0; memcpy(&desc.cid, params + GSPRM_CHAN_CID, 4); memcpy(&desc.instance_pa, params + GSPRM_CHAN_INSTANCE_MEM, 8); memcpy(&desc.userd_pa, params + GSPRM_CHAN_USERD_MEM, 8); @@ -2443,51 +2822,80 @@ static void gsp_rm_handle_rm_alloc(struct gsp_emu_context *ctx, * notifyIndex, data}). The driver allocates one per channel and waits * for the GPU to post the event before it considers submitted work * complete - remember them so completed work can be signalled back. + * + * The table is append+retire: FREE retires records (valid=false) but + * never shrinks events_n, so long-lived boots with many client + * create/destroy cycles would otherwise exhaust the 128 slots and + * SILENTLY drop new registrations - after which the completion + * wake-ups (post_channel_event / post_all_events) fail with EINVAL + * and a later client waits forever for a completion that can no + * longer be notified (observed r162: `completion hint rc=-22` + * repeating from the first post-teardown client onward). Reclaim the + * first retired slot for a new registration, and log loudly when the + * table is genuinely full of live entries. */ - if (class_id == 0x79 && params_size >= 24 && rm->events_n < GSPRM_MAX_EVENTS) { + if (class_id == 0x79 && params_size >= 24) { u32 h_src = 0, notify_index = 0, hclass = 0; u64 data64 = 0; int hc_off = -1, i; + u32 slot = rm->events_n; - /* - * Locate hClass (0x79) inside the params instead of trusting a - * fixed offset: NV0005_ALLOC_PARAMETERS is - * {hParentClient, hSrcResource, hClass, notifyIndex, data}, - * with hSrcResource immediately before hClass. - */ - for (i = 0; i + 4 <= (int)params_size; i += 4) { - u32 v; - - memcpy(&v, params + i, 4); - if (v == 0x79) { - hc_off = i; + for (i = 0; i < (int)rm->events_n; i++) { + if (!rm->events[i].valid) { + slot = (u32)i; break; } } - if (hc_off >= 4) { - memcpy(&h_src, params + hc_off - 4, 4); - memcpy(&hclass, params + hc_off, 4); - if (hc_off + 8 <= (int)params_size) - memcpy(¬ify_index, params + hc_off + 4, 4); - if (hc_off + 16 <= (int)params_size) - memcpy(&data64, params + hc_off + 8, 8); - } else { - memcpy(&h_src, params + 4, 4); - memcpy(¬ify_index, params + 12, 4); - memcpy(&data64, params + 16, 8); + if (slot >= GSPRM_MAX_EVENTS) + nv_vgpu_err(ctx->vgpu, + "vgsp-rm: event table full (%u live records); dropping registration hObject=0x%x - completion wake-ups will not reach this client", + rm->events_n, ho); + else { + /* + * Locate hClass (0x79) inside the params instead of + * trusting a fixed offset: NV0005_ALLOC_PARAMETERS is + * {hParentClient, hSrcResource, hClass, notifyIndex, + * data}, with hSrcResource immediately before hClass. + */ + for (i = 0; i + 4 <= (int)params_size; i += 4) { + u32 v; + + memcpy(&v, params + i, 4); + if (v == 0x79) { + hc_off = i; + break; + } + } + if (hc_off >= 4) { + memcpy(&h_src, params + hc_off - 4, 4); + memcpy(&hclass, params + hc_off, 4); + if (hc_off + 8 <= (int)params_size) + memcpy(¬ify_index, + params + hc_off + 4, 4); + if (hc_off + 16 <= (int)params_size) + memcpy(&data64, + params + hc_off + 8, 8); + } else { + memcpy(&h_src, params + 4, 4); + memcpy(¬ify_index, params + 12, 4); + memcpy(&data64, params + 16, 8); + } + rm->events[slot].h_src = h_src; + rm->events[slot].h_event = ho; + rm->events[slot].h_client = hc; + rm->events[slot].notify_index = notify_index; + rm->events[slot].data = (u32)data64; + rm->events[slot].valid = true; + if (slot == rm->events_n) + rm->events_n++; + if (gsp_sema_dbg) + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: event registered hEvent=0x%x hSrc=0x%x notifyIndex=%u (off %d, slot %u) params=%*phN", + ho, h_src, notify_index, hc_off, + slot, + (int)min(params_size, (u32)24), + params); } - rm->events[rm->events_n].h_src = h_src; - rm->events[rm->events_n].h_event = ho; - rm->events[rm->events_n].h_client = hc; - rm->events[rm->events_n].notify_index = notify_index; - rm->events[rm->events_n].data = (u32)data64; - rm->events[rm->events_n].valid = true; - rm->events_n++; - if (gsp_sema_dbg) - nv_vgpu_info(ctx->vgpu, - "vgsp-rm: event registered hEvent=0x%x hSrc=0x%x notifyIndex=%u (off %d) params=%*phN", - ho, h_src, notify_index, hc_off, - (int)min(params_size, (u32)24), params); } /* Echo the fixed alloc header + as many param bytes as are contiguous @@ -2679,6 +3087,115 @@ gsp_rm_handle_promote_fault_method_buffers(struct gsp_emu_context *ctx, } /* + * r519 experiment switch, independent of the P3 whitelist: forward ONLY the + * BINAPI reserved-interface query 0x20810107 to the live GSP session. The P3 + * whitelist may be off (its mixed real/emulated device answers perturb the + * boot); this one command is what the GL surface init needs answered. + */ +static bool gsp_rm_binapi07_fwd_param; +module_param_named(gsp_rm_binapi07_fwd, gsp_rm_binapi07_fwd_param, bool, 0644); +MODULE_PARM_DESC(gsp_rm_binapi07_fwd, + "Forward only the guest GL core's BINAPI 0x20810107 query to the live GSP session (default 0; diagnostic/experiment)"); + +/* + * The live session's oracle client/object for the 0x20810107 experiment, + * created on demand; 0 until the first query builds it. One device per module + * load, so file-statics are enough; reset by unload. + */ +static u32 gsp_rm_binapi07_be_client; +static u32 gsp_rm_binapi07_be_object; + +/* + * One GSP_RM_ALLOC (fn 103) against the live backend, built exactly like the + * rm-call ladder's helper (rpc_gsp_rm_alloc_v03_00: hClient, hParent, + * hObject, hClass, status, paramsSize, flags, reserved[4] + params). + * object_is_out selects the reply read: a ROOT alloc returns the + * server-chosen client handle in params[0] (payload +32), anything else + * echoes the client-chosen hObject at +8; status is at +16 in both cases. + */ +static int gsp_rm_binapi07_backend_alloc(u32 h_client, u32 h_parent, + u32 h_object, u32 h_class, + const void *params, u32 params_len, + bool object_is_out, + u32 *out_object, u32 *out_status) +{ + u8 *areq; + void *arsp = NULL; + u32 arsp_len = 0, st = 0xffffffff, ho = 0; + int ret; + + areq = kzalloc(32 + params_len, GFP_KERNEL); + if (!areq) + return -ENOMEM; + memcpy(areq + 0, &h_client, 4); + memcpy(areq + 4, &h_parent, 4); + memcpy(areq + 8, &h_object, 4); + memcpy(areq + 12, &h_class, 4); + memcpy(areq + 20, ¶ms_len, 4); + if (params_len) + memcpy(areq + 32, params, params_len); + + ret = gsp_phys_backend_rpc(GSPRM_RPC_FUNC_GSP_RM_ALLOC, areq, + 32 + params_len, &arsp, &arsp_len, 500); + kfree(areq); + if (!ret && arsp) { + if (arsp_len >= 20) + memcpy(&st, (u8 *)arsp + 16, 4); + if (object_is_out) { + if (arsp_len >= 36) + memcpy(&ho, (u8 *)arsp + 32, 4); + } else if (arsp_len >= 12) { + memcpy(&ho, (u8 *)arsp + 8, 4); + } + } + kfree(arsp); + if (out_object) + *out_object = ho; + if (out_status) + *out_status = st; + return ret; +} + +/* + * Build the oracle object on the live session: a class-0x2081 child of the + * session's internal subdevice, since the server dispatches the method from + * the OBJECT's class (the subdevice attempt alone answered + * NV_ERR_NOT_SUPPORTED, 0x56, and a fresh root client at runtime answered + * NV_ERR_INVALID_CLIENT, 0x23 - the ladder "adopts" the server's own + * internal client instead of creating one). A small candidate list of + * likely-free handles in that client's namespace is tried; the first + * success wins. Returns 0 with *out_object set. + */ +static int gsp_rm_binapi07_backend_setup(struct gsp_emu_context *ctx, + struct gsp_phys_backend_static_config *snap, + u32 *out_object) +{ + static const u32 cand[] = { 0xabcd2081u, 0xabcd2001u, 0xabcd2a2au, + 0xabcd5f5fu, 0xabcd1234u }; + u8 p4[4] = { 0, 0, 0, 0 }; + u32 st = 0xffffffff, obj = 0; + int i, ret; + + for (i = 0; i < (int)ARRAY_SIZE(cand); i++) { + u32 want = cand[i]; + + obj = want; + ret = gsp_rm_binapi07_backend_alloc(snap->h_internal_client, + snap->h_internal_subdevice, + want, 0x2081, p4, 4, + false, &obj, &st); + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: BINAPI-EXPERIMENT oracle BINAPI alloc attempt %d: hObject=0x%x ret=%d status=0x%x", + i, want, ret, st); + if (!ret && st == 0 && obj) { + *out_object = obj; + return 0; + } + } + return -EIO; +} + +/* * Bounded logging for the P3 forward counters: the first four fallbacks (so a * session that cannot forward is visible) and every 64th event. Never called * when the parameter is off. @@ -2727,12 +3244,37 @@ static bool gsp_rm_ctrl_forward(struct gsp_emu_context *ctx, u32 out_params_size; int ret; + bool guest_handles = false; + /* Wave 1 whitelist: compile-time side-effect-free queries only. */ switch (req->cmd) { case TU104_CTRL_BUS_GET_PCI_BAR_INFO: case TU104_CTRL_BUS_GET_INFO_V2: case TU104_CTRL_FB_GET_INFO_V2: case TU104_CTRL_INTERNAL_GMMU_GET_STATIC_INFO: + if (!gsp_phys_ctrl_forward_enabled()) + return false; + break; + case 0x20810107: /* NV2081_BINAPI reserved-interface method 7 */ + /* + * EXPERIMENT (r519): the guest GL core allocates a BINAPI + * object (class 0x2081) and sends this 2-byte query during + * 3D-surface init; our {0,0} generic reply leaves the + * driver's feature flag clear and the EGL surface init tears + * down (EGL_BAD_ALLOC at eglMakeCurrent). The guest-side + * handles (hClient 0xc1d..., a 0xbeef object) have no backend + * counterpart: allocate a real BINAPI object (class 0x2081) on + * the live session once, then send the query to it. The first + * attempt (subdevice translation) came back NV_ERR_NOT_SUPPORTED + * (0x56) - the server dispatches the method by the OBJECT's + * class, so a subdevice cannot answer a BINAPI method. The + * reply is logged either way; an error falls back to the + * emulated generic success until the semantics are known. + * Gated by its own parameter so the P3 whitelist can stay off. + */ + if (!gsp_rm_binapi07_fwd_param) + return false; + guest_handles = true; break; default: return false; @@ -2744,24 +3286,44 @@ static bool gsp_rm_ctrl_forward(struct gsp_emu_context *ctx, if (req->paramsSize > GSPRM_MAX_SINGLE_ELEM_DATA - GSP_RM_CTRL_HDR_SIZE) goto fallback; + /* The experiment is scoped to the observed shape: 2-byte in/out. */ + if (guest_handles && req->paramsSize != 2) + goto fallback; + /* A valid snapshot implies a READY session and the adopted handles. */ if (gsp_phys_backend_static_get(&snap)) goto fallback; + if (guest_handles && !gsp_rm_binapi07_be_object) { + u32 obj = 0; + + if (gsp_rm_binapi07_backend_setup(ctx, &snap, &obj)) + goto fallback; + /* + * The object lives in the internal client's namespace; the + * control is sent as that client (snap.h_internal_client). + */ + gsp_rm_binapi07_be_client = snap.h_internal_client; + gsp_rm_binapi07_be_object = obj; + } + /* Translate the guest's emulated internal handles; any other handle * domain stays emulated. */ - if (req->hClient != TU104_H_INTERNAL_CLIENT || - (req->hObject != TU104_H_INTERNAL_DEVICE && - req->hObject != TU104_H_INTERNAL_SUBDEVICE)) + if (!guest_handles && + (req->hClient != TU104_H_INTERNAL_CLIENT || + (req->hObject != TU104_H_INTERNAL_DEVICE && + req->hObject != TU104_H_INTERNAL_SUBDEVICE))) goto fallback; fwd_req = kmalloc(GSP_RM_CTRL_HDR_SIZE + req->paramsSize, GFP_KERNEL); if (!fwd_req) goto fallback; memset(&fwd_hdr, 0, sizeof(fwd_hdr)); - fwd_hdr.hClient = snap.h_internal_client; - fwd_hdr.hObject = req->hObject == TU104_H_INTERNAL_DEVICE ? - snap.h_internal_device : snap.h_internal_subdevice; + fwd_hdr.hClient = guest_handles ? gsp_rm_binapi07_be_client : + snap.h_internal_client; + fwd_hdr.hObject = guest_handles ? gsp_rm_binapi07_be_object : + (req->hObject == TU104_H_INTERNAL_DEVICE ? + snap.h_internal_device : snap.h_internal_subdevice); fwd_hdr.cmd = req->cmd; fwd_hdr.paramsSize = req->paramsSize; fwd_hdr.flags = req->flags; @@ -2781,6 +3343,27 @@ static bool gsp_rm_ctrl_forward(struct gsp_emu_context *ctx, } /* + * r519 experiment evidence: log the live GSP's own answer to the + * BINAPI query (status + first params bytes) before deciding. + */ + if (req->cmd == 0x20810107) { + u32 st = 0xffffffff, ps = 0xffffffff; + u8 b[8] = { 0 }; + + if (rsp && rsp_len >= GSP_RM_CTRL_HDR_SIZE) { + memcpy(&st, (u8 *)rsp + 12, 4); + memcpy(&ps, (u8 *)rsp + 16, 4); + if (ps <= 8 && ps <= rsp_len - GSP_RM_CTRL_HDR_SIZE) + memcpy(b, (u8 *)rsp + GSP_RM_CTRL_HDR_SIZE, + ps); + } + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: BINAPI-EXPERIMENT 0x20810107 forwarded to live GSP: rsp_len=%u status=0x%x outSize=%u bytes=%02x %02x %02x %02x %02x %02x %02x %02x", + rsp_len, st, ps, b[0], b[1], b[2], b[3], b[4], + b[5], b[6], b[7]); + } + + /* * The reply is rpc_gsp_rm_control_v03_00: status at +12, paramsSize * at +16, echoed params at +24 - the same 24-byte prefix the emulated * handler builds below. The RPC-level rpc_result was already checked @@ -2826,6 +3409,101 @@ fallback: return false; } +/* + * r489: a GSP_RM_CONTROL reply whose rpc record exceeds the driver's + * pRpc->maxRpcSize must be sent with the GSP's continuation-record protocol + * (nv gsp_rpc.c _rpcWriteContinuationEntries): the FIRST record carries + * exactly maxRpcSize bytes of [rpc header + ctrl header + params]; each + * following NV_VGPU_MSG_FUNCTION_CONTINUATION_RECORD record carries up to + * maxRpcSize - sizeof(hdr) bytes of the remainder. The driver's + * _issueRpcLarge consumes the first frame as min(bufSize, maxRpcSize) and + * then requires one continuation record per remaining chunk - without them + * it times out ("rpcRecvPoll timedout for fn 71 continuation record") and + * the control fails (NV_ERR_INVALID_PARAMETER), which is how the GL VASpace + * construction died on cmd 0x2080a0a4 (paramsSize 67392 > maxRpcSize). + * + * maxRpcSize is the guest driver's GSP_MSG_QUEUE_RPC_SIZE_MAX; the value is + * pinned by the driver's own arithmetic: for the 0x2080a0a4 reply + * (full_param_size = 32 + 24 + 67392 = 67448) it reported remainingSize + * 0x940 = 2368 after the first frame, i.e. maxRpcSize = 67448 - 2368 = + * 65080. A build whose constant differs shows up immediately as a + * mismatched continuation count in the guest log; adjust here. + */ +#define GSPRM_CRM_CONT_MAXRPC 65080u + +static int gsp_rm_send_rm_control(struct gsp_emu_context *ctx, u32 sequence, + const void *data, u32 data_len) +{ + u32 total = sizeof(struct gsprm_rpc_header) + data_len; + const u8 *p = data; + u32 off = 0; + int ret; + + if (total <= GSPRM_CRM_CONT_MAXRPC) + return gsp_rm_send(ctx, GSPRM_RPC_FUNC_GSP_RM_CONTROL, + sequence, 0, data, data_len); + + /* Frame 1: rpc.length = maxRpcSize. */ + ret = gsp_rm_send(ctx, GSPRM_RPC_FUNC_GSP_RM_CONTROL, sequence, 0, + p, GSPRM_CRM_CONT_MAXRPC - + sizeof(struct gsprm_rpc_header)); + if (ret) + return ret; + off = GSPRM_CRM_CONT_MAXRPC - sizeof(struct gsprm_rpc_header); + + /* Continuations: rpc.length = hdr + chunk. */ + while (off < data_len) { + u32 chunk = data_len - off; + + if (chunk > GSPRM_CRM_CONT_MAXRPC - + sizeof(struct gsprm_rpc_header)) + chunk = GSPRM_CRM_CONT_MAXRPC - + sizeof(struct gsprm_rpc_header); + ret = gsp_rm_send(ctx, GSPRM_RPC_FUNC_CONTINUATION_RECORD, + sequence, 0, p + off, chunk); + if (ret) + return ret; + off += chunk; + } + return 0; +} + +/* + * Real-GSP KGR static-data snapshots (mode-35 boot capture; see + * gsp_phys_kgr_snap_get in gsp_phys_booter.c). The physical backend asks the + * live GSP the real client's KGR query set at boot and retains the replies; + * serve the identical bytes for the same commands so the guest's GL/3D stack + * reads the device's true GR static values. Concrete consumer: the X + * driver's 3D-engine selection maps GR_GET_INFO index SM_VERSION (0x0c) + * through a table and fails with "Failed to select a 3D engine" when it + * reads back as SM_VERSION_NONE (0); the live GSP's KGR_GET_INFO reply + * carries the real value (0x75 on TU104). Returns true when the reply was + * served from the snapshot (out_params_size set); false falls through to the + * emulated handler unchanged. + */ +static bool gsp_rm_kgr_snap_serve(struct gsp_emu_context *ctx, u32 cmd, + u32 req_params_size, + u8 *out_params, u32 out_len, + u32 *out_params_size) +{ + u32 n = 0; + int ret; + + ret = gsp_phys_kgr_snap_get(cmd, out_params, out_len, &n); + if (ret || !n) + return false; + + /* Never answer with more bytes than the guest asked for. */ + if (n > req_params_size) + return false; + + *out_params_size = n; + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: KGR cmd=0x%x served from the real-GSP snapshot (%u bytes)", + cmd, n); + return true; +} + static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx, const u8 *elem, u32 sequence) { @@ -2860,13 +3538,26 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx, * return means nothing was sent and the emulated switch below runs * byte-identically to the parameter-off path. */ - if (gsp_phys_ctrl_forward_enabled() && + if ((gsp_phys_ctrl_forward_enabled() || gsp_rm_binapi07_fwd_param) && gsp_rm_ctrl_forward(ctx, &req, req_data + GSP_RM_CTRL_HDR_SIZE, reply, reply_data_len, sequence)) { kfree(reply); return; } + /* + * KGR static-data snapshots (mode-35 boot capture): the live GSP's own + * replies to the real client's KGR query set are served byte-identically + * for the same commands, so the guest's GL/3D stack sees the device's + * true GR static values. Falls through to the emulated switch when no + * snapshot exists (backend not READY, other boot mode, cmd not captured). + */ + if (gsp_rm_kgr_snap_serve(ctx, req.cmd, req.paramsSize, + reply + GSP_RM_CTRL_HDR_SIZE, + reply_data_len - GSP_RM_CTRL_HDR_SIZE, + &out_params_size)) + goto rm_control_reply; + switch (req.cmd) { case TU104_CTRL_FIFO_GET_DEVICE_INFO_TABLE: gsp_rm_handle_fifo_device_table(ctx, sequence, &req, @@ -2876,7 +3567,8 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx, break; case 0x90f10106: /* NV90F1_CTRL_CMD_VASPACE_COPY_SERVER_RESERVED_PDES */ - gsp_rm_handle_copy_server_pdes(ctx, req_data + GSP_RM_CTRL_HDR_SIZE, + gsp_rm_handle_copy_server_pdes(ctx, req.hObject, + req_data + GSP_RM_CTRL_HDR_SIZE, req.paramsSize); out_status = 0; out_params_size = 0; @@ -3187,6 +3879,27 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx, break; } + /* + * r487/r489 THE GL/VULKAN VASPACE WALL (Xorg/EGL/GZDOOM, every + * session to date): the reply must follow the GSP's own large-reply + * framing. r487 clamped the echo to fit one 64 KB record; the guest + * then refused the record ("Incorrect message length 65528") and, + * when it fit, waited for CONTINUATION records that never came + * ("rpcRecvPoll timedout for fn 71 continuation record + * (remainingSize=0x940)") - rpc.c's _issueRpcLarge consumes the + * first frame up to pRpc->maxRpcSize (65080) and then requires + * NV_VGPU_MSG_FUNCTION_CONTINUATION_RECORD frames for the rest + * (the GSP's own _rpcWriteContinuationEntries). The size is left + * unclamped here; gsp_rm_send_rm_control() frames it per that + * protocol. Root evidence: cmd 0x2080a0a4 (PERF_VPSTATES_INFO, + * paramsSize 67392, FINN-serialized) sent during Xorg VASpace + * construction; its failure made vmmCreateVaspace return ffff and + * killed GL - and, after the length was fixed, the 3D-engine + * selection ("Failed to select a 3D engine") until the framing was + * correct. + */ + +rm_control_reply: /* Build the reply header */ memcpy(reply + 0, &req.hClient, 4); memcpy(reply + 4, &req.hObject, 4); @@ -3206,8 +3919,8 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx, out_status, dt_us); } - if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_GSP_RM_CONTROL, sequence, 0, - reply, GSP_RM_CTRL_HDR_SIZE + out_params_size)) + if (gsp_rm_send_rm_control(ctx, sequence, reply, + GSP_RM_CTRL_HDR_SIZE + out_params_size)) nv_vgpu_err(ctx->vgpu, "vgsp-rm: RM_CONTROL reply send failed cmd=0x%x", req.cmd); else @@ -3217,6 +3930,61 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx, } /* + * r510gl: DUP_OBJECT (function 21) - NVOS55_PARAMETERS echo with status=NV_OK. + * + * The guest driver dups an object into a second client at EGL/GLX + * MakeCurrent time (memory and VASpace objects: the GL context's client + * shares the device's allocations), and the historical generic-SUCCESS + * default replied with an EMPTY payload - the driver's reply must carry the + * params struct back (rpc_dup_object_v03_00 = NVOS55_PARAMETERS + * {hClient, hParent, hObject, hClientSrc, hObjectSrc, flags, status}), so + * an empty reply failed the call and the driver tore the whole EGL session + * down (guest witness: 'RPC UNKNOWN(21) seq=0 len=60 -> generic SUCCESS', + * immediately followed by the session's FREE cascade and + * eglMakeCurrent -> EGL_BAD_ALLOC (0x3003), 2026-10-23 r510). + * + * The emulation does not model per-object state for these classes (controls + * and allocs are answered from the request), so the echo with NV_OK is the + * complete answer; the dup'd handle is used by later calls exactly like the + * source handle. + */ +static void gsp_rm_handle_dup_object(struct gsp_emu_context *ctx, + const u8 *elem, u32 sequence) +{ + struct gsp_rm_emu *rm = ctx->rm_emu; + const struct gsprm_rpc_header *hdr = + (const struct gsprm_rpc_header *)(elem + GSPRM_ELEM_HDR_SIZE); + const u8 *req = elem + GSPRM_ELEM_HDR_SIZE + sizeof(*hdr); + u8 params[28]; /* NVOS55_PARAMETERS_v03_00 */ + u32 hDstClient = 0, hDstObject = 0, hSrcClient = 0, hSrcObject = 0; + + if (hdr->length >= sizeof(*hdr) + sizeof(params)) + memcpy(params, req, sizeof(params)); + else + memset(params, 0, sizeof(params)); + + memcpy(&hDstClient, params + 0, 4); + memcpy(&hDstObject, params + 8, 4); + memcpy(&hSrcClient, params + 12, 4); + memcpy(&hSrcObject, params + 16, 4); + memset(params + 24, 0, 4); /* status = NV_OK */ + + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: DUP_OBJECT hDstClient=0x%x hDstObject=0x%x hSrcClient=0x%x hSrcObject=0x%x seq=%u -> NV_OK echo", + hDstClient, hDstObject, hSrcClient, hSrcObject, sequence); + + /* Small delay before the reply: same reply-matching race mitigation as + * FREE/UNLOADING. */ + udelay(2000); + + if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_DUP_OBJECT, sequence, 0, + params, sizeof(params))) + nv_vgpu_err(ctx->vgpu, "vgsp-rm: DUP_OBJECT reply send failed"); + else + rm->rpcs_replied++; +} + +/* * GSP_RM_FREE (function 10) - NVOS00_PARAMETERS echo with status=NV_OK. * * rpc_gsp_free: { hRoot, hObjectParent, hObjectOld, status }. Echo the @@ -3244,7 +4012,7 @@ static void gsp_rm_handle_free(struct gsp_emu_context *ctx, const u8 *elem, memcpy(&hObjOld, params + 8, 4); memset(params + 12, 0, 4); /* status = NV_OK */ - gsp_sema_service_channel_del(hObjOld); + gsp_sema_service_channel_del(ctx->vgpu, hObjOld); /* * Retire event records that refer to the object being freed. Posting an @@ -3253,11 +4021,19 @@ static void gsp_rm_handle_free(struct gsp_emu_context *ctx, const u8 *elem, * client's notifier events (hSrcResource 0) linger across runs and the * fallback wake-up nudge would hit them. Also retire a client's events * when its root object is freed (hObjOld == hRoot). + * + * m176: the hEvent match must also match the owning client. Handle + * values are recycled across runs, so a late FREE of an old client's + * hEvent must never retire a new client's record with the same numeric + * handle - the new client's completion wake-up would be lost and it + * would wait forever (late-client stall class). hRoot is the freeing + * client's own handle in NVOS00_PARAMETERS. */ for (i = 0; i < rm->events_n; i++) { if (!rm->events[i].valid) continue; - if (rm->events[i].h_event == hObjOld || + if ((rm->events[i].h_event == hObjOld && + (!hRoot || rm->events[i].h_client == hRoot)) || (hObjOld && hObjOld == hRoot && rm->events[i].h_client == hRoot)) { rm->events[i].valid = false; @@ -3455,14 +4231,18 @@ static void gsp_rm_handle_element(struct gsp_emu_context *ctx, const u8 *elem) return; case GSPRM_RPC_FUNC_CONTINUATION_RECORD: - /* CONTINUATION_RECORD: driver expects a reply. Send SUCCESS. */ - nv_vgpu_info(ctx->vgpu, "vgsp-rm: async RPC %s(%u) seq=%u len=%u -> SUCCESS reply", - gsp_rm_rpc_name(function), function, sequence, - hdr->length); - if (gsp_rm_send(ctx, function, sequence, 0, NULL, 0)) - nv_vgpu_err(ctx->vgpu, "vgsp-rm: reply send failed fn=%u", function); - else - rm->rpcs_replied++; + /* + * r489: a CONTINUATION_RECORD from the driver is ONE-WAY data + * - the tail of a multi-frame request (its _issueRpcLarge + * sends frame 1 then continuation records). The old reply + * here was a guess ("driver expects a reply"); it injected an + * extra fn-71 record into the reply stream that the driver + * drained as an event ("Unexpected RPC event from GPU0: 0x47") + * and it corrupted the framing of our own large replies + * (which use fn-71 continuations the other way). Consume + * silently: the driver waits only for the reply to frame 1. + */ + rm->rpcs_replied++; return; case GSPRM_RPC_FUNC_SET_GUEST_SYSTEM_INFO: @@ -3485,6 +4265,10 @@ static void gsp_rm_handle_element(struct gsp_emu_context *ctx, const u8 *elem) gsp_rm_handle_free(ctx, elem, sequence); return; + case GSPRM_RPC_FUNC_DUP_OBJECT: + gsp_rm_handle_dup_object(ctx, elem, sequence); + return; + case GSPRM_RPC_FUNC_SET_PAGE_DIRECTORY: gsp_rm_handle_set_page_directory(ctx, elem, sequence, hdr->length); return; @@ -3657,8 +4441,40 @@ void gsp_rm_emu_fini(struct gsp_emu_context *ctx) void gsp_rm_emu_reset(struct gsp_emu_context *ctx) { struct gsp_rm_emu *rm = ctx->rm_emu; + u32 i, stale; - if (!rm || !rm->parsed) + if (!rm) + return; + + /* + * m176: a reset means the emulated GSP is gone and every RM client + * with it (probe/unload, engine reset). Event records left valid from + * the previous session would be posted to with stale + * (hClient, hEvent) pairs - the guest RM asserts in CliGetEventInfo + * and drops the record - and a recycled hEvent can absorb a wake-up + * meant for the new client. Drop them regardless of parse state. + */ + stale = 0; + for (i = 0; i < GSPRM_MAX_EVENTS; i++) { + if (rm->events[i].valid) { + rm->events[i].valid = false; + stale++; + } + } + if (stale) + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: reset dropped %u stale event record(s)", + stale); + + /* + * m177: RM reload means a new guest boot: the guest's direct-map base + * and every content-scanned translation may have changed. Raise the + * boot-scoped wipe (executed on the sema svc thread; client session + * flushes intentionally keep that state - see svc_session_flush). + */ + gsp_sema_service_guest_reset(); + + if (!rm->parsed) return; nv_vgpu_info(ctx->vgpu, "vgsp-rm: reset (rpcs=%llu replied=%llu unknown=%llu)", diff --git a/kernel/gsp_rm_emulation.h b/kernel/gsp_rm_emulation.h index e464809..a0bd4c7 100644 --- a/kernel/gsp_rm_emulation.h +++ b/kernel/gsp_rm_emulation.h @@ -45,6 +45,7 @@ struct gsp_emu_context; #define GSPRM_RPC_FUNC_GSP_RM_CONTROL 76 #define GSPRM_RPC_FUNC_GSP_RM_ALLOC 103 #define GSPRM_RPC_FUNC_FREE 10 +#define GSPRM_RPC_FUNC_DUP_OBJECT 21 #define GSPRM_RPC_FUNC_UNLOADING_GUEST_DRIVER 47 #define GSPRM_RPC_FUNC_UPDATE_BAR_PDE 70 #define GSPRM_RPC_EVENT_POST_EVENT 0x1003 @@ -197,13 +198,34 @@ struct gsp_rm_emu { u32 notify_index; u32 data; bool valid; + unsigned long last_post; /* m171: jiffies of the last posted record */ } events[GSPRM_MAX_EVENTS]; u32 events_n; u64 events_posted; u64 events_dropped; - u32 event_burst; /* event records written since the last reply */ + u32 event_burst; /* event records written in the current window */ + unsigned long event_burst_at; /* m171: burst window start (time-based reset) */ }; +/* + * r576 DIAG: per-boot witness budget for GSP event-post payloads (default 0 + * = off). Nonzero logs every EVENT_POST_EVENT record actually sent (slot, + * hClient, hEvent, notifyIndex, data, send rc) and the sentinel case of a + * post_all_events call that found nothing postable - the evidence needed + * when a client parks waiting for a completion wake-up that never arrives. + */ +extern unsigned gsp_rm_ev_log_max; + +/* + * r577 experiment knobs (runtime-writable; default 0 = legacy). See the + * definitions in gsp_rm_emulation.c: event record sequence, bNotifyList, + * hClient override - the A/B space for the guest-RM event drop. + */ +extern unsigned gsp_rm_ev_seq; +extern unsigned gsp_rm_ev_nlist; +extern unsigned gsp_rm_ev_cl; +extern unsigned gsp_rm_ev_func; + int gsp_rm_emu_init(struct nv_vgpu_device *vgpu, struct gsp_emu_context *ctx); void gsp_rm_emu_fini(struct gsp_emu_context *ctx); int gsp_rm_post_channel_event(struct nv_vgpu_device *vgpu, u32 h_src); diff --git a/kernel/gsp_rm_tu104.h b/kernel/gsp_rm_tu104.h index b5ee4d7..3382939 100644 --- a/kernel/gsp_rm_tu104.h +++ b/kernel/gsp_rm_tu104.h @@ -754,7 +754,18 @@ static const struct tu104_internal_device_info tu104_internal_device_table[] = { TU104_CE_CAPS0_SYSMEM | \ TU104_CE_CAPS0_P2P | \ TU104_CE_CAPS0_BL_GT_64K) -#define TU104_CE_CAPS1 0x00 +/* + * Caps byte 1: the 2D/3D copy path requires a CE that supports the + * blocklinear access modes (hardware bug 200288104): the X driver's + * nvCEChooseEngine skips every CE without CE_SUPPORTS_NONPIPELINED_BL + * (byte1 0x01; and PIPELINED_BL 0x02 for the preferred pass) and then + * fails with "Failed to find a suitable copy object" - the observed wall + * after the DMA pushbuffer test started passing. Turing's CEs are fully + * capable (pipelined + non-pipelined BL + pitchlinear): byte1 = 0x03. + * (Reference: ctrl2080ce.h NV2080_CTRL_CE_CAPS_CE_SUPPORTS_*_BL 1:0x01/ + * 1:0x02.) + */ +#define TU104_CE_CAPS1 0x03 /* ENGINE_TYPE_COPY0 = 0x9, COPY(i) = COPY0 + i (cl2080_notification.h) */ #define TU104_ENGINE_TYPE_COPY0 0x00000009 diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 20dfc54..741fdc2 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -93,6 +93,13 @@ static inline u64 svc_fb_shadow_size(const struct nv_vgpu_device *vgpu) #define KVA_HI 0xffffc00000000000ULL /* + * r534: guest-read granularity for the full-RAM discovery scan + * (svc_scan_pass). Hoisted here because the scan is defined before the + * push-apply window that used to own the define. + */ +#define SVC_SCAN_CHUNK (64 * 1024) /* 16 pages per guest read */ + +/* * INC_METHOD header for the CE semaphore group. Observed value 0x20038090: * bits [31:28] opcode/const, [23:16] count = 3, lower bits encode * (address >> 2) == 0x240 plus subchannel flags (0x8000 in the observed @@ -160,8 +167,27 @@ struct sema_map_entry { #define SVC_USERD_GP_GET 0x88 #define SVC_USERD_GP_PUT 0x8c +/* + * r496b: guest-RAM ring content scan (see gr_ram_cursor in struct + * svc_channel) and the VRAM USERD shape scan (svc_chan_userd). One scan + * slice per service pass so the service thread stays responsive. + */ +#define SVC_GR_RAM_SLICE (512ULL << 20) /* guest RAM per pass */ +#define SVC_GR_RAM_PASS_MAX 16 /* slices before giving up */ +#define SVC_GR_RAM_WIN (2ULL << 20) /* allocation window pinned */ + struct svc_channel { bool valid; + u32 session_gen; /* m176: session generation this entry was + * created in; learned PA state (counter_pa/ + * release_pa) is only inherited on a re- + * registration within the same generation, so + * a recycled handle from a previous client can + * never keep the dead session's write targets */ + struct nv_vgpu_device *vgpu; /* owning mdev (guest session); the + * table is keyed by (vgpu, h_object) + * because two guests' stock allocators + * choose identical handles */ u32 h_object; u32 engine_type; u32 cid; /* RM channel id (doorbell token vector) */ @@ -184,6 +210,28 @@ struct svc_channel { * reset (PUT dropping below put_prev) resyncs * last_get to slot 0 so the live first push is * processed instead of skipped. */ + bool gr_pub_armed; /* mm2 (r207/r208 second-client hang): a GR + * channel's completion word may only receive a + * GP_PUT publish after the channel has been + * proven to belong to the CURRENT session. A + * GR ring/USERD surviving in its fixed VRAM + * slot from the previous session reads a + * residual put at discovery (e.g. 109/112); + * publishing that residual put into the + * channel's own word (8c634144 made the target + * release_pa - the client's waited word) while + * the stale ring is consumed poisons the fresh + * client's UVM tracking semaphore: its + * completed_value observes a far-forward + * payload, the later reset/release sequence is + * a >MAX_JUMP (2*entries) downward observation, + * and the fresh session's first waits wedge + * (matmul: one GR push per channel then 242 s + * silent). Armed when a GR channel is seen with + * put <= 1 (pristine ring) or when the fresh- + * put-drop rewind proves a session switch; the + * rewind also voids any publish state the + * stale pass advanced. */ bool logged; /* one state log per channel */ bool pool_rel_seen; /* ISSUE-0002 r70: a pool release word for * this channel has been read through a @@ -228,6 +276,10 @@ struct svc_channel { * window (observed: one 8 s grace per push, * serializing the runtime init) */ DECLARE_BITMAP(slot_done_epoch, SVC_GPFIFO_MAX_BITS); + u8 slot_pend_tries[SVC_GPFIFO_MAX_BITS]; /* fail4 r308: bounded + * re-holds of a denied device-dst copy slot */ + u32 counter_gen; /* r311: session generation that authored the + * completion word (guards the residue zero) */ /* GPFIFO slots already consumed in the current * put epoch (the client's put value since it * last wrapped). A pin-batch rewind replays @@ -298,6 +350,10 @@ struct svc_channel { * publish targets counter_pa instead, so both * families of the counter page get written * (SA-COUNTER FORENSICS-grv7-counter F1) */ + u64 release_va; /* r378: the release VA release_pa was learned + * from, kept so a late write (stale-recover + * zero) can re-validate the target through + * gsp_sema_write_guard before it lands */ u32 counter_last; /* last value published to counter_pa */ u64 pushpg_va; /* last push page logged as pin-vs-window * divergent (rate limit: one line per page) */ @@ -343,7 +399,107 @@ struct svc_channel { u32 gr_hold_idx; /* GPFIFO slot the window was armed for (identity: * an entry holds at most once, then is consumed * even if still unresolved - bounded zeros, never - * a hang) */ + * a hang); m156: re-held while the window is + * live so a busy forward gets its real retry */ + /* + * r509: one-shot verdict of the absolute bound. The expiry call + * pushes the deadline (so the hold window restarts) and latches this + * flag; the r351 retry-forward block consumes it within the same + * pass. Without the latch the two call sites in one pass could not + * both observe the expiry (the first push of the deadline made the + * second call see a fresh window - r213-r215's arming bug in + * reverse). + */ + bool gr_hold_bound_hit; + unsigned long gr_hold_deadline; /* GRFIX FIX1b: hard expiry of the data- + * stalled hold for gr_hold_dead_va. + * Fixed once on the first hold of that + * launch push VA and NEVER moved while + * the launch stays held; when it passes + * the launch proceeds (pre-8b058884 + * behaviour), so the guest can never + * wait unboundedly. Keyed by the launch + * VA, not the GPFIFO slot: pin-batch + * rewinds re-decode one launch at + * different slot indices (r213: + * entry 107 -> 107 -> 99), which + * re-armed a slot-keyed deadline on + * every pass. */ + u64 gr_hold_dead_va; /* launch push VA the bound belongs to */ + unsigned long gr_hold_dead_since; /* first-hold stamp (waited_ms) */ + bool gr_hold_exp_logged; /* one GRFILL-EXPIRED line per launch */ + u64 opdef_hold_dead_va; /* r361: launch push VA the opdef bound + * belongs to (slot-keyed `since` is + * defeated by rewind replays) */ + unsigned long opdef_hold_deadline; /* absolute opdef hold expiry */ + bool opdef_hold_exp_logged; /* one OPDEF-EXPIRED line per launch */ + u32 gr_rehold_idx; /* m156: slot whose next decode must force a + * fresh launch capture (a held entry is + * re-decoded as a replay, which suppresses + * the capture; forcing it lets a deferred + * launch be forwarded once the engine's + * in-flight work completes) */ + u32 gr_nocap_idx; /* m164 CLASS-2: slot in the no-capture hold + * (a readable launch push whose decode saw + * QMD words but built no candidate) */ + u32 gr_nocap_replay;/* 1 = window opened on a replay decode */ + u64 gr_nocap_va; /* push VA the window was opened for */ + unsigned long gr_nocap_since; /* absolute first-seen stamp */ + /* + * r496b (X/GL kernel-RM sysmem channel): ring discovery by content in + * guest RAM. The CPU RM maps sysmem objects locally (no + * SET_PAGE_DIRECTORY, no rm_mem record carrying the ring VA), so + * svc_lookup_trusted cannot name the ring page; the driver's kickoff + * GPFIFO entries {pushbuffer VA, len} identify it instead. The scan + * is sliced (SVC_GR_RAM_SLICE per pass) and cursor-resumed so the + * service thread stays responsive. + */ + u64 gr_ram_cursor; /* resume point of the content scan */ + u8 gr_ram_passes; /* slices walked so far */ + bool gr_ram_logged; /* one discovery line per channel */ + /* + * r496e: true when the ring was found by the RAM content scan (a + * kernel-RM sysmem channel - the X/GL 3D channel). Such a channel's + * pushes are 2D/3D method streams the emulation does not execute; + * the correct model is consume-and-apply-releases: no QMD capture, + * no launch forward, no no-capture hold. Also allows an all-zero + * NOP push (the driver's pushbuffer sanity test writes exactly one + * zero dword) to be consumed as the no-op it is. + */ + bool gr_ram_ring; + /* + * r496b: the FB-scan USERD (see svc_userd_fb_scan). Latched when the + * registered/ring-derived candidates all fail; the scan validates the + * hit against the ring entries the put claims, so a foreign channel's + * USERD cannot be adopted. Bounded retries: the driver kicks off a + * few passes after channel registration, so a scan before the first + * put must not retire the discovery. + */ + u8 userd_scans; + /* + * r471: the completion-counter publish must never advance the + * guest's word past a push whose content was still unwritten when + * the entry was consumed (the streams_events staging-reuse + * corruption: the publish satisfied the client's D2H wait before + * its copy ran, and the readback drained a staging buffer that + * still held the PREVIOUS copy's bytes). Lowest ring slot consumed + * with all-zero content; 0 = none. Cleared when a later decode of + * a GR push reads non-zero content (the content landed - the rewinds + * re-read entries constantly, so this heals within a pass or two). + */ + u32 gr_bad_put; + /* + * STALLFREE r225: stale-epoch / stale-replay wedge watchdog. A + * channel that cannot make forward progress (last_get pinned behind + * put, or a residual session put that never drops) is detected here + * and recovered instead of spinning. All fields MUST be in the pump + * snapshot write-back list (GRFIX NOTE). + */ + u64 wd_last_get; /* last_get at the window start */ + u64 wd_put; /* put at the window start */ + u64 wd_sub; /* submissions at the window start */ + unsigned long wd_since; /* window start (0 = no window) */ + unsigned long wd_rewind_off_until; /* pin-batch rewind cooldown */ }; /* @@ -399,6 +555,12 @@ struct svc_vaspace { u64 pdb; u32 entries; /* root entry count */ int ap; /* SVC_APERTURE_* of the directory itself */ + unsigned long stamp; /* r459: last registration/use; LRU evict + * when the table is full (a stale entry from + * a dead client otherwise blocks every new + * client after the 8th registration: the + * mid-boot walk failure that made the 10th+ + * client's copies fail rc=-2) */ }; struct svc_lat { @@ -449,11 +611,136 @@ static u32 svc_ring_n; * client spins). One clean boot always works, so make the next session look * like one: when the channel set empties, request a flush and let the next * rebuild start from the same empty state as a fresh boot. + * + * Dual-guest staging (goal 1.0): the discovery tables are still shared + * between guests (per-guest learned tables are the staged G4 remainder), so + * the flush is requested only when the channel set is globally empty. The + * active decision is made per guest (svc_guests below): one guest closing + * its last channel never flushes while another guest still has live + * channels - otherwise the departing guest would wipe the live guest's + * pins/maps and stall it. Single-guest behaviour is unchanged: the one + * guest's last-channel close is also the global empty transition. */ static u8 svc_sess_flush; +static u8 svc_guest_reset_req; /* m177: guest RM reload; wiped on svc thread */ static u8 svc_sess_active; /* a channel of a session has registered (until flush) */ +static u32 svc_sess_gen; /* m176: bumped by every executed session flush; + * stamps channels/scrubbers so per-session learned + * state is never inherited by a recycled handle */ static struct completion svc_sess_flush_done; /* a pending flush has executed */ static unsigned long svc_ring_done; /* CE ranks whose ring is pinned (session state) */ + +/* Per-guest session records (one per open mdev; see SVC_MAX_CHANNELS for the channel cap). */ +#define SVC_MAX_GUESTS 8 +struct svc_guest_sess { + struct nv_vgpu_device *vgpu; /* NULL = free slot */ + bool active; /* a channel of this guest is + * registered until this guest's + * last channel closes */ + u64 last_build; /* per-guest svc_rebuild freshness */ +}; +static struct svc_guest_sess svc_guests[SVC_MAX_GUESTS]; +static DEFINE_SPINLOCK(svc_guest_lock); + +/* + * Find (or, with create, allocate) the session record for one mdev. The + * records are pointer-keyed; a removed mdev's record is freed by + * gsp_sema_service_vgpu_cleanup() so the pointer can never match a newly + * allocated device. + */ +static struct svc_guest_sess *svc_guest_get(struct nv_vgpu_device *vgpu, + bool create) +{ + struct svc_guest_sess *sess = NULL; + unsigned long flags; + int i; + + if (!vgpu) + return NULL; + spin_lock_irqsave(&svc_guest_lock, flags); + for (i = 0; i < SVC_MAX_GUESTS; i++) { + if (svc_guests[i].vgpu == vgpu) { + sess = &svc_guests[i]; + break; + } + if (create && !sess && !svc_guests[i].vgpu) + sess = &svc_guests[i]; + } + if (sess && create && !sess->vgpu) { + memset(sess, 0, sizeof(*sess)); + sess->vgpu = vgpu; + } + spin_unlock_irqrestore(&svc_guest_lock, flags); + return sess; +} + +/* + * CeUtils scrubber channel (the guest PMA's internal CE channel created at + * driver init). When the PMA must hand out scrubbed pages it waits on the + * channel's finish payload (channel buffer offset finishPayloadOffset = + * channelPbSize + GPFIFO_SIZE + 4; for the regular scrubber = 0x6c004), + * which the physical CE would advance as the scrub work completes. In this + * emulation nothing authors it, so the third+ CUDA session's cudaMalloc + * freezes in scrubCheckAndWaitForSize. + * + * The scrubber channel is identified by its GPFIFO entry count + * (NUM_COPY_BLOCKS = 4096; client channels use 1024); its USERD PA is known + * from the channel ALLOC registration. The channel buffer's PA cannot be + * resolved through the global VA map (the kernel VA space collides + * numerically with the client's CE push-buffer window), so the guest + * scanner (vmallocinfo + kcore page-table walk) registers it through + * sema_scrubbuf_add. The service then authors, from the channel's own + * GP_PUT: USERD GP_GET = put, host semaphore = put (so the guest's + * channelWaitForFreeEntry never blocks), and finish payload = submitted + * block count (so the scrub wait completes). Idempotent, forward-only. + */ +struct svc_scrub { + struct nv_vgpu_device *vgpu; /* owning mdev (NULL = free slot) */ + bool userd_ok; /* a 4096-entry channel was registered */ + u32 h_object; /* handle of the registered scrubber channel */ + u64 userd_pa; + u32 userd_as; + u64 gpfifo_va; + u32 entries; + bool pa_ok; /* scanner registered fin/hsem PAs */ + u64 fin_pa; /* finish payload word (u32) */ + u64 hsem_pa; /* channel host semaphore word (u32) */ + bool seeded; + u32 put_last; + u64 submitted; /* blocks submitted since seeding */ + u64 authored; /* last authored finish value */ + u32 pa_gen; /* scanner registration generation (logged) */ + u32 session_gen; /* m176: session generation of the last registration */ + u32 log_budget; +}; +/* + * One scrubber record per guest: every guest's PMA has its own 4096-entry + * scrubber channel, and the authored words must be written through the + * owning guest's address space (writing guest A's PA through guest B's VFIO + * device would corrupt B). Keyed by vgpu pointer, freed by + * gsp_sema_service_vgpu_cleanup(). + */ +static struct svc_scrub svc_scrubs[SVC_MAX_GUESTS]; + +static struct svc_scrub *svc_scrub_get(struct nv_vgpu_device *vgpu, bool create) +{ + struct svc_scrub *s = NULL; + int i; + + if (!vgpu) + return NULL; + for (i = 0; i < SVC_MAX_GUESTS; i++) { + if (svc_scrubs[i].vgpu == vgpu) + return &svc_scrubs[i]; + if (create && !s && !svc_scrubs[i].vgpu) + s = &svc_scrubs[i]; + } + if (s) { + memset(s, 0, sizeof(*s)); + s->vgpu = vgpu; + } + return s; +} static u8 svc_pb_passes; static u64 svc_pb_last; /* @@ -591,7 +878,23 @@ struct sema_svc { u64 sem256_log; /* 256-byte op trace budget */ u64 op_tr_log; /* operand targeted-resolve budget */ u32 op_tr_pending; /* an operand's targeted scan is in progress */ + u32 op_owed; /* r540: a denied transfer op + * was never re-run because the + * per-slot pend cap was hit; the + * slot must stay INCOMPLETE so a + * later trusted/injected pass + * runs it (the r539 5-minute + * device-init stall class: the + * client's GRCE H2D was denied + * while untrusted, capped, then + * replay-suppressed forever and + * the copy never executed) */ + u64 op_owed_log; /* r540: bounded owe witness */ u64 copy_watch_log; /* sema_copy_watch log budget */ + u64 copy_dst_hold; /* m169: device-dst copies held for + * want of a pin (stale-table guard) */ + u64 copy_dst_relaxed; /* m169b: guards released without + * a pin after the bounded wait */ u64 copy_pat_log; /* sema_copy_pat log budget */ u64 copy_dst_log; /* sema_copy_dst log budget */ u32 op_tr_src; /* provenance of last svc_op_translate */ @@ -599,6 +902,20 @@ struct sema_svc { u64 push_pat_log; /* push-content pattern budget */ u64 dma_skip_log; /* unrecognized-DMA budget */ u64 xfer_raw_log; /* operandless transfer raw-dump budget */ + u64 fill_ram_log; /* r361 physical-RAM fill build budget */ + u64 pushtrace_all; /* fail4: de-windowed push decode + * trace (every push, bounded) */ + u64 opadd_log; /* fail4: every op appended */ + u64 op2d_raw_log; /* r458: raw 2D-shaped copy pushes */ + u64 dmatrace_log; /* fail4: denied-transfer trace */ + u64 pend_dbg_log; /* fail4 r305: pending/hold witness */ + u64 snap_gate_log; /* fail4 r311: capture-gate witness */ + u64 copyent_log; /* fail4: svc_copy entry trace */ + u64 bigrun_log; /* fail4: big method-data run headers */ + u64 lid_hdr_log; /* fail4: LOAD_INLINE_DATA headers */ + u64 skiprun_log; /* fail4: silently skipped data runs */ + u64 mystery_log; /* fail4: moderate op=1 runs on big pushes */ + u64 next_hdr_log; /* fail4: header after each big LID run */ u64 grce_log; /* GRCE inline-write budget */ u32 cur_ch; /* channel of the op being run */ u64 entry_log; /* per-push summary budget */ @@ -615,6 +932,11 @@ struct sema_svc { u64 sem_write_reresolve; /* writes re-resolved after read-back */ u64 pool_alt_applied; /* pool releases fanned to a * displaced pin (ISSUE-0002) */ + u64 pool_dual_applied; /* r539: pool releases fanned + * to the secondary candidate page + * (pin/walk disagreement; the + * r538 init-stall class) */ + u64 pool_dual_log; /* r539 fan-out log budget */ u64 pool_residue; /* fresh-channel first releases whose * pool word read far ahead of the * payload (previous session's @@ -644,10 +966,19 @@ struct sema_svc { * expectation advanced, write queued */ u64 gpfifo_held; /* passes an entry was left pending * for a translation/grace window */ + u64 gr_nocap_held; /* m164: GR holds opened for a + * launch push whose decode saw QMD + * words but built no candidate */ + u64 gr_nocap_consumed; /* m164: no-capture windows that + * expired and consumed the entry */ + u64 gr_owed_resolved; /* r548: unwound deferred launches + * software-completed at hold expiry + * instead of consumed silently */ u64 gpfifo_starved; /* entries consumed without a * deliverable exact slot */ u64 counter_rel_applied; /* releases to registered counters */ u64 counter_rel_dbg; /* rel-apply diagnostic budget */ + u64 crel_entry_dbg; /* C1: rel-apply entry budget, 512/session */ u64 counter_noslot_dbg; /* "no slot registered" budget, * separate so it cannot hide the * written/skip lines */ @@ -669,9 +1000,13 @@ struct sema_svc { u64 opdef_held; u64 opdef_applied; u64 opdef_expired; + u64 opdef_delivered_late; /* stall fix: set delivered at consume */ + u64 opdef_drop_lost; /* withheld releases dropped unapplied */ u64 vidmem_writes; /* semaphores written in vidmem */ u64 gpfifo_log; /* channel-state log budget */ u64 gpfifo_entry_log; /* per-entry log budget */ + u64 walk_dbg; /* C4: GR put/walk witness budget */ + u64 gr_rel_auth_log; /* F1: GR-REL-AUTHOR budget, 256/session */ u64 doorbell_count; /* NV_VIRTUAL_FUNCTION_DOORBELL pokes */ u32 doorbell_token; /* last token written */ u32 doorbell_log; /* token log budget */ @@ -773,6 +1108,12 @@ struct sema_svc { u64 fwd_gr_fallback; u64 fwd_gr_refused; u64 fwd_gr_defer; /* m150: held for data-arg retry */ + u64 fwd_gr_dstpend; /* r541: held for an unexecuted + * device write on a data page */ + u32 fwd_gr_dstpend_log; + u64 fwd_gr_argdefer; /* r543: held for a partial + * argument set (unresolved arg) */ + u32 fwd_gr_argdefer_log; u64 fwd_gr_timeout; u64 fwd_gr_fault; u64 fwd_gr_bridge; @@ -783,6 +1124,22 @@ struct sema_svc { * abort, or a release0 page the backend could not observe). */ u64 fwd_gr_sw_rel; + /* + * STALLFREE: failed forwarded launches the pass-top outcome check + * completed in software because no hold owned their release (the + * bounded-completion guarantee; zero on a green run). + */ + u64 fwd_gr_stall_swrel; + /* + * STALLFREE r225: GR channels recovered by the stale-epoch / + * stale-replay wedge watchdog (zero on a green run). + */ + u64 gr_stale_wedge; + u64 fwd_gr_hold_exp; /* GRFIX FIX1: data-stalled holds + * that reached the hard bound and + * were consumed with the software + * release (proof the bound fires + * instead of a stall) */ /* Distinct stalled VA tracking for rate-limited logging. */ #define SVC_STALLED_MAX 256 @@ -950,6 +1307,13 @@ static bool svc_gr_content_rearm; static u32 svc_cepin_log; /* + * m154: injected pins dropped because they collide with a live channel's + * GPFIFO ring range (see svc_handle_map_add). Counted + logged so the + * defensive drop is visible in the boot log. + */ +static u32 svc_ring_pin_skips; + +/* * m148 (second-client wedge): re-arm the per-rank one-shot ring/push-buffer * latches for a (re-)registered channel. The guest driver rebuilds the GSP * session for every client context and RM reuses the same object handles and @@ -995,6 +1359,7 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) if (!desc->engine_type) { WRITE_ONCE(svc_gr_content_rearm, true); svc_cepin_log = 0; + svc_ring_pin_skips = 0; } /* @@ -1019,7 +1384,8 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) } spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { - if (svc.chan[i].valid && svc.chan[i].h_object == desc->h_object) { + if (svc.chan[i].valid && svc.chan[i].vgpu == desc->vgpu && + svc.chan[i].h_object == desc->h_object) { ch = &svc.chan[i]; break; } @@ -1030,9 +1396,21 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) int k; u64 keep_counter_pa = 0; u64 keep_release_pa = 0; + u64 keep_release_va = 0; u8 keep_gr_index = 0, keep_ce_index = 0; + bool keep_pub_armed = false; + u32 keep_counter_gen = 0; DECLARE_BITMAP(keep_rel_done, SVC_GPFIFO_MAX_BITS); - bool same = ch->valid && ch->h_object == desc->h_object; + bool same = ch->valid && ch->vgpu == desc->vgpu && + ch->h_object == desc->h_object; + /* m176: only a same-generation re-registration may inherit the + * learned write targets. After a session flush the same numeric + * handle belongs to a fresh client whose counter/release slots + * live in a brand-new page; inheriting the dead session's PAs + * made every release land on the old page (and be skipped + * against its stale contents) while the new client polled its + * own slot forever. */ + bool same_gen = same && ch->session_gen == svc_sess_gen; /* * A re-registration of the same channel must not lose the @@ -1041,32 +1419,60 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) * counter layout does not change). Same for the increment * release marks: losing them would re-apply pushes. */ - if (same) { + if (same_gen) { keep_counter_pa = ch->counter_pa; keep_release_pa = ch->release_pa; + keep_release_va = ch->release_va; keep_gr_index = ch->gr_index; keep_ce_index = ch->ce_index; + /* + * r311: a same-session re-registration must not + * forget that this session already proved itself + * (gr_pub_armed) - losing it re-arms the residual + * watchdog, and its recovery zeroes a completion + * word this live session authored, which the + * client's UVM reads as a downward jump and never + * completes on (r311 vector_add: pub-adv/zero + * 3 s cycles with the client polling forever). + */ + keep_pub_armed = ch->gr_pub_armed; + keep_counter_gen = ch->counter_gen; bitmap_copy(keep_rel_done, ch->rel_done, SVC_GPFIFO_MAX_BITS); } memset(ch, 0, sizeof(*ch)); - if (same) + ch->gr_hold_idx = ~0u; /* m156: no hold armed yet */ + ch->gr_hold_dead_va = 0; /* GRFIX: no launch bound armed */ + ch->gr_hold_exp_logged = false; + ch->opdef_hold_dead_va = 0; /* r361: no opdef bound armed */ + ch->opdef_hold_exp_logged = false; + ch->gr_rehold_idx = ~0u; + ch->gr_nocap_idx = ~0u; /* m164: no no-capture window armed */ + ch->session_gen = svc_sess_gen; /* m176 */ + if (same_gen) { bitmap_copy(ch->rel_done, keep_rel_done, SVC_GPFIFO_MAX_BITS); + ch->gr_pub_armed = keep_pub_armed; + ch->counter_gen = keep_counter_gen; + } /* - * Rank among the live CE channels. The RM hands out object - * handles in creation order and the guest's UVM allocates one - * push buffer, ring and tracking semaphore per channel in that - * same order, so the rank is what ties a channel to its ring - * offset and its push-buffer base (see svc_ring_autopin / - * svc_pb_pair). + * Rank among the live CE channels OF THIS GUEST. The RM hands + * out object handles in creation order and the guest's UVM + * allocates one push buffer, ring and tracking semaphore per + * channel in that same order, so the rank is what ties a + * channel to its ring offset and its push-buffer base (see + * svc_ring_autopin / svc_pb_pair). The rank must not count the + * other guest's channels: their handles are independent and + * their allocators interleave. */ if (desc->engine_type) { u8 rank = 0; for (k = 0; k < SVC_MAX_CHANNELS; k++) - if (svc.chan[k].valid && svc.chan[k].engine_type && + if (svc.chan[k].valid && + svc.chan[k].vgpu == desc->vgpu && + svc.chan[k].engine_type && svc.chan[k].h_object < desc->h_object) rank++; ch->ce_index = rank; @@ -1074,13 +1480,16 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) u8 rank = 0; for (k = 0; k < SVC_MAX_CHANNELS; k++) - if (svc.chan[k].valid && !svc.chan[k].engine_type && + if (svc.chan[k].valid && + svc.chan[k].vgpu == desc->vgpu && + !svc.chan[k].engine_type && svc.chan[k].h_object < desc->h_object) rank++; ch->gr_index = rank; } ch->valid = true; svc_sess_active = 1; + ch->vgpu = desc->vgpu; ch->h_object = desc->h_object; ch->engine_type = desc->engine_type; ch->cid = desc->cid; @@ -1095,9 +1504,18 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) if (same) { ch->counter_pa = keep_counter_pa; ch->release_pa = keep_release_pa; + ch->release_va = keep_release_va; ch->gr_index = keep_gr_index; ch->ce_index = keep_ce_index; } + /* Per-guest session flag: this guest has a live channel. */ + { + struct svc_guest_sess *sess = + svc_guest_get(desc->vgpu, true); + + if (sess) + sess->active = true; + } } spin_unlock_irqrestore(&svc.chan_lock, flags); @@ -1110,6 +1528,48 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) */ svc_chan_rearm_rank(ch); svc.last_chan_reg = jiffies; + /* + * CeUtils scrubber channel: NUM_COPY_BLOCKS = 4096 GPFIFO + * entries (client channels use 1024). Stash its USERD and ring + * geometry for the scrub pass; the channel itself is left + * entirely alone (no rank change, no pump touch changes) - the + * pass authors the finish payload from the USERD alone. A + * colliding numeric-handle FREE may still unregister the + * channel; the stashed geometry survives, so the scrub path + * does not depend on the channel-table identity at all. + */ + if (desc->gpfifo_entries >= 4096 && desc->gpfifo_va && + desc->userd_pa) { + struct svc_scrub *s = svc_scrub_get(desc->vgpu, true); + + if (!s) + goto scrub_done; /* table full: max guests */ + /* Re-seed the counting on a fresh scrubber object or a + * new client session (m176: the counter/submitted state + * is per-session; a recycled scrubber handle after a + * flush must start from the new channel's GPPut, not + * carry the dead session's counts). */ + if (!s->userd_ok || + s->session_gen != svc_sess_gen || + s->h_object != desc->h_object || + s->gpfifo_va != desc->gpfifo_va) { + s->seeded = false; + s->submitted = 0; + } + s->session_gen = svc_sess_gen; + s->userd_ok = true; + s->h_object = desc->h_object; + s->userd_pa = desc->userd_pa; + s->userd_as = desc->userd_as ? desc->userd_as + : SVC_APERTURE_SYS; + s->gpfifo_va = desc->gpfifo_va; + s->entries = desc->gpfifo_entries; + pr_info("sema-svc: scrubber channel 0x%x detected (gpfifo=0x%llx entries=%u userd=0x%llx as=%u)\n", + desc->h_object, desc->gpfifo_va, + desc->gpfifo_entries, desc->userd_pa, + s->userd_as); + } +scrub_done: pr_info("sema-svc: channel 0x%x registered gpfifo=0x%llx entries=%u userd=0x%llx eng=%u cid=%u notif=0x%llx\n", desc->h_object, desc->gpfifo_va, desc->gpfifo_entries, desc->userd_pa, desc->engine_type, desc->cid, @@ -1121,7 +1581,8 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) } /* P6b wave-1.5 hold state is defined with the forwarding section below. */ -static void svc_fwd_hold_abort(u32 h_object); +static void svc_fwd_hold_abort(struct nv_vgpu_device *vgpu, u32 h_object); +static void svc_fwd_hold_clear(void); /* m176: session flush drops a stale hold */ /* * P6b wave-2 inline software fallback for a withheld (skipped) candidate; * defined with the forwarding section below. Service-thread only. @@ -1139,8 +1600,43 @@ static bool svc_opdef_release(struct nv_vgpu_device *vgpu, static void svc_opdef_drop(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u32 entry_idx); static void svc_opdef_reset_all(void); +static void svc_opdef_finish_ch(struct svc_channel *ch, u32 idx); +static u32 svc_opdef_log; /* m176: forward decl for the session flush */ +/* + * One-cycle GAP-1/GAP-2 instrumentation budgets (agent-datapath plan C1-C10): + * small, reset per session where they are service-thread state, so a later + * client's strand is never blacked out by the first client's spend. + */ +static u32 svc_relx_log; /* C2: deferred-launch apply entry witness */ +static u32 svc_opdef_clr_log; /* C3: opdef-clear caller witness */ +static u32 svc_busy_defer_log; /* C8: GR busy-defer timeout witness */ +static u32 svc_hold_fb_log; /* C7: hold failure/timeout fallback witness */ +static u32 svc_cand_ovw_log; /* C9: candidate overwrite witness */ +static u32 svc_inline_recs_log; /* C10: inline-recs digest witness */ +static u32 svc_gr_await_log; /* r351: launch-held-for-trust witness */ +static u32 svc_gr_retry_log; /* r351: retry-forward/proceed witness */ +static bool svc_hold_bypass(void); /* r563: stall self-heal forward decl */ -void gsp_sema_service_channel_del(u32 h_object) +/* + * r551: releases DELIVERED by the opdef hold-expiry paths (flush/sweep) are + * remembered by (channel, entry, va, payload) for a short window, so a + * re-decode of the parked entry does not withhold the (monotonic, + * forward-only) release again - see the helpers with the opdef machinery. + * Declared here because svc_session_flush() resets them per session. + */ +#define SVC_OPDEF_DONE_SLOTS 256 +#define SVC_OPDEF_DONE_TTL_MS 60000u +struct svc_opdef_done_rec { + u32 h_object; + u32 entry_idx; + u64 va; + u32 payload; + unsigned long j; +}; +static struct svc_opdef_done_rec svc_opdef_done[SVC_OPDEF_DONE_SLOTS]; +static u32 svc_opdef_done_next; + +void gsp_sema_service_channel_del(struct nv_vgpu_device *vgpu, u32 h_object) { unsigned long flags; int i; @@ -1150,7 +1646,8 @@ void gsp_sema_service_channel_del(u32 h_object) spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { - if (svc.chan[i].valid && svc.chan[i].h_object == h_object) { + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].h_object == h_object) { svc.chan[i].valid = false; pr_info("sema-svc: channel 0x%x unregistered (submissions=%llu)\n", h_object, svc.chan[i].submissions); @@ -1160,19 +1657,27 @@ void gsp_sema_service_channel_del(u32 h_object) /* * ISSUE-0002: when the last channel closes, the next client is a new * session in a new UVM VA space; its discovery state must not inherit - * this session's map/pins/page-directory latch. Request the flush - * here; it executes at the next rebuild (the only writer of that - * state) just like a fresh boot would start empty. Only the transition - * from "a session existed" to "empty" may request it: redundant - * deletes for handles that never registered would otherwise re-flush - * (and wipe) a session that started after the real one ended. + * this session's map/pins/page-directory latch. The active decision is + * per guest (svc_guests) and the flush request is raised only when NO + * guest has a live channel left: the discovery tables are still shared + * between guests (staged G4), so flushing while another guest is live + * would wipe that guest's pins and stall it. One guest: the last-channel + * close IS the global empty transition, so behaviour is unchanged. */ if (svc_sess_active) { + struct svc_guest_sess *sess = svc_guest_get(vgpu, false); + bool any = false, own = false; + for (i = 0; i < SVC_MAX_CHANNELS; i++) { - if (svc.chan[i].valid) - break; + if (!svc.chan[i].valid) + continue; + any = true; + if (svc.chan[i].vgpu == vgpu) + own = true; } - if (i == SVC_MAX_CHANNELS) { + if (sess && !own) + sess->active = false; + if (!any) { svc_sess_active = 0; svc_sess_flush = 1; } @@ -1187,9 +1692,10 @@ void gsp_sema_service_channel_del(u32 h_object) * P6b wave 1.5: a deleted channel's held releases must not wait out * the timeout; mark the hold aborted so the pump applies them (or * drops them if the snapshot already lost the channel) on its next - * pass. + * pass. The hold is matched per (vgpu, handle): the other guest's + * identical handle must not abort this guest's hold. */ - svc_fwd_hold_abort(h_object); + svc_fwd_hold_abort(vgpu, h_object); } /* @@ -1197,6 +1703,132 @@ void gsp_sema_service_channel_del(u32 h_object) * the page tables itself and hands their base address to the GPU). Store it so * the service can translate that space's VAs directly. */ +/* + * r496j: unverified page-directory candidates from COPY_SERVER_PDES. A + * kernel-RM internal VAS publishes its chain there, but registering every + * client's directory directly lets a stale kernel directory answer another + * client's VA in the walker's first-success order (the r459 stale-directory + * class; observed as a CUDA client hanging after DEVICE). Candidates are + * parked here and registered only after a walk of the client's own + * content-validated ring VA through the candidate reproduces that ring's PA + * (svc_gr_ram_ring_scan's latch calls svc_vas_candidates_verify). + */ +#define SVC_VAS_CAND_MAX 16 +static struct { + bool valid; + u32 h_vaspace; + u64 pdb; + u32 entries; + bool vidmem; + unsigned long stamp; /* r537: park time for the bounded fallback */ +} svc_vas_cand[SVC_VAS_CAND_MAX]; + +static int svc_gmmu_walk(struct nv_vgpu_device *vgpu, u64 pdb, u32 entries, + u64 va, int tap, u64 *addr, int *ap, bool trace); + +void gsp_sema_service_vaspace_candidate_add(u32 h_vaspace, u64 pdb, + u32 entries, bool vidmem) +{ + int i; + + if (!pdb || entries < 2 || entries > 4096) + return; + for (i = 0; i < SVC_VAS_CAND_MAX; i++) { + if (svc_vas_cand[i].valid && + svc_vas_cand[i].h_vaspace == h_vaspace && + svc_vas_cand[i].pdb == pdb) + return; /* already parked */ + if (!svc_vas_cand[i].valid) { + svc_vas_cand[i].valid = true; + svc_vas_cand[i].h_vaspace = h_vaspace; + svc_vas_cand[i].pdb = pdb; + svc_vas_cand[i].entries = entries; + svc_vas_cand[i].vidmem = vidmem; + svc_vas_cand[i].stamp = jiffies; + return; + } + } +} + +/* + * r537 bounded fallback: register a parked candidate that has waited for + * its content-verified ring walk longer than SVC_VAS_CAND_AGE_MS. The + * verification (svc_vas_candidates_verify) needs the client's ring to be + * found by the RAM content scan; on a boot where that scan never anchors + * the ring (observed: pd_found=0 and every launch operand walk rc=-2 for + * minutes, the client spinning with its data VAs unresolvable) the + * candidate stays parked forever and the client can never proceed. The + * candidate itself came from the client's own RM publish (its VAS + * handle), and the r459 newest-first walk order keeps an older stale + * directory from winning, so after the bounded wait the registration is + * the lesser evil; it is logged loudly for the record. + */ +#define SVC_VAS_CAND_AGE_MS 15000u + +static void svc_vas_candidates_age_out(struct nv_vgpu_device *vgpu) +{ + int i; + + for (i = 0; i < SVC_VAS_CAND_MAX; i++) { + struct sema_vaspace_desc d; + + if (!svc_vas_cand[i].valid) + continue; + if (time_before(jiffies, svc_vas_cand[i].stamp + + msecs_to_jiffies(SVC_VAS_CAND_AGE_MS))) + continue; + memset(&d, 0, sizeof(d)); + d.h_vaspace = svc_vas_cand[i].h_vaspace; + d.pdb = svc_vas_cand[i].pdb; + d.num_entries = svc_vas_cand[i].entries; + d.b_vidmem = svc_vas_cand[i].vidmem; + pr_warn("sema-svc: VAS 0x%x page directory registered UNVERIFIED after %u ms parked (pdb=0x%llx entries=%u; the content-verified ring walk never anchored this boot)\n", + d.h_vaspace, SVC_VAS_CAND_AGE_MS, d.pdb, + d.num_entries); + gsp_sema_service_vaspace_add(&d); + svc_vas_cand[i].valid = false; + } +} + +/* + * Verify each parked candidate against one content-validated translation + * (the ring VA -> PA the RAM scan just proved): walk the ring page through + * the candidate and register only the directory that reproduces it. Runs on + * the service thread from the ring-scan latch; the walk itself is cheap (a + * few table reads). + */ +static void svc_vas_candidates_verify(struct nv_vgpu_device *vgpu, u64 va, + u64 pa) +{ + int i; + + for (i = 0; i < SVC_VAS_CAND_MAX; i++) { + u64 wpa = 0; + int wap = 0; + int ap = svc_vas_cand[i].vidmem ? SVC_APERTURE_FB + : SVC_APERTURE_SYS; + struct sema_vaspace_desc d; + + if (!svc_vas_cand[i].valid) + continue; + if (svc_gmmu_walk(vgpu, svc_vas_cand[i].pdb, + svc_vas_cand[i].entries, va & ~0xfffULL, + ap, &wpa, &wap, false) || + (wpa & ~0xfffULL) != (pa & ~0xfffULL)) + continue; + memset(&d, 0, sizeof(d)); + d.h_vaspace = svc_vas_cand[i].h_vaspace; + d.pdb = svc_vas_cand[i].pdb; + d.num_entries = svc_vas_cand[i].entries; + d.b_vidmem = svc_vas_cand[i].vidmem; + gsp_sema_service_vaspace_add(&d); + svc_vas_cand[i].valid = false; + pr_info("sema-svc: VAS 0x%x page directory registered after verified ring walk (pdb=0x%llx entries=%u ap=%d)\n", + d.h_vaspace, d.pdb, d.num_entries, ap); + break; + } +} + void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc) { struct svc_vaspace *vs = NULL; @@ -1215,12 +1847,40 @@ void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc) if (!vs && !svc.vas[i].valid) vs = &svc.vas[i]; } + /* + * r459: the table is 8 slots and the guest driver does not always + * send a teardown for a client's VA space (observed: 23 distinct + * registrations, 0 drops in one boot). Once all slots hold a dead + * client's directory, a NEW client's registration found no slot and + * every walk of its VAs failed (copies rc=-2, launches reading + * stale pages - the 10th-client mid-boot degradation). Evict the + * least-recently-registered entry instead of refusing the new one; + * the old client is gone and its directory pointer is dead guest + * memory, so the eviction can only remove stale authority. + */ + if (!vs) { + unsigned long oldest = 0; + int oi = 0; + + for (i = 0; i < SVC_MAX_VASPACES; i++) { + if (!svc.vas[i].valid) + continue; + if (!oldest || time_before(svc.vas[i].stamp, oldest)) { + oldest = svc.vas[i].stamp; + oi = i; + } + } + vs = &svc.vas[oi]; + pr_info("sema-svc: VA space table full; evicting 0x%x (pdb=0x%llx) for 0x%x\n", + vs->h_vaspace, vs->pdb, desc->h_vaspace); + } if (vs) { vs->valid = true; vs->h_vaspace = desc->h_vaspace; vs->pdb = desc->pdb; vs->entries = desc->num_entries; vs->ap = desc->b_vidmem ? SVC_APERTURE_FB : SVC_APERTURE_SYS; + vs->stamp = jiffies; /* A new directory invalidates what we learned from the old one. */ memset(svc.lat, 0, sizeof(svc.lat)); } @@ -1444,8 +2104,18 @@ static int svc_gmmu_walk(struct nv_vgpu_device *vgpu, u64 pdb, u32 entries, return -ENOENT; } - /* PD0: one dual entry per index (big half then small half). */ + /* + * PD0: one dual entry per index (big half then small half). + * Geometry per the driver's own GMMU format (gp10x family): + * PD3 [48:47], PD2 [46:38], PD1 [37:29], PD0 [28:21], PT + * [20:12]. PD0 is 8 bits - an earlier 0x1ff experiment here was + * reverted; the observed +2 MB client-page phase is NOT a PD0 + * index error. + */ idx = (va >> 21) & 0xff; + if (trace) + pr_info("sema-walk: PD0 table=0x%llx idx=%u va=0x%llx tap=%d\n", + table, idx, va, tap); if (svc_read_ap(vgpu, table + 16 * idx, tap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS, @@ -1455,6 +2125,9 @@ static int svc_gmmu_walk(struct nv_vgpu_device *vgpu, u64 pdb, u32 entries, : SVC_APERTURE_SYS, &small, sizeof(small))) return -ENOENT; + if (trace) + pr_info("sema-walk: PD0 big=0x%016llx small=0x%016llx\n", + big, small); /* Small half: 4 KB page table under it. */ sap = svc_pde_aperture(((small >> 1) & 0x3)); @@ -1466,6 +2139,12 @@ static int svc_gmmu_walk(struct nv_vgpu_device *vgpu, u64 pdb, u32 entries, *addr += va & 0xfff; return 0; } + if (trace) + pr_info("sema-walk: small miss sap=%d t=0x%llx\n", + sap, t); + } else if (trace) { + pr_info("sema-walk: small skip sap=%d small&1=%llu\n", + sap, small & 1); } /* Big half: a 2 MB page here, or a large (64 KB) page table. */ @@ -1498,6 +2177,9 @@ static int svc_gmmu_walk(struct nv_vgpu_device *vgpu, u64 pdb, u32 entries, return 0; } } + if (trace) + pr_info("sema-walk: no leaf va=0x%llx big=0x%llx small=0x%llx\n", + va, big, small); return -ENOENT; } @@ -1526,6 +2208,26 @@ static int svc_walk_vaspaces(struct nv_vgpu_device *vgpu, u64 va, } spin_unlock_irqrestore(&svc.vas_lock, flags); + /* + * r459: try the NEWEST registration first. A dead client's stale + * directory (the table outlives clients; only 8 slots) can still + * resolve a VA - through guest memory the driver has since reused - + * to a wrong page, and the first success wins. Ordering by the + * registration stamp makes the live client's directory the + * authority and only falls through to older ones when it cannot + * answer. + */ + for (i = 1; i < n; i++) { + struct svc_vaspace k = local[i]; + int j = i - 1; + + while (j >= 0 && time_before(local[j].stamp, k.stamp)) { + local[j + 1] = local[j]; + j--; + } + local[j + 1] = k; + } + for (i = 0; i < n; i++) { int alt; @@ -1651,8 +2353,23 @@ static void svc_pd_discover(struct nv_vgpu_device *vgpu) } if (n < SVC_PD_MIN_HITS) { n = 0; - if (svc.map_n < SVC_PD_SAMPLES) - return; /* no evidence yet: retry when pairs appear */ + /* + * r510: a LARGE map is a mixed set of VA spaces cached over + * the whole guest boot (the comment above already names the + * consequence: no single directory reproduces a mixed set), so + * scanning with its stride subsample burns the attempts budget + * on evidence that cannot succeed - and by the time the + * push-buffer pairs (the strongest samples) exist, the budget + * is spent (r508/r509 witness: 'not found (base=0x0 ap=ram + * cands=39621 best=0/8)' during the first client, eight RAM + * walks consumed, every later client's high argument pointers + * unresolvable - the terminal class). Only a SMALL map, which + * is one session's own pairs, still falls back; otherwise wait + * for the push-buffer pairs and keep the attempts. + */ + if (svc.map_n > (u32)(SVC_PD_SAMPLES * 2) || + svc.map_n < SVC_PD_SAMPLES) + return; /* no coherent evidence yet: retry later */ step = svc.map_n / SVC_PD_SAMPLES; for (i = 0; i < SVC_PD_SAMPLES; i++) { sva[i] = svc.map[i * step].gpu_page; @@ -1725,13 +2442,133 @@ static void svc_pd_discover(struct nv_vgpu_device *vgpu) svc_pd_cands, svc_pd_best, n); } +/* + * m177c: a SECOND discovered-directory slot for the UVM SYS VA space (the + * 0x2xxxxxxx family: GR rings/pushes, QMD windows, counter pages). The + * primary slot (above) is validated against rm_mem-content pairs - the + * 0x1xxxxxxx pool family - and cannot walk the UVM space, so without pins + * the counter page (0x20400fXXX) never resolved: the guest's KMD/UVM + * epoch waited on completions that could never be written and re-ran its + * ~103 s retry loop forever (r194/r195: 'rel-apply ... slot pending' x8, + * then silence). Samples for this slot are content-VERIFIED + * (push_va -> push_pa) pairs from the GR content discovery and the CE + * push pairing: the push bytes themselves are the proof, and they live in + * exactly the space this directory maps. + */ +#define SVC_PD_RING_SAMPLES 8 +static u64 svc_pd_r_v[SVC_PD_RING_SAMPLES]; +static u64 svc_pd_r_pa[SVC_PD_RING_SAMPLES]; +static u32 svc_pd_r_n; +static u64 svc_pd_r_base; +static int svc_pd_r_ap; +static u8 svc_pd_r_found; +static u8 svc_pd_r_attempts; +static u64 svc_pd_r_ev_hash; + +static void svc_pd_ring_note(u64 va, u64 pa) +{ + u32 i; + + if (!va || !pa) + return; + va &= ~0xfffULL; + pa &= ~0xfffULL; + for (i = 0; i < svc_pd_r_n; i++) + if (svc_pd_r_v[i] == va) + return; + if (svc_pd_r_n < SVC_PD_RING_SAMPLES) { + svc_pd_r_v[svc_pd_r_n] = va; + svc_pd_r_pa[svc_pd_r_n] = pa; + svc_pd_r_n++; + } +} + +static void svc_pd_ring_discover(struct nv_vgpu_device *vgpu) +{ + u64 chunk = 64 * 1024; + u8 *buf; + u64 gpa; + u64 ev = svc_pd_r_n; + u32 i; + + if (svc_pd_r_found || svc_pd_r_attempts >= SVC_PD_ATTEMPTS_MAX) + return; + if (svc_pd_r_n < SVC_PD_MIN_HITS) + return; + for (i = 0; i < svc_pd_r_n; i++) + ev = (ev + svc_pd_r_v[i] + (svc_pd_r_pa[i] << 12)) * 1000003; + if (svc_pd_r_attempts && ev == svc_pd_r_ev_hash) + return; + svc_pd_r_ev_hash = ev; + svc_pd_r_attempts++; + + buf = kmalloc(chunk, GFP_KERNEL); + if (!buf) + return; + for (gpa = 0; gpa < SEMA_PA_MAX && !svc_pd_r_found; gpa += chunk) { + u32 len = (u32)min_t(u64, chunk, SEMA_PA_MAX - gpa); + u64 off; + + if (svc_read(vgpu, gpa, buf, len)) + continue; + for (off = 0; off + 8 <= len && !svc_pd_r_found; + off += SEMA_PAGE_SIZE) { + u64 e; + u64 a = 0; + int eap = 0; + u32 hits = 1; + + memcpy(&e, buf + off, sizeof(e)); + if ((e & 1) || !((e >> 1) & 0x3)) + continue; + /* First sample rejects almost every candidate. */ + if (svc_gmmu_walk(vgpu, gpa + off, 4, svc_pd_r_v[0], + SVC_APERTURE_SYS, &a, &eap, false) || + (a & ~0xfffULL) != svc_pd_r_pa[0]) + continue; + for (i = 1; i < svc_pd_r_n; i++) { + if (!svc_gmmu_walk(vgpu, gpa + off, 4, + svc_pd_r_v[i], + SVC_APERTURE_SYS, &a, &eap, + false) && + (a & ~0xfffULL) == svc_pd_r_pa[i]) + hits++; + } + if (hits >= SVC_PD_MIN_HITS) { + svc_pd_r_base = gpa + off; + svc_pd_r_ap = SVC_APERTURE_SYS; + svc_pd_r_found = 1; + pr_info("sema-svc: discovered UVM ring-space page directory 0x%llx (ram), %u/%u content-verified samples\n", + svc_pd_r_base, hits, svc_pd_r_n); + } + } + } + kfree(buf); +} + static int svc_walk_discovered(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, int *ap) { - if (!svc_pd_found) - return -ENOENT; - return svc_gmmu_walk(vgpu, svc_pd_base, svc_pd_entries, va, - svc_pd_ap, addr, ap, false); + int rc = -ENOENT; + + if (svc_pd_found) + rc = svc_gmmu_walk(vgpu, svc_pd_base, svc_pd_entries, va, + svc_pd_ap, addr, ap, false); + if (rc && svc_pd_r_found) + rc = svc_gmmu_walk(vgpu, svc_pd_r_base, 4, va, + svc_pd_r_ap, addr, ap, false); + /* + * m177c: the ring-space directory discovery runs on new samples + * only (fingerprint + attempt cap inside). A first call with + * enough verified ring/push samples performs one bounded scan; + * after that this is a flag check. + */ + if (rc && !svc_pd_r_found && svc_pd_r_n >= SVC_PD_MIN_HITS) + svc_pd_ring_discover(vgpu); + if (rc && svc_pd_r_found) + rc = svc_gmmu_walk(vgpu, svc_pd_r_base, 4, va, + svc_pd_r_ap, addr, ap, false); + return rc; } /* Learned translations: a walk costs a handful of guest reads, so cache the @@ -2495,6 +3332,29 @@ static inline bool svc_pool_hold_active(void) } /* + * m165 (D-F2a): the blanket pool hold exists to stop a release decoded + * before the guest's own injection lands from being applied through a + * stale translation. Once a host-verified injected pin for THIS VA is + * present, that premise is gone: the pin source is the resolver's first + * choice (read-back proven) and the ipin table is wiped at session flush, + * so it is this session's ground truth. r156 measured every pool release + * waiting the full registration+5.00 s with the pin ready all along. + */ +static bool svc_pool_hold_active_va(u64 va) +{ + u64 ipa = 0; + int iap = 0; + + if (!svc_pool_hold_active()) + return false; + if (!gsp_sema_pool_hold_fresh) + return true; + if (svc_ipin_lookup(va, &ipa, &iap) == 0) + return false; + return true; +} + +/* * Verified pool-semaphore translations, keyed by page VA. An entry is only * created from an injected pin or from a targeted rm_mem resolve whose page * was read back successfully; the learned cache and the discovered map are @@ -2518,6 +3378,16 @@ static u32 svc_pool_pa_n; static u32 svc_pool_pa_next; static u64 svc_pool_pa_logged[SVC_POOL_PA_LOG_MAX]; static u32 svc_pool_pa_logged_n; +/* + * fail4: the completion-hint window was 10 s. It outlived short-lived + * clients: after a client freed its RM events the window kept re-posting + * hint events every 0.5 s, and a post already queued in the status ring + * could be read after the guest's own free processing - the guest ISR + * then ran a freed RC callback and the guest panicked in interrupt + * (rm_run_rc_callback+0x0). 1.5 s still covers a waiter that arrives + * after the edge while shrinking the post-free exposure ~7x. + */ +#define SVC_POST_HINT_MS 1500 static u64 svc_pool_pa_miss_log; static u32 svc_pool_pa_hit_log; static u32 svc_pool_wr_log; @@ -2566,6 +3436,14 @@ static void svc_pool_wr_note(u32 h, u64 va, u64 pa, const char *src, } } +/* r372: landing-table entries repaired from a fresh pin (stale-ledger class). */ +static u64 svc_ledger_pin_repairs; +/* r372: pool alt fan-outs skipped because the primary pin moved on. */ +static u64 svc_pool_alt_skipped; +/* r372: bounded page-note resolution diagnostics (va/pa/ap/branch). */ +static u32 svc_gr_pagenote_log; +static u32 svc_fbhpa_log; + /* * Fan a pool release out to the last displaced translation of the same page * (see svc_ipin_alt): the primary target may be the stale one when two host @@ -2589,6 +3467,27 @@ static void svc_pool_alt_apply(struct nv_vgpu_device *vgpu, u32 h, u64 va, return; if ((pa & ~(u64)(SEMA_PAGE_SIZE - 1)) == page) return; /* never write an identity translation */ + /* + * r372 (stale-ledger corruption class): fan out ONLY while the + * current primary pin for this VA still names the displaced page. + * Once a newer pin has displaced it, the old page belongs to a + * buffer the guest has freed - writing it lands on whatever the + * guest kernel reused the page for. A missing current pin is + * equally unprovable: skip. The primary release write already + * succeeded either way, so this is a lost best-effort, not a + * failure. + */ + { + u64 cpa = 0; + int cap = SVC_APERTURE_SYS; + + if (svc_ipin_lookup(va, &cpa, &cap) || + ((cpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1)))) { + svc_pool_alt_skipped++; + return; + } + } if (svc_read_ap(vgpu, pa, svc_ap_of(ap), &cur, sizeof(cur))) return; if (cur >= val) @@ -2621,6 +3520,12 @@ static inline bool svc_pool_pa_identity(u64 va, u64 pa) (va & ~(u64)(SEMA_PAGE_SIZE - 1)); } +static void svc_pool_boot_store(u64 va, u64 pa, int ap); /* m168 */ +static bool svc_dst_guard_hold(u64 va); /* m169b */ +static bool svc_dst_guard_pending(u64 va); /* r231 */ +static u32 svc_read_order_log; /* r231 budget */ +static u32 svc_atrace_log; /* r234 A-TRACE budget */ + static int svc_pool_pa_lookup(u64 va, u64 *pa, int *ap) { u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); @@ -2651,6 +3556,8 @@ static void svc_pool_pa_store(u64 va, u64 pa, int ap) /* Never cache an identity translation (see svc_pool_pa_identity). */ if (svc_pool_pa_identity(page, pa)) return; + /* m168: every proven resolution also feeds the boot-scoped cache. */ + svc_pool_boot_store(va, pa, ap); for (i = 0; i < svc_pool_pa_n; i++) { if (svc_pool_pa[i].va_page == page) { svc_pool_pa[i].pa_page = @@ -2680,6 +3587,207 @@ static void svc_pool_pa_forget(u64 va) } /* + * m168 (D-F2c): boot-scoped verified pool-page cache. Within one guest boot + * the UVM semaphore pool (gpu->semaphore_pool, a 4 KB sysmem rm_mem at + * 0x121000000 whose 1024 words are the per-channel tracker slots) belongs + * to the shared GPU object; clients of the same boot reuse the pages while + * their per-process vaspaces come and go. The session flush wipes + * svc_pool_pa deliberately (a VA may name a new allocation for the next + * client), but wiping the POOL page forces every new client through the + * slow resolver: r170's clients 4-6 starved 2+ minutes on + * "pending release starved ... pool/waiter page" while the scanner learned + * the live tracker sems. This cache survives the flush; it is filled ONLY + * from read-back-proven resolutions (pin/cache/walk/vaspaces/targeted) and + * every use re-proves the exact offset readable, so a page that moved + * fails the proof and is dropped. A fresh injected pin always overwrites + * it (pins are the resolver's first source and run before this cache). + */ +struct svc_pool_boot_pa { + u64 va_page; + u64 pa_page; + int ap; + /* + * r539: secondary candidate. The pool VA can be answered by two + * read-back-proven sources that disagree (the injected pin - the + * guest kernel's own tracker->page->rm_mem chain - and the host + * GMMU walk of the client's page tables). Either can be the page + * the client actually polls: the r536 fix preferred the walk and + * the r538 init-stall boot showed the pin and the walk flapping + * every resolve while a client sat in uvm channel_pool_add -> + * uvm_push_end_and_wait. A release written only to the losing page + * strands the tracker. Keep BOTH pages and fan every pool release + * out to the secondary as well: the word is a forward-only + * monotonic counter, so a duplicate write is absorbed. The + * secondary is dropped when it stops being readable and ages out + * after SVC_POOL_DUAL_TTL_MS of not being re-proven by any source + * (a page the guest has since freed must not be written). + */ + u64 pa2_page; + int ap2; + unsigned long seen; /* last proof time of the primary */ + unsigned long seen2; /* last proof time of the secondary */ +}; + +static struct svc_pool_boot_pa svc_pool_boot[SVC_POOL_PA_MAX]; +static u32 svc_pool_boot_n; +static u32 svc_pool_boot_next; +static u32 svc_pool_boot_conflicts; +static u32 svc_pool_boot_hits; +/* r539: fresh-secondary window; the flapping sources (the watcher's ~2 s + * pool re-inject, the per-resolve walk) re-prove both candidates well inside + * this, while a page the guest has since freed and reused stops being + * re-proven and ages out before any write can land on it (the r372 + * stale-page corruption rule). */ +#define SVC_POOL_DUAL_TTL_MS 10000u + +static int svc_pool_boot_lookup(u64 va, u64 *pa, int *ap) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 i; + + for (i = 0; i < svc_pool_boot_n; i++) { + if (svc_pool_boot[i].va_page == page) { + if (svc_pool_pa_identity(page, + svc_pool_boot[i].pa_page)) + return -ENOENT; + *pa = svc_pool_boot[i].pa_page | + (va & (SEMA_PAGE_SIZE - 1)); + *ap = svc_pool_boot[i].ap; + return 0; + } + } + return -ENOENT; +} + +static void svc_pool_boot_store(u64 va, u64 pa, int ap) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 i; + + if (svc_pool_pa_identity(page, pa)) + return; + for (i = 0; i < svc_pool_boot_n; i++) { + if (svc_pool_boot[i].va_page == page) { + if (svc_pool_boot[i].pa_page == pa_page) { + svc_pool_boot[i].ap = ap; + svc_pool_boot[i].seen = jiffies; + return; + } + if (svc_pool_boot[i].pa2_page == pa_page) { + /* The previously-secondary page is the + * freshest proof: promote it, demote the + * old primary. No conflict line: both + * candidates stay live and both receive + * the fan-out. */ + u64 op = svc_pool_boot[i].pa_page; + int oap = svc_pool_boot[i].ap; + unsigned long oseen = svc_pool_boot[i].seen; + + svc_pool_boot[i].pa_page = pa_page; + svc_pool_boot[i].ap = ap; + svc_pool_boot[i].seen = jiffies; + svc_pool_boot[i].pa2_page = op; + svc_pool_boot[i].ap2 = oap; + svc_pool_boot[i].seen2 = oseen; + return; + } + /* + * r539: a second, DIFFERENT page. Keep the old + * primary as the secondary candidate instead of + * discarding it - the r538 init-stall boot had the + * pin and the walk flapping forever, and the client + * polls whichever page its own mapping names. + */ + svc_pool_boot_conflicts++; + pr_info_ratelimited("sema-svc: pool-boot second candidate va=0x%llx primary=0x%llx -> 0x%llx (old primary kept as secondary)\n", + page, svc_pool_boot[i].pa_page, pa_page); + svc_pool_boot[i].pa2_page = svc_pool_boot[i].pa_page; + svc_pool_boot[i].ap2 = svc_pool_boot[i].ap; + svc_pool_boot[i].seen2 = svc_pool_boot[i].seen; + svc_pool_boot[i].pa_page = pa_page; + svc_pool_boot[i].ap = ap; + svc_pool_boot[i].seen = jiffies; + return; + } + } + if (svc_pool_boot_n < SVC_POOL_PA_MAX) { + i = svc_pool_boot_n++; + } else { + i = svc_pool_boot_next++ % SVC_POOL_PA_MAX; + } + svc_pool_boot[i].va_page = page; + svc_pool_boot[i].pa_page = pa_page; + svc_pool_boot[i].ap = ap; + svc_pool_boot[i].pa2_page = 0; + svc_pool_boot[i].ap2 = 0; + svc_pool_boot[i].seen = jiffies; + svc_pool_boot[i].seen2 = 0; +} + +/* + * r539: fan one applied pool release out to the secondary candidate page + * (see struct svc_pool_boot_pa). Forward-only and read-back proven, exactly + * like the primary write; a failure is counted, never propagated - the + * primary write already landed. A secondary that stopped being readable is + * dropped; one that was not re-proven by any source for + * SVC_POOL_DUAL_TTL_MS is left alone (the guest may have reused the page). + */ +static void svc_pool_boot_fanout(struct nv_vgpu_device *vgpu, u32 h, u64 va, + u32 val) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 pa = 0; + int ap = SVC_APERTURE_SYS; + u32 cur = 0; + u32 back = 0; + u32 i; + + for (i = 0; i < svc_pool_boot_n; i++) + if (svc_pool_boot[i].va_page == page) + break; + if (i == svc_pool_boot_n || !svc_pool_boot[i].pa2_page) + return; + if (!svc_pool_boot[i].seen2 || + time_after(jiffies, svc_pool_boot[i].seen2 + + msecs_to_jiffies(SVC_POOL_DUAL_TTL_MS))) + return; + pa = svc_pool_boot[i].pa2_page | (va & (SEMA_PAGE_SIZE - 1)); + ap = svc_pool_boot[i].ap2; + if (svc_pool_pa_identity(va, pa)) + return; + if (svc_read_ap(vgpu, pa, svc_ap_of(ap), &cur, sizeof(cur))) { + /* The page is gone: drop the stale secondary. */ + svc_pool_boot[i].pa2_page = 0; + return; + } + if (cur >= val) + return; /* forward-only: nothing to add */ + if (svc_write_ap(vgpu, pa, svc_ap_of(ap), &val, sizeof(val))) + return; + if (svc_read_ap(vgpu, pa, svc_ap_of(ap), &back, sizeof(back))) + return; + if (back != val) + return; + svc.pool_dual_applied++; + if (svc.pool_dual_log < 128) { + svc.pool_dual_log++; + pr_info("vgsp-sema: pool-fanout ch=0x%x va=0x%llx pa2=0x%llx cur=%u -> val=%u (secondary candidate received the release)\n", + h, va, pa, cur, val); + } +} + +static void svc_pool_boot_forget(u64 va) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 i; + + for (i = 0; i < svc_pool_boot_n; i++) + if (svc_pool_boot[i].va_page == page) + svc_pool_boot[i].va_page = 0; +} + +/* * Resolve one pool-semaphore target the way the successful deferred apply did: * injected pin (host-verified ground truth), the verified cache, then the * targeted rm_mem record scan, whose result is read back before it is cached. @@ -2712,20 +3820,140 @@ static int svc_pool_pa_resolve(struct nv_vgpu_device *vgpu, u32 h, u64 va, * the injected page is readable before trusting it; a failed * proof falls through to the read-back-validated cache, the * structural walk and the targeted rm_mem record. + * + * r536: the read-back proof does NOT catch a moved pool whose + * OLD page is still readable - the write then lands on the old + * page while the client polls the new one and the tracker + * starves (observed: "pending release starved va=0x12100000x + * tries=64" for ~45 s after the pool moved, with the stale pin + * pa=0x120b30000 still readable; the fresh pin pa=0x130e9d000 + * arrived 45 s later). The client's OWN page tables are the + * GPU/UVM ground truth for where the tracker reads, so when a + * structural walk resolves a DIFFERENT page than the pin, the + * walk wins (both are read-back proven at the exact offset). */ u32 probe = 0; if (!svc_read_ap(vgpu, pa, svc_ap_of(a), &probe, sizeof(probe))) { + u64 wpa = 0; + int wap = SVC_APERTURE_SYS; + bool walk_hit = false; + + if (!svc_walk_discovered(vgpu, page, &wpa, &wap) || + !svc_walk_vaspaces(vgpu, page, &wpa, &wap)) + walk_hit = true; + if (walk_hit && + (wpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1)) && + !svc_pool_pa_identity(va, wpa)) { + u32 wprobe = 0; + u64 exact = (wpa & ~(u64)(SEMA_PAGE_SIZE - 1)) | + (va & (SEMA_PAGE_SIZE - 1)); + + if (!svc_read_ap(vgpu, exact, svc_ap_of(wap), + &wprobe, sizeof(wprobe))) { + /* + * r539: record the PIN as the + * secondary candidate before the walk + * page becomes the primary. The walk + * is not infallible (the r538 boot: + * pin and walk flapped on every + * resolve and the client polled the + * pin's page); keeping both means the + * release fan-out reaches whichever + * page the client actually polls. + */ + svc_pool_boot_store(va, pa, a); + svc_pool_pa_store(va, exact, wap); + svc_tr_clear_failure(page); + if (src) + *src = "walk-over-pin"; + svc_pool_pa_note(h, va, exact, + "walk-over-pin"); + *addr = exact; + *ap = wap; + return 0; + } + } svc_tr_clear_failure(page); if (src) *src = "pin"; svc_pool_pa_note(h, va, pa, "pin"); + svc_pool_boot_store(va, pa, a); *addr = pa; *ap = a; return 0; } } + /* + * m168: boot-scoped page, survives the session flush. Every use + * re-proves the exact offset readable: a page that moved fails the + * proof and is dropped (the fresh pin path above always wins when a + * pin exists). + */ + if (!svc_pool_boot_lookup(va, &pa, &a)) { + u32 bprobe = 0; + + /* + * r580: the boot cache survives session flushes, but a pool + * page MOVES between sessions (the client's new allocation) + * while the old page often stays readable - the read-back + * proof cannot tell. The client's own page tables are the + * ground truth for where ITS tracker reads, so try the + * structural walk BEFORE trusting the cache: a differing, + * read-proven walk page wins and the cached page stays as the + * fan-out's secondary candidate. Live evidence (r579 boot): + * releases were written to the stale cache pa=0x11e6d9000 + * while the current page was 0x134730000 (the pin) and the + * client's tracker starved; the pool-wr line said + * src=bootcache. + */ + { + u64 wpa2 = 0; + int wap2 = SVC_APERTURE_SYS; + bool walk_hit = false; + + if (!svc_walk_discovered(vgpu, page, &wpa2, &wap2) || + !svc_walk_vaspaces(vgpu, page, &wpa2, &wap2)) + walk_hit = true; + if (walk_hit && + (wpa2 & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1)) && + !svc_pool_pa_identity(va, wpa2)) { + u64 exact = (wpa2 & ~(u64)(SEMA_PAGE_SIZE - 1)) | + (va & (SEMA_PAGE_SIZE - 1)); + + if (!svc_read_ap(vgpu, exact, svc_ap_of(wap2), + &bprobe, sizeof(bprobe))) { + svc_pool_boot_store(va, pa, a); + svc_pool_pa_store(va, exact, wap2); + svc_tr_clear_failure(page); + if (src) + *src = "walk-over-boot"; + svc_pool_pa_note(h, va, exact, + "walk-over-boot"); + *addr = exact; + *ap = wap2; + return 0; + } + } + } + + if (!svc_read_ap(vgpu, pa, svc_ap_of(a), &bprobe, + sizeof(bprobe))) { + svc_tr_clear_failure(page); + svc_pool_boot_hits++; + if (src) + *src = "bootcache"; + *addr = pa; + *ap = a; + return 0; + } + pr_info("sema-svc: pool-boot stale va=0x%llx pa=0x%llx (read-back failed; dropping)\n", + va, pa); + svc_pool_boot_forget(va); + } if (!svc_pool_pa_lookup(va, &pa, &a)) { svc_tr_clear_failure(page); if (svc_pool_pa_hit_log < SVC_POOL_PA_HIT_LOG_MAX) { @@ -2900,6 +4128,17 @@ static int svc_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val) } static void svc_pending_add(u64 va, u32 payload, u32 op); +/* r34x: defined with the pending ring / opdef slots (later in the file); + * the acquire gate drains/appraises withheld releases before it holds. */ +static void svc_apply_pending(struct nv_vgpu_device *vgpu); +static bool svc_wait_pend_release(u64 va); + +/* + * F1: GR release-word authorship checks. Defined next to the svc_fwd_req + * slot; forward-declared because the sweep writer runs earlier in the file. + */ +static bool svc_gr_rel_match(u64 va, u32 *payload); +static bool svc_gr_rel_inflight(u64 va, u32 *payload); static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) { @@ -2982,6 +4221,7 @@ static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) if (is_pool) { svc_pool_wr_note(h, va, addr, src, cur, val, "ok"); svc_pool_alt_apply(vgpu, h, va, val); + svc_pool_boot_fanout(vgpu, h, va, val); } return 0; } @@ -3008,6 +4248,7 @@ static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) svc_pool_wr_note(h, va, addr, "targeted", cur, val, "reresolve"); svc_pool_alt_apply(vgpu, h, va, val); + svc_pool_boot_fanout(vgpu, h, va, val); } svc.sem_write_reresolve++; pr_info_ratelimited("vgsp-sema: sem-write re-resolve va=0x%llx pa=0x%llx old=0x%llx new=0x%x\n", @@ -3020,6 +4261,7 @@ static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) svc_pool_wr_note(h, va, old_addr, src, cur, val, "advisory"); svc_pool_alt_apply(vgpu, h, va, val); + svc_pool_boot_fanout(vgpu, h, va, val); } return 0; /* the original write landed; read-back is advisory */ } @@ -3056,76 +4298,92 @@ static void svc_scan_pass(struct nv_vgpu_device *vgpu, u8 *buf, u64 gpa; u32 nonz = 0; - for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SEMA_PAGE_SIZE) { - u32 off; + /* + * r534: read guest RAM in SVC_SCAN_CHUNK (64 KB) blocks instead of + * one 4 KB page per vfio_dma_rw call. The full 6 GB scan is the + * rebuild's whole cost (1.5 M guest reads ~= 10-11 s per rebuild, + * once per 10 s: the pump got one pass per rebuild and the second + * client's resolutions could wait minutes). A 64 KB read amortizes + * the iommu walk + pin over 16 pages, cutting the scan to well under + * a second with identical pair/base detection (the inner page loop + * is byte-for-byte the old per-page body). + */ + for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SVC_SCAN_CHUNK) { + u32 po; if (READ_ONCE(svc_sess_flush)) break; /* ISSUE-0002: a flush is waiting on this scan */ - if (svc_read(vgpu, gpa, buf, SEMA_PAGE_SIZE)) + if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK)) continue; - for (off = 0; off + 16 <= SEMA_PAGE_SIZE; off += 8) { - u64 a, b; + for (po = 0; po < SVC_SCAN_CHUNK; po += SEMA_PAGE_SIZE) { + u8 *page = buf + po; + u64 pg = gpa + po; + u32 off; - memcpy(&a, buf + off, 8); - memcpy(&b, buf + off + 8, 8); + for (off = 0; off + 16 <= SEMA_PAGE_SIZE; off += 8) { + u64 a, b; - /* (a) self-pointer -> base candidate */ - if (a >= KVA_LO && a < KVA_HI && - (a & 0xfff) == ((gpa + off) & 0xfff)) { - u64 cb = a - gpa - off; - u32 i; + memcpy(&a, page + off, 8); + memcpy(&b, page + off + 8, 8); - if ((cb & 0x3fffffffULL) == 0) { - for (i = 0; i < *ncand; i++) { - if (cands[i].base == cb) { - cands[i].count++; - break; + /* (a) self-pointer -> base candidate */ + if (a >= KVA_LO && a < KVA_HI && + (a & 0xfff) == ((pg + off) & 0xfff)) { + u64 cb = a - pg - off; + u32 i; + + if ((cb & 0x3fffffffULL) == 0) { + for (i = 0; i < *ncand; i++) { + if (cands[i].base == cb) { + cands[i].count++; + break; + } + } + if (i == *ncand && *ncand < 8) { + cands[*ncand].base = cb; + cands[*ncand].count = 1; + (*ncand)++; } - } - if (i == *ncand && *ncand < 8) { - cands[*ncand].base = cb; - cands[*ncand].count = 1; - (*ncand)++; } } - } - /* (b) {cpu_va, gpu_va} pair (page aligned: real mappings) */ - if (a >= KVA_LO && a < KVA_HI && - b >= SEMA_VA_LO && b < SEMA_VA_HI && - (a & 0xfff) == 0 && (b & 0xfff) == 0 && - *npairs < PAIR_MAX) { - u8 ok = 0; + /* (b) {cpu_va, gpu_va} pair (page aligned: real mappings) */ + if (a >= KVA_LO && a < KVA_HI && + b >= SEMA_VA_LO && b < SEMA_VA_HI && + (a & 0xfff) == 0 && (b & 0xfff) == 0 && + *npairs < PAIR_MAX) { + u8 ok = 0; - /* - * UVM's uvm_rm_mem_t stores vas[UVM_ID_CPU] - * (offset 48) directly before vas[UVM_ID_GPU0] - * (offset 56), so a pair taken from a live rm_mem - * is preceded by the header {u32 type; bitmap - * mapped_on}. Validate it when it fits in the same - * page: freed allocations and unrelated tables - * that happen to hold KVA/VA pairs do not match, - * and a stale entry for the same VA must not - * shadow the live one. - */ - if (off >= 48) { - u32 type; - u64 mapped_on; - - memcpy(&type, buf + off - 48, 4); - memcpy(&mapped_on, buf + off - 40, 8); - if ((type == 0 || type == 1) && - (mapped_on & 0x3) == 0x3) - ok = 1; + /* + * UVM's uvm_rm_mem_t stores vas[UVM_ID_CPU] + * (offset 48) directly before vas[UVM_ID_GPU0] + * (offset 56), so a pair taken from a live rm_mem + * is preceded by the header {u32 type; bitmap + * mapped_on}. Validate it when it fits in the same + * page: freed allocations and unrelated tables + * that happen to hold KVA/VA pairs do not match, + * and a stale entry for the same VA must not + * shadow the live one. + */ + if (off >= 48) { + u32 type; + u64 mapped_on; + + memcpy(&type, page + off - 48, 4); + memcpy(&mapped_on, page + off - 40, 8); + if ((type == 0 || type == 1) && + (mapped_on & 0x3) == 0x3) + ok = 1; + } + pairs[*npairs].cpu_va = a; + pairs[*npairs].gpu_va = b; + pairs[*npairs].valid_hdr = ok; + (*npairs)++; } - pairs[*npairs].cpu_va = a; - pairs[*npairs].gpu_va = b; - pairs[*npairs].valid_hdr = ok; - (*npairs)++; } + nonz++; } - nonz++; } } @@ -3285,6 +4543,133 @@ static int svc_map_lookup_range(u64 gpu_va, u32 from, u32 to, u64 *pa) static void svc_session_flush(void); /* defined next to the pending queue */ +/* + * Ledger sizing + counters hoisted above svc_rebuild so the periodic diag + * can report the table's fill state: a full table degrades the write/read + * authority silently (home replacement evicts another VA's landing), which + * is exactly the D2H "reads back prior content" window. + */ +#define SVC_LEDGER_MAX (1u << 14) +#define SVC_LEDGER_MASK (SVC_LEDGER_MAX - 1u) +#define SVC_LEDGER_PROBE 16u +/* + * STALLFREE: eviction burst threshold for the cap-and-degrade reset. A + * cluster keeps 16 same-home VA pages; EXIT_RESET evictions inside one session + * means the table is thrashing (pin churn / leftover crashed clients), and + * the half-evicted table silently strands the evicted pages' landing + * authority. At the threshold the cache is cleared and rebuilt from live + * writes (uniform resolver fallback instead of silent partial authority). + */ +#define SVC_LEDGER_EVICT_RESET 256u +static u32 svc_ledger_n; +static u64 svc_ledger_diverges; +static u64 svc_ledger_pinwins; /* r336: pin-backed divergence wins */ +static u32 svc_d2hmap_log; /* D2H-MAP per-page destination budget */ +/* + * r336/r34x: host-semaphore acquire (C56F SEM_EXECUTE op 0/2/3/4/5) wait + * state. The decoder captures EVERY acquire of the ring entry currently + * being decoded (svc_waits[], reset per entry at the top of + * svc_decode_stream), and svc_run_ops honors them before running any of + * the entry's work, so a copy cannot execute ahead of the dependency its + * push waits on (the streams2 readback-before-scale class). Per-entry + * capture is essential: the wait state used to leak across entries (a + * later launch's self-release wait, e.g. need=85 while the held push + * releases 84, gated an earlier entry that carries no acquire at all - + * the r340 over-hold). The re-decode re-parses the entry, so the state + * is refreshed on every pass. + * + * Semantics per op (class contract, see svc_wait_satisfied): + * 0 ACQUIRE = (v == p); 2 ACQ_STRICT_GEQ = v >= p; 3 ACQ_CIRC_GEQ = + * circular (s32)(v - p) >= 0; 4 ACQ_AND = (v & p) != 0; + * 5 ACQ_NOR = ~(v | p) != 0. + */ +#define SVC_WAIT_MAX 8 +/* + * r401: bounded acquire wait (fail-open), now a module parameter. The + * original 100 ms bound was shorter than a single GR launch's completion + * latency in this emulation (observed 0.5-1.6 s per launch), so a copy + * that waited on a launch's tracker (streams2: the D2H readback's + * ACQ_CIRC_GEQ on the scale channel's progress counter) gave up and ran + * BEFORE the launch wrote its output - the host read pre-scale data + * (bad=32767, c2 all wrong). 4 s covers several launch latencies; the + * bound still guarantees the entry is never parked forever (a genuinely + * unsatisfiable acquire fails open late, after the dependency's own work + * would have landed - late-but-correct instead of early-and-wrong). + */ +unsigned gsp_sema_wait_hold_ms = 4000; +struct svc_wait_op { + u64 va; + u32 payload; + u8 op; /* C56F SEM_EXECUTE operation */ + u32 off; /* method offset in the entry (order) */ +}; +static struct svc_wait_op svc_waits[SVC_WAIT_MAX]; +static u32 svc_wait_n; /* acquires captured this entry */ +/* + * r484 THE STAGING-DRAIN COMPLETION EDGE: when the acquire gate holds an + * entry, the op can still have executed on an earlier decode whose gate did + * not trigger (svc_wait_n was 0 on that pass), and the once-only done + * bitmap then suppresses it on the first pass whose gate finally passes - + * the pass at which the dependency just became installed and the op's + * inputs changed. The release then applies and the guest drains a staging + * buffer holding the PRE-dependency bytes (streams_events s=2: the D2H + * copy read dc2 before the vadd_into wrote it, ran early on gate-less + * passes, was once-only-skipped on the passing pass, and the client + * drained a2+b2 instead of a2+b2+c1). Remember the last held (channel, + * entry) and force exactly ONE re-run of its ops on the first passing + * pass. + */ +static u32 svc_wait_hold_h; +static u32 svc_wait_hold_idx = 0xffffffffu; +static bool svc_wait_repass; +static u32 svc_wait_ovf; /* acquires dropped past the array */ +static u64 svc_wait_held; +static u64 svc_wait_pass; +static u64 svc_wait_giveup; +static u32 svc_wait_log; /* per-session held-row print budget */ +static u32 svc_wait_pass_log; /* per-session resolved-row budget */ +static u32 svc_remapconst_log; /* r343 FILL: SET_REMAP_CONST_A rows */ +static u32 svc_pitch_log; /* r402: 2D (pitched) op builds */ +static u32 svc_fill_raw_log; /* r365 FILL-RAW: raw dropped-fill dumps */ +static u32 svc_lowfill_log; /* r365b LOWFILL-WIT: below-floor fills */ +static u32 svc_tabcopy_log; /* r367 TABLE-COPY: table-page copies */ +static u32 svc_fill_noremap; /* r343 FILL: fail-closed holds */ +static u64 svc_wait_rel_va; /* this entry's own release (mirror) */ +static u32 svc_wait_rel_payload; +static u32 svc_wait_rel_off; /* its method offset (order check) */ +static bool svc_wait_rel_have; +/* + * Per-channel first-hold clock for the bounded wait: a gate evaluation + * that cannot satisfy the acquire arms a 100 ms budget per hold episode; + * expiry fails open (ops run) with a loud 'give-up' row. The table is + * tiny (a handful of channels), service-thread only, cleared per pass + * resolution / session flush. + */ +struct svc_wait_hold { + bool active; + u32 h; + unsigned long since; +}; +#define SVC_WAIT_HOLD_MAX 16 +static struct svc_wait_hold svc_wait_holds[SVC_WAIT_HOLD_MAX]; +static u32 svc_dst_div_log; /* r402: copy-dst ledger/probe divergence */ +static u64 svc_ledger_stores; +static u64 svc_ledger_evictions; +static u64 svc_ledger_evict_since_reset; +static u64 svc_ledger_evict_resets; +static u64 svc_ledger_rewrites; +static u64 svc_ledger_reloc_diverges; +static u32 svc_ledger_store_log; +/* + * STALLFREE: pending-ring overflow telemetry (the negative queue silently + * overwrote its oldest entry when full). Hoisted here so svc_rebuild's diag + * can print it; the ring itself is defined with the pending queue below. + */ +static u64 svc_pending_ovf; +static u32 svc_pending_ovf_log; +/* STALLFREE: rescue scans that hit their wall-clock cap (bounded service). */ +static u64 svc_rescue_scan_capped; + static void svc_rebuild(struct nv_vgpu_device *vgpu) { static u8 *buf; @@ -3307,7 +4692,8 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) return; if (!buf) { - buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL); + /* r534: sized for svc_scan_pass's 64 KB chunked guest reads. */ + buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); if (!buf) return; } @@ -3387,18 +4773,24 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) svc.false_trans, svc.gpfifo_skipped, svc.gpfifo_rescued, svc.releases_faked, svc.releases_recovered, svc.counter_rel_applied); - pr_info("sema-svc: drain held=%llu starved=%llu forwarded=%llu faked=%llu rec=%llu ops_once=%llu reident=%llu opdef=%llu/%llu/%llu\n", + pr_info("sema-svc: drain held=%llu starved=%llu forwarded=%llu faked=%llu rec=%llu ops_once=%llu reident=%llu opdef=%llu/%llu/%llu gr_nocap=%llu/%llu gr_owed=%llu gr_hold_exp=%llu gr_swrel=%llu gr_wedge=%llu pend_ovf=%llu\n", svc.gpfifo_held, svc.gpfifo_starved, svc.releases_forwarded, svc.releases_faked, svc.releases_recovered, svc.ops_once_skipped, svc.ops_reident, - svc.opdef_held, svc.opdef_applied, svc.opdef_expired); - pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu altw=%llu residue=%llu\n", + svc.opdef_held, svc.opdef_applied, svc.opdef_expired, + svc.gr_nocap_held, svc.gr_nocap_consumed, + svc.gr_owed_resolved, + svc.fwd_gr_hold_exp, svc.fwd_gr_stall_swrel, + svc.gr_stale_wedge, svc_pending_ovf); + pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu altw=%llu residue=%llu wait=%llu/%llu giveup=%llu ovf=%u\n", svc.counter_pub_calls, svc.gr_rel_applied, svc.sem_write_reresolve, svc.pushpg_fallback, - svc.pool_alt_applied, svc.pool_residue); - if (svc.resc_run) - pr_info("sema-svc: rescue_run=%llu (multi-page content pins)\n", - svc.resc_run); + svc.pool_alt_applied, svc.pool_residue, + svc_wait_held, svc_wait_pass, svc_wait_giveup, + svc_wait_ovf); + if (svc.resc_run || svc_rescue_scan_capped) + pr_info("sema-svc: rescue_run=%llu scan_capped=%llu (multi-page content pins)\n", + svc.resc_run, svc_rescue_scan_capped); pr_info("sema-svc: gpfifo done=%llu nomap=%llu vidmem_writes=%llu ring_auto=%u pb=%u/%u pins=%u/%llu ipins=%u/%llu\n", svc.gpfifo_entries_done, svc.gpfifo_nomap, svc.vidmem_writes, svc_ring_n, svc_pb_n, svc_pb_passes, @@ -3407,6 +4799,20 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) svc_vas_n, SVC_VAS_SLOTS, svc.vas_writes, svc.vas_reads, svc.vas_shadowed); /* + * D2H-source authority visibility (r220): a full ledger degrades to + * home-slot replacement, which silently evicts another VA's landing + * and lets readers fall back to the resolver - the "reads back prior + * content" window. n approaching SVC_LEDGER_MAX or evict > 0 in a + * client's window is the tell. + */ + pr_info("sema-svc: ledger n=%u/%u stores=%llu evict=%llu resets=%llu diverges=%llu pinwins=%llu reloc_div=%llu rewrites=%llu pinrep=%llu alt_skip=%llu dual=%llu/%u\n", + svc_ledger_n, SVC_LEDGER_MAX, svc_ledger_stores, + svc_ledger_evictions, svc_ledger_evict_resets, + svc_ledger_diverges, svc_ledger_pinwins, + svc_ledger_reloc_diverges, svc_ledger_rewrites, + svc_ledger_pin_repairs, svc_pool_alt_skipped, + svc.pool_dual_applied, svc_pool_boot_conflicts); + /* * P6b wave 1: guest-forwarding summary. Printed whenever any counter * moved (not only on requests) so a run that captured no candidate is * still diagnosable from the log. @@ -3464,6 +4870,9 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) #define CE_M_OFFSET_OUT_UPPER 0x408 #define CE_M_OFFSET_OUT_LOWER 0x40c #define CE_M_LINE_LENGTH_IN 0x418 +#define CE_M_LINE_COUNT 0x41c +#define CE_M_PITCH_IN 0x410 +#define CE_M_PITCH_OUT 0x414 #define CE_M_SET_REMAP_CONST_A 0x700 #define CE_M_SET_REMAP_CONST_B 0x704 #define CE_M_SET_REMAP_COMPONENTS 0x708 @@ -3538,6 +4947,19 @@ struct svc_memset_op { u64 bytes; /* copy length */ u32 src_phys; u32 src_aperture; + /* + * r402: 2D (pitched) transfers. rows > 1 selects row-wise mode: + * the op runs `rows` times with the destination advancing by + * dpitch (and the source by spitch) per row, each row carrying + * `bytes` (copy) or elem*count (fill) bytes. cudaMemset2D and + * cudaMemcpy2D issue exactly this shape (LINE_COUNT rows, + * PITCH_OUT/PITCH_IN in bytes); without it the fill/copy ran flat + * and the partial-width rows corrupted the neighbouring columns + * (victory_memset2d bad>12M, memcpy_roundtrip pitched class). + */ + u32 rows; + u32 dpitch; + u32 spitch; }; /* Decoder state carried across the methods of one push. */ @@ -3549,9 +4971,19 @@ struct svc_push_state { u32 have_in, have_out; u32 line_length; u32 gr_line_count; /* GRCE LINE_COUNT */ + /* + * r402: 2D transfer geometry (CE class): PITCH_IN/PITCH_OUT in + * bytes and LINE_COUNT rows. A pitched fill/copy whose pitch + * exceeds the row length must run row-wise; a flat transfer leaves + * pitch == line_length (or no pitch method at all). + */ + u32 pitch_in, pitch_out; + u32 have_pitch_in, have_pitch_out; + u32 line_count; u32 sem_va_hi, sem_va_lo, sem_payload; u32 have_sem; u32 have_remap; + u32 have_remap_a, have_remap_b, have_remap_c; u32 release_dma; u32 sem_inc; u32 rs_upper, rs_lower, rs_payload; @@ -3682,7 +5114,18 @@ static int svc_op_translate(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, * withholding the release corrupted the next session (r59: the late * memset overwrote refilled data after the client had passed the * release), so the two must move together. + * + * r383 (streams2 big-copy class): name the missed VA through the + * same stalled-VA demand channel the GR fill uses. The userspace pin + * stream watches the "stalled VA" row and pushes the VA to the + * in-guest scanner, so the pin lands on the next scanner pass + * instead of waiting out a full scan cadence. The hold itself is + * unchanged; this only shortens the window in which the entry sits + * op-pending (r377: a held H2D copy executed ~1.5 s late and read a + * staging page the client had since refilled with the next copy's + * bytes - the neighbor-data class). */ + svc_log_stalled_va(va & ~(u64)(SEMA_PAGE_SIZE - 1)); svc.op_tr_pending = 1; return -ENOENT; } @@ -3707,19 +5150,281 @@ static bool svc_dst_watched(u64 dst) } /* + * COPYAUDIT shared budget (see the svc_copy comment for the contract): + * every copy at/above SVC_COPY_AUDIT_MIN is logged once with want/done and + * gets a first/last-16-bytes destination readback vs the written bytes + * (COPY-MISMATCH). One budget covers every copy path in this file so a + * whole cycle stays visible. + */ +#define SVC_COPY_AUDIT_MIN (32u << 10) /* >32 KB copies audited */ +#define SVC_COPY_AUDIT_MAX 2048u /* per-boot line budget */ +#define SVC_CEAUDIT_MAX 4096u /* per-boot CEAUDIT budget */ +static u64 svc_copy_audit_n; +static u64 svc_ceaudit_n; + +/* COPYAUDIT for a finished fill: log want/done and check the first 16 + * destination bytes against the repeated pattern. The readback uses the + * accessor matching the aperture the fill wrote through. */ +static void svc_fill_audit(struct nv_vgpu_device *vgpu, u64 va0, u64 addr, + u32 aperture, size_t total, const u8 *pat, u32 elem) +{ + u8 exp[16], got[16] = {0}; + size_t i; + bool ok = false; + + if (total < SVC_COPY_AUDIT_MIN || svc_copy_audit_n >= SVC_COPY_AUDIT_MAX) + return; + /* + * The fill writes the 8-byte pattern group repeatedly (pat is the + * element zero-padded to 8 bytes), so the expectation mirrors that: + * byte i of the destination equals pat[i % 8]. + */ + for (i = 0; i < sizeof(exp); i++) + exp[i] = pat[i % 8]; + if (aperture == CE_APERTURE_VID) { + u8 *base = vgpu->bars[1].vaddr; + + if (base && addr + sizeof(got) <= svc_fb_shadow_size(vgpu)) { + memcpy(got, base + addr, sizeof(got)); + ok = true; + } + } else if (!svc_read(vgpu, addr, got, sizeof(got))) { + ok = true; + } + svc_copy_audit_n++; + pr_info("vgsp: COPYAUDIT path=sema-fill va=0x%llx len=0x%llx done=0x%llx\n", + va0, (u64)total, (u64)total); + if (ok && memcmp(exp, got, sizeof(exp))) + pr_info_ratelimited("vgsp: COPY-MISMATCH va=0x%llx off=0x0 path=sema-fill want=%*phN got=%*phN\n", + va0, 16, exp, 16, got); +} + +/* * Write `count` elements of `value` (element size `elem`) starting at * `addr`, in vidmem or guest RAM depending on the aperture DST_TYPE * selected. Returns 0 on success. */ +static void svc_ledger_store(u64 va, u64 pa, u32 ap); /* R2 ledger, defined below */ +static int svc_ledger_lookup(u64 va, u64 *pa, u32 *ap); /* R2 ledger, defined below */ +static void svc_ledger_clear(void); /* STALLFREE: cap-and-degrade reset */ + +/* + * r384 (the guest boot-corruption lottery, root-caused 2026-10-20): + * page-table pages of the guest kernel's direct map were found ZEROED + * (two adjacent 4 KiB PTE pages at gpa 0x12d3c2000/0x3000 => two 2 MiB + * holes in the guest physmap, kernel objects inside them unreachable, + * multi-oops panic). The only emulation writers of page-sized ZERO runs + * are the virtual FILL executor (svc_fill / CE memsets), whose large + * >= SVC_VAS_FLOOR destinations resolve through the walk/scan when no + * pin and no ledger answer exists - and that resolver is demonstrably + * non-deterministic (four executions of the same 8 KiB engine-window + * clear resolved to four different guest pages, all refused by the + * below-floor guard). + * + * Two nets, mirroring the m169/r378 copy guards: + * 1. svc_fill_guard_hold(): a LARGE (>= 64 KiB) virtual fill whose only + * authority is the walk/scan (no pin, no ledger) is held through a + * short bounded window and its VA pushed to the pin-demand channel + * (the userspace pin stream injects the guest-pagemap truth), so the + * retried decode resolves through a real pin. Fail-open after the + * bound with the historical behaviour, like the copy guard. + * 2. svc_fill_target_is_pt(): a ZERO fill about to land in guest RAM on + * a page whose first 64 qwords are all PTE-shaped (present, NX, + * physical address in range) is REFUSED loudly. A page table page is + * never a legitimate zero-fill destination; this directly blocks the + * observed corruption shape even when some other resolver approved + * the target. Pin-backed resolutions are exempt (the guest's own + * pagemap is truth - if it says the VA maps the page, obey it). + */ +unsigned gsp_sema_fill_guard_ms = 800; +bool gsp_sema_fill_guard = true; +#define SVC_FILL_GUARD_SLOTS 16 +static struct { + u64 va_page; + unsigned long first; +} svc_fill_guard[SVC_FILL_GUARD_SLOTS]; +static u32 svc_fill_guard_next; +static u32 svc_fill_held; +static u32 svc_fill_pt_refused; +static u32 svc_fill_exec_log; /* r384 FILL-EXEC witness budget */ + +/* true = the guard may hold this fill; false = fail-open, run it. */ +static bool svc_fill_guard_hold(u64 va) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + unsigned long now = jiffies; + u32 i, slot = SVC_FILL_GUARD_SLOTS; + + for (i = 0; i < SVC_FILL_GUARD_SLOTS; i++) { + if (svc_fill_guard[i].va_page == page) { + if (time_before(now, svc_fill_guard[i].first + + msecs_to_jiffies( + gsp_sema_fill_guard_ms))) + return true; + if (svc_fill_guard[i].first) { + svc_fill_guard[i].first = 0; + pr_info("sema-svc: fill guard released without a pin va=0x%llx (using the walk translation)\n", + page); + } + return false; + } + if (!svc_fill_guard[i].va_page) + slot = i; + } + if (slot == SVC_FILL_GUARD_SLOTS) + slot = svc_fill_guard_next++ % SVC_FILL_GUARD_SLOTS; + svc_fill_guard[slot].va_page = page; + svc_fill_guard[slot].first = now; + return true; +} + +/* + * A guest-kernel page table page: the first SVC_PT_SCAN qwords are all + * present PTEs (bit 0), non-executable (bit 63 - kernel data mappings), + * with a physical address inside the 34-bit guest range. Random payload + * qwords pass all three conditions with probability far below the 75% + * threshold at SVC_PT_SCAN=64; a genuine table passes at 100%. + */ +#define SVC_PT_SCAN 64u +#define SVC_PT_MIN 48u +static bool svc_fill_target_is_pt(struct nv_vgpu_device *vgpu, u64 addr) +{ + u64 ents[SVC_PT_SCAN]; + u32 i, hits = 0; + + if (addr & (SEMA_PAGE_SIZE - 1)) + return false; /* only a page-aligned table shape */ + if (svc_read(vgpu, addr, ents, sizeof(ents))) + return false; + for (i = 0; i < SVC_PT_SCAN; i++) { + u64 e = ents[i]; + + if (!(e & 1ull)) + continue; + if (!(e & (1ull << 63))) + continue; + if ((e & 0x000ffffffffff000ull) == 0) + continue; + if ((e & 0x000ffffffffff000ull) >= (1ull << 34)) + continue; + hits++; + } + return hits >= SVC_PT_MIN; +} + +/* + * r391 per-page destination resolution for a VIRTUAL fill (the r390 + * guest-panic class). A device VA's pages are independent guest allocations + * - the pin stream shows split runs (va=0x7c36c7400000 -> 29 pages at + * 0x131b63000, the next run at 0x131a80000) - so resolving only the fill's + * first page and writing `total` bytes contiguously past it writes over + * whatever memory follows the first run. On r390 a 383620-byte zero fill + * did exactly that: the contiguous write covered the guest kernel's + * page-table pages (0x131ba1000/0x131ba2000/0x131bab000), the direct map + * grew holes, and the guest died with "Fatal exception in interrupt" + * (unmap_page_range / cpuacct / psi faults). svc_copy has always translated + * virtual operands per 4 KB page; the fill must do the same. + * + * Authority order mirrors the top-level resolution (and svc_copy's per-page + * dst): a fresh pin (guest pagemap truth) wins and repairs the landing + * table, then the landing table (the page a previous write of this VA + * landed on), then the resolver chain. A page with no resolution returns + * -ENOENT: the caller fails the fill loudly instead of writing through a + * contiguous guess. + */ +static int svc_fill_dst_page_pa(struct nv_vgpu_device *vgpu, u64 va, + u64 *pa_out, u32 *ap_out, u32 *prov_out) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 pa = 0, lpa = 0; + u32 aper = 0, lap = 0; + + if (!svc_pin_lookup_any(page, &pa, &aper)) { + *prov_out = 1; + if (!svc_ledger_lookup(page, &lpa, &lap) && + (((lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != page) || + (u32)lap != (u32)aper)) { + svc_ledger_pin_repairs++; + if (svc_ledger_pin_repairs <= 32) + pr_info("sema-ledger: PIN-WINS fill-page va=0x%llx ledger=0x%llx/%u pin=0x%llx/%u\n", + page, + lpa & ~(u64)(SEMA_PAGE_SIZE - 1), + lap, pa, aper); + svc_ledger_store(page, pa, aper); + } + *pa_out = pa; + *ap_out = aper; + return 0; + } + if (!svc_ledger_lookup(page, &pa, &aper)) { + *prov_out = 2; + *pa_out = pa; + *ap_out = aper; + return 0; + } + if (!svc_op_translate(vgpu, page, &pa, &aper)) { + *prov_out = 3; + *pa_out = pa; + *ap_out = aper; + return 0; + } + return -ENOENT; +} + +static int svc_fill_1d(struct nv_vgpu_device *vgpu, u64 addr, u64 value, + u32 elem, u32 count, u32 phys, u32 aperture, + u64 *done_out); +static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, + size_t total, u32 dst_phys, u32 dst_aperture, + u32 src_phys, u32 src_aperture, u64 *done_out, + u32 owner_ch, u32 owner_idx, u64 owner_hash); + +/* + * r402 row-wise wrapper: a 2D (pitched) fill runs the 1D fill once per + * row with the destination advancing by dpitch. Each row keeps the whole + * per-page resolution, ledger-store and guard machinery of the 1D path + * (no new resolution semantics), and a failed row's retry simply re-runs + * the already-written rows with the same pattern - idempotent for fills. + */ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, - u32 count, u32 phys, u32 aperture) + u32 count, u32 phys, u32 aperture, u64 *done_out, + u32 rows, u32 dpitch) +{ + if (rows > 1 && dpitch) { + u64 done = 0; + int rc = 0; + u32 r; + + for (r = 0; r < rows; r++) { + u64 row_done = 0; + + rc = svc_fill_1d(vgpu, addr + (u64)r * dpitch, + value, elem, count, phys, aperture, + &row_done); + done += row_done; + if (rc) + break; + } + if (done_out) + *done_out = done; + return rc; + } + return svc_fill_1d(vgpu, addr, value, elem, count, phys, aperture, + done_out); +} + +static int svc_fill_1d(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, + u32 count, u32 phys, u32 aperture, u64 *done_out) { u8 pat[8]; u8 bounce[256]; size_t total = (size_t)elem * count; size_t o; bool shadow = false; + u64 va0 = addr; /* original destination VA (fills, C2) */ + if (done_out) + *done_out = 0; if (elem != 1 && elem != 2 && elem != 4 && elem != 8) return -EINVAL; if (!count || total > (256ULL << 20)) @@ -3730,21 +5435,151 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, if (!phys) { /* Virtual destination: the VA shadow first (its pages stay - * authoritative), then the pinned map and the VA->PA map. */ + * authoritative), then a live pin (the guest's newest + * statement - r372: a re-pinned VA must not be written + * through a stale landing), then the unified landing table + * (the page a previous write of this VA landed on - always + * the authoritative location of the content), then the + * pinned map and the VA->PA map. */ u64 pa; u32 aper; + u32 prov = 0; /* r384: 1 pin, 2 ledger, 3 walk/scan */ if (svc_vas_has(addr)) { shadow = true; + } else if (!svc_pin_lookup_any(addr, &pa, &aper)) { + u64 lpa = 0; + u32 lap = 0; + + prov = 1; + /* + * r372 pin-wins: repair the landing table when it + * still names the pre-re-pin page, so every later + * reader follows the pin too. + */ + if (!svc_ledger_lookup(addr, &lpa, &lap) && + (((lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1))) || + (u32)lap != (u32)aper)) { + svc_ledger_pin_repairs++; + if (svc_ledger_pin_repairs <= 32) + pr_info("sema-ledger: PIN-WINS fill-dst va=0x%llx ledger=0x%llx/%u pin=0x%llx/%u\n", + addr, + lpa & ~(u64)(SEMA_PAGE_SIZE - 1), + lap, pa, aper); + svc_ledger_store(addr, + pa & ~(u64)(SEMA_PAGE_SIZE - 1), + aper); + } + addr = pa; + aperture = (aper == SVC_APERTURE_FB) ? + CE_APERTURE_VID : CE_APERTURE_SYS; + } else if (!svc_ledger_lookup(addr, &pa, &aper)) { + prov = 2; + addr = pa; + aperture = (aper == SVC_APERTURE_FB) ? + CE_APERTURE_VID : CE_APERTURE_SYS; } else if (svc_op_translate(vgpu, addr, &pa, &aper)) { if (!svc_vas_can_shadow(addr)) return -ENOENT; shadow = true; svc.vas_shadowed++; } else { + prov = 3; addr = pa; aperture = aper; } + /* + * r365b LOWFILL-WIT: a below-floor virtual fill (the guest's + * vidmem-window page-table writes) is a corruption risk: its + * resolution must be proven before the op is allowed to run + * (r365: 16-byte and 32 KB zero fills landed in guest RAM and + * killed the guest kernel). Log every resolution bounded. + */ + if (va0 < SVC_VAS_FLOOR && svc_lowfill_log < 64) { + svc_lowfill_log++; + pr_info("sema-lowfill: va=0x%llx shadow=%d resolved=0x%llx ap=%u elem=%u count=%u value=0x%llx\n", + va0, shadow, addr, aperture, elem, count, + value); + } + /* + * r369: a below-floor fill may ONLY land in vidmem. The + * 0x204020000 engine-window clear resolved to a DIFFERENT + * guest-RAM page on every execution (9 runs, 9 pages) and + * the guest died with its task_struct corrupted (gmain NULL + * deref in __fdget) - each wrong resolution zeroed 8 KB of + * guest kernel memory. Vidmem resolutions (the page-table + * region, ap=VID) are the emulation's own shadow and stay + * allowed; a SYS resolution for a window VA is refused and + * logged loudly. + */ + if (va0 < SVC_VAS_FLOOR && !shadow && + aperture != CE_APERTURE_VID) { + pr_err_ratelimited("vgsp-sema: LOWFILL-REFUSED va=0x%llx pa=0x%llx ap=%u elem=%u count=%u (below-floor fill resolved outside vidmem)\n", + va0, addr, aperture, elem, count); + return -ENOENT; + } + /* + * r384 net 1 (pin-demand hold for unproven large fills): a + * >= 64 KiB virtual fill whose only authority is the + * walk/scan resolver (no pin, no landing-table entry) AND + * which would land in GUEST RAM (SYS aperture) is the class + * that zeroed the guest's page-table pages (see the guard + * commentary above svc_fill_guard_hold). Demand the VA + * through the pin channel and hold the op through a short + * bounded window so the retried decode resolves through the + * guest's own pagemap; fail open with the historical + * behaviour after the bound. Vidmem-shadow resolutions are + * the emulation's own memory and stay immediate. + */ + if (gsp_sema_fill_guard && va0 >= SVC_VAS_FLOOR && !shadow && + total >= (64u << 10) && prov >= 3 && + aperture == CE_APERTURE_SYS) { + svc_log_stalled_va(va0); + if (svc_fill_guard_hold(va0)) { + svc_fill_held++; + if (svc_fill_held <= 32) + pr_info("sema-svc: FILL-HOLD va=0x%llx resolved=0x%llx ap=%u total=%zu value=0x%llx (unproven large fill; pin demanded)\n", + va0, addr, aperture, total, + value); + return -ENOENT; + } + } + /* + * r384 net 2 (page-table-shape refusal): a ZERO fill about + * to land in guest RAM on a page whose first 64 qwords are + * all present+NX+in-range PTEs is refused loudly - a page + * table page is never a legitimate zero-fill destination. + * This blocks the observed corruption shape (two adjacent + * PTE pages zeroed => two 2 MiB holes in the guest physmap) + * even when some other resolver approved the target, and + * irrespective of how wrong the resolution is. Pin-backed + * resolutions are exempt: the guest's own pagemap is truth + * there. + */ + if (gsp_sema_fill_guard && va0 >= SVC_VAS_FLOOR && !shadow && + value == 0 && total >= SEMA_PAGE_SIZE && + aperture == CE_APERTURE_SYS && prov != 1 && + svc_fill_target_is_pt(vgpu, addr)) { + svc_fill_pt_refused++; + if (svc_fill_pt_refused <= 32) + pr_err("vgsp-sema: FILL-PT-REFUSED va=0x%llx pa=0x%llx prov=%u total=%zu (zero fill target is a page table)\n", + va0, addr, prov, total); + svc_log_stalled_va(va0); + return -ENOENT; + } + /* + * r384 witness: where every >= 4 KiB virtual fill actually + * lands (va, pa, aperture, resolution provenance, pattern) - + * the decisive line for the boot-corruption forensics (the + * large zero fills were invisible until now). Bounded. + */ + if (va0 >= SVC_VAS_FLOOR && !shadow && + total >= SEMA_PAGE_SIZE && svc_fill_exec_log < 64) { + svc_fill_exec_log++; + pr_info("sema-svc: FILL-EXEC va=0x%llx pa=0x%llx ap=%u prov=%u total=%zu value=0x%llx\n", + va0, addr, aperture, prov, total, value); + } } if (gsp_sema_dbg && !shadow && svc_dst_watched(addr) && @@ -3759,10 +5594,40 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, svc.op_tr_src, elem, pb, svc.cur_ch); } - if (shadow) - return svc_vas_fill(addr, pat, elem, total); + if (shadow) { + int rc = svc_vas_fill(addr, pat, elem, total); - if (aperture == CE_APERTURE_VID) { + if (!rc && done_out) + *done_out = (u64)total; + return rc; + } + + /* + * C2 (data class) + m180 unification: a large virtual fill is a + * write/read pair with the memset's readers like any H2D copy - + * record its landing pages so the engine fill and the D2H read of + * this VA use exactly the page the fill wrote. Only the VID + * (shadow) case is contiguous; a SYS landing is recorded per page + * in the loop below (r391: the contiguous `addr + po` store also + * poisoned the table with pages the fill never wrote). + */ + if (!phys && !shadow && va0 >= SVC_VAS_FLOOR && + total >= (64u << 10) && aperture == CE_APERTURE_VID) { + u64 po; + + for (po = 0; po < total; po += SEMA_PAGE_SIZE) + svc_ledger_store(va0 + po, addr + po, + SVC_APERTURE_FB); + } + + /* + * r391: a virtual device destination is translated per 4 KB page + * (see svc_fill_dst_page_pa) - never written through the first + * page's PA contiguously. The whole-range shadow write below only + * applies to a resolution that is contiguous by construction. + */ + if (aperture == CE_APERTURE_VID && + !(va0 >= SVC_VAS_FLOOR && !phys && !shadow)) { u8 *base = vgpu->bars[1].vaddr; size_t size = svc_fb_shadow_size(vgpu); @@ -3772,6 +5637,9 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, memcpy(base + addr + o, pat, 8); if (o < total) memcpy(base + addr + o, pat, total - o); + if (done_out) + *done_out = (u64)total; + svc_fill_audit(vgpu, va0, addr, aperture, total, pat, elem); return 0; } @@ -3779,17 +5647,107 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, for (o = 0; o < total; ) { size_t chunk = total - o; size_t i; + u64 wpa = addr; + u32 wap = aperture; + bool per_page = !phys && !shadow && va0 >= SVC_VAS_FLOOR; if (chunk > sizeof(bounce)) chunk = sizeof(bounce); + /* + * r391: resolve THIS page's own landing (see + * svc_fill_dst_page_pa) and clamp the chunk to the page + * boundary - a virtual VA's pages are independent guest + * allocations and must never be written through the first + * page's PA contiguously. + */ + if (per_page) { + u32 off = (u32)((va0 + o) & (SEMA_PAGE_SIZE - 1)); + u64 ppa = 0; + u32 pap = 0, prov = 0; + int prc; + + chunk = min_t(size_t, chunk, SEMA_PAGE_SIZE - off); + prc = svc_fill_dst_page_pa(vgpu, va0 + o, &ppa, + &pap, &prov); + if (prc) { + if (done_out) + *done_out = (u64)o; + /* + * r391: demand the exact page through the pin + * channel (the userspace stream pushes this VA + * to the in-guest scanner) and hold the fill + * for a bounded retry - never write through a + * contiguous guess past a missing page. + */ + svc_log_stalled_va((va0 + o) & + ~(u64)(SEMA_PAGE_SIZE - 1)); + pr_err_ratelimited("vgsp: FILL-TRUNC va=0x%llx want=0x%llx done=0x%llx path=sema-fill (page 0x%llx unresolved)\n", + va0, (u64)total, (u64)o, + (va0 + o) & + ~(u64)(SEMA_PAGE_SIZE - 1)); + return -ENOENT; + } + wap = (pap == SVC_APERTURE_FB) ? + CE_APERTURE_VID : CE_APERTURE_SYS; + wpa = (ppa & ~(u64)(SEMA_PAGE_SIZE - 1)) + off; + /* + * r391 net: the r384 page-table-shape refusal + * applied to EVERY page of a virtual zero fill. + * The r390 fill's first page was the client's + * buffer, the later pages were the guest's PTE + * pages - the first-page-only check could not see + * them. A page-table-shaped guest-RAM page is never + * a legitimate zero-fill target: skip it loudly + * instead of zeroing it. Pin-backed resolutions + * are exempt (the guest's pagemap is truth there). + */ + if (wap == CE_APERTURE_SYS && value == 0 && prov != 1 && + svc_fill_target_is_pt(vgpu, + wpa & ~(u64)(SEMA_PAGE_SIZE - 1))) { + svc_fill_pt_refused++; + if (svc_fill_pt_refused <= 32) + pr_err("vgsp-sema: FILL-PT-REFUSED va=0x%llx pa=0x%llx prov=%u total=%zu (zero fill page is a page table; page skipped)\n", + va0 + o, wpa, prov, total); + o += chunk; + continue; + } + } for (i = 0; i + 8 <= chunk; i += 8) memcpy(bounce + i, pat, 8); if (i < chunk) memcpy(bounce + i, pat, chunk - i); - if (svc_write(vgpu, addr + o, bounce, chunk)) + if (wap == CE_APERTURE_VID) { + u8 *base = vgpu->bars[1].vaddr; + size_t size = svc_fb_shadow_size(vgpu); + + if (!base || wpa + chunk > size) { + if (done_out) + *done_out = (u64)o; + return -EINVAL; + } + memcpy(base + wpa, bounce, chunk); + } else if (svc_write(vgpu, wpa, bounce, chunk)) { + if (done_out) + *done_out = (u64)o; + if (total >= SVC_COPY_AUDIT_MIN) + pr_err_ratelimited("vgsp: COPY-TRUNC va=0x%llx want=0x%llx done=0x%llx path=sema-fill rc=%d\n", + va0, (u64)total, (u64)o, -EIO); return -EIO; + } + /* + * r391: record this page's landing (the readers follow the + * table) - SYS and VID alike, exactly like the copy path. + */ + if (per_page) + svc_ledger_store(va0 + o, + wpa & ~(u64)(SEMA_PAGE_SIZE - 1), + (wap == CE_APERTURE_VID) ? + SVC_APERTURE_FB : SVC_APERTURE_SYS); o += chunk; } + if (done_out) + *done_out = (u64)total; + svc_fill_audit(vgpu, va0, addr, aperture, total, pat, elem); return 0; } @@ -3844,13 +5802,23 @@ static struct svc_fwd_cand svc_fwd_cand; #define SVC_GR_POINTER_MAX 8 #define SVC_GR_PAGES_MAX 16 /* + * relfix PART 3 (fill candidate superset): maximum distinct candidate pages + * per collection pass. Raised from 16 because the full-page scans added + * below (aligned with the rewrite's own whole-page scan) can legitimately + * carry more pointer-shaped words than the argument region alone; the old + * cap silently dropped the extra words, which then resurfaced as RELOC-GAP + * at rewrite time. It remains a hard bound with the historical silent-drop + * semantics beyond it (n_cand is logged in the fill summary). + */ +#define SVC_GR_CAND_MAX 24u +/* * m99: guest data identity-map capacity. The forwarded kernel's global data * pages (discovered by scanning the param/const-0 page for device VAs) are - * resolved on the service thread and identity-mapped into the engine tree. - * 16K pages = 64 MB of window coverage, comfortably above the demo buffers - * (2 x 16 MB observed) with headroom. + * resolved on the service thread and relocated into the engine-visible + * region. B-2: 32K records = 2x the grown region's 16384 pages of headroom, + * so the hand-off array is never the binding limit. */ -#define SVC_GR_DATA_MAX 16384 +#define SVC_GR_DATA_MAX 32768 #define SVC_GR_PUSH_WORDS 256 #define SVC_GR_QMD_WORD_NONE 0xffffffffu /* svc_gr_cand.rel_flags bits (decoded from qmd[24] and qmd[4]). */ @@ -3894,9 +5862,18 @@ struct svc_gr_data_win { u64 reloc_base; /* relocated window base */ }; -struct svc_gr_cand { +struct svc_gr_cand_rec { bool valid; bool offered; /* consumed by a request or a fallback */ + /* + * STALLFREE: exactly-once latch for the candidate's QMD completion + * release when the launch is consumed without ever being forwarded + * (a data-stall/busy/nocap hold fell through). Zeroed with the rest + * of the candidate at capture. + */ + bool rel_done; + unsigned long captured_jiffies; /* STALLFREE: capture stamp for the + * GR-STALL waited_ms accounting */ u32 h_object; u64 push_va; u32 push_len; @@ -3921,7 +5898,105 @@ struct svc_gr_cand { u32 pointer_word[SVC_GR_POINTER_MAX]; /* QMD word of the pair */ }; -static struct svc_gr_cand svc_gr_cand; +/* + * fail4: the capture was a single slot; two concurrent launches (streams2's + * vadd pair on two channels) overwrote each other's candidate while the + * first was kept for a retry (the candidate-overwrite witness) and the + * engines ran crossed parameters (c1 == a2+b2). Keep up to four candidates + * side by side in a ring; the accessor macro addresses the current slot, + * so every existing use keeps working on the selected candidate, and + * capture/consume select the slot instead of clobbering it. + */ +#define SVC_GR_CAND_RING 4 +static struct svc_gr_cand_rec svc_gr_cand_ring[SVC_GR_CAND_RING]; +static u32 svc_gr_cand_pos; +#define svc_gr_cand (svc_gr_cand_ring[svc_gr_cand_pos]) +/* + * r383: push-VA history of recent captures, beyond the (four-slot) + * candidate ring's lifetime. A replayed decode of a complete QMD asks + * "has this launch ever been captured?"; the ring alone can have rotated + * past it, and a wrong "no" would re-capture (and re-execute) an already + * forwarded launch. Bounded, service thread only. + */ +#define SVC_GR_CAP_HIST 8 +static u64 svc_gr_cap_hist[SVC_GR_CAP_HIST]; +static u32 svc_gr_cap_hist_pos; + +/* + * r401: robust session-scoped "already captured" memory for the r383 + * replay-capture path. The 4-slot candidate ring + 8-VA history evicted a + * real capture within ~12 later captures, so a true replay of an + * already-forwarded launch (identical push VA and content) was + * re-captured and re-executed: the r399 soak log shows the same nine + * launch pushes re-submitted every ~10-11 s for minutes (idempotent + * kernels survive it; accumulators would not). The open-addressed table + * keeps the (push_va, push_hash) identity of every capture of the + * session, so a true replay always classifies as "seen" (LAUNCH-SKIP, + * exactly once) while a genuinely new push - even at the same VA with + * the same entry words - is captured through the normal + * `trusted && !replay` path and never consults this table. Cleared by + * svc_session_flush (a new client's identical-payload pushes restart + * their release counters, so a boot-scoped table would wrongly suppress + * them). Service-thread only. + */ +#define SVC_GR_SEEN_SLOTS 2048 +#define SVC_GR_SEEN_PROBE 8 +static struct { + u64 push_va; + u64 hash; +} svc_gr_seen[SVC_GR_SEEN_SLOTS]; +static u64 svc_gr_seen_stale; /* probe-chain evictions (telemetry) */ + +static u32 svc_gr_seen_mix(u64 push_va, u64 hash) +{ + u64 k = push_va * 0x9E3779B97F4A7C15ULL; + + k ^= hash + 0x9E3779B97F4A7C15ULL + (k << 6) + (k >> 2); + return (u32)(k % SVC_GR_SEEN_SLOTS); +} + +static bool svc_gr_seen_lookup(u64 push_va, u64 hash) +{ + u32 i, base = svc_gr_seen_mix(push_va, hash); + + if (!push_va) + return false; + for (i = 0; i < SVC_GR_SEEN_PROBE; i++) { + u32 k = (base + i) % SVC_GR_SEEN_SLOTS; + + if (!svc_gr_seen[k].push_va) + return false; + if (svc_gr_seen[k].push_va == push_va && + svc_gr_seen[k].hash == hash) + return true; + } + return false; +} + +static void svc_gr_seen_store(u64 push_va, u64 hash) +{ + u32 i, base = svc_gr_seen_mix(push_va, hash); + + if (!push_va) + return; + for (i = 0; i < SVC_GR_SEEN_PROBE; i++) { + u32 k = (base + i) % SVC_GR_SEEN_SLOTS; + + if (!svc_gr_seen[k].push_va || + (svc_gr_seen[k].push_va == push_va && + svc_gr_seen[k].hash == hash)) { + svc_gr_seen[k].push_va = push_va; + svc_gr_seen[k].hash = hash; + return; + } + } + /* Full probe chain: replace the tail (oldest-ish) entry. */ + svc_gr_seen_stale++; + svc_gr_seen[(base + SVC_GR_SEEN_PROBE - 1) % SVC_GR_SEEN_SLOTS].push_va = + push_va; + svc_gr_seen[(base + SVC_GR_SEEN_PROBE - 1) % SVC_GR_SEEN_SLOTS].hash = + hash; +} static_assert(SVC_GR_QMD_WORDS == GSP_PHYS_GR_QMD_WORDS, "the sema GR QMD word count must match the backend ABI"); @@ -3957,7 +6032,7 @@ static_assert(sizeof(struct svc_gr_page) == 24, static u64 svc_push_hash(const u8 *p, u32 len); /* QMD field decode: the exact shape the host CUDA bridge uses. */ -static void svc_gr_qmd_decode(struct svc_gr_cand *c) +static void svc_gr_qmd_decode(struct svc_gr_cand_rec *c) { const u32 *q = c->qmd; u32 i; @@ -3996,13 +6071,50 @@ static void svc_gr_cand_capture(struct svc_channel *ch, { u32 i, n; + /* + * C9 (GAP-2 A4): a capture over a still-valid candidate (never + * offered) destroys an unforwarded launch - the single-slot + * overwrite hazard. Bounded, unconditional (gsp_sema_dbg is defined + * later in the file and this site may not reference it). + */ + /* + * fail4: never overwrite a DIFFERENT launch's unconsumed candidate - + * move to a free ring slot instead (same-push re-captures still + * refresh in place). + */ + if (svc_gr_cand.valid && !svc_gr_cand.offered && + svc_gr_cand.push_va != push_va) { + u32 ri; + + for (ri = 1; ri < SVC_GR_CAND_RING; ri++) { + u32 rp = (svc_gr_cand_pos + ri) % SVC_GR_CAND_RING; + + if (!svc_gr_cand_ring[rp].valid || + svc_gr_cand_ring[rp].offered) { + svc_gr_cand_pos = rp; + break; + } + } + } + if (svc_gr_cand.valid && svc_cand_ovw_log < 64) { + svc_cand_ovw_log++; + pr_info("sema-gr: CAND-OVERWRITE old_h=0x%x old_push=0x%llx new_push=0x%llx\n", + svc_gr_cand.h_object, svc_gr_cand.push_va, push_va); + } memset(&svc_gr_cand, 0, sizeof(svc_gr_cand)); svc_gr_cand.valid = true; + svc_gr_cand.captured_jiffies = jiffies; svc_gr_cand.h_object = ch->h_object; svc_gr_cand.push_va = push_va; svc_gr_cand.push_len = push_len; + /* r383: remember the identity beyond this ring slot's lifetime. */ + svc_gr_cap_hist[svc_gr_cap_hist_pos % SVC_GR_CAP_HIST] = push_va; + svc_gr_cap_hist_pos++; if (push && push_len) svc_gr_cand.push_hash = svc_push_hash(push, push_len); + /* r401: the session-wide exactly-once memory (survives ring rotation; + * see svc_gr_seen_lookup). */ + svc_gr_seen_store(push_va, svc_gr_cand.push_hash); if (push && push_len >= 4) { u32 total = push_len / 4; u32 n = total < SVC_GR_PUSH_WORDS ? total : SVC_GR_PUSH_WORDS; @@ -4095,8 +6207,14 @@ static bool svc_fwd_defer_arm(void) { if (!gsp_phys_guest_mem_enabled()) return false; - if (svc_gr_cand.valid && gsp_phys_gr_forward_enabled()) - return true; + if (gsp_phys_gr_forward_enabled()) { + u32 ri; + + for (ri = 0; ri < SVC_GR_CAND_RING; ri++) + if (svc_gr_cand_ring[ri].valid && + !svc_gr_cand_ring[ri].offered) + return true; + } return svc_fwd_cand.valid && gsp_phys_work_defer_release_enabled(); } @@ -4155,8 +6273,434 @@ static bool svc_fwd_defer_armed; #define SVC_COPY_FORWARDED 1 /* Byte copy between two CE addresses (src/dst decoding mirrors svc_fill). */ +/* + * m171 (AGENT-F R2): write/read consistency ledger. A large H2D write to a + * device VA records {va_page -> pa, aperture} for every page it wrote; the + * later read of the same VA (the launch data fill, or a D2H copy) prefers + * the recorded page whenever its own resolution disagrees. The pipeline + * then reads exactly what was written regardless of which of the two + * resolutions is stale - the r174/r175 first-page divergence (the write + * landed on pa X, the fill read a stale page Y, and the engine computed on + * old bytes) can no longer split a write/read pair. Session-scoped: cleared + * with the flush; zero cost while empty. Apertures are stored in the SVC_ + * encoding, the canonical space for the svc accessors. + */ +struct svc_ledger_ent { + u64 va_page; + u64 pa_page; + u32 ap; +}; +/* + * Unified VA->landing-page authority ("R2 ledger"). + * + * m172 grew the original 256-entry ring to 2048, but a ring still evicts + * unrelated VAs wholesale: one 4 MiB H2D plus one launch's data-page + * record set is 4-6 K inserts, so an 8 MiB machine operand's early pages + * were evicted before the launch fill looked them up, and the fill fell + * back to its own (possibly stale) resolution - the exact "write path and + * read path pick different pages" class. + * + * m180: keyed hash table instead of a ring. One entry per VA page, O(1) + * update/lookup, no cross-VA eviction, and the insert of a page already + * present rewrites it in place (a write is a new landing and is always + * the truth). 16384 slots cover a full multi-buffer client's working set + * (8 MB in + 8 MB out + engine records ~ 12 K pages); hash collisions + * only occur between VA pages 64 MiB apart or across clients whose pages + * do not co-reside in time. + * + * COPYAUDIT r220: SVC_LEDGER_MAX/MASK/PROBE and the counters live above + * svc_rebuild (hoisted for the diag line); only the table and the home + * helper stay here. + */ +static struct svc_ledger_ent svc_ledger[SVC_LEDGER_MAX]; + +static inline u32 svc_ledger_home(u64 page) +{ + return (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & + SVC_LEDGER_MASK; +} + +static void svc_ledger_store(u64 va, u64 pa, u32 ap) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 home = svc_ledger_home(page); + u32 p; + + for (p = 0; p < SVC_LEDGER_PROBE; p++) { + struct svc_ledger_ent *e = &svc_ledger[(home + p) & + SVC_LEDGER_MASK]; + + if (e->va_page == page) { + /* + * A write path moving an existing page: its landing + * is the new truth. Logged (bounded) because a move + * is the write-side half of the DIVERGE class. + */ + if (e->pa_page != pa_page) { + svc_ledger_rewrites++; + if (svc_ledger_rewrites <= 16) + pr_info("sema-ledger: rewrite va=0x%llx old=0x%llx new=0x%llx ap=%u\n", + page, e->pa_page, pa_page, + ap); + } + e->pa_page = pa_page; + e->ap = ap; + return; + } + if (!e->va_page) { + e->va_page = page; + e->pa_page = pa_page; + e->ap = ap; + svc_ledger_n++; + svc_ledger_stores++; + if (svc_ledger_store_log < 64) { + svc_ledger_store_log++; + pr_info("sema-ledger: store va=0x%llx pa=0x%llx ap=%u n=%u\n", + page, pa_page, ap, svc_ledger_n); + } + return; + } + } + /* + * Cluster full (16 same-home VA pages co-resident): replace the home + * slot. Bounded, rare, and only affects VAs whose pages hash to the + * identical home slot. r220: this is a SILENT loss of the evicted + * VA's landing authority (its later readers fall back to the + * resolver, possibly stale), so the first occurrences are logged and + * counted into the diag line. + * + * STALLFREE cap-and-degrade: a sustained eviction burst means the + * table is being thrashed (a pin storm / leftover crashed client + * churning VA pages faster than the working set), and a half-evicted + * table silently strands authority for exactly the evicted pages. + * Past SVC_LEDGER_EVICT_RESET evictions in one session the cache is + * cleared wholesale and rebuilt from live writes: every reader + * uniformly falls back to the resolver until its page is rewritten, + * which removes the stranded-authority class at a known, logged + * cost. The clear is a cache operation only (no control flow, no + * wait); the current store then lands in the empty table. + */ + svc_ledger_evictions++; + svc_ledger_evict_since_reset++; + if (svc_ledger_evict_since_reset >= SVC_LEDGER_EVICT_RESET) { + svc_ledger_evict_since_reset = 0; + svc_ledger_evict_resets++; + pr_warn("sema-ledger: LEDGER-RESET evict=%llu resets=%llu (cluster thrash under churn; cache cleared, readers fall back to the resolver until rewritten)\n", + svc_ledger_evictions, svc_ledger_evict_resets); + svc_ledger_clear(); + } + if (svc_ledger_evictions <= 32) + pr_warn("sema-ledger: EVICT home=%u old_va=0x%llx new_va=0x%llx (cluster full; old landing authority lost until rewritten)\n", + home, (u64)svc_ledger[home].va_page, page); + svc_ledger[home].va_page = page; + svc_ledger[home].pa_page = pa_page; + svc_ledger[home].ap = ap; + svc_ledger_stores++; +} + +static int svc_ledger_lookup(u64 va, u64 *pa, u32 *ap) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 home = svc_ledger_home(page); + u32 p; + + for (p = 0; p < SVC_LEDGER_PROBE; p++) { + const struct svc_ledger_ent *e = + &svc_ledger[(home + p) & SVC_LEDGER_MASK]; + + if (!e->va_page) + return -ENOENT; + if (e->va_page == page) { + *pa = e->pa_page | (va & (SEMA_PAGE_SIZE - 1)); + *ap = e->ap; + return 0; + } + } + return -ENOENT; +} + +static void svc_ledger_clear(void) +{ + memset(svc_ledger, 0, sizeof(svc_ledger)); + svc_ledger_n = 0; + svc_ledger_rewrites = 0; + svc_ledger_reloc_diverges = 0; + svc_ledger_evict_since_reset = 0; +} + +/* + * COPYAUDIT (2026-10-18 copy-completeness audit): every copy at/above + * SVC_COPY_AUDIT_MIN is logged once with its want/done byte counts, and + * its first/last 16 destination bytes are read back immediately and + * compared against the bytes just written (COPY-MISMATCH on difference). + * COPY-TRUNC marks a copy that failed after partially writing (want>done), + * so a partial transfer can never be silent. Pure diagnostics: no control + * flow, locking or timing change; the readback uses the same accessor the + * write just used. Definitions live with svc_fill (first user). + */ +static void svc_copy_audit_chunk(struct nv_vgpu_device *vgpu, u64 dst_va, + u64 o, const u8 *bounce, size_t chunk, + bool shadow, u64 pa_eff, u32 aper, bool tail) +{ + size_t n = chunk < 16 ? chunk : 16; + size_t so = tail ? chunk - n : 0; /* true end sample */ + u8 rb[16] = {0}; + + if (shadow) { + if (svc_vas_read(dst_va + o + so, rb, n)) + return; + } else if (aper == CE_APERTURE_VID) { + u8 *base = vgpu->bars[1].vaddr; + + if (!base || pa_eff + so + n > svc_fb_shadow_size(vgpu)) + return; + memcpy(rb, base + pa_eff + so, n); + } else if (svc_read(vgpu, pa_eff + so, rb, n)) { + return; + } + if (memcmp(rb, bounce + so, n)) { + pr_info_ratelimited("vgsp: COPY-MISMATCH va=0x%llx off=0x%llx path=ce-soft-copy want=%*phN got=%*phN\n", + dst_va + o + so, o + so, (int)n, + bounce + so, (int)n, rb); + pr_info_ratelimited("vgsp: CEAUDIT-MISMATCH op=copy va=0x%llx off=0x%llx path=ce-soft-copy want=%*phN got=%*phN\n", + dst_va + o + so, o + so, (int)n, + bounce + so, (int)n, rb); + } +} + +#define SVC_COPY_TRUNC_RET(rc_) \ + do { \ + if (done_out) \ + *done_out = (u64)o; \ + if (total >= SVC_COPY_AUDIT_MIN) \ + pr_err_ratelimited("vgsp: COPY-TRUNC va=0x%llx want=0x%llx done=0x%llx path=ce-soft-copy rc=%d\n", \ + dst_orig, (u64)total, (u64)o, (rc_)); \ + return (rc_); \ + } while (0) + +/* + * fail4: pipeline-order source snapshot for held copies. The m169 + * device-dst guard defers a staged H2D (host->device) for up to 3 s; + * the driver's ping-pong staging buffers are overwritten by the next + * chunk meanwhile, so the deferred read returns the content of a later + * chunk (vector_add: device a[i] pages held a[i+2 MB] - exactly the + * 2x1 MB staging stride). A held copy snapshots its source at hold + * time and the retried execution copies from the snapshot, matching + * the hardware's in-order read of the staging. + */ +#define SVC_COPY_SNAP_N 16 +#define SVC_COPY_SNAP_MAX (4u << 20) +#define SVC_COPY_SNAP_TTL_MS 30000u +struct svc_copy_snap { + u64 dst, src; + size_t total; + /* + * Owner identity (r444): the channel, ring entry and push-content + * hash of the decode pass that captured this snapshot. A snapshot + * exists to serve ONE entry's retried copy (the held staging read); + * keying it by (dst,src,total) alone let a snapshot captured for an + * earlier entry poison a later copy with the same operand triple - + * the wedge_ab "bad = mutated-element count" class: step N's H2D + * executed from step N-1's snapshot because the re-decode of the + * completed step N-1 entry re-captured its (unchanged) source after + * its copy had already run, and step N's copy then consumed it. + * The pass hash changes whenever the guest rewrites the push, so a + * ring slot reused by a fresh push can never match a stale owner. + */ + u64 owner_hash; + u32 owner_ch; + u32 owner_idx; + u8 *buf; + unsigned long stamp; +}; +static struct svc_copy_snap svc_copy_snaps[SVC_COPY_SNAP_N]; + +static u8 *svc_copy_snap_lookup(u64 dst, u64 src, size_t total, + u32 owner_ch, u32 owner_idx, u64 owner_hash) +{ + u32 i; + + for (i = 0; i < SVC_COPY_SNAP_N; i++) + if (svc_copy_snaps[i].buf && + svc_copy_snaps[i].dst == dst && + svc_copy_snaps[i].src == src && + svc_copy_snaps[i].total == total && + svc_copy_snaps[i].owner_ch == owner_ch && + svc_copy_snaps[i].owner_idx == owner_idx && + svc_copy_snaps[i].owner_hash == owner_hash) + return svc_copy_snaps[i].buf; + return NULL; +} + +static void svc_copy_snap_free(u64 dst, u64 src, size_t total, + u32 owner_ch, u32 owner_idx, u64 owner_hash) +{ + u32 i; + + for (i = 0; i < SVC_COPY_SNAP_N; i++) + if (svc_copy_snaps[i].buf && + svc_copy_snaps[i].dst == dst && + svc_copy_snaps[i].src == src && + svc_copy_snaps[i].total == total && + svc_copy_snaps[i].owner_ch == owner_ch && + svc_copy_snaps[i].owner_idx == owner_idx && + svc_copy_snaps[i].owner_hash == owner_hash) { + kfree(svc_copy_snaps[i].buf); + svc_copy_snaps[i].buf = NULL; + return; + } +} + +/* + * r456/r457: the snapshot lifecycle is capture-once -> free on the copy's + * own successful consumption (svc_copy) -> free on ring-slot reuse with a + * new push hash (svc_copy_snap_capture) -> free when a re-decode skips an + * already-done op (svc_run_ops) -> TTL. The old per-entry owner free was + * removed from every opdef_clear caller: an entry whose release was + * delivered can still be REPLAYED (the pin-batch rewind) with its copy + * still owed, and freeing at that point let the replay re-capture the + * driver's refilled ping-pong staging (the r457 mode-C b1 H2D wrote b2's + * bytes into db1). A stale snapshot can only ever serve its own + * (ch, idx, hash) owner, so the r444 cross-entry poisoning stays closed. + */ +static void __maybe_unused svc_copy_snap_free_owner(u32 owner_ch, u32 owner_idx) +{ + u32 i; + + for (i = 0; i < SVC_COPY_SNAP_N; i++) + if (svc_copy_snaps[i].buf && + svc_copy_snaps[i].owner_ch == owner_ch && + svc_copy_snaps[i].owner_idx == owner_idx) { + kfree(svc_copy_snaps[i].buf); + svc_copy_snaps[i].buf = NULL; + } +} + +/* Capture the source of a virtual-operand copy about to be held. */ +static void svc_copy_snap_capture(struct nv_vgpu_device *vgpu, u64 dst, + u64 src, size_t total, u32 owner_ch, + u32 owner_idx, u64 owner_hash) +{ + struct svc_copy_snap *s = NULL; + u8 *buf; + size_t o; + u32 i; + + if (!total || total > SVC_COPY_SNAP_MAX) + return; + if (svc_copy_snap_lookup(dst, src, total, owner_ch, owner_idx, + owner_hash)) + return; /* already captured */ + /* + * r456: evict any snapshot left by the SAME ring slot under a + * DIFFERENT push hash (the slot was reused by a fresh push). The + * surviving-across-retries snapshot (svc_opdef_clear APPLY no + * longer frees it) must never be readable by the next occupant of + * the slot; the pass hash changes whenever the guest rewrites the + * push, so a same-slot different-hash snapshot is by definition + * stale for the new push. + */ + for (i = 0; i < SVC_COPY_SNAP_N; i++) { + if (svc_copy_snaps[i].buf && + svc_copy_snaps[i].owner_ch == owner_ch && + svc_copy_snaps[i].owner_idx == owner_idx && + svc_copy_snaps[i].owner_hash != owner_hash) { + kfree(svc_copy_snaps[i].buf); + svc_copy_snaps[i].buf = NULL; + } + } + for (i = 0; i < SVC_COPY_SNAP_N; i++) { + if (!svc_copy_snaps[i].buf) { + s = &svc_copy_snaps[i]; + break; + } + if (time_after(jiffies, svc_copy_snaps[i].stamp + + msecs_to_jiffies(SVC_COPY_SNAP_TTL_MS))) { + kfree(svc_copy_snaps[i].buf); + svc_copy_snaps[i].buf = NULL; + s = &svc_copy_snaps[i]; + break; + } + } + if (!s) + return; /* all slots live: keep the historical behaviour */ + buf = kmalloc(total, GFP_KERNEL); + if (!buf) + return; + for (o = 0; o < total; ) { + size_t chunk = min_t(size_t, 0x1000 - ((src + o) & 0xfff), + total - o); + u64 pa = 0; + u32 ap = 0; + + /* The source is a virtual host VA; resolve per page first. */ + if (svc_op_translate(vgpu, src + o, &pa, &ap) || + svc_read(vgpu, pa, buf + o, chunk)) { + kfree(buf); + return; + } + o += chunk; + } + s->dst = dst; + s->src = src; + s->total = total; + s->owner_ch = owner_ch; + s->owner_idx = owner_idx; + s->owner_hash = owner_hash; + s->stamp = jiffies; + s->buf = buf; + pr_info_ratelimited("vgsp-sema: copy snapshot captured dst=0x%llx src=0x%llx total=%zu owner=0x%x/%u/%016llx first4=%08x %08x %08x %08x\n", + dst, src, total, owner_ch, owner_idx, + owner_hash, + total >= 4 ? *(const u32 *)buf : 0, + total >= 8 ? *(const u32 *)(buf + 4) : 0, + total >= 12 ? *(const u32 *)(buf + 8) : 0, + total >= 16 ? *(const u32 *)(buf + 12) : 0); +} + +static int svc_fwd_operand_hpa(struct nv_vgpu_device *vgpu, u8 kind, + u64 pa, u32 off, u32 bytes, u64 *hpa, + u32 *out_off); + +/* r402 row-wise wrapper: see svc_fill. The source advances by spitch. */ +/* r402: read-only resolver probe (defined with the GR fill diagnostics). */ +static const char *svc_gr_src_probe(struct nv_vgpu_device *vgpu, u64 va, + u64 *pa_out, u32 *ap_out); static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, - u32 dst_phys, u32 dst_aperture, u32 src_phys, u32 src_aperture) + u32 dst_phys, u32 dst_aperture, u32 src_phys, u32 src_aperture, + u64 *done_out, u32 rows, u32 dpitch, u32 spitch, + u32 owner_ch, u32 owner_idx, u64 owner_hash) +{ + if (rows > 1 && dpitch) { + u64 done = 0; + int rc = 0; + u32 r; + + for (r = 0; r < rows; r++) { + u64 row_done = 0; + + rc = svc_copy_1d(vgpu, dst + (u64)r * dpitch, + src + (u64)r * spitch, total, + dst_phys, dst_aperture, src_phys, + src_aperture, &row_done, + owner_ch, owner_idx, owner_hash); + done += row_done; + if (rc) + break; + } + if (done_out) + *done_out = done; + return rc; + } + return svc_copy_1d(vgpu, dst, src, total, dst_phys, dst_aperture, + src_phys, src_aperture, done_out, + owner_ch, owner_idx, owner_hash); +} + +static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, + u32 dst_phys, u32 dst_aperture, u32 src_phys, u32 src_aperture, + u64 *done_out, u32 owner_ch, u32 owner_idx, u64 owner_hash) { u8 bounce[256]; size_t o; @@ -4166,10 +6710,29 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, u64 dst_orig = dst, src_orig = src; u32 dst_prov = 0, src_prov = 0; bool bigcopy_this = false; + bool audit = total >= SVC_COPY_AUDIT_MIN; + u8 *snap = NULL; static u64 bigcopy_log; + bool trace64 = (total == 65536u); + const char *dir = "?"; + if (done_out) + *done_out = 0; /* 0 until the first byte lands */ if (!total || total > (256ULL << 20)) return -EINVAL; + /* + * fail4: name every copy the executor is asked to run (pre- + * resolution), so a copy that fails or is held before any witness + * still has an entry record. + */ + if (gsp_sema_dbg && svc.copyent_log < 128) { + svc.copyent_log++; + pr_info("sema-copyent: h=0x%x dst=0x%llx src=0x%llx total=%zu dphys=%u dap=%u sphys=%u sap=%u\n", + svc.cur_ch, dst, src, total, dst_phys, dst_aperture, + src_phys, src_aperture); + } + snap = svc_copy_snap_lookup(dst_orig, src_orig, total, owner_ch, + owner_idx, owner_hash); if (!dst_phys) { u64 pa; @@ -4177,17 +6740,96 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, if (svc_vas_has(dst)) { dst_shadow = true; + } else if (!svc_ledger_lookup(dst, &pa, &aper)) { + /* + * r402 diagnostic (bounded): name the case where the + * ledger's landing page for a copy DESTINATION + * disagrees with the live resolver (pin/walk). The + * D2H staging writes failed on the class this + * witness detects: the writer follows the ledger + * while the guest's reader holds a different page, + * so the client reads stale data (streams2 c1 + * zeros / smiley 3-of-4 passes). Read-only probe + * (svc_gr_src_probe stores nothing). + */ + if (gsp_sema_dbg && svc_dst_div_log < 64 && + dst >= SVC_VAS_FLOOR) { + u64 wpa = 0; + u32 wap = 0; + const char *src = + svc_gr_src_probe(vgpu, dst, &wpa, &wap); + + if (wpa && + (((wpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1))) || + ((wap == SVC_APERTURE_FB) != + (aper == SVC_APERTURE_FB)))) { + svc_dst_div_log++; + pr_err_ratelimited("vgsp-sema: DST-DIVERGE va=0x%llx ledger=0x%llx/%u probe=%s 0x%llx/%u (write follows ledger)\n", + dst, pa, aper, src, wpa, wap); + } + } + /* + * m180 unification: a previous write of this VA is + * the authority for where the content lives (and so + * for where a new write lands); it also spares the + * m169 stale-walk hold below, which exists precisely + * because no pin was known. + * + * r376 pin-agreement: a FRESH pin that disagrees is + * newer than the recorded landing - the guest freed + * or moved the buffer - so the pin wins and the + * entry is repaired (the same rule the fills and the + * launch paths apply; without it a stale entry + * writes a freed page - the guest-corruption class). + */ + { + u64 ipa = 0; + int iap = SVC_APERTURE_SYS; + + if (!svc_ipin_lookup(dst, &ipa, &iap) && + (((ipa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1))) || + (u32)iap != (u32)aper)) { + svc_ledger_pin_repairs++; + if (svc_ledger_pin_repairs <= 64) + pr_info("sema-ledger: PIN-WINS copy-dst va=0x%llx ledger=0x%llx/%u pin=0x%llx/%d\n", + dst, + pa & ~(u64)(SEMA_PAGE_SIZE - 1), + aper, ipa, iap); + svc_ledger_store(dst, + ipa & ~(u64)(SEMA_PAGE_SIZE - 1), + (u32)iap); + pa = ipa; + aper = (u32)iap; + } + } + dst = pa; + dst_aperture = (aper == SVC_APERTURE_FB) ? + CE_APERTURE_VID : CE_APERTURE_SYS; + dst_prov = 1; + dst_virt = true; } else if (svc_op_translate(vgpu, dst, &pa, &aper)) { - if (!svc_vas_can_shadow(dst)) { + u64 lpa = 0; + u32 lap = 0; + + if (!svc_ledger_lookup(dst, &lpa, &lap)) { + dst = lpa; + dst_aperture = (lap == SVC_APERTURE_FB) ? + CE_APERTURE_VID : CE_APERTURE_SYS; + dst_prov = 1; + dst_virt = true; + } else if (!svc_vas_can_shadow(dst)) { if (gsp_sema_dbg && svc.op_tr_log < 64) { svc.op_tr_log++; pr_info("sema-tr-miss: DST va=0x%llx h=0x%x\n", dst, svc.cur_ch); } return -ENOENT; + } else { + dst_shadow = true; + svc.vas_shadowed++; } - dst_shadow = true; - svc.vas_shadowed++; } else { dst = pa; dst_aperture = aper; @@ -4195,24 +6837,177 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, dst_virt = true; } } + /* + * m169 (AGENT-F R1): a device-lane destination translated only by a + * page-directory walk (prov 3/4) or the RAM scan (5) can be a stale + * PTE from the previous session - the table CONTENT is not wiped by + * the session flush - and a wrong-but-valid vidmem write is silent. + * r172 streams2_diag: the stream-1 a H2D ran rc=0 through a + * walk-resolved destination while the launch (pin-resolved) read the + * true page still holding the previous client's h_a; the engine + * computed from stale inputs and the client verified 32768 bad. + * Since the resolver tries pins FIRST, prov >= 3 means no pin existed + * at resolve time: hold the copy (-ENOENT feeds the m163 bounded + * hold), which re-decodes after the client's pins land (seconds) and + * is loud on expiry - never silent corruption. Device destinations + * only; sysmem copies keep the historical sources. + */ + if (dst_virt && dst_aperture == CE_APERTURE_VID && + dst_prov >= 3 && dst_orig >= SVC_VAS_FLOOR && + gsp_sema_copy_dst_pin_guard && + svc_dst_guard_hold(dst_orig)) { + svc.copy_dst_hold++; + if (svc.copy_dst_hold <= 16) + pr_info("sema-svc: device-dst copy held: dst=0x%llx prov=%u (no pin yet; stale-table guard)\n", + dst_orig, dst_prov); + /* + * fail4: the staging source is valid NOW; snapshot it so + * the deferred execution copies the pipeline-order bytes. + */ + if (!src_phys && src_orig >= SVC_VAS_FLOOR) + svc_copy_snap_capture(vgpu, dst_orig, src_orig, + total, owner_ch, owner_idx, + owner_hash); + return -ENOENT; + } + /* + * r378: the same stale-table class for SYS destinations resolved by + * the walk/scan (prov >= 3): a FRESH injected pin that disagrees is + * the guest's newest statement, so writing through the resolved page + * lands on a page the guest has freed/re-homed (the guest-corruption + * class). Hold through the same bounded re-decode path instead of + * writing; a missing pin keeps the historical behaviour (the + * resolution is then the write authority). + */ + if (dst_virt && dst_aperture == CE_APERTURE_SYS && + dst_prov >= 3 && dst_orig >= SVC_VAS_FLOOR && + gsp_sema_copy_dst_pin_guard) { + u64 ipa = 0; + int iap = SVC_APERTURE_SYS; + + if (!svc_ipin_lookup(dst_orig, &ipa, &iap) && + (((ipa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (dst & ~(u64)(SEMA_PAGE_SIZE - 1))) || + svc_ap_of(iap) != SVC_APERTURE_SYS) && + svc_dst_guard_hold(dst_orig)) { + svc.copy_dst_hold++; + if (svc.copy_dst_hold <= 16) + pr_info("sema-svc: device-dst copy held: dst=0x%llx prov=%u pin=0x%llx (stale SYS target)\n", + dst_orig, dst_prov, ipa); + if (!src_phys && src_orig >= SVC_VAS_FLOOR) + svc_copy_snap_capture(vgpu, dst_orig, + src_orig, total, + owner_ch, owner_idx, + owner_hash); + return -ENOENT; + } + } + /* + * r231 READ-ORDER guard/witness: a device-src copy whose source VA + * has a PENDING device-dst write hold reads the PRE-write content - + * the writer's bytes land only when the guard resolves while the + * reader executes immediately (the ctaid_probe RAWA round trip is + * the reproducer). Witnessed always (bounded); deferred only with + * sema_read_order_guard, through the same bounded-hold re-decode. + */ + if (src_orig >= SVC_VAS_FLOOR && svc_dst_guard_pending(src_orig)) { + if (svc_read_order_log < 16) { + svc_read_order_log++; + pr_info("sema-svc: READ-ORDER-WIT device-src copy va=0x%llx reads while a device-dst write to the same VA is still held%s\n", + src_orig, gsp_sema_read_order_guard ? + " (deferring)" : " (guard off)"); + } + if (gsp_sema_read_order_guard) + return -ENOENT; + } if (!src_phys) { u64 pa; u32 aper; if (svc_vas_has(src)) { src_shadow = true; + } else if (src_orig >= SVC_VAS_FLOOR && + !svc_ledger_lookup(src, &pa, &aper)) { + /* + * r375 read-after-write authority: the landing table + * records where every write of this VA landed (writes + * store it, fresh pins repair it). A reader MUST read + * exactly that page or the engine's output is invisible + * (the sum=0 class). The walk/pin sources can answer + * differently - e.g. the same numeric pa in a different + * aperture - so the table wins for device VAs; only a + * VA with no recorded landing falls through to the + * resolver below. + * + * r376 pin-agreement: unless a FRESH pin disagrees - + * then the pin is newer (the guest re-pinned the VA) + * and wins, and the entry is repaired. + */ + { + u64 ipa = 0; + int iap = SVC_APERTURE_SYS; + + if (!svc_ipin_lookup(src, &ipa, &iap) && + (((ipa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1))) || + (u32)iap != (u32)aper)) { + svc_ledger_pin_repairs++; + if (svc_ledger_pin_repairs <= 64) + pr_info("sema-ledger: PIN-WINS copy-src va=0x%llx ledger=0x%llx/%u pin=0x%llx/%d\n", + src, + pa & ~(u64)(SEMA_PAGE_SIZE - 1), + aper, ipa, iap); + svc_ledger_store(src, + ipa & ~(u64)(SEMA_PAGE_SIZE - 1), + (u32)iap); + pa = ipa; + aper = (u32)iap; + } + } + src = pa; + src_aperture = (aper == SVC_APERTURE_FB) ? + CE_APERTURE_VID : CE_APERTURE_SYS; + src_prov = 1; + src_virt = true; } else if (svc_op_translate(vgpu, src, &pa, &aper)) { - if (gsp_sema_dbg && svc.op_tr_log < 64) { - svc.op_tr_log++; - pr_info("sema-tr-miss: SRC va=0x%llx h=0x%x\n", - src, svc.cur_ch); + u64 lpa = 0; + u32 lap = 0; + + /* + * m180 unification: read the page the last host write + * of this VA landed on when the resolver can no longer + * answer (same rule as the per-page loop below), instead + * of failing the whole copy. + */ + if (!svc_ledger_lookup(src, &lpa, &lap)) { + src = lpa; + src_aperture = (lap == SVC_APERTURE_FB) ? + CE_APERTURE_VID : CE_APERTURE_SYS; + src_prov = 1; + src_virt = true; + } else { + if (gsp_sema_dbg && svc.op_tr_log < 64) { + svc.op_tr_log++; + pr_info("sema-tr-miss: SRC va=0x%llx h=0x%x\n", + src, svc.cur_ch); + } + return -ENOENT; } - return -ENOENT; } else { src = pa; src_aperture = aper; src_prov = svc.op_tr_src; src_virt = true; + /* + * r375b: do NOT record a resolver answer into the + * landing table here. The resolver can return a stale + * PTE page (the m169 class) and the table is the WRITE + * authority - a poisoned entry makes every later write + * of this VA land on the stale page (r375 boot: slab + * GPF in kmem_cache_alloc at guest t=77s, before any + * client). The table is fed by writes and fresh pins + * only. + */ } } @@ -4222,8 +7017,35 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, * 16 bytes on each side before the copy. The vadd data path is exactly * these copies; this separates a wrong resolution from a wrong copy. */ - if (total >= (1u << 20) && bigcopy_log < 24) { + if ((total >= SVC_COPY_AUDIT_MIN || + (dst_virt && dst_orig >= SVC_VAS_FLOOR)) && + (bigcopy_log < gsp_sema_bigcopy_log_max || total == 65536u)) { u8 sb[16] = {0}, db[16] = {0}; + /* + * r235: 64 KB copies are exempt from the bigcopy budget - the + * probe's A upload/readbacks must ALWAYS appear (the budget + * belongs to the multi-MB traffic and had absorbed every 64 KB + * probe copy in the r235 capture). + */ + bool trace64 = (total == 65536u); + const char *dir = + (dst_aperture == CE_APERTURE_VID && + src_aperture == CE_APERTURE_SYS) ? "H2D" : + (src_aperture == CE_APERTURE_VID && + dst_aperture == CE_APERTURE_SYS) ? "D2H" : + src_shadow ? "shadow->" : "phys"; + + /* + * r235: 64 KB copies are exempt from the bigcopy budget - the + * probe's A upload/readbacks must ALWAYS appear (the budget + * belongs to the multi-MB traffic and had absorbed every 64 KB + * probe copy in the r235 capture). + */ + dir = (dst_aperture == CE_APERTURE_VID && + src_aperture == CE_APERTURE_SYS) ? "H2D" : + (src_aperture == CE_APERTURE_VID && + dst_aperture == CE_APERTURE_SYS) ? "D2H" : + src_shadow ? "shadow->" : "phys"; bigcopy_log++; bigcopy_this = true; @@ -4247,12 +7069,55 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, } else { svc_read(vgpu, dst, db, sizeof(db)); } - pr_info("sema-bigcopy: dst_va=0x%llx -> pa=0x%llx ap=%u prov=%u shadow=%u | src_va=0x%llx -> pa=0x%llx ap=%u prov=%u shadow=%u | bytes=%zu\n", + pr_info("sema-bigcopy: dst_va=0x%llx -> pa=0x%llx ap=%u prov=%u shadow=%u | src_va=0x%llx -> pa=0x%llx ap=%u prov=%u shadow=%u | bytes=%zu vbase=%px\n", dst_orig, dst, dst_aperture, dst_prov, dst_shadow, src_orig, src, src_aperture, src_prov, src_shadow, - total); + total, (void *)vgpu->bars[1].vaddr); + /* + * r348 diagnostic: for a high-VA SYS source name the exact + * host PA this read will touch (gpa_to_hpa) - the direct + * comparison against the svc-fwd-hpa writeback line. + */ + if (src_orig >= SVC_VAS_FLOOR && + src_aperture == CE_APERTURE_SYS) { + u64 shpa = 0; + u32 soff = 0; + + if (!svc_fwd_operand_hpa(vgpu, SVC_APERTURE_SYS, + src, 0, SEMA_PAGE_SIZE, + &shpa, &soff) && !soff) + pr_info("sema-bigcopy: src-hpa va=0x%llx gpa=0x%llx -> hpa=0x%llx\n", + src_orig, src, shpa); + else + pr_info("sema-bigcopy: src-hpa va=0x%llx gpa=0x%llx -> UNMAPPED\n", + src_orig, src); + } pr_info("sema-bigcopy: src16=%16phN dst16_before=%16phN\n", sb, db); + /* + * r234 A-TRACE: one labeled line per 64 KB copy so the probe A + * upload/readback is greppable end-to-end. pt=P1 = H2D source + * read (the uploaded bytes as ints), pt=P4 = D2H source read + * (the engine-authored page as ints). The expected content is + * ints 0,1,2,3,... for the probe A buffer. + */ + if (trace64 && svc_atrace_log < 128u) { + u32 i0, i1, i2, i3, j0 = 0, j1 = 0, j2 = 0, j3 = 0; + + svc_atrace_log++; + memcpy(&i0, sb, 4); + memcpy(&i1, sb + 4, 4); + memcpy(&i2, sb + 8, 4); + memcpy(&i3, sb + 12, 4); + memcpy(&j0, db, 4); + memcpy(&j1, db + 4, 4); + memcpy(&j2, db + 8, 4); + memcpy(&j3, db + 12, 4); + pr_info("vgsp: A-TRACE pt=%s dir=%s dst_va=0x%llx dst_pa=0x%llx dst_prov=%u | src_va=0x%llx src_pa=0x%llx src_prov=%u | src_int4=%d %d %d %d dst_int4_before=%d %d %d %d\n", + !strcmp(dir, "H2D") ? "P1" : "P4", dir, + dst_orig, dst, dst_prov, src_orig, src, + src_prov, i0, i1, i2, i3, j0, j1, j2, j3); + } } /* @@ -4301,6 +7166,17 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, svc_fwd_cand.bytes = (u32)total; /* + * m180 unification: record the destination landing + * even when the copy itself is deferred to the + * forwarded-request path (the software copy below + * never runs then, so the in-loop store would be + * skipped and the page would have no authority for + * the fill/D2H readers). + */ + if (dst_orig >= SVC_VAS_FLOOR) + svc_ledger_store(dst_orig, dst_page, dst_kind); + + /* * P6b wave 2: with the hardware-primary path armed * (parameter on and all three prerequisites on, so the * request can queue as a guest copy and its releases @@ -4434,64 +7310,267 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, chunk = min_t(size_t, chunk, 0x1000 - ((dst_orig + o) & 0xfff)); - if (src_shadow) { + if (snap) { + /* + * r314: consume the decode-time staging snapshot. + * The capture machinery existed since cb50282b but + * the execution never read it (lookup at svc_copy's + * top, free on success, NO consumer) - every copy + * read the live staging, which the driver had + * refilled once the push's fence fired (the + * deterministic 786335-bad vector_add output, + * identical across boots). Reading from the + * snapshot also needs no live source translation, + * so the copies execute on their first allowed + * pass instead of storming 60+ rc=-2 retries. + */ + memcpy(bounce, snap + o, chunk); + } else if (src_shadow) { if (svc_vas_read(src_orig + o, bounce, chunk)) - return -EIO; + SVC_COPY_TRUNC_RET(-EIO); } else if (src_virt) { u64 pa = 0; u32 aper = 0; + u64 lpa = 0; + u32 lap = 0; - if (svc_op_translate(vgpu, src_orig + o, &pa, &aper)) - return -ENOENT; + /* + * r375: the landing table is the read-after-write + * authority for device VAs (see the top-level src + * resolve). Per page, the table wins whenever it has + * an entry; the resolver below only answers a VA the + * table has never seen. r376: a fresh disagreeing pin + * is newer than the table and wins (repaired). + */ + if (src_orig >= SVC_VAS_FLOOR && + !svc_ledger_lookup(src_orig + o, &lpa, &lap)) { + u64 ipa = 0; + int iap = SVC_APERTURE_SYS; + + if (!svc_ipin_lookup(src_orig + o, &ipa, &iap) && + (((ipa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (lpa & ~(u64)(SEMA_PAGE_SIZE - 1))) || + (u32)iap != (u32)lap)) { + svc_ledger_pin_repairs++; + svc_ledger_store(src_orig + o, + ipa & ~(u64)(SEMA_PAGE_SIZE - 1), + (u32)iap); + lpa = ipa & ~(u64)(SEMA_PAGE_SIZE - 1); + lap = (u32)iap; + } + pa = lpa; + aper = (lap == SVC_APERTURE_FB) ? + CE_APERTURE_VID : CE_APERTURE_SYS; + } else if (svc_op_translate(vgpu, src_orig + o, &pa, &aper)) { + /* + * m180 unification: a per-page resolution miss + * must not fail the whole copy when the unified + * table knows where this VA's content landed. + */ + if (svc_ledger_lookup(src_orig + o, &lpa, &lap)) + SVC_COPY_TRUNC_RET(-ENOENT); + pa = lpa; + aper = (lap == SVC_APERTURE_FB) ? + CE_APERTURE_VID : CE_APERTURE_SYS; + } else if (src_orig >= SVC_VAS_FLOOR && + !svc_ledger_lookup(src_orig + o, &lpa, &lap) && + ((lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1)) || + (lap == SVC_APERTURE_FB) != + (aper == CE_APERTURE_VID))) { + /* + * m171 (AGENT-F R2): read-after-write - when + * a large H2D wrote this device VA page + * through a different translation, read the + * page the write landed on. + */ + svc_ledger_diverges++; + if (svc_ledger_diverges <= 32) + pr_info("sema-ledger: DIVERGE read va=0x%llx wrote=0x%llx read=0x%llx\n", + src_orig + o, lpa, pa); + pa = lpa; + aper = (lap == SVC_APERTURE_FB) ? + CE_APERTURE_VID : CE_APERTURE_SYS; + } if (aper == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; if (!base || pa + chunk > svc_fb_shadow_size(vgpu)) - return -EINVAL; + SVC_COPY_TRUNC_RET(-EINVAL); memcpy(bounce, base + pa, chunk); } else if (svc_read(vgpu, pa, bounce, chunk)) { - return -EIO; + SVC_COPY_TRUNC_RET(-EIO); } } else if (src_aperture == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; size_t size = svc_fb_shadow_size(vgpu); if (!base || src + o + chunk > size) - return -EINVAL; + SVC_COPY_TRUNC_RET(-EINVAL); memcpy(bounce, base + src + o, chunk); } else { if (svc_read(vgpu, src + o, bounce, chunk)) - return -EIO; + SVC_COPY_TRUNC_RET(-EIO); + } + + /* + * r367 TABLE-COPY witness: a page-table page/entry copy is + * the guest's authoritative table content; dump the carried + * bytes so a table that later walks as ZERO can be traced to + * the copy that built it (or proven the source was already + * zero). Bounded; table-shaped sizes only. + */ + if (!o && (total == 4096 || total == 16) && + svc_tabcopy_log < 64) { + svc_tabcopy_log++; + pr_info("sema-tabcopy: dst_orig=0x%llx src_orig=0x%llx total=%zu dphys=%u dvirt=%d svirt=%d r_dst=0x%llx r_src=0x%llx first16=%16phN\n", + dst_orig, src_orig, total, dst_phys, dst_virt, + src_virt, dst, src, bounce); } if (dst_shadow) { if (svc_vas_write(dst_orig + o, bounce, chunk)) - return -ENOMEM; + SVC_COPY_TRUNC_RET(-ENOMEM); + if (audit && (!o || o + chunk == total)) + svc_copy_audit_chunk(vgpu, dst_orig, o, bounce, + chunk, true, 0, 0, + (o != 0)); } else if (dst_virt) { u64 pa = 0; u32 aper = 0; + u64 lpa = 0; + u32 lap = 0; + u32 wprov = 0; - if (svc_op_translate(vgpu, dst_orig + o, &pa, &aper)) - return -ENOENT; + if (svc_op_translate(vgpu, dst_orig + o, &pa, &aper)) { + /* + * m180 unification: a per-page resolution miss + * lands on the known page instead of failing + * the whole copy (readers already use it). + */ + if (svc_ledger_lookup(dst_orig + o, &lpa, &lap)) + SVC_COPY_TRUNC_RET(-ENOENT); + pa = lpa; + aper = (lap == SVC_APERTURE_FB) ? + CE_APERTURE_VID : CE_APERTURE_SYS; + wprov = 0xF1; + } else if (dst_orig >= SVC_VAS_FLOOR && + !svc_ledger_lookup(dst_orig + o, &lpa, &lap) && + ((lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1)) || + (lap == SVC_APERTURE_FB) != + (aper == CE_APERTURE_VID))) { + svc_ledger_diverges++; + if (svc.op_tr_src == 1) { + /* + * r336 PIN-WINS (the r335 vector_add + * guest-panic class): a pin-backed + * translation (guest pagemap truth, + * op_tr_src == 1) that disagrees + * with the ledger means the guest + * remapped or reused this VA. The + * ledger page is where an earlier + * write landed; the guest has since + * freed or moved it, so writing it + * can land on memory the guest + * kernel now owns (observed: RM + * object fields overwritten with + * client c-data, slab freelist + * corruption, guest panic). Trust + * the pin for the write and re-home + * the ledger (the store after the + * write records the fresh page); + * readers then converge on the same + * page too. + */ + svc_ledger_pinwins++; + if (svc_ledger_diverges <= 512) + pr_info("sema-ledger: DIVERGE write va=0x%llx prev=0x%llx cur=0x%llx (pin wins)\n", + dst_orig + o, lpa, pa); + wprov = svc.op_tr_src; + } else { + /* + * m180 unification: keep ONE + * authoritative page per VA. A new + * write goes to the page the + * previous content landed on, and + * the fill/D2H readers (which + * prefer the table) stay consistent + * with it. Non-pin translations + * (2..5) can carry stale PTEs + * (m169), so the write history + * stays the authority there. + */ + if (svc_ledger_diverges <= 512) + pr_info("sema-ledger: DIVERGE write va=0x%llx prev=0x%llx cur=0x%llx\n", + dst_orig + o, lpa, pa); + pa = lpa; + aper = (lap == SVC_APERTURE_FB) ? + CE_APERTURE_VID : CE_APERTURE_SYS; + wprov = 0xF0; + } + } else { + wprov = svc.op_tr_src; + } + /* + * D2H-MAP (diagnostic): per-page destination + * resolution of large host-dest copies - the direct + * witness for which guest pages a readback lands on. + * Bounded; one line per 4 KB page of copies >= 1 MB. + */ + if (total >= (1u << 20) && !(o & (SEMA_PAGE_SIZE - 1)) && + svc_d2hmap_log < 4096u) { + svc_d2hmap_log++; + pr_info("sema-d2hmap: va=0x%llx o=%zu pa=0x%llx ap=%u prov=%u\n", + dst_orig + o, o, pa, aper, wprov); + } if (aper == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; if (!base || pa + chunk > svc_fb_shadow_size(vgpu)) - return -EINVAL; + SVC_COPY_TRUNC_RET(-EINVAL); memcpy(base + pa, bounce, chunk); } else if (svc_write(vgpu, pa, bounce, chunk)) { - return -EIO; + SVC_COPY_TRUNC_RET(-EIO); } + if (audit && (!o || o + chunk == total)) + svc_copy_audit_chunk(vgpu, dst_orig, o, bounce, + chunk, false, pa, aper, + (o != 0)); + /* + * m171/m180: a large device-dest write records its + * landing page - SYS and VID alike, because the + * sysmem-backed UVM buffers are the dominant + * device-visible case and were the half the table + * never saw. The later kernel fill / D2H read of the + * same VA then uses exactly this page even if its own + * resolution is stale. + */ + if (dst_orig >= SVC_VAS_FLOOR && total >= (64u << 10)) + svc_ledger_store(dst_orig + o, pa, + (aper == CE_APERTURE_VID) ? + SVC_APERTURE_FB : + SVC_APERTURE_SYS); } else if (dst_aperture == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; size_t size = svc_fb_shadow_size(vgpu); if (!base || dst + o + chunk > size) - return -EINVAL; + SVC_COPY_TRUNC_RET(-EINVAL); memcpy(base + dst + o, bounce, chunk); + if (audit && (!o || o + chunk == total)) + svc_copy_audit_chunk(vgpu, dst_orig, o, bounce, + chunk, false, dst + o, + CE_APERTURE_VID, + (o != 0)); } else { if (svc_write(vgpu, dst + o, bounce, chunk)) - return -EIO; + SVC_COPY_TRUNC_RET(-EIO); + if (audit && (!o || o + chunk == total)) + svc_copy_audit_chunk(vgpu, dst_orig, o, bounce, + chunk, false, dst + o, + CE_APERTURE_SYS, + (o != 0)); } (void)i; o += chunk; @@ -4517,6 +7596,23 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, } pr_info("sema-bigcopy: dst16_after=%16phN (dst_va=0x%llx pa=0x%llx)\n", da, dst_orig, dst); + /* + * r234 A-TRACE pt=P2/P4R: the destination content right after + * the copy - for H2D this is the landed upload (P2), for D2H + * the host-side readback result (P4R). + */ + if (trace64 && svc_atrace_log < 128u) { + u32 i0, i1, i2, i3; + + svc_atrace_log++; + memcpy(&i0, da, 4); + memcpy(&i1, da + 4, 4); + memcpy(&i2, da + 8, 4); + memcpy(&i3, da + 12, 4); + pr_info("vgsp: A-TRACE pt=%s dir=%s dst_va=0x%llx dst_pa=0x%llx dst_int4_after=%d %d %d %d\n", + !strcmp(dir, "H2D") ? "P2" : "P4R", dir, + dst_orig, dst, i0, i1, i2, i3); + } } if (gsp_sema_dbg && !src_shadow && !dst_shadow && @@ -4553,6 +7649,34 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, pr_info("sema-copy-dst: after dst=0x%llx dst16=%*phN\n", dst, n, da); } + if (audit && svc_copy_audit_n < SVC_COPY_AUDIT_MAX) { + svc_copy_audit_n++; + pr_info("vgsp: COPYAUDIT path=ce-soft-copy va=0x%llx len=0x%llx done=0x%llx\n", + dst_orig, (u64)total, (u64)o); + } + /* + * m180 parity with svc_fill: a large virtual-destination copy is a + * write/read pair with the launch rec-build readers - record its + * landing pages so the GR fill follows the page the copy actually + * wrote. Without this, a rec table built before the pins landed + * kept the stale pre-pin pages while the copy landed on the pinned + * page (streams2 r254: the b1 rec range carried the a2 pages and + * the vadd read a2's pattern as its second input). + */ + if (!dst_phys && dst_virt && dst_orig >= SVC_VAS_FLOOR && + total >= (64u << 10)) { + u32 cap = (dst_aperture == CE_APERTURE_VID) ? + SVC_APERTURE_FB : SVC_APERTURE_SYS; + size_t po; + + for (po = 0; po < total; po += SEMA_PAGE_SIZE) + svc_ledger_store(dst_orig + po, dst + po, cap); + } + if (snap) + svc_copy_snap_free(dst_orig, src_orig, total, owner_ch, + owner_idx, owner_hash); + if (done_out) + *done_out = (u64)o; /* o == total on the success path */ return 0; } @@ -4583,6 +7707,23 @@ static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va, svc.memsets_done++; return 0; } + /* + * m180 unification: a page with a known landing (previous H2D/fill or + * engine write-back) is authoritative - write the inline data there + * so the engine fill / D2H readers that follow the table see it. + */ + { + u64 lpa = 0; + u32 lap = 0; + + if (!svc_ledger_lookup(dst_va, &lpa, &lap)) { + pa = lpa & ~0xfffULL; + ap = (lap == SVC_APERTURE_FB) ? + SVC_APERTURE_FB : SVC_APERTURE_SYS; + svc_pin_store(dst_va & ~0xfffULL, pa, ap); + goto write_landing; + } + } if (svc_lookup_trusted(vgpu, dst_va & ~0xfffULL, &pa, &ap)) { if (!svc_vas_write(dst_va, data, bytes)) { svc.vas_shadowed++; @@ -4602,6 +7743,7 @@ static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va, return -ENOENT; } svc_pin_store(dst_va & ~0xfffULL, pa, ap); +write_landing: pa += dst_va & 0xfff; if (gsp_sema_dbg && svc.grce_log < (gsp_sema_ops_log > 64 ? gsp_sema_ops_log : 64)) { @@ -4624,24 +7766,477 @@ static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va, for (o = 0; o < bytes; o += 4096) { u32 chunk = bytes - o > 4096 ? 4096 : bytes - o; - if (svc_write(vgpu, pa + o, (void *)(data + o), chunk)) + if (svc_write(vgpu, pa + o, (void *)(data + o), chunk)) { + if (bytes >= SVC_COPY_AUDIT_MIN) + pr_err_ratelimited("vgsp: COPY-TRUNC va=0x%llx want=0x%llx done=0x%llx path=grce-inline rc=%d\n", + dst_va, (u64)bytes, + (u64)o, -EIO); return -EIO; + } } } + /* + * COPYAUDIT (see the svc_copy comment): log the big inline write and + * read the first 16 destination bytes back through the aperture it + * was written to, comparing against the source bytes. + */ + if (bytes >= SVC_COPY_AUDIT_MIN && svc_copy_audit_n < SVC_COPY_AUDIT_MAX) { + u8 rb[16] = {0}; + bool ok = false; + + if (ap == SVC_APERTURE_FB) { + u8 *base = vgpu->bars[1].vaddr; + + if (base && pa + sizeof(rb) <= svc_fb_shadow_size(vgpu)) { + memcpy(rb, base + pa, sizeof(rb)); + ok = true; + } + } else if (!svc_read(vgpu, pa, rb, sizeof(rb))) { + ok = true; + } + svc_copy_audit_n++; + pr_info("vgsp: COPYAUDIT path=grce-inline va=0x%llx len=0x%llx done=0x%llx\n", + dst_va, (u64)bytes, (u64)bytes); + if (ok && memcmp(rb, data, sizeof(rb))) + pr_info_ratelimited("vgsp: COPY-MISMATCH va=0x%llx off=0x0 path=grce-inline want=%*phN got=%*phN\n", + dst_va, 16, data, 16, rb); + } + /* + * m180: record the landing so the fill/D2H readers of this VA follow + * the bytes the engine's inline path wrote, not their own resolution. + */ + if (dst_va >= SVC_VAS_FLOOR) + svc_ledger_store(dst_va, pa, (u32)ap); svc.memsets_done++; return 0; } +/* defined below (svc_handle_map_add); the CE decode path drains the queued + * scanner injections before resolving copy operands, exactly like the GR + * path's m143 drain. */ +static void svc_handle_map_add(void); + /* Execute a decoded operation list (fills and copies). `ops_done` (may be * NULL) is the per-entry bitmap of ops that already ran for the entry's * identity; `ops_base` is the sequence number of ops[0] within the entry. */ + +/* + * r34x: evaluate a C56F SEM_EXECUTE acquire against the semaphore word. + * The class defines the opcodes (clc56f.h NVC56F_SEM_EXECUTE_OPERATION: + * 0 ACQUIRE, 1 RELEASE, 2 ACQ_STRICT_GEQ, 3 ACQ_CIRC_GEQ, 4 ACQ_AND, + * 5 ACQ_NOR, 6 REDUCTION) and the CUDA driver documents the wait + * conditions each op must implement (cuda_device.h CUIsemaphoreAquire- + * Flags_enum: GEQ default = circular >=, EQUAL = ==, AND = (v & p) != 0, + * NOR = ~(v | p) != 0, STRICT_GEQ = >= with no wrap; ampere_semaphore.c + * maps those flags onto the opcodes: EQUAL->0, STRICT_GEQ->2, GEQ->3, + * AND->4, NOR->5). Note op 0 is EQUALITY, not a >=: only op 2/3 are + * ordering compares. 32-bit payloads: every observed CUDA acquire is + * PAYLOAD_SIZE 32BIT (SEM_PAYLOAD_HI == 0); a 64-bit acquire is captured + * and judged on the low word only (its high word is not modelled). + */ +static bool svc_wait_satisfied(u32 cur, u8 op, u32 need) +{ + switch (op) { + case 0: /* ACQUIRE: wait for v == p */ + return cur == need; + case 2: /* ACQ_STRICT_GEQ: v >= p, no wrap */ + return cur >= need; + case 3: /* ACQ_CIRC_GEQ: v in [p, p + 2^31 - 1] (wrap allowed) */ + return (s32)(cur - need) >= 0; + case 4: /* ACQ_AND: (v & p) != 0 */ + return (cur & need) != 0; + case 5: /* ACQ_NOR: ~(v | p) != 0 */ + return ~(cur | need) != 0; + default: + return true; + } +} + +/* r34x: the per-channel first-hold clock for the bounded wait. */ +static struct svc_wait_hold *svc_wait_hold_get(u32 h, bool create) +{ + struct svc_wait_hold *free_slot = NULL; + int i; + + for (i = 0; i < SVC_WAIT_HOLD_MAX; i++) { + struct svc_wait_hold *e = &svc_wait_holds[i]; + + if (e->active && e->h == h) + return e; + if (!e->active && !free_slot) + free_slot = e; + } + if (!create || !free_slot) + return NULL; + memset(free_slot, 0, sizeof(*free_slot)); + free_slot->active = true; + free_slot->h = h; + free_slot->since = jiffies; + return free_slot; +} + +/* + * r506w/r507a: bounded per-VA failure latch for virtual ops. A virtual op + * that keeps failing -ENOENT (an unresolvable VA) re-runs on every decode + * pass and sets svc.op_tr_pending each time, so its push holds for the full + * SVC_OPDEF_HOLD_MS opdef bound - a client whose stream carries such an op + * (r505/r506 witness: the RM's channel-progress zeroing at 0x2000c0xxx + * after the first client, dst never translated) ground its whole stream + * at ~10 s per push ("hang after DEVICE" for matmul/multikernel). + * + * The window must still cover the legitimate pin-late retry (observed + * pin latency 0.5-2 s: the H2D copy's operands land a moment after the + * push, r507 matmul's 1 MB UVA copy at 0x7136... needed >8 fast passes), + * so the latch is TIME-based: the FIRST failure of a VA arms a + * SVC_OPDEF_HOLD_MS window during which the op holds its entry exactly as + * before; after a full bound with no success the op still attempts the + * write but no longer forces the hold - the entry consumes promptly and + * its withheld release applies, exactly the outcome the expired bound + * already sanctioned (op stays skipped, release delivered), minus the + * 10 s tax per subsequent pass. A success clears the record (a later + * failure gets a fresh window). Service-thread only, like every other op + * path. + */ +#define SVC_OPFAIL_MAX 32 +/* + * r507a: the op-failure window rides the opdef hold bound (SVC_OPDEF_HOLD_MS + * defined further down). Keep the literal here so the latch (used by + * svc_run_ops) precedes that define. + * + * r549: 10000 -> 3000. The window is the SECOND wait an unresolvable op + * costs: fail #1 arms this record, the entry is held for the opdef bound, + * the retry's fail #2 latches the op out of the hold. r548 evidence: a + * client whose init sequence carries ~65 device-dst ops with destinations + * no resolver covers walked one entry per 10 s (OPDEF-EXPIRED every + * 10.5 s, idx 7 -> 18) - the client spun at 100 % CPU for 10+ minutes and + * the battery timed out. Ops that DO resolve show 0.5-2 s pin latency, so + * 3 s keeps the m163 protection and cuts the dead-op cost to ~3 s per + * distinct destination. + */ +#define SVC_OPFAIL_WINDOW_MS 3000u +static struct { + u64 va; + unsigned long since; +} svc_opfail[SVC_OPFAIL_MAX]; +static u32 svc_opfail_log; + +static void svc_opfail_clear(u64 va) +{ + u32 i; + + for (i = 0; i < SVC_OPFAIL_MAX; i++) + if (svc_opfail[i].va == va) + svc_opfail[i].va = 0; +} + +/* + * r541: device-destination copies that decoded but have not executed yet. + * + * smiley live evidence (r539): passes 2 and 4 verified a + (previous b) - + * the client's cudaMemcpy(b) completed (its release landed) while the H2D + * op was still held, the client submitted the launch, the GR forward read + * the shadow page BEFORE the H2D executed (b's copy ran at 294026.818, + * launch 2's engine read at ~294026.3), and the engine computed with the + * PREVIOUS pass's b (bad=65024). The same shape is the streams2 / + * streams_events readback lottery: a device page whose writer has not run. + * + * The guard: when a device-dst copy (or fill) op fails to execute this + * pass, its destination VA page is recorded here with a stamp. A GR launch + * whose data pages overlap a fresh record is NOT forwarded (the pump's + * bounded busy-defer retries it; the record clears when the op finally + * executes or the SVC_DSTPEND_MS window expires, at which point the + * historical forward stands). Keyed by VA page: the page identity is what + * the launch operand and the copy share, and it works even when neither + * side resolves. + */ +#define SVC_DSTPEND_MAX 64 +#define SVC_DSTPEND_MS 10000u +static struct { + u64 va_page; + unsigned long since; +} svc_dstpend[SVC_DSTPEND_MAX]; +static u32 svc_dstpend_n; +static u32 svc_dstpend_log; + +static void svc_dstpend_note(u64 va) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 i; + + if (!page) + return; + for (i = 0; i < svc_dstpend_n; i++) { + if (svc_dstpend[i].va_page == page) { + /* + * r553: the window is ABSOLUTE from the first failed + * attempt for this page, never re-freshened by the + * retries. The retried decode of a copy that cannot + * resolve re-notes its destination on every pass; with + * a sliding stamp the record never expired and the r541 + * launch guard deferred the launch forever - r551 soak: + * victory_strided repeated "GR busy-defer timeout -> + * hold 8000 ms" every ~4 s for 427 s with the launch + * never forwarded (the copies' -ENOENT retries kept the + * note fresh). A record older than the window is + * refreshed only when a NEW failure episode starts. + */ + if (time_before(jiffies, svc_dstpend[i].since + + msecs_to_jiffies(SVC_DSTPEND_MS))) + return; + svc_dstpend[i].since = jiffies; + return; + } + } + if (svc_dstpend_n < SVC_DSTPEND_MAX) + i = svc_dstpend_n++; + else + i = (u32)(jiffies % SVC_DSTPEND_MAX); + svc_dstpend[i].va_page = page; + svc_dstpend[i].since = jiffies; +} + +/* + * r566: note the destinations of every COLLECTED device-dst copy that did + * not execute this pass, whenever the entry is being held (op_tr_pending). + * The r541 launch guard could only see copies that FAILED with -ENOENT + * (svc_dstpend_note at the failure site); the r568 mail failure is the + * missing case: the b H2D's COPY op was collected but never attempted while + * its entry was held for another operand, the launch's copy-in ran 0.6 s + * before the copy executed, and the engine computed c = a + 0 (the + * readback page held a's values). With the destination noted, the launch + * guard defers until the copy runs (its success path clears the record), + * bounded by the same dstpend TTL + busy-defer machinery as every other + * guard. + */ +static void svc_dstpend_note_ops(struct svc_memset_op *ops, u32 n, u32 base, + u64 *ops_done) +{ + u32 i; + + for (i = 0; i < n; i++) { + struct svc_memset_op *m = &ops[i]; + u32 seq = base + i; + + if (m->type != 1 || m->phys || m->dst < SVC_VAS_FLOOR) + continue; + if (ops_done && seq < 64 && ((*ops_done >> seq) & 1)) + continue; /* executed (this pass or earlier) */ + svc_dstpend_note(m->dst); + } +} + +static void svc_dstpend_clear(u64 va) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 i; + + for (i = 0; i < svc_dstpend_n; i++) + if (svc_dstpend[i].va_page == page) + svc_dstpend[i].va_page = 0; +} + +static bool svc_dstpend_hit(u64 va) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 i; + + for (i = 0; i < svc_dstpend_n; i++) { + if (svc_dstpend[i].va_page != page) + continue; + if (time_after_eq(jiffies, svc_dstpend[i].since + + msecs_to_jiffies(SVC_DSTPEND_MS))) { + svc_dstpend[i].va_page = 0; + return false; + } + return true; + } + return false; +} + +static bool svc_opfail_latched(u64 va) +{ + u32 i; + u32 free_i = SVC_OPFAIL_MAX; + + for (i = 0; i < SVC_OPFAIL_MAX; i++) { + if (svc_opfail[i].va == va) { + if (time_after_eq(jiffies, + svc_opfail[i].since + + msecs_to_jiffies(SVC_OPFAIL_WINDOW_MS))) { + if (svc_opfail_log < 32) { + svc_opfail_log++; + pr_info("sema-svc: OPFAIL-LATCH va=0x%llx (>=%u ms of failures; op no longer holds its entry)\n", + va, SVC_OPFAIL_WINDOW_MS); + } + return true; + } + return false; + } + if (!svc_opfail[i].va && free_i == SVC_OPFAIL_MAX) + free_i = i; + } + if (free_i < SVC_OPFAIL_MAX) { + svc_opfail[free_i].va = va; + svc_opfail[free_i].since = jiffies; + } + return false; +} + static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, - u32 n_ops, u32 ch_h, u64 *ops_done, u32 ops_base) + u32 n_ops, u32 ch_h, u64 *ops_done, u32 ops_base, + u32 entry_idx, u64 pass_hash) { u32 i; static u64 bigop_log; svc.cur_ch = ch_h; + /* r484: one-shot per call; set only by the hold->pass transition. */ + svc_wait_repass = false; + /* + * m163: apply every queued scanner injection before resolving this + * pass's copy operands. Without this, a CE copy decoded in the window + * between the client's push and its pins being processed resolves + * through the previous client's stale translation (or fails), while + * its release is still applied - the client's cudaMemcpy "completes" + * with no data and the next GR launch computes from the stale vidmem + * page (streams2 r163: a1 held the previous client's h_a; the vadd + * output was a_stale + b_fresh). + */ + svc_handle_map_add(); + /* + * r336/r34x: honor every host-semaphore acquire (C56F SEM_EXECUTE + * op 0/2/3/4/5) captured for THIS ring entry before any of its work + * runs, so a copy cannot execute ahead of the dependency its push + * waits on (the streams2 readback-before-scale class). Decision + * ladder per wait: + * 1. saved value satisfies the op's condition -> pass; + * 2. the entry's own release, when its method offset precedes the + * acquire's (release-then-acquire, channel.c crossEngineSync), + * counts as the candidate value - the release method has + * already executed by the time the acquire is evaluated; + * an acquire that precedes its own trailing tracking release + * does NOT count it (nothing has installed that value yet); + * 3. drain the pending-translation release queue (releases OUR + * machinery is merely late about) and re-read once; + * 4. still unsatisfied -> hold the entry (release withheld through + * the op-pending machinery) for at most gsp_sema_wait_hold_ms + * per episode, then fail open loudly ('give-up': the value + * cannot be installed by firmware alone - it sits behind an + * upstream opdef/operand hold - so the old pass-through + * behaviour is safer than parking the channel forever). + * Unreadable semaphore: fail-open (the pre-r336 behaviour). + * The pass/hold/give-up decisions are logged as 'sema-wait' rows + * (op, payload, observed value, decision). + */ + if (svc_wait_n && gsp_sema_wait_gate) { + struct svc_wait_hold *gh; + bool hold = false; + u32 wi; + /* + * Kill switch: a wait that never satisfies (a counter our + * emulation does not advance the way the driver expects) + * would otherwise park entries until the sweep expires each + * one. After 2048 holds in a boot the gate fails open + * permanently (the pre-r336 behaviour) and says so once. + */ + if (svc_wait_held > 2048) { + if (svc_wait_log < 4096) { + svc_wait_log++; + pr_info("sema-wait: gate disabled after %llu holds (fail-open)\n", + svc_wait_held); + } + svc_wait_pass++; + goto svc_wait_done; + } + gh = svc_wait_hold_get(ch_h, true); + for (wi = 0; wi < svc_wait_n; wi++) { + const struct svc_wait_op *w = &svc_waits[wi]; + u32 cur = 0, waited_ms = 0; + bool ok = true, same_rel = false, pend = false; + + if (gh) + waited_ms = jiffies_to_msecs(jiffies - gh->since); + if (svc_sem_read(vgpu, w->va, &cur)) + continue; /* unreadable: fail-open */ + ok = svc_wait_satisfied(cur, w->op, w->payload); + if (!ok && svc_wait_rel_have && + svc_wait_rel_va == w->va && + svc_wait_rel_off < w->off) { + same_rel = true; + ok = svc_wait_satisfied(svc_wait_rel_payload, + w->op, w->payload); + } + if (!ok) { + svc_apply_pending(vgpu); + if (!svc_sem_read(vgpu, w->va, &cur)) + ok = svc_wait_satisfied(cur, w->op, + w->payload); + } + if (ok) { + if (svc_wait_pass_log < 256) { + svc_wait_pass_log++; + pr_info("sema-wait: pass ch=0x%x va=0x%llx op=%u pay=%u cur=%u same_rel=%u waited_ms=%u -> ops run this pass\n", + ch_h, w->va, w->op, w->payload, + cur, same_rel, waited_ms); + } + continue; + } + pend = svc_wait_pend_release(w->va); + if (waited_ms >= gsp_sema_wait_hold_ms) { + /* + * Bounded episode expired: fail open for the + * WHOLE entry (one row, then the ops run). A + * per-wait continue here would ping-pong + * with a second unsatisfied wait: the first + * gave up and reset the shared clock before + * the second was judged, and the pair would + * hold forever 100 ms at a time. + */ + svc_wait_giveup++; + if (svc_wait_log < 2048) { + svc_wait_log++; + pr_info("sema-wait: give-up ch=0x%x va=0x%llx op=%u pay=%u cur=%u pend=%u waited_ms=%u -> fail-open (cannot be satisfied by firmware alone; ops run)\n", + ch_h, w->va, w->op, w->payload, + cur, pend, waited_ms); + } + if (gh) + gh->active = false; + hold = false; + svc_wait_pass++; + goto svc_wait_done; + } + hold = true; + if (svc_wait_log < 512) { + svc_wait_log++; + pr_info("sema-wait: hold ch=0x%x va=0x%llx op=%u pay=%u cur=%u same_rel=%u pend=%u waited_ms=%u -> entry held (acquire not satisfied)\n", + ch_h, w->va, w->op, w->payload, cur, + same_rel, pend, waited_ms); + } + break; /* one unsatisfied wait holds the entry */ + } + if (hold) { + svc_wait_held++; + svc_wait_hold_h = ch_h; + svc_wait_hold_idx = entry_idx; + svc.op_tr_pending = 1; + return; + } + /* + * r484: the gate just PASSED for an entry it previously held - + * the dependency is installed now, so this pass's ops must + * re-run even if the once-only bitmap says they already did + * (they ran against the pre-dependency state). + */ + if (ch_h == svc_wait_hold_h && + entry_idx == svc_wait_hold_idx) { + svc_wait_repass = true; + svc_wait_hold_idx = 0xffffffffu; + } + svc_wait_pass++; + if (gh) + gh->active = false; + } +svc_wait_done: for (i = 0; i < n_ops; i++) { struct svc_memset_op *m = &ops[i]; u32 seq = ops_base + i; @@ -4654,18 +8249,70 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, * ran; re-running a CE copy would read a staging buffer the * client has since refilled and overwrite finished data. */ - if (ops_done && seq < 64 && ((*ops_done >> seq) & 1)) { + if (!svc_wait_repass && ops_done && seq < 64 && + ((*ops_done >> seq) & 1)) { svc.ops_once_skipped++; + /* + * r456: this op already ran in an earlier pass. Any + * staging snapshot a later re-decode pass re-captured + * for it (the capture sites run on every decode) is + * a leak that the r444 APPLY-free used to sweep; with + * that free removed (r456 keeps the snapshot alive + * across retries) drop it here so it can never serve + * a re-push of the ring slot. + */ + if (m->type == 1 && m->bytes <= SVC_COPY_SNAP_MAX) + svc_copy_snap_free(m->dst, m->src, m->bytes, + ch_h, entry_idx, pass_hash); continue; } - if (m->type == 1) - rc = svc_copy(vgpu, m->dst, m->src, m->bytes, - m->phys, m->aperture, - m->src_phys, m->src_aperture); - else - rc = svc_fill(vgpu, m->dst, m->value, m->elem, - m->count, m->phys, m->aperture); + { + u64 ce_len = (m->type == 1) ? m->bytes : + (u64)m->elem * m->count; + u64 ce_done = 0; + + if (m->type == 1) + rc = svc_copy(vgpu, m->dst, m->src, m->bytes, + m->phys, m->aperture, + m->src_phys, m->src_aperture, + &ce_done, m->rows, m->dpitch, + m->spitch, ch_h, entry_idx, + pass_hash); + else + rc = svc_fill(vgpu, m->dst, m->value, m->elem, + m->count, m->phys, m->aperture, + &ce_done, m->rows, m->dpitch); + /* + * CEAUDIT (2026-10-18 r220): per-op audit of the CE + * data path - the software executor of every guest + * cudaMemcpy - at/above SVC_COPY_AUDIT_MIN, including + * the rc != 0 attempts the copy-level COPYAUDIT cannot + * see (the pre-loop resolution misses return before + * any byte moves: done=0x0). src/dst are the guest + * VAs of the op; done is the byte count that landed. + * Bounded per boot. + */ + if (ce_len >= SVC_COPY_AUDIT_MIN && + svc_ceaudit_n < SVC_CEAUDIT_MAX) { + svc_ceaudit_n++; + /* + * A failing op is retried by the hold and would + * spam: throttle the rc != 0 repeats; successful + * big ops log unconditionally (budget-bounded). + */ + if (rc) + pr_info_ratelimited("vgsp: CEAUDIT op=%s src=0x%llx dst=0x%llx len=0x%llx done=0x%llx rc=%d\n", + m->type == 1 ? "copy" : "fill", + m->type == 1 ? m->src : 0ull, + m->dst, ce_len, ce_done, rc); + else + pr_info("vgsp: CEAUDIT op=%s src=0x%llx dst=0x%llx len=0x%llx done=0x%llx rc=%d\n", + m->type == 1 ? "copy" : "fill", + m->type == 1 ? m->src : 0ull, + m->dst, ce_len, ce_done, rc); + } + } if ((rc == 0 || rc == SVC_COPY_FORWARDED) && ops_done && seq < 64) *ops_done |= 1ULL << seq; @@ -4692,11 +8339,28 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, */ if (rc == SVC_COPY_FORWARDED) continue; - if (rc == 0) + if (rc == 0) { svc.memsets_done++; - else if (rc == -ENOENT) + /* r507a: a success re-arms the failure window. */ + svc_opfail_clear(m->dst); + /* r541: the device page's writer has run; lift the + * launch-ordering guard for it. */ + svc_dstpend_clear(m->dst); + } else if (rc == -ENOENT) { + /* + * r541: a copy/fill that cannot run yet leaves its + * target page stale; a GR launch must not read it + * (see svc_dstpend_note). Record every unexecuted + * target - the VA-page key keeps the guard narrow. + */ + svc_dstpend_note(m->dst); + if (svc_dstpend_log < 64) { + svc_dstpend_log++; + pr_info("sema-svc: DSTPEND note va=0x%llx type=%u ap=%u (device write unexecuted; GR launches touching this page hold)\n", + m->dst, m->type, m->aperture); + } svc.memsets_nomap++; - else + } else svc.memsets_fail++; /* * ISSUE-0002 (r71/r72): a failed LARGE op leaves work owed on @@ -4713,9 +8377,20 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, * (observed r72 c2) must not withhold a release that gates a GR * launch with a 5 s poll deadline. On expiry the release is * flushed and the op stays skipped (the pre-r62 fallback). + * m163 extends the hold to copy ops whose destination is a + * device buffer: a lost small H2D that silently "completes" + * corrupts the next launch's input (streams2 r163: the 128 KB + * a1 copy's release was applied with no copy op ever run; the + * vadd computed a_stale + b and the client read the wrong c). + * A small failing device copy now costs a bounded hold and the + * retried decode runs it once the pins land - never silent + * corruption. Small failing memsets keep the old trade. */ if (rc != 0 && rc != SVC_COPY_FORWARDED && - m->bytes >= (256u << 10)) + (m->bytes >= (256u << 10) || + (m->type == 1 && m->dst >= SVC_VAS_FLOOR) || + (m->type == 0 && !m->phys && m->dst >= SVC_VAS_FLOOR)) && + !svc_opfail_latched(m->dst)) svc.op_tr_pending = 1; if (gsp_sema_dbg && m->bytes == 256 && svc.sem256_log < 64) { svc.sem256_log++; @@ -4726,7 +8401,7 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, } if (gsp_sema_dbg && ((rc == 0 && svc.memsets_done <= gsp_sema_ops_log) || - (rc != 0 && svc.memsets_fail + svc.memsets_nomap <= 64))) + (rc != 0 && svc.memsets_fail + svc.memsets_nomap <= 2048))) pr_info("sema-svc: %s dst=0x%llx value=0x%llx elem=%u count=%u bytes=%llu phys=%u ap=%u src=0x%llx srcphys=%u srcap=%u rc=%d h=0x%x\n", m->type == 1 ? "memcopy" : "memset", m->dst, m->value, m->elem, m->count, @@ -4744,7 +8419,7 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, * read-back signature: always keep room for it past the * general trace budget so the DtoH's src/dst/rc is on record. */ - if ((svc.op_trace_log < 160 && + if ((svc.op_trace_log < (gsp_sema_dbg ? 32768 : 2048) && (time_before(jiffies, svc.op_trace_until) || m->dst >= SVC_VAS_FLOOR || (m->type == 1 && !m->src_phys && m->src >= SVC_VAS_FLOOR))) || @@ -4873,6 +8548,8 @@ static u64 svc_rel_va_seen; * on every decoded release regardless of its target. */ static u32 svc_rel_count; +static u32 svc_fake_rel_log; /* r576b DIAG: sema-fake witness budget */ +static u32 svc_ce_foreign_log; /* r577: foreign-release refusal witness */ static u32 svc_dbg_streams; static u32 svc_chan_put_log; /* ISSUE-0002 (r55): per-rank put-snapshot throttle (ch is a local copy). */ @@ -4973,6 +8650,8 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h, bool skip; int i; u32 dbg = 0; + u64 rel_pa = 0; + u64 cnt_pa = 0; if (!h || !payload) { /* @@ -5000,7 +8679,8 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h, spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { - if (svc.chan[i].valid && svc.chan[i].h_object == h) { + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].h_object == h) { /* * The release targets the slot the push itself named * (release_pa learned from the release VA); the @@ -5009,12 +8689,23 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h, * FORENSICS-grv7-counter F1). Fall back to counter_pa * when the release slot could not be resolved. */ - pa = svc.chan[i].release_pa ? - svc.chan[i].release_pa : svc.chan[i].counter_pa; + rel_pa = svc.chan[i].release_pa; + cnt_pa = svc.chan[i].counter_pa; + pa = rel_pa ? rel_pa : cnt_pa; break; } } spin_unlock_irqrestore(&svc.chan_lock, flags); + /* + * C1 (GAP-1 H3/H4): was the apply entered, and with which slot? + * Budget 512/session (reset in svc_session_flush); the per-branch + * prints below stay on their own counter_rel_dbg budget. + */ + if (gsp_sema_dbg && svc.crel_entry_dbg < 512) { + svc.crel_entry_dbg++; + pr_info("sema-diag: crel-entry h=0x%x va=0x%llx pay=%u inc=%d rel_pa=0x%llx cnt_pa=0x%llx\n", + h, sem_va, payload, inc ? 1 : 0, rel_pa, cnt_pa); + } if (!pa) { /* * Pool (UVM tracking) semaphores have no counter slot. The @@ -5051,7 +8742,7 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h, wrc = svc_sem_write(vgpu, h, sem_va, payload); if (!wrc) { svc.releases_applied++; - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); if (gsp_sema_dbg && svc.counter_rel_dbg < 256) { svc.counter_rel_dbg++; pr_info("sema-chan: rel-apply h=0x%x va=0x%llx pay=%u cur=%u -> pool rescue\n", @@ -5072,6 +8763,7 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h, spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { if (svc.chan[i].valid && + svc.chan[i].vgpu == vgpu && svc.chan[i].h_object == h) { if (payload > svc.chan[i].pending_rel_pay) { @@ -5124,6 +8816,15 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h, return; } nv = inc ? cur + payload : payload; + /* + * r378: the slot PA was learned from the release VA (or registered + * from the client's pagemap) earlier; re-validate it against the + * current pin before the 4-byte release lands, so a slot the guest + * has since freed/re-homed is never written through the stale page. + */ + if (sem_va && !gsp_sema_write_guard(sem_va, pa, SVC_APERTURE_SYS, + "counter-rel")) + return; if (svc_write_ap(vgpu, pa, SVC_APERTURE_SYS, &nv, sizeof(nv))) { if (gsp_sema_dbg && svc.counter_rel_dbg < 256) { svc.counter_rel_dbg++; @@ -5135,7 +8836,7 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h, svc.releases_applied++; svc.counter_rel_applied++; svc.gr_rel_applied++; - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); if (gsp_sema_dbg && svc.counter_rel_dbg < 256) { svc.counter_rel_dbg++; pr_info("sema-chan: rel-apply h=0x%x pa=0x%llx pay=%u cur=%u -> written\n", @@ -5151,7 +8852,7 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h, * harmless for an absolute release (its guard drops it), but an increment * would advance the slot twice. */ -static void svc_counter_pending_clear(u32 h) +static void svc_counter_pending_clear(struct nv_vgpu_device *vgpu, u32 h) { unsigned long flags; int i; @@ -5160,7 +8861,8 @@ static void svc_counter_pending_clear(u32 h) return; spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { - if (svc.chan[i].valid && svc.chan[i].h_object == h) { + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].h_object == h) { svc.chan[i].pending_rel_pay = 0; svc.chan[i].pending_rel_inc = 0; break; @@ -5194,13 +8896,13 @@ static void svc_counter_flush_pending(struct nv_vgpu_device *vgpu, if (payload > 0x100000) { ch->pending_rel_pay = 0; ch->pending_rel_inc = 0; - svc_counter_pending_clear(ch->h_object); + svc_counter_pending_clear(ch->vgpu, ch->h_object); return; } } else if (payload <= cur || payload - cur > 0x100000) { ch->pending_rel_pay = 0; ch->pending_rel_inc = 0; - svc_counter_pending_clear(ch->h_object); + svc_counter_pending_clear(ch->vgpu, ch->h_object); if (cur > ch->counter_last) ch->counter_last = cur; return; @@ -5211,13 +8913,13 @@ static void svc_counter_flush_pending(struct nv_vgpu_device *vgpu, return; ch->pending_rel_pay = 0; ch->pending_rel_inc = 0; - svc_counter_pending_clear(ch->h_object); + svc_counter_pending_clear(ch->vgpu, ch->h_object); if (nv > ch->counter_last) ch->counter_last = nv; svc.releases_applied++; svc.counter_rel_applied++; svc.gr_rel_applied++; - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); if (gsp_sema_dbg && svc.counter_rel_dbg < 256) { svc.counter_rel_dbg++; pr_info("sema-chan: rel-flush h=0x%x pa=0x%llx pay=%u cur=%u -> written\n", @@ -5259,13 +8961,13 @@ static struct { u64 pa_page; } svc_cnt_der[SVC_CNT_DER_MAX]; static u32 svc_cnt_der_n; +static u32 svc_cnt_der_log; static int svc_counter_derive(struct nv_vgpu_device *vgpu, u64 va, u64 *pa_out) { u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); u64 off = va & (SEMA_PAGE_SIZE - 1); u64 pa = 0; - int ap = SVC_APERTURE_SYS; u32 probe = 0; u32 i; bool fresh = true; @@ -5278,13 +8980,57 @@ static int svc_counter_derive(struct nv_vgpu_device *vgpu, u64 va, u64 *pa_out) } } if (fresh) { - if (svc_walk_vaspaces(vgpu, page, &pa, &ap) && - svc_walk_discovered(vgpu, page, &pa, &ap) && - svc_targeted_resolve(vgpu, page, &pa)) - return -ENOENT; - if (ap != SVC_APERTURE_SYS) + u64 p = 0; + int a = SVC_APERTURE_SYS; + bool ok = false; + const char *src = "none"; + + /* + * m177b: first READABLE source wins. The previous chain kept + * the first source that merely PRODUCED a translation and then + * failed -EIO on an unreadable page without trying the rest: + * the published directory can yield a stale/incorrect chain + * for a counter VA while the rm_mem-content scan would resolve + * it (r194: the counter page 0x20400f000 never resolved; its + * walk hit probed unreadable and targeted was never reached). + */ + if (!svc_walk_vaspaces(vgpu, page, &p, &a) && + a == SVC_APERTURE_SYS && + !svc_read_ap(vgpu, + (p & ~(u64)(SEMA_PAGE_SIZE - 1)) | off, + SVC_APERTURE_SYS, &probe, sizeof(probe))) { + ok = true; + src = "vaspaces"; + } + if (!ok && !svc_walk_discovered(vgpu, page, &p, &a) && + a == SVC_APERTURE_SYS && + !svc_read_ap(vgpu, + (p & ~(u64)(SEMA_PAGE_SIZE - 1)) | off, + SVC_APERTURE_SYS, &probe, sizeof(probe))) { + ok = true; + src = "discovered"; + } + if (!ok && !svc_targeted_resolve(vgpu, page, &p) && + !svc_read_ap(vgpu, + (p & ~(u64)(SEMA_PAGE_SIZE - 1)) | off, + SVC_APERTURE_SYS, &probe, sizeof(probe))) { + ok = true; + src = "targeted"; + } + if (!ok) { + if (svc_cnt_der_log < 16) { + svc_cnt_der_log++; + pr_info("sema-chan: counter page 0x%llx unreadable via vaspaces/discovered/targeted; slot stays pending\n", + page); + } return -ENOENT; - pa = (pa & ~(u64)(SEMA_PAGE_SIZE - 1)) | off; + } + pa = (p & ~(u64)(SEMA_PAGE_SIZE - 1)) | off; + if (svc_cnt_der_log < 16) { + svc_cnt_der_log++; + pr_info("sema-chan: counter page 0x%llx resolved via %s (pa=0x%llx)\n", + page, src, pa); + } } /* Read the slot before trusting the mapping: an unreadable page is * not registered, so the caller keeps the "slot pending" fallback. */ @@ -5302,14 +9048,22 @@ static int svc_counter_derive(struct nv_vgpu_device *vgpu, u64 va, u64 *pa_out) /* * Auto-register a GR channel's completion-counter slot from the observed - * layout, without waiting for a readable push: the counter records form a - * 16-byte-strided array in the client's counter page, allocated in channel - * order from the low end (SA-COUNTER FORENSICS-grv7-counter: the ascending - * slot fec0 + r*0x10 is the polled completion word; the push's REPORT_SEM - * release writes a DIFFERENT (descending) family and is tracked separately - * in release_pa). The slot PA comes from a host-verified pin or from the - * in-kernel structural derivation (svc_counter_derive), so the write target - * is a page the client actually polls. + * layout, without waiting for a readable push: the client zips its GR + * channels in ascending handle order onto DESCENDING 16-byte slots, so the + * rank-r channel's own completion word - the slot its pushes' REPORT_SEM + * release names, verified for all 20 ranks - is + * 0x20400fec0 + (SVC_GR_COUNTER_TOTAL-1-r)*0x10. The ascending formula + * (F1/eea5ea93) registered the completion word of the PAIRED rank (19-r) + * instead, so svc_counter_publish wrote each channel's ring PUT into + * another channel's waited word: the busy ctx-init channel rank 16 + * (h=0x5c000059, PUT~94) overwrote rank 3's completion word 0x20400ffc0 + * from 1 to 94 (witness `pub-adv h=0x5c000059 pa=...fc0 val=94 cur=1`), + * after which the rank-3 stream's own release (payload 1) was + * forward-only-skipped and its sync never completed (evtest3 + * cudaStreamSynchronize(st[2]); victory_streams_events). The slot PA comes + * from a host-verified pin or from the in-kernel structural derivation + * (svc_counter_derive), so the write target is a page the client actually + * polls. */ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu, struct svc_channel *ch, int n_gr) @@ -5322,7 +9076,8 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu, return; if (ch->gr_index >= n_gr) return; - va = 0x20400fec0ULL + (u64)ch->gr_index * 0x10ULL; + va = 0x20400fec0ULL + + (u64)(SVC_GR_COUNTER_TOTAL - 1 - ch->gr_index) * 0x10ULL; if (ch->counter_pa) { /* * FORENSICS-v2 anomaly 6: a slot derived in an earlier pass @@ -5350,6 +9105,7 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu, ch->counter_pa = ipa & ~0x3ULL; for (j = 0; j < SVC_MAX_CHANNELS; j++) { if (svc.chan[j].valid && + svc.chan[j].vgpu == vgpu && svc.chan[j].h_object == ch->h_object) { svc.chan[j].counter_pa = ch->counter_pa; @@ -5398,6 +9154,7 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu, spin_lock_irqsave(&svc.chan_lock, flags); for (j = 0; j < SVC_MAX_CHANNELS; j++) { if (svc.chan[j].valid && + svc.chan[j].vgpu == vgpu && svc.chan[j].h_object != ch->h_object && svc.chan[j].counter_pa == pa) { collision = true; @@ -5423,6 +9180,7 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu, ch->counter_pa = pa; for (j = 0; j < SVC_MAX_CHANNELS; j++) { if (svc.chan[j].valid && + svc.chan[j].vgpu == vgpu && svc.chan[j].h_object == ch->h_object) { svc.chan[j].counter_pa = pa; break; @@ -5448,12 +9206,14 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu, /* * Learn a GR channel's REPORT_SEM release slot from a release VA the push * carried (0x20400fXX observed). This is the slot the release write - * targets (release_pa); it is deliberately NOT the completion-counter slot - * (counter_pa), which the publish advances and which the client polls - * (SA-COUNTER FORENSICS-grv7-counter F1: green wrote both families of the - * counter page). An injected pin is the authoritative source; on a miss the - * slot is derived in-kernel (svc_counter_derive) so the release is authored - * immediately rather than waiting for the external pin watcher. + * targets (release_pa) and - per the descending layout the autoreg above + * documents - the channel's own polled completion word, the same slot the + * autoreg registers as counter_pa; keeping it in its own field preserves + * the push-named truth even when the registration derive fails, and + * svc_counter_publish prefers it. An injected pin is the authoritative + * source; on a miss the slot is derived in-kernel (svc_counter_derive) so + * the release is authored immediately rather than waiting for the external + * pin watcher. */ static void svc_counter_learn(struct nv_vgpu_device *vgpu, u32 h, u64 sem_va) { @@ -5482,7 +9242,8 @@ static void svc_counter_learn(struct nv_vgpu_device *vgpu, u32 h, u64 sem_va) spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { - if (svc.chan[i].valid && svc.chan[i].h_object == h) { + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].h_object == h) { if (svc.chan[i].release_pa != (pa & ~0x3ULL)) { svc.chan[i].release_pa = pa & ~0x3ULL; svc.chan[i].relearn_n++; @@ -5497,6 +9258,13 @@ static void svc_counter_learn(struct nv_vgpu_device *vgpu, u32 h, u64 sem_va) pr_info("sema-chan: release learned h=0x%x pa=0x%llx (from release va=0x%llx)\n", h, pa & ~0x3ULL, sem_va); } + /* + * r378: keep the VA the slot was learned from so a + * late write to release_pa (the stale-recover zero) + * can re-validate the target through + * gsp_sema_write_guard before it lands. + */ + svc.chan[i].release_va = sem_va; break; } } @@ -5529,7 +9297,7 @@ static bool svc_rel_apply_906f(struct nv_vgpu_device *vgpu, cur = (op == HOST_SEM_OP_REDUCTION) ? cur + payload : payload; if (!svc_sem_write(vgpu, ch->h_object, va, cur)) { svc.releases_applied++; - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); return true; } } @@ -5557,7 +9325,7 @@ static bool svc_rel_apply_reportsem(struct nv_vgpu_device *vgpu, svc.pushes_pending++; if (!svc_sem_write(vgpu, ch->h_object, va, payload)) { svc.releases_applied++; - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); return true; } return false; @@ -5568,7 +9336,33 @@ static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu, u32 payload, u32 cur, bool pending, bool inc, bool inc_apply) { - if (pending || inc_apply) { + /* + * P4 (GAP-1 H3): the exact-slot counter delivery runs before every + * path that can early-return, so a GR completion-counter release can + * never be skipped by the generic write's outcome. The generic + * write/learn tail below is skipped entirely for counter VAs: an + * increment release is not idempotent, so it must apply exactly once + * (the early return here is that exactly-once edge). + */ + if (ch->engine_type == 0 && svc_counter_va(sem_va)) { + if (pending || inc_apply) + svc.pushes_pending++; + svc_counter_learn(vgpu, ch->h_object, sem_va); + svc_counter_release_apply(vgpu, ch->h_object, sem_va, payload, + inc && inc_apply); + return false; + } + /* + * r506w: generic absolute releases are forward-only by VALUE, whichever + * pass discovers the gap. The recorded pending bit came from the + * decode's read of the release word, and that read can be blinded by a + * stale/crossed landing (r506 witness: a withheld tracker release went + * to the flush with pending=0 while the fresh read still showed 0, and + * the release was then dropped by the counter fallback below - the + * client's tracker never advanced and its stream ground at the 10 s + * bound per push). Increments keep the one-shot claim gate (`inc_apply`). + */ + if (pending || inc_apply || (!inc && payload > cur)) { svc.pushes_pending++; if (inc) cur += payload; @@ -5579,19 +9373,41 @@ static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu, * generic translation can be a stale content-scanned page; * those releases go through the injected-pin-backed exact slot * (svc_counter_learn / svc_counter_release_apply) below. + * Counter VAs are handled by the early block above; the check + * stays as belt-and-braces for the generic write. */ if (ch->engine_type == 0 && svc_counter_va(sem_va)) { /* handled by the exact-slot path */ } else if (!svc_sem_write(vgpu, ch->h_object, sem_va, cur)) { svc.releases_applied++; - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); return true; } } - if (ch->engine_type == 0) - svc_counter_learn(vgpu, ch->h_object, sem_va); - svc_counter_release_apply(vgpu, ch->h_object, sem_va, payload, - inc && inc_apply); + /* + * r506w: the exact-slot counter fallback is for COUNTER VAs only. Its + * `release_pa ?: counter_pa` fallback writes (and forward-only-skips + * against) the channel's REGISTERED slot, which belongs to a different + * word - the r506 witness: releases to the tracker VA 0x20480fef0 were + * skipped against the polluted PUT counter (cur=96) and lost. For a + * non-counter VA the generic write above is the delivery path. + */ + if (svc_counter_va(sem_va)) { + if (ch->engine_type == 0) + svc_counter_learn(vgpu, ch->h_object, sem_va); + svc_counter_release_apply(vgpu, ch->h_object, sem_va, payload, + inc && inc_apply); + } + /* + * Stall-class fix (R2): when the decode said this absolute release is + * genuinely pending and every write attempt above failed without + * queueing, the release must not vanish - retry it through the pending + * pass. Increments stay excluded (their one-shot claim makes + * re-queueing unsafe until a diag run proves otherwise). + */ + if (pending && !inc && !(ch->engine_type == 0 && + svc_counter_va(sem_va))) + svc_pending_add(sem_va, payload, 0); return false; } @@ -5695,6 +9511,93 @@ static void svc_fwd_rel_direct_note(u32 h, u64 va, u32 payload) * mapped to its register offset. Each completed QMD resets the accumulator, * so a push may carry more than one launch. */ +/* + * m164: per-decode QMD witness for the CLASS-2 no-capture hold. Set to the + * max number of QMD words any accumulator saw during the current + * svc_decode_stream() call; reset at its top and read by the entry loop's + * no-capture check. Service-thread only. + */ +static u32 svc_gr_qmd_seen; +static u64 svc_gr_nocap_log; + +/* + * r383: has a launch with this push VA ever been captured? Scans the live + * candidate ring, the recent-capture history and the r401 session-wide + * (push_va, hash) seen table. Used by the replayed-QMD dispatch below: a + * replay whose push was never captured is a launch that would otherwise be + * silently dropped (the streams2 second-vadd class); a replay of a + * captured/offered push is the exactly-once no-op. The hash is required + * for the seen table so a later, DIFFERENT push at the same VA (a loop + * iteration: same entry words, new content) is never misclassified as + * seen - the caller only consults this on a true replay anyway. + */ +static bool svc_gr_cand_push_seen(u64 push_va, u64 hash) +{ + u32 i; + + if (!push_va) + return false; + for (i = 0; i < SVC_GR_CAND_RING; i++) + if (svc_gr_cand_ring[i].valid && + svc_gr_cand_ring[i].push_va == push_va) + return true; + for (i = 0; i < SVC_GR_CAP_HIST; i++) + if (svc_gr_cap_hist[i] == push_va) + return true; + return svc_gr_seen_lookup(push_va, hash); +} + +/* + * r391f: once-only identity for GRCE inline writes. A push's + * LOAD_INLINE_DATA runs author bytes into the vidmem shadow; a re-decode of + * the same push (replay/rescue/rehold) must not re-run them - the target + * page may have been superseded in between (victory_reduce: the client's + * cudaMemset inline write re-zeroed the accumulator after the launch's + * copy-out, the sum=0 class). Keyed by (push_va, push content hash, dst, + * bytes): the release payload inside every push makes the hash unique per + * push, so a new client's identical-looking write never collides. Session + * scoped (cleared by svc_session_flush). Service-thread only. + * + * Sizing (r398): the ring must outlive the gap between a push's first + * execution and its next re-decode. The observed failing case had 76 + * inline writes between the two, which evicted a 64-slot table and let + * the memset re-run; 8192 slots (256 KB) gives a ~100x margin over the + * busiest observed burst. + */ +#define SVC_INLINE_ONCE_SLOTS 8192 +static struct { + u64 push_va; + u64 hash; + u64 dst; + u32 bytes; +} svc_inline_once[SVC_INLINE_ONCE_SLOTS]; +static u32 svc_inline_once_next; + +static bool svc_inline_once_seen(u64 push_va, u64 hash, u64 dst, u32 bytes) +{ + u32 i; + + if (!push_va) + return false; + for (i = 0; i < SVC_INLINE_ONCE_SLOTS; i++) + if (svc_inline_once[i].push_va == push_va && + svc_inline_once[i].hash == hash && + svc_inline_once[i].dst == dst && + svc_inline_once[i].bytes == bytes) + return true; + return false; +} + +static void svc_inline_once_store(u64 push_va, u64 hash, u64 dst, u32 bytes) +{ + u32 i = svc_inline_once_next++ % SVC_INLINE_ONCE_SLOTS; + + svc_inline_once[i].push_va = push_va; + svc_inline_once[i].hash = hash; + svc_inline_once[i].dst = dst; + svc_inline_once[i].bytes = bytes; +} + static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, struct svc_push_state *st, u32 addr, u32 count, const u8 *data, const u8 *push, u32 push_len, @@ -5724,6 +9627,8 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, } } } + if (st->qmd_n > svc_gr_qmd_seen) + svc_gr_qmd_seen = st->qmd_n; if (st->qmd_have == 3) st->qmd_dev_va = (((u64)st->qmd_hi << 32) | st->qmd_lo) << 8; if (st->qmd_n == SVC_QMD_WORDS) { @@ -5757,13 +9662,70 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, memcpy(recs[n].data, rec->data, rec->len); n++; } - if (gsp_phys_gr_forward_enabled() && trusted && !replay) - svc_gr_cand_capture(ch, st, recs, n, push, push_len, - st->push_va); - else if (!replay || !gsp_phys_gr_forward_enabled()) + /* + * C10 (GAP-2 A5): duplicate destinations in the newest-first + * inline capture decide which record wins the backend's + * overlay; bounded and unconditional (gsp_sema_dbg is defined + * later in the file and this site may not reference it). + */ + if (svc_inline_recs_log < 64) { + u32 dup = 0, a2, b2; + + for (a2 = 0; a2 < n; a2++) + for (b2 = a2 + 1; b2 < n; b2++) + if (recs[a2].dst_va == + recs[b2].dst_va) + dup++; + svc_inline_recs_log++; + pr_info("sema-gr: inline recs n=%u dup_dst=%u\n", + n, dup); + } + if (gsp_phys_gr_forward_enabled()) { + u64 push_hash = (push && push_len) ? + svc_push_hash(push, push_len) : 0; + + if (trusted && !replay) { + svc_gr_cand_capture(ch, st, recs, n, push, + push_len, st->push_va); + } else if (trusted && replay && + !svc_gr_cand_push_seen(st->push_va, + push_hash)) { + /* + * r383 (streams2 second-vadd class): a + * complete QMD first seen on a replay pass + * fell through BOTH arms below - the + * forward path skips replays and the + * bridge arm needs !replay - so the + * launch was silently dropped (zero + * engine writes, client read stale/zero + * outputs). The push VA was never + * captured, so capture it now (trusted + * content only, as the first-sight path); + * a replay of an already captured/offered + * launch stays the exactly-once no-op. + */ + pr_info("sema-gr: REPLAY-CAPTURE h=0x%x push=0x%llx (complete QMD first seen on a replay pass; no capture on record)\n", + ch->h_object, st->push_va); + svc_gr_cand_capture(ch, st, recs, n, push, + push_len, st->push_va); + } else if (replay && svc_gr_nocap_log < 64) { + svc_gr_nocap_log++; + pr_info("sema-gr: LAUNCH-SKIP h=0x%x push=0x%llx (replay; already captured=%d trusted=%d)\n", + ch->h_object, st->push_va, + svc_gr_cand_push_seen(st->push_va, + push_hash), + trusted); + } else if (!replay) { + gsp_cuda_bridge_launch(vgpu, ch->h_object, + st->qmd, + st->qmd_dev_va, + st->push_va, recs, n); + } + } else if (!replay) { gsp_cuda_bridge_launch(vgpu, ch->h_object, st->qmd, st->qmd_dev_va, st->push_va, recs, n); + } st->qmd_n = 0; st->qmd_bits = 0; st->qmd_have = 0; @@ -5772,6 +9734,89 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, } } +static u64 svc_pass_hash; /* r391f: this decode pass's push content hash */ + +/* + * r481 RISK-REDUCTION diagnostic (default 0): dump the full decoded method + * stream of the first few large GR-class pushes (len >= 1024) so the launch + * push's acquire/release method groups (or their absence) are visible + * verbatim. The streams_events class evidence shows the st2 vadd's copy-in + * running ~115 ms BEFORE its stream's H2D soft copies; the ordering the + * driver relied on is only provable by reading the push's own methods. + * Bounded: gsp_sema_push_raw dumps, at most once per push VA. + */ +unsigned gsp_sema_push_raw; +static u64 svc_pushraw_last_va; +static u32 svc_pushraw_n; + +static void svc_push_raw_dump(const u8 *buf, u32 len, u64 push_va, + u32 h_object) +{ + u32 off = 0, gi = 0; + + if (!gsp_sema_push_raw || svc_pushraw_n >= gsp_sema_push_raw) + return; + if (len < 512 || len > 8192) + return; + if (push_va == svc_pushraw_last_va) + return; + svc_pushraw_last_va = push_va; + svc_pushraw_n++; + pr_info("sema-pushraw: BEGIN h=0x%x va=0x%llx len=%u n=%u\n", + h_object, push_va, len, svc_pushraw_n); + while (off + 4 <= len && gi < 160) { + u32 hdr, op, cnt, subch, addr, v[4] = {0,0,0,0}, k, n; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 0x7; + cnt = (hdr >> 16) & 0xfff; + subch = (hdr >> 13) & 0x7; + addr = (hdr & 0x1fff) << 2; + n = cnt > 4 ? 4 : cnt; + for (k = 0; k < n && off + 8 + 4 * k <= len; k++) + memcpy(&v[k], buf + off + 4 + 4 * k, 4); + pr_info("sema-pushraw: [%u] off=0x%x op=%u sub=%u addr=0x%x cnt=%u v0=%08x v1=%08x v2=%08x v3=%08x\n", + gi, off, op, subch, addr, cnt, + v[0], v[1], v[2], v[3]); + if (op == 3 && cnt) { + off += 4 + cnt * 4; + gi++; + continue; + } + if (!cnt) { + gi++; + off += 4; + continue; + } + if (op == 1) { + off += 4 + cnt * 4; + gi++; + continue; + } + { + u32 p; + bool found = false; + + for (p = off + 4; p + 4 <= len; p += 4) { + u32 h2; + + memcpy(&h2, buf + p, 4); + if (((h2 >> 29) & 0x7) == 1 && + ((h2 >> 16) & 0xfff)) { + off = p; + found = true; + break; + } + } + if (!found) + off = len; + gi++; + } + } + pr_info("sema-pushraw: END h=0x%x va=0x%llx consumed=0x%x/%u\n", + h_object, push_va, off, len); +} + static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u32 len, u64 page_pa, bool allow_ops, struct svc_channel *ch, u32 entry_idx, @@ -5779,6 +9824,35 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u64 *ops_done) { struct svc_push_state st; + + /* r391f: the inline-write once-only key needs the push's identity. */ + svc_pass_hash = svc_push_hash(buf, len); + + /* + * r316: pending releases record the channel they were decoded for; + * the verified pool resolver is channel-keyed (see svc_pending_add + * and the matmul r322 starved/hung note). + */ + svc.cur_ch = ch->h_object; + /* + * r34x: the acquire-wait capture is PER RING ENTRY. It used to be + * reset only at SET_OBJECT, so a decode pass over a later entry (or + * a set of entries, since a held entry is re-decoded after the + * sweep walks ahead) leaked its acquire into an earlier entry's + * release-site gate: r340 held a D2H copy entry that carries no + * acquire at all on a later launch's 'wait for the next tracking + * value' (need=85 while the held entry releases 84) - a circular + * wait only our own withheld release could ever satisfy, so every + * hold expired via the 10 s fallback (wait=4542 etc.). Reset here, + * per entry; SET_OBJECT still resets at the push boundary, and the + * re-decode re-captures the same waits, so held entries re-evaluate + * identically. + */ + svc_wait_n = 0; + svc_wait_rel_va = 0; + svc_wait_rel_payload = 0; + svc_wait_rel_off = 0; + svc_wait_rel_have = false; /* * Static for the same reason as svc_handle_chan_dump: a 32-entry ops * array is a ~2 KB frame, and this is called from the single service @@ -5790,6 +9864,9 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u32 ops_seq = 0; /* ops appended this pass (sequence numbers) */ u32 off = 0; + /* m164: fresh CLASS-2 QMD witness for this decode pass. */ + svc_gr_qmd_seen = 0; + /* * A false VA->PA pair (the discovered map is content-scanned) can * point a ring entry's push VA at an unrelated page - observed: @@ -5804,6 +9881,13 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, allow_ops = false; } + /* + * r481: verbatim method-stream dump for large GR-class pushes + * (default 0 = off). See svc_push_raw_dump. + */ + if (gsp_sema_push_raw && ch && !ch->engine_type && len >= 512) + svc_push_raw_dump(buf, len, push_va, ch->h_object); + memset(&st, 0, sizeof(st)); st.push_va = push_va; /* @@ -5822,21 +9906,52 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * capture: the pump consumes a capture immediately after the pass that * built it, so anything still here is stale. Replay/rescue decodes * must not wipe a capture the pump has not consumed yet. + * + * r548: a valid, never-offered candidate is NOT stale - it is a launch + * whose forward was DEFERRED (r541/r543 guards, backend busy): the + * pump deliberately left it for a bounded retry. Wiping it here let + * the retry passes re-decode the entry as a replay (candidate already + * captured) and then consume it without any forward or completion + * (the r547 saxpy stall: both launches deferred, ring wiped by an + * interleaved fresh decode, zero GR LAUNCH / GR-COMPLETE-SW for 16+ + * minutes). Keep it; the capture path (svc_gr_cand_capture) rotates to + * a free ring slot when a different launch needs the current one. */ - if (!replay && trusted && ch->engine_type == 0) + if (!replay && trusted && ch->engine_type == 0 && + (!svc_gr_cand.valid || svc_gr_cand.offered)) memset(&svc_gr_cand, 0, sizeof(svc_gr_cand)); svc_fwd_rel_n = 0; svc_fwd_defer_armed = false; svc.pages_decoded++; /* Diagnostic: name every push decoded in the post-completion window. */ - if (svc.op_trace_log < 160 && + if (svc.op_trace_log < (gsp_sema_dbg ? 32768 : 2048) && time_before(jiffies, svc.op_trace_until)) { svc.op_trace_log++; pr_info("sema-pushtrace: h=0x%x eng=%u va=0x%llx len=%u allow=%d trusted=%d\n", ch->h_object, ch->engine_type, push_va, len, allow_ops, trusted); } - if (gsp_sema_dbg && ch->engine_type && svc_dbg_streams < 96) { + /* + * fail4: de-windowed decode trace - every push the service looks at, + * bounded per session. The windowed/identified-channel diagnostics + * above can miss a whole client's push stream (unidentified channel, + * no bridge completion to arm the window); this record cannot. + */ + /* + * fail4 r302: with sema_dbg on the boot-time driver init spends the + * 512-line budget long before the run under diagnosis, leaving the + * interesting push stream (a later client's copies) invisible. The + * debug flag raises the bound so one boot covers init + run. + */ + if (svc.pushtrace_all < (gsp_sema_dbg ? 32768 : 512)) { + svc.pushtrace_all++; + pr_info("sema-pushall: h=0x%x eng=%u va=0x%llx pa=0x%llx len=%u allow=%d trusted=%d first8=%08x %08x\n", + ch->h_object, ch->engine_type, push_va, page_pa, len, + allow_ops, trusted, + len >= 4 ? *(const u32 *)(buf + 0) : 0, + len >= 8 ? *(const u32 *)(buf + 4) : 0); + } + if (gsp_sema_dbg && svc_dbg_streams < 96) { svc_dbg_streams++; pr_info("sema-dbg: stream h=0x%x eng=%u va=0x%llx len=%u allow=%d w=%08x %08x %08x %08x %08x %08x %08x %08x\n", ch->h_object, ch->engine_type, push_va, len, allow_ops, @@ -5884,28 +9999,195 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, addr = (hdr & 0x1fff) << 2; /* + * fail4: name every big method-data run and every run whose + * address is the LOAD_INLINE_DATA method (any op/subch - the + * encoding of the 64 KB H2D upload is what we are pinning). + */ + if (gsp_sema_dbg && len >= 8192 && count >= 1024 && + svc.bigrun_log < 128) { + svc.bigrun_log++; + pr_info("sema-bigrun: h=0x%x push=0x%llx off=0x%x hdr=0x%08x op=%u cnt=%u subch=%u addr=0x%x have_out=%u out=0x%llx line=%u lcnt=%u w=%08x %08x\n", + ch->h_object, push_va, off, hdr, op, count, subch, + addr, st.have_out, st.offset_out, st.line_length, + st.gr_line_count, + off + 8 <= len ? *(const u32 *)(buf + off + 4) : 0, + off + 12 <= len ? *(const u32 *)(buf + off + 8) : 0); + } + if (gsp_sema_dbg && len >= 8192 && + (addr == GRCE_M_LOAD_INLINE_DATA || addr == 0x1b8) && + svc.lid_hdr_log < 64) { + svc.lid_hdr_log++; + pr_info("sema-lid-hdr: h=0x%x push=0x%llx off=0x%x hdr=0x%08x op=%u cnt=%u subch=%u addr=0x%x trusted=%d allow=%d\n", + ch->h_object, push_va, off, hdr, op, count, subch, + addr, trusted, allow_ops); + } + + /* * GRCE LOAD_INLINE_DATA: a data run (opcode 3) that carries * the DMA source bytes in the push itself. Run the write to * the destination the preceding methods selected. */ if (op == 3 && subch == 1 && addr == GRCE_M_LOAD_INLINE_DATA && count && off + 4 + count * 4 <= len) { - if ((trusted || allow_ops) && !gsp_sema_skip_ops && - st.have_out && st.line_length) { - u32 bytes = st.line_length * - (st.gr_line_count ? st.gr_line_count : 1); - u32 have = count * 4; + /* + * fail4 slot rule: in the UVM upload pushes each + * LOAD_INLINE_DATA run fills a whole 32 KB slot - its + * data runs from the header to the next run header + * (or to the next 0x8000 boundary when this is the + * last run), regardless of the count field. A compact + * push (no room for a following slot) keeps the + * count*4 reading (the small parameter banks). The + * old count*4 reading truncated the big runs to half + * their slot and left the tail to be misparsed as + * methods (the split-upload bug). + */ + bool slot_run = (off + 0x8000 < len); + u32 slot_end = off + 0x8000; + u32 slot_bytes = 0; + /* + * fail4: the bridge's argument-block search needs the + * run's START as its destination; the cursor advance + * below must not leak into the rec (an end-of-run + * dst_va makes the QMD param patch miss and the + * engine never consumes the launch). + */ + u64 run_dst = st.offset_out; + + if (slot_run) { + u32 p; + + for (p = off + 4 + count * 4; + p + 4 <= slot_end && p + 4 <= len; p += 4) { + u32 h2; + + memcpy(&h2, buf + p, 4); + if (((h2 >> 29) & 0x7) == 3 && + ((h2 >> 13) & 0x7) == 1 && + ((h2 & 0x1fff) << 2) == + GRCE_M_LOAD_INLINE_DATA && + ((h2 >> 16) & 0xfff)) { + slot_end = p; + break; + } + } + slot_bytes = slot_end - (off + 4); + if (slot_bytes > 0x8000 - 4) + slot_bytes = 0x8000 - 4; + } + /* + * r391f once-only (the sum=0 class): the same + * inline write must not run twice for one push + * identity. The ops are gated by allow_ops + * (entry_replay clears it), but the inline-write + * gate `trusted || allow_ops` re-ran the write on + * every re-decode (a replay is trusted by + * construction): victory_reduce's + * cudaMemset(d_sum,0,8) is exactly such an inline + * write (observed: `inline write + * dst=0x7845ef000000 -> pa=0x7c00000 bytes=8 + * data8=0` firing 86 us before the D2H readback, + * zeroing the accumulator the launch had just + * written - the client then read zeros). A coarse + * !replay gate is wrong (a push whose first pass + * was untrusted must still author its bytes on the + * later trusted pass); key the write by + * (push_va, push content hash, dst, bytes) - a new + * push's release payload makes the hash unique - + * and skip only a genuine duplicate. The table is + * session-scoped (a new client starts clean). + */ + { + u32 bytes = slot_run ? slot_bytes : + st.line_length * + (st.gr_line_count ? + st.gr_line_count : 1); + u32 have = slot_run ? slot_bytes : count * 4; if (bytes > have) bytes = have; - svc_grce_inline_write(vgpu, st.offset_out, - buf + off + 4, bytes, - ch->h_object); - } else if (gsp_sema_dbg && svc.grce_log < 16) { - svc.grce_log++; - pr_info("sema-grce: inline data not run (trusted=%d allow=%d have_out=%d line=%u) h=0x%x\n", - trusted, allow_ops, st.have_out, - st.line_length, ch->h_object); + /* + * r391f once-only (the sum=0 class): the same + * inline write must not run twice for one push + * identity. The ops are gated by allow_ops + * (entry_replay clears it), but the + * inline-write gate `trusted || allow_ops` + * re-ran the write on every re-decode (a replay + * is trusted by construction): victory_reduce's + * cudaMemset(d_sum,0,8) is exactly such an + * inline write (observed: `inline write + * dst=0x7845ef000000 -> pa=0x7c00000 bytes=8 + * data8=0` firing 86 us before the D2H + * readback, zeroing the accumulator the launch + * had just written - the client then read + * zeros). A coarse !replay gate is wrong (a push + * whose first pass was untrusted must still + * author its bytes on the later trusted pass); + * key the write by (push_va, push content hash, + * dst, bytes) - the release payload inside every + * push makes the hash unique per push - and skip + * only a genuine duplicate. Session-scoped: a new + * client starts clean. + */ + if ((trusted || allow_ops) && !gsp_sema_skip_ops && + st.have_out && st.line_length && bytes && + !svc_inline_once_seen(push_va, svc_pass_hash, + st.offset_out, + bytes)) { + svc_inline_once_store(push_va, + svc_pass_hash, + st.offset_out, + bytes); + svc_grce_inline_write(vgpu, st.offset_out, + buf + off + 4, + bytes, + ch->h_object); + /* + * fail4: name the header that follows + * each big LID run - the split 64 KB + * upload's other payload carriers sit + * exactly there. + */ + if (gsp_sema_dbg && len >= 4096 && + svc.next_hdr_log < 24) { + u32 noff = off + 4 + count * 4; + u32 nh = 0, nw = 0; + + if (noff + 4 <= len) + memcpy(&nh, buf + noff, 4); + if (noff + 8 <= len) + memcpy(&nw, buf + noff + 4, + 4); + svc.next_hdr_log++; + pr_info("sema-next-hdr: h=0x%x push=0x%llx after=0x%x cnt=%u nxt_hdr=0x%08x nxt_w0=%08x cursor=0x%llx\n", + ch->h_object, push_va, + off, count, nh, nw, + st.offset_out); + } + } else if (gsp_sema_dbg && svc.grce_log < 64) { + svc.grce_log++; + pr_info("sema-grce: inline data not run (trusted=%d allow=%d have_out=%d line=%u) h=0x%x\n", + trusted, allow_ops, st.have_out, + st.line_length, ch->h_object); + } + /* + * fail4 (r526): hardware semantics - a + * LOAD_INLINE_DATA run advances the destination + * cursor by the bytes it carries whether or not + * the once-only gate let the write run. Without + * this, the consecutive runs of a split upload + * all land at the first destination (the old + * RAWA bug), and worse: on a re-decode pass where + * the gate suppresses the FIRST run, the cursor + * stays at the first run's destination and the + * SECOND run executes there - its data (the + * buffer's bytes 8..16) overwriting offset 0 + * (the observed tgemm XDIFF: X[0..1] <- + * X[2..3], deterministic, every pass). The gate + * suppresses the write, not the method stream's + * positional semantics. + */ + if (bytes) + st.offset_out += bytes; } /* * Record the destination and the payload bytes for @@ -5915,21 +10197,43 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * kernel arguments in the QMD parameter bank, and the * destination VA may not be readable later. */ - if (st.have_out && st.offset_out) { + if (st.have_out && run_dst) { struct svc_grce_rec *rec = &svc_grce_dst_ring[svc_grce_dst_ring_next % SVC_GRCE_DST_MAX]; - rec->dst_va = st.offset_out; - rec->len = min_t(u32, count * 4, + rec->dst_va = run_dst; + rec->len = min_t(u32, + slot_run ? slot_bytes : count * 4, GSP_BRIDGE_INLINE_BYTES); memcpy(rec->data, buf + off + 4, rec->len); svc_grce_dst_ring_next++; } - off += 4 + count * 4; + /* + * fail4 slot rule: hop to the next slot's run header + * instead of walking the slot's raw tail as methods. + */ + off += slot_run ? (slot_end - off) : 4 + count * 4; continue; } + /* + * fail4: name the moderate op=1 method runs on big pushes - + * the split 64 KB upload's payload carriers beyond the LID + * class land here (their addr selects the write port). + */ + if (gsp_sema_dbg && len >= 4096 && op == 1 && + count >= 16 && count <= 1023 && svc.mystery_log < 48) { + svc.mystery_log++; + pr_info("sema-mystery: h=0x%x push=0x%llx off=0x%x hdr=0x%08x cnt=%u subch=%u addr=0x%x have_out=%u out=0x%llx line=%u lcnt=%u w=%08x %08x %08x\n", + ch->h_object, push_va, off, hdr, count, subch, + addr, st.have_out, st.offset_out, + st.line_length, st.gr_line_count, + off + 8 <= len ? *(const u32 *)(buf + off + 4) : 0, + off + 12 <= len ? *(const u32 *)(buf + off + 8) : 0, + off + 16 <= len ? *(const u32 *)(buf + off + 12) : 0); + } + if (op == 4 || op == 7) { /* Header-only methods (immediate data in the header * for op 4; end-of-push segment for op 7). */ @@ -5994,6 +10298,13 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * held INC-shaped words queued a bogus LAUNCH_DMA). * No release is decoded from these encodings. */ + if (gsp_sema_dbg && len >= 8192 && count >= 1024 && + svc.skiprun_log < 64) { + svc.skiprun_log++; + pr_info("sema-skiprun: h=0x%x push=0x%llx off=0x%x hdr=0x%08x op=%u cnt=%u subch=%u addr=0x%x trusted=%d allow=%d\n", + ch->h_object, push_va, off, hdr, op, + count, subch, addr, trusted, allow_ops); + } off += 4 + count * 4; continue; } @@ -6060,6 +10371,12 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, /* SET_OBJECT on the channel subchannel: a push starts */ n_ops = 0; memset(&st, 0, sizeof(st)); + /* r336/r34x: new push, new acquire set + release mirror */ + svc_wait_n = 0; + svc_wait_rel_va = 0; + svc_wait_rel_payload = 0; + svc_wait_rel_off = 0; + svc_wait_rel_have = false; off += 4 + count * 4; continue; } @@ -6078,6 +10395,10 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, svc.pushes_seen++; st.rel_va = va; st.rel_payload = payload; + svc_wait_rel_va = va; + svc_wait_rel_payload = payload; + svc_wait_rel_off = off; + svc_wait_rel_have = true; st.have_rel = 1; svc_rel_va_seen = va; svc_rel_count++; @@ -6085,10 +10406,17 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * REPORT_SEM comment below. */ if (allow_ops && n_ops && !gsp_sema_skip_ops) { svc_run_ops(vgpu, ops, n_ops, ch->h_object, - ops_done, ops_seq - n_ops); + ops_done, ops_seq - n_ops, + entry_idx, svc_pass_hash); svc.ring_ops++; ops_runs++; } + /* r566: a held entry's unexecuted device-dst + * copies must be visible to the launch guard. */ + if (svc.op_tr_pending) + svc_dstpend_note_ops(ops, n_ops, + ops_seq - n_ops, + ops_done); n_ops = 0; /* * A release to a GR completion counter must go @@ -6130,7 +10458,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, or.va = va; or.payload = payload; or.entry_idx = entry_idx; - if (svc_fwd_defer_arm()) { + if (svc_fwd_defer_arm() && + !svc.op_tr_pending) { svc_opdef_drop(vgpu, ch, entry_idx); if (svc_fwd_rel_n < SVC_FWD_REL_MAX) { @@ -6175,6 +10504,42 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, va, payload, op); } } + } else if (op != HOST_SEM_OP_RELEASE && + va >= SEMA_VA_LO && va < SEMA_VA_HI) { + /* + * r336/r34x: C56F SEM_EXECUTE acquire ops + * (0 = ACQUIRE/equal, 2 = strict GEQ, + * 3 = circular GEQ, 4 = AND, 5 = NOR). + * Core CUDA starts its compute-channel + * pushes with one of these to wait for a + * progress counter (observed: + * [LO=0x0400ffd0][HI=2][pay=4][0][0x1003] on + * the D2H readback push - ACQ_CIRC_GEQ on + * the channel counter 0x20400ffd0). + * svc_run_ops honors every captured acquire + * of this entry before running its work, so + * a copy cannot execute ahead of the + * dependency it waits on (the streams2 + * readback-before-scale class). Capturing + * into the per-entry list (instead of a + * single last-wait global) is what stops a + * later entry's acquire from gating an + * earlier entry (the r340 over-hold). + * PAYLOAD_SIZE 64BIT (execute bit 24) is + * captured on the low word only; every + * observed CUDA acquire is 32-bit. + */ + if (svc_wait_n < SVC_WAIT_MAX) { + struct svc_wait_op *w = + &svc_waits[svc_wait_n++]; + + w->va = va; + w->payload = payload; + w->op = (u8)op; + w->off = off; + } else { + svc_wait_ovf++; + } } off += 4 + count * 4; continue; @@ -6224,6 +10589,10 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, svc.pushes_seen++; st.rel_va = va; st.rel_payload = st.rs_payload; + svc_wait_rel_va = va; + svc_wait_rel_payload = st.rs_payload; + svc_wait_rel_off = off; + svc_wait_rel_have = true; st.have_rel = 1; svc_rel_va_seen = va; svc_rel_count++; @@ -6244,10 +10613,15 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, */ if (allow_ops && n_ops && !gsp_sema_skip_ops) { svc_run_ops(vgpu, ops, n_ops, ch->h_object, - ops_done, ops_seq - n_ops); + ops_done, ops_seq - n_ops, + entry_idx, svc_pass_hash); svc.ring_ops++; ops_runs++; } + if (svc.op_tr_pending) + svc_dstpend_note_ops(ops, n_ops, + ops_seq - n_ops, + ops_done); n_ops = 0; /* * ISSUE-0002 (r62/r65): a captured @@ -6359,7 +10733,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * operand state, so one run covers both the * C5B5 and the GRCE copy shapes. */ - if (ch->h_object && gsp_sema_dbg && + if (gsp_sema_dbg && svc.xfer_raw_log < 48) { u32 dl = len < 192 ? len : 192; @@ -6401,11 +10775,61 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, st.have_sem = 1; break; case CE_M_SET_REMAP_CONST_A: + /* + * r343 FILL instrument: a fill-shaped LAUNCH_DMA + * (REMAP_ENABLE + destination PITCH, OFFSET_OUT and + * no OFFSET_IN) needs this constant to build its op. + * The 8MB-memset class (memcpy_roundtrip all-bad, + * victory_memset2d) shows fill-shaped launches with + * have_remap=0 that decode nothing and are consumed + * silently. Log every decode (bounded) so one run + * captures the real run shape even when the count + * guard below cannot use it. + */ + if (svc_remapconst_log < 256) { + svc_remapconst_log++; + pr_info("sema-remapconst: h=0x%x push=0x%llx off=0x%x count=%u v0=0x%x v1=0x%x v2=0x%x trusted=%d allow=%u\n", + ch->h_object, push_va, off, count, + v0, count >= 2 ? v1 : 0u, + count >= 3 ? v2 : 0u, trusted, allow_ops); + } if (count >= 3) { st.remap_a = v0; st.remap_b = v1; st.remap_comp = v2; st.have_remap = 1; + st.have_remap_a = 1; + st.have_remap_b = 1; + st.have_remap_c = 1; + } else if (count >= 1) { + /* + * r365: the driver also pushes the fill constants + * as SEPARATE count=1 methods (the CE + * page-table fills carry only SET_REMAP_CONST_A + * with A=0). Capture the individual parts but do + * NOT set have_remap: r365b proved that letting + * these pass the fill guard executes below-floor + * window fills (0x2000c0xxx, 0x204020000) whose + * svc_fill resolution landed in GUEST RAM and + * zeroed slab pages (guest kernel GPF in + * kmem_cache_alloc -> panic at ~75 s). The + * window fills only run behind gsp_sema_winfill + * with the correct resolution proven first. + */ + st.remap_a = v0; + st.have_remap_a = 1; + } + break; + case CE_M_SET_REMAP_CONST_B: + if (count >= 1) { + st.remap_b = v0; + st.have_remap_b = 1; + } + break; + case CE_M_SET_REMAP_COMPONENTS: + if (count >= 1) { + st.remap_comp = v0; + st.have_remap_c = 1; } break; case CE_M_SET_SRC_PHYS_MODE: @@ -6418,12 +10842,42 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, break; case CE_M_LINE_LENGTH_IN: st.line_length = v0; + /* + * r458 THE memset2d root cause: the driver encodes + * LINE_LENGTH_IN (0x418) and the adjacent LINE_COUNT + * (0x41c) as ONE count=2 method group - observed raw + * push bytes `20028106 00001000 00000100` = header + * (subch 4, count 2, addr 0x418), line length 4096, + * line count 0x100 = 256. Reading only v0 left + * line_count = 0, so the 2D-copy builder below took + * the single-row path: one 4 KB op per push, and a + * 4 MB cudaMemcpy2D delivered exactly one page per + * 1 MB block (memset2d probe: per-MB ok=4096, + * bad=12.5M/12.58M). The GRCE variant below has + * always decoded the count=2 form; the CE path did + * not. + */ + if (count >= 2) + st.line_count = v1; + break; + /* r402: 2D geometry (cudaMemset2D / cudaMemcpy2D). */ + case CE_M_LINE_COUNT: + st.line_count = v0; + break; + case CE_M_PITCH_IN: + st.pitch_in = v0 & 0xffffff; + st.have_pitch_in = 1; + break; + case CE_M_PITCH_OUT: + st.pitch_out = v0 & 0xffffff; + st.have_pitch_out = 1; break; case CE_M_LAUNCH_DMA: { u32 dma = v0; u32 transfer = DMA_DATA_TRANSFER_TYPE(dma); + u32 n_ops_before = n_ops; - if (gsp_sema_dbg && ch->h_object && + if (gsp_sema_dbg && svc.lda_log < (gsp_sema_ops_log > 48 ? gsp_sema_ops_log : 48)) { svc.lda_log++; pr_info("sema-lda: h=0x%x eng=%u push=0x%llx inj=%u allow=%u dma=0x%08x xfer=%u remap=%u dpitch=%u dstphys=%u srcphys=%u sem=%u out=0x%llx in=0x%llx line=%u have_out=%u have_in=%u have_remap=%u have_sem=%u ops=%u\n", @@ -6440,7 +10894,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, } /* Diagnostic: full operand state in the post-completion * window (or for high device VAs), bounded. */ - if (svc.op_trace_log < 160 && ch->h_object && + if (svc.op_trace_log < (gsp_sema_dbg ? 32768 : 2048) && ch->h_object && (time_before(jiffies, svc.op_trace_until) || st.offset_out >= SVC_VAS_FLOOR || st.offset_in >= SVC_VAS_FLOOR)) { @@ -6464,7 +10918,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * bounded number of such launches so the operand * encoding (or its absence) is on record. */ - if (transfer != 0 && ch->h_object && gsp_sema_dbg && + if (transfer != 0 && gsp_sema_dbg && svc.xfer_raw_log < 48 && (!st.have_out || !st.line_length)) { u32 dl = len < 192 ? len : 192; @@ -6479,25 +10933,182 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, if (allow_ops && transfer != 0 && st.line_length && st.have_out) { - if (DMA_REMAP_ENABLE(dma) && st.have_remap && + /* + * r361: a fill-shaped transfer whose + * destination is PHYSICAL guest RAM (the + * driver zeroes freshly-allocated PTE / + * page-directory pages with exactly this + * push shape) needs no translation: the + * destination already IS the GPA. The old + * `offset_out >= SVC_VAS_FLOOR` guard + * compared the GPA against the VA floor + * and blocked every one of these, so the + * PD0 pages the GMMU walk later reads + * stayed stale and the first CUDA client + * of a fresh boot dropped operand pages + * (GRFILL-DROP). SEMA_PA_MAX bounds the + * range so a ramfill can only touch guest + * RAM; virtual engine-window fills + * (0x2040xxxxx) stay excluded. + */ + bool ramfill = DMA_DST_TYPE_PHYSICAL(dma) && + st.have_dst_mode && + st.dst_phys_mode == CE_APERTURE_SYS && + st.offset_out + (u64)st.line_length <= + SEMA_PA_MAX; + /* r365: vidmem-window fill (see gsp_sema_winfill). */ + bool winfill = gsp_sema_winfill && + st.offset_out >= SVC_VIDMEM_VA_LO && + st.offset_out < SVC_VIDMEM_VA_BASE + + svc_fb_shadow_size(vgpu); + + if (DMA_REMAP_ENABLE(dma) && + (st.have_remap || + (st.offset_out >= SVC_VAS_FLOOR) || + ramfill || winfill) && DMA_DST_MEMORY_LAYOUT_PITCH(dma) && n_ops < SVC_MAX_MEMSETS) { - u32 csz = (st.remap_comp >> 16) & 0x3; - u32 ndst = (st.remap_comp >> 24) & 0x3; + u32 csz = st.have_remap ? + ((st.remap_comp >> 16) & 0x3) : 0; + u32 ndst = st.have_remap ? + ((st.remap_comp >> 24) & 0x3) : 0; + u32 dst_x = st.have_remap ? + (st.remap_comp & 0x7) : 4; struct svc_memset_op *m = &ops[n_ops]; m->type = 0; m->dst = st.offset_out; - m->value = ndst >= 1 ? - (((u64)st.remap_b << 32) | - st.remap_a) : st.remap_b; - m->elem = (csz + 1) * (ndst + 1); - m->count = st.line_length; + /* + * r367: the driver's own memset push + * (channel_utils.c) proves the + * encoding: LINE_LENGTH_IN is the + * fill SIZE IN BYTES, and + * SET_REMAP_COMPONENTS selects + * COMPONENT_SIZE ([17:16], 0..3 -> + * 1/2/3/4 bytes) and + * NUM_DST_COMPONENTS ([25:24], n+1 + * components). elem = + * (csz+1)*(ndst+1) is the pattern + * width; count must therefore be + * line_length/elem - the old + * count=line_length wrote + * elem x too much (r365: a 383620- + * byte memset wrote 1534480 bytes, + * clobbering the bytes after it). + * The value comes from the mapped + * constants: X<-CONST_A (4) and + * Y<-CONST_B (5); with two + * components the pattern is + * B:A 64-bit. + */ + if (st.have_remap) + m->value = ndst >= 1 ? + (((u64)st.remap_b << 32) | + st.remap_a) : + (dst_x == 5 ? st.remap_b : + st.remap_a); + else + /* + * r357: a fill-shaped transfer + * (REMAP_ENABLE + DST PITCH, + * no source) to a CLIENT DEVICE + * VA whose value constant did + * not decode is a ZERO fill: + * the driver's page-table + * clears and cudaMemset(0) + * issue exactly this shape with + * no SET_REMAP_CONST method. + * Building no op left the + * guest's GPU page tables + * stale (the GMMU walk then + * missed freshly-mapped + * operand pages and the first + * CUDA client of every fresh + * boot ran with dropped + * operands). The >= floor + * guard keeps engine-window + * fills (0x2040xxxxx init + * clears) OUT of the executor: + * executing those clobbered + * emulated engine state and + * hung client init (r356). + */ + m->value = 0; + m->elem = st.have_remap ? + (csz + 1) * (ndst + 1) : 1; + if (m->elem > 1 && + st.line_length % m->elem) { + /* not pattern-divisible: exact + * byte count wins */ + m->elem = 1; + m->count = st.line_length; + } else { + m->count = st.line_length / + m->elem; + } + /* + * r402: a 2D fill (cudaMemset2D): + * LINE_COUNT rows of + * LINE_LENGTH_IN bytes at + * PITCH_OUT stride. A pitch WIDER + * than the row selects row mode; a + * flat fill carries pitch == row + * length (or no pitch method) and + * keeps the historical single-run + * path. + * + * r420c: LINE_COUNT is + * authoritative for the flat case + * too - cudaMemset2D with pitch == + * width (victory_memset2d, + * pitch=4096/w=4096) is CONTIGUOUS + * rows, so the single run must + * cover rows*line bytes. The + * historical path filled one row + * and the readback lost the rest + * (bad ~= 12.5M of 12.58M). + */ + if (st.line_count > 1) { + u32 fpitch = st.have_pitch_out ? + st.pitch_out : + st.line_length; + + if (fpitch > st.line_length) { + m->rows = + st.line_count; + m->dpitch = fpitch; + } else if (st.line_length % + m->elem) { + m->elem = 1; + m->count = + st.line_length * + st.line_count; + } else { + m->count = + (st.line_length / + m->elem) * + st.line_count; + } + } + if (m->rows > 1 && svc_pitch_log < 32) { + svc_pitch_log++; + pr_info("sema-2d: fill rows=%u dpitch=%u line=%u dst=0x%llx h=0x%x push=0x%llx\n", + m->rows, m->dpitch, + st.line_length, m->dst, + ch->h_object, push_va); + } m->phys = DMA_DST_TYPE_PHYSICAL(dma); m->aperture = st.have_dst_mode ? st.dst_phys_mode : CE_APERTURE_VID; n_ops++; ops_seq++; + if (ramfill && svc.fill_ram_log < 24) { + svc.fill_ram_log++; + pr_info("vgsp-sema: RAMFILL-EXEC push=0x%llx dst=0x%llx line=%u ap=%u value=0x%llx (physical guest-RAM fill built)\n", + push_va, st.offset_out, + st.line_length, + m->aperture, m->value); + } } else if (!DMA_REMAP_ENABLE(dma) && st.have_in && n_ops < SVC_MAX_MEMSETS) { @@ -6514,6 +11125,80 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, DMA_SRC_TYPE_PHYSICAL(dma); m->src_aperture = st.have_src_mode ? st.src_phys_mode : CE_APERTURE_VID; + /* + * r402: a 2D copy (cudaMemcpy2D): + * LINE_COUNT rows of + * LINE_LENGTH_IN bytes with + * PITCH_IN/PITCH_OUT strides. + * Row mode when a pitch is + * wider than the row. + * + * r420c: pitch == row length + * (or no pitch method) makes the + * rows CONTIGUOUS, so the single + * flat run must cover + * rows*line bytes - the + * historical path copied one row + * (victory_memset2d's pitched + * D2H/H2D at pitch 4096 = width + * 4096 lost all but the first + * row). + */ + if (st.line_count > 1) { + u32 dpitch = st.have_pitch_out ? + st.pitch_out : + st.line_length; + u32 spitch = st.have_pitch_in ? + (st.pitch_in ? + st.pitch_in : + dpitch) : + dpitch; + + if (dpitch == st.line_length && + spitch == st.line_length) { + m->bytes = + st.line_length * + st.line_count; + } else { + m->rows = + st.line_count; + m->dpitch = dpitch; + m->spitch = spitch; + } + } + /* + * r458 memset2d forensics: the + * driver's cudaMemcpy2D pushes decode + * LINE_LENGTH=4096 with PITCH_IN/OUT + * present but LINE_COUNT absent + * (lc=0), so only one 4 KB row per + * push executes and a 4 MB copy + * delivers 4 pages. Dump the raw push + * (bounded) for every pitch-shaped + * copy so the method stream that + * carries the row count is on record. + */ + if (gsp_sema_dbg && + (st.have_pitch_out || + st.have_pitch_in) && + svc.op2d_raw_log < 12) { + u32 dl = len < 220 ? len : 220; + + svc.op2d_raw_log++; + pr_info("sema-op2d-raw: h=0x%x push=0x%llx len=%u off=0x%x line=%u lc=%u po=%u/%u pi=%u/%u out=0x%llx in=0x%llx bytes=%u dma=0x%08x\n", + ch->h_object, push_va, + len, off, + st.line_length, + st.line_count, + st.have_pitch_out, + st.pitch_out, + st.have_pitch_in, + st.pitch_in, + m->dst, m->src, + (u32)m->bytes, dma); + pr_info("sema-op2d-raw: raw %*phN\n", + dl, buf); + } n_ops++; ops_seq++; } else if (gsp_sema_dbg && svc.dma_skip_log < 32) { @@ -6542,6 +11227,388 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, } /* + * r567: a trusted, transfer-shaped push that decoded + * NO op this pass (its operands are not resolved yet: + * the r569 mail b-copy needed have_in, which the + * staging upload had not produced) must (a) keep its + * slot incomplete so a later pass runs the copy + * instead of replay-suppressing it forever (op_owed, + * the r540 rule), and (b) make its stale destination + * visible to the r541 launch guard, which defers any + * launch reading that page until the copy runs or the + * dstpend window expires. The release still applies + * (no hold is armed: a push whose trust never arrives + * must not wedge the pump) - the guard, not the + * release, provides the ordering. + */ + if (allow_ops && transfer != 0 && trusted && ch && + n_ops == n_ops_before && st.have_out && + st.offset_out >= SVC_VAS_FLOOR) { + svc_dstpend_note(st.offset_out); + svc.op_owed = 1; + if (svc.op_owed_log < 64) { + svc.op_owed_log++; + pr_info("sema-pend: OWED-PUSH h=0x%x push=0x%llx out=0x%llx line=%u (transfer-shaped, no op; dst noted for the launch guard)\n", + ch->h_object, push_va, + st.offset_out, + st.line_length); + } + } + + /* + * r365 FILL-RAW: the fill class that still decodes no + * op (window VA, no remap const decoded) - dump the + * raw push and decode its method stream so the true + * value/component encoding is on record. Bounded. + */ + if (allow_ops && transfer != 0 && + DMA_REMAP_ENABLE(dma) && + DMA_DST_MEMORY_LAYOUT_PITCH(dma) && + st.have_out && !st.have_in && !st.have_remap && + svc_fill_raw_log < 48) { + u32 dl = len < 160 ? len : 160; + u32 mo = 0, mn = 0; + + svc_fill_raw_log++; + pr_info("sema-fill-raw: h=0x%x push=0x%llx out=0x%llx line=%u dma=0x%08x dmode=%u have_dstmode=%u trusted=%d allow=%d\n", + ch->h_object, push_va, st.offset_out, + st.line_length, dma, st.dst_phys_mode, + st.have_dst_mode, trusted, allow_ops); + pr_info("sema-fill-raw: raw len=%u %*phN\n", + len, dl, buf); + /* decode the first method pairs */ + while (mo + 8 <= len && mn < 14) { + u32 h2, c2, a2; + + memcpy(&h2, buf + mo, 4); + c2 = (h2 >> 16) & 0xfff; + a2 = (h2 & 0x1fff) << 2; + if ((h2 >> 29) != 1 || !c2 || + mo + 4 + c2 * 4 > len) + break; + if (c2 >= 2) + pr_info("sema-fill-raw: m[%u] addr=0x%x cnt=%u v0=0x%08x v1=0x%08x\n", + mn, a2, c2, + *(const u32 *)(buf + mo + 4), + *(const u32 *)(buf + mo + 8)); + else + pr_info("sema-fill-raw: m[%u] addr=0x%x cnt=%u v0=0x%08x\n", + mn, a2, c2, + *(const u32 *)(buf + mo + 4)); + mo += 4 + c2 * 4; + mn++; + } + } + + /* + * r343 FILL diagnostic (LOG-ONLY, no hold): a + * fill-shaped transfer (REMAP_ENABLE + destination + * PITCH, OFFSET_OUT and no OFFSET_IN) whose value + * constant did not decode currently decodes no op + * and is consumed silently - the guest's cudaMemset + * then reports success with zero bytes written + * (r341 memcpy_roundtrip: 8MB memset readback + * all-bad; victory_memset2d). An op_tr_pending hold + * here WEDGED client init (r343: first cudaMalloc + * retried forever - driver init issues fill-shaped + * CE pushes, the hold re-armed every decode pass + * and never expired). So: log loudly only; the + * sema-remapconst rows above capture the real + * decode shape to build the proper fill decoder. + */ + if (allow_ops && transfer != 0 && + st.line_length >= 1024 && st.have_out && + !st.have_in && DMA_REMAP_ENABLE(dma) && + DMA_DST_MEMORY_LAYOUT_PITCH(dma) && + !st.have_remap && n_ops == n_ops_before && + svc_fill_noremap < 8) { + svc_fill_noremap++; + pr_err("vgsp: FILL-NOREMAP h=0x%x push=0x%llx dma=0x%08x line=%u out=0x%llx phys=%u dstmode=%u: fill-shaped launch, value source not decoded (log-only diagnostic)\n", + ch->h_object, push_va, dma, + st.line_length, st.offset_out, + DMA_DST_TYPE_PHYSICAL(dma), + st.have_dst_mode ? st.dst_phys_mode : 99); + } + + /* + * fail4: name a transfer-shaped launch that decoded NO + * op because the pass was denied (allow_ops=0) - the + * op-chain above (and its dma-skip dump) is skipped + * wholesale in that case, so this class was invisible. + */ + if (gsp_sema_dbg && transfer != 0 && !allow_ops && + svc.dmatrace_log < (gsp_sema_dbg ? 4096 : 64)) { + svc.dmatrace_log++; + pr_info("sema-dmatrace: denied h=0x%x push=0x%llx dma=0x%08x line=%u in=0x%llx out=0x%llx have_in=%u have_out=%u have_remap=%u trusted=%d replay=%d eng=%u pend_would=%d\n", + ch->h_object, push_va, dma, + st.line_length, st.offset_in, + st.offset_out, st.have_in, st.have_out, + st.have_remap, trusted, replay, + ch->engine_type, + (!allow_ops && trusted && + !ch->engine_type && transfer != 0 && + st.line_length && st.have_out && + st.have_in && + !DMA_REMAP_ENABLE(dma) && + st.offset_out >= SVC_VAS_FLOOR && + entry_idx != SVC_ENTRY_NONE && + entry_idx < SVC_GPFIFO_MAX_BITS && + ch->slot_pend_tries[entry_idx] < 4) ? 1 : 0); + } + /* + * fail4 r302: a denied device-destination copy that + * WOULD have built a type=1 op (the exact branch shape + * above: no remap, both offsets, device dst) must not + * be consumed silently. This is the sw-copy client + * mode: the GR-classified GRCE channel's ops run only + * once the pin/trust stream (injected) reaches the + * entry, and the first pass over a chunk push lands + * before that - the copy then never ran and the guest + * computed on stale VRAM (vector_add r302: chunk + * copies cc..d1 skipped; only the tail entries were + * re-decoded by a rewind and ran). Mark the pass + * op-pending so the pump holds the entry through the + * m163/opdef machinery and the retry runs the copy as + * soon as the trust (or the operand translation) + * arrives; the bounded SVC_OPDEF_HOLD_MS expiry keeps + * the pre-existing skip as the last resort. + */ + if (!allow_ops && trusted && + !ch->engine_type && transfer != 0 && + st.line_length && st.have_out && st.have_in && + !DMA_REMAP_ENABLE(dma) && + st.offset_out >= SVC_VAS_FLOOR) { + /* + * r541: the denied device-dst copy has NOT + * run; whatever release edge later consumes + * this entry, a GR launch must not read the + * target page until the copy executes (see + * svc_dstpend_note). + */ + svc_dstpend_note(st.offset_out); + /* + * r314 (CONT.57 lead 2): capture the staging + * source on the DENIED pass too. This is the + * pipeline-order moment: the push was just + * decoded, the driver has not seen this + * push's fence (it applies later in this same + * pass at the earliest), so the staging still + * holds this chunk's bytes. The old capture + * gate lived in the op-build block, which + * only runs on ALLOWED passes - by then the + * fence had long fired and the staging was + * refilled (r302-r313: captures 0-3/8, the + * late executions read refilled staging, the + * deterministic 786335-bad output). Same + * translation-probe guard as the build-time + * gate: only an SYS-resolving source is + * host staging. + */ + if (st.line_length <= SVC_COPY_SNAP_MAX) { + u64 spa = 0; + u32 sap = 0; + + if (!svc_op_translate(vgpu, + st.offset_in, + &spa, &sap) && + sap == CE_APERTURE_SYS) + svc_copy_snap_capture(vgpu, + st.offset_out, + st.offset_in, + st.line_length, + ch->h_object, + entry_idx, + svc_pass_hash); + } + /* + * r308: the `!replay` gate above kept this + * dead on the sw-copy channel: after the first + * consumed pass every later pass classifies as + * a replay, so the denied chunk copies were + * never retried (vector_add: 6 of 8 chunks + * skipped). A replay pass that marks the slot + * pending un-completes it (slot_ops_complete + * is written from svc.op_tr_pending below), so + * the next pass is treated fresh and the copy + * runs once trust/translation arrives. The + * per-slot counter caps the re-holds so a slot + * whose trust never arrives cannot wedge the + * channel in 10 s hold cycles. + */ + if (entry_idx != SVC_ENTRY_NONE && + entry_idx < SVC_GPFIFO_MAX_BITS) { + if (ch->slot_pend_tries[entry_idx] < 4) { + ch->slot_pend_tries[entry_idx]++; + svc.op_tr_pending = 1; + if (gsp_sema_dbg && + svc.pend_dbg_log < 512) { + svc.pend_dbg_log++; + pr_info("sema-pend: h=0x%x push=0x%llx idx=%u out=0x%llx in=0x%llx line=%u replay=%d trusted=%d tries=%u -> op_tr_pending\n", + ch->h_object, push_va, + entry_idx, st.offset_out, + st.offset_in, + st.line_length, + replay, trusted, + ch->slot_pend_tries[entry_idx]); + } + } else { + /* + * r540: the cap stopped the + * re-hold, but the op is still + * owed. Leaving the slot + * complete would classify every + * later pass as a replay and + * suppress the copy forever - + * the client waits on a copy + * that never ran (the r539 + * ~5-minute device-init stall: + * a GRCE H2D denied while + * untrusted, capped at 4 tries, + * then replay-suppressed after + * the pin arrived). Mark the + * pass owed so the slot stays + * INCOMPLETE and the next + * fresh pass (injected or + * translated) executes the + * copy. No hold is armed, so a + * slot whose trust never arrives + * still cannot wedge the pump. + */ + svc.op_owed = 1; + if (svc.op_owed_log < 64) { + svc.op_owed_log++; + pr_info("sema-pend: OWED h=0x%x push=0x%llx idx=%u out=0x%llx in=0x%llx line=%u trusted=%d (cap hit; slot stays incomplete for a fresh retry)\n", + ch->h_object, push_va, + entry_idx, st.offset_out, + st.offset_in, + st.line_length, + trusted); + } + } + } + } + /* + * fail4: name every op appended to this pass (copy or + * fill) with its resolved operand fields, so the + * executor's population is on record even when the + * 64 KB copy witness below never fires for it. + */ + if (n_ops != n_ops_before) { + struct svc_memset_op *lm = &ops[n_ops - 1]; + + /* + * fail4 CONT.53: H2D-shaped copies (virtual + * device destination, virtual host source) + * read the driver's ping-pong staging. The + * driver paces restaging on the copy's + * completion fence; our execution may run + * later than the push's processing position, + * so capture the source NOW (bounded pool) + * and let svc_copy execute from the capture - + * the hardware's in-order staging read. + */ + /* + * H2D shape ONLY: device destination (VID + * aperture), host source (SYS). A D2H readback + * (host dst) captured here would freeze + * pre-kernel zeros (observed in the r300 + * captures) and break readbacks. + */ + if (lm->type == 1 && !lm->phys && + lm->dst >= SVC_VAS_FLOOR && + lm->aperture == CE_APERTURE_VID && + !lm->src_phys && + lm->src >= SVC_VAS_FLOOR && + lm->bytes <= SVC_COPY_SNAP_MAX) { + /* + * r311: the src_aperture equality + * gate never matched the sw-copy + * client (the driver sends no + * SET_SRC_PHYS_MODE, so src_ap + * defaults to VID=0 while the + * source is host staging; r302-r310 + * logged zero captures and the + * late executions read refilled + * staging). The translation is the + * truth: capture only when the + * first source page resolves + * through the SYS aperture. + */ + u64 spa = 0; + u32 sap = 0; + + if (!svc_op_translate(vgpu, lm->src, + &spa, &sap) && + sap == CE_APERTURE_SYS) + svc_copy_snap_capture(vgpu, + lm->dst, + lm->src, + lm->bytes, + ch->h_object, + entry_idx, + svc_pass_hash); + if (gsp_sema_dbg && + svc.snap_gate_log < 256) { + svc.snap_gate_log++; + pr_info("sema-snapgate: dst=0x%llx src=0x%llx trc=%d sap=%u\n", + lm->dst, lm->src, + svc_op_translate(vgpu, lm->src, &spa, &sap), + sap); + } + } + + /* + * fail4 CE linear auto-advance, BOTH sides: the + * driver re-sets only ONE offset per op and the + * copy engine advances the other by the transfer + * size. H2D ops show an explicit advancing DST + * with a FIXED SRC base (every 1 MB chunk read + * the source base again: vector_add's device a + * held h_a[0..1MB) x4 -> 75% mismatch; the + * readbacks are the mirror image). Advance both + * registers; an explicit re-set simply overrides. + */ + { + u64 adv = (lm->type == 1) ? lm->bytes : + (u64)lm->elem * lm->count; + + st.offset_out += adv; + if (lm->type == 1) + st.offset_in += adv; + } + if (svc.opadd_log < 2048) { + svc.opadd_log++; + pr_info("sema-opadd: h=0x%x type=%u dst=0x%llx src=0x%llx bytes=%llu elem=%u count=%u ap=%u src_ap=%u phys=%u src_phys=%u\n", + ch->h_object, lm->type, + lm->dst, lm->src, lm->bytes, + lm->elem, lm->count, + lm->aperture, lm->src_aperture, + lm->phys, lm->src_phys); + /* + * r420d: 2D geometry of every copy op + * (the memset2d readback forensics - + * the driver decomposes cudaMemcpy2D + * into per-push ops whose LINE_COUNT/ + * PITCH decode must be visible to + * tell "one row per op" from "N rows + * in one op"). Bounded. + */ + if (lm->type == 1) + pr_info("sema-op2d: h=0x%x dst=0x%llx src=0x%llx bytes=%u rows=%u dpitch=%u spitch=%u line=%u lc=%u have_po=%u po=%u have_pi=%u pi=%u\n", + ch->h_object, lm->dst, + lm->src, lm->bytes, + lm->rows, lm->dpitch, + lm->spitch, + st.line_length, + st.line_count, + st.have_pitch_out, + st.pitch_out, + st.have_pitch_in, + st.pitch_in); + } + } + + /* * Both release encodings write the payload word first * (NvGpuSemaphore.data[0] for the four-word format, the * single word for the one-word format), and the client's @@ -6567,6 +11634,10 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, st.sem_va_lo; st.rel_va = sem_va; st.rel_payload = st.sem_payload; + svc_wait_rel_va = sem_va; + svc_wait_rel_payload = st.sem_payload; + svc_wait_rel_off = off; + svc_wait_rel_have = true; st.have_rel = 1; if (st.have_sem && st.sem_payload && @@ -6664,6 +11735,22 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, ch->entry_want = st.sem_payload; have_cur = true; pending = 1; + /* + * fail4: apply this seed ABSOLUTE. + * The generic applier still sees the + * increment flag (inc), so leaving + * the residue in cur makes it write + * cur+payload (observed 134+1=135) + * - a completed value far past the + * guest's queued count, which + * trips the guest UVM assert + * (uvm_channel.c:204 completed <= + * queued) and kills the device. + * cur=0 makes inc-apply land the + * payload itself, exactly the + * re-seed the client polls. + */ + cur = 0; svc.pool_residue++; if (svc.pool_residue_log < 32) { svc.pool_residue_log++; @@ -6780,7 +11867,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, gsp_sema_skip_ops); } svc_run_ops(vgpu, ops, n_ops, ch->h_object, - ops_done, ops_seq - n_ops); + ops_done, ops_seq - n_ops, + entry_idx, svc_pass_hash); svc.ring_ops++; ops_runs++; } @@ -6876,7 +11964,44 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, */ svc.fwd_rel_overflow++; svc_fwd_soft_copy_inline(vgpu); - if (svc_rel_apply_launch(vgpu, ch, + /* + * fail4 CONT.53: never clear a + * push's fence before its ops + * read their source. The pass's + * ops run at the pass end; the + * driver's ping-pong staging + * restages as soon as the fence + * clears, and a later op read + * then returns chunk+N bytes + * (vector_add +2 MB). Queue the + * release through the pending + * pass instead. + */ + if (n_ops > 0 || + svc.op_tr_pending) { + /* + * r316 (CONT.60): + * op_tr_pending covers + * the denied pass - the + * pass built no op + * (allow=0) but a copy is + * still owed; applying + * the fence here (the old + * n_ops==0 path) let the + * driver refill the + * ping-pong staging before + * the copy ever ran - the + * cold-run capture gap + * (vector_add run 1 + * bad=157696/193536 vs + * bad=0 warm). + */ + svc_pending_add(sem_va, + st.sem_payload, + inc ? 1u : 0u); + svc_rel_va_seen = sem_va; + svc_rel_count++; + } else if (svc_rel_apply_launch(vgpu, ch, sem_va, st.sem_payload, cur, pending, @@ -6910,7 +12035,20 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, svc_fwd_rel_direct_note(ch->h_object, sem_va, st.sem_payload); - if (svc_rel_apply_launch(vgpu, ch, sem_va, + /* + * fail4 CONT.53: same ops-order + * guard as above - a pass with + * pending copy ops must run + * them before its fence clears. + */ + if (n_ops > 0) { + svc_pending_add(sem_va, + st.sem_payload, + inc ? 1u : 0u); + svc_rel_va_seen = sem_va; + svc_rel_count++; + } else if (svc_rel_apply_launch(vgpu, ch, + sem_va, st.sem_payload, cur, pending, inc, inc_apply)) { @@ -6930,7 +12068,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, off += 4 + count * 4; } - if (gsp_sema_dbg && ch->h_object && svc.entry_log < 512) { + if (gsp_sema_dbg && ch->h_object && + svc.entry_log < 16384) { svc.entry_log++; { u8 tail[24]; @@ -6996,6 +12135,12 @@ struct svc_pending_release { u64 va; u32 payload; u32 op; /* 0 = RELEASE, 1 = REDUCTION (add payload) */ + u32 h; /* r316: channel the release was decoded for; the + * verified pool resolver is channel-keyed and h=0 + * never matches its per-channel entries (matmul + * r322: pool-rel cand resolved at decode (rrc=0, + * h=channel) while svc_apply_pending with h=0 + * starved forever and hung the client). */ /* * Resolution attempts that found no translation for this VA. * Telemetry for ISSUE-0001: a pending release that keeps failing to @@ -7012,25 +12157,67 @@ static struct svc_pending_release svc_pending[SVC_PENDING_MAX]; static u32 svc_pending_n; static u32 svc_pending_next; +static u64 svc_pending_rescued; /* stall-class: entries the old >= dedup would have dropped */ +static u32 svc_pending_rescued_log; +/* + * STALLFREE: the pending ring silently overwrote its oldest entry when full + * (an evicted release is a client wait that never clears). The counters are + * hoisted next to the ledger counters so the periodic diag (above) can print + * them; the ring is a bounded queue, so under churn this is the back-pressure + * signal, not an error path. + */ + static void svc_pending_add(u64 va, u32 payload, u32 op) { u32 i; if (!payload) return; - for (i = 0; i < svc_pending_n; i++) - if (svc_pending[i].va == va && - svc_pending[i].payload >= payload) + for (i = 0; i < svc_pending_n; i++) { + if (svc_pending[i].va != va) + continue; + /* + * Stall-class fix (R1): only an EXACT duplicate may be + * dropped. The old rule dropped every release behind an + * already-queued higher payload; in a burst the queue could + * end up holding only the last value (observed: payloads 7..34 + * decoded, queue held {7,34}, the guest word stopped short of + * the waiter's value and the client spun forever with + * starved=0). Distinct releases must always queue - the apply + * pass already stale-drops values the word has passed. + */ + if (svc_pending[i].payload == payload && + svc_pending[i].op == op) return; + if (svc_pending[i].payload >= payload) { + svc_pending_rescued++; + if (svc_pending_rescued_log < 32) { + svc_pending_rescued_log++; + pr_info("sema-svc: pending rescue va=0x%llx pay=%u op=%u (queued; ahead entry pay=%u op=%u)\n", + va, payload, op, + svc_pending[i].payload, + svc_pending[i].op); + } + } + } if (svc_pending_n < SVC_PENDING_MAX) { i = svc_pending_n++; } else { i = svc_pending_next++ % SVC_PENDING_MAX; + svc_pending_ovf++; + if (svc_pending_ovf_log < 8) { + svc_pending_ovf_log++; + pr_warn("sema-svc: PENDING-OVF ring full (%u); overwriting oldest entry va=0x%llx pay=%u op=%u (n=%llu)\n", + SVC_PENDING_MAX, svc_pending[i].va, + svc_pending[i].payload, svc_pending[i].op, + svc_pending_ovf); + } } svc_pending[i].va = va; svc_pending[i].payload = payload; svc_pending[i].op = op; svc_pending[i].tries = 0; + svc_pending[i].h = svc.cur_ch; } /* @@ -7049,6 +12236,35 @@ static void svc_session_flush(void) unsigned long flags; u32 i; + /* + * m167: never wipe a LIVE session. The flush is requested when a + * channel close leaves the table empty, but that transition fires + * during ordinary UVM channel cycling and the deferred execution can + * land after the same client (or the next client, when its chan_add + * raced the requested flush) has registered channels again - r169: + * the flush executed 2.5 s after smiley's launch with its channels + * live (registered 7 s earlier), wiped the queued completion releases, + * and the client spun forever in uvm_channel_is_value_completed + * (release never applied; pool word frozen). A live channel's session + * owns the tables - skipping is always safe because a genuine session + * end re-arms the flush when its own channels close. + */ + { + bool live = false; + + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) + if (svc.chan[i].valid) { + live = true; + break; + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + if (live) { + pr_info("sema-svc: session flush skipped: live channel(s) present (deferred flush raced a new session registration)\n"); + return; + } + } + svc_gr_diag_reset(); /* m150: per-session diagnostic budgets */ svc.map_n = 0; svc_pb_n = 0; @@ -7057,7 +12273,26 @@ static void svc_session_flush(void) svc_pd_best = 0; svc_pd_cands = 0; svc_pd_ev_hash = 0; - svc.direct_base = 0; + /* + * m177c: the UVM ring-space directory is per-client guest memory + * (and its samples come from this session's rings): reset with the + * session like the primary slot. + */ + svc_pd_r_found = 0; + svc_pd_r_attempts = 0; + svc_pd_r_base = 0; + svc_pd_r_ev_hash = 0; + svc_pd_r_n = 0; + /* + * m177: svc.direct_base is NOT cleared here. It is the guest kernel's + * direct-map base - a per-BOOT constant, identical for every client + * session of the same guest boot - and it is the gate and input of + * the pin-free resolution paths (svc_targeted_resolve). Clearing it + * per session forced every client to wait for a rebuild scan while + * its first cudaMalloc polled, and made the module depend on the + * host pin tools entirely. It is wiped on a guest reboot by + * svc_guest_reset_wipe() (gsp_sema_service_guest_reset()). + */ svc_pending_n = 0; svc_pending_next = 0; /* @@ -7068,6 +12303,18 @@ static void svc_session_flush(void) svc_opdef_reset_all(); svc_pool_pa_n = 0; svc_pool_pa_next = 0; + /* + * m168 addendum (r171): the UVM semaphore pool is per-CLIENT, not + * per-boot. The boot cache's conflict line proved it: the same VA + * resolved to a different PA for the next client + * (cached=0x1247b5000 new=0x1321fa000). A cache surviving the flush + * therefore served the previous client's dead page: the release was + * applied (read-back passes on a dead-but-readable page), dropped + * from pending, and the live client spun forever. Session-scope the + * cache; the scanner's per-client resolution is the source of truth. + */ + svc_pool_boot_n = 0; + svc_pool_boot_next = 0; svc_retry_next = 0; svc_fwd_rel_n = 0; svc_grce_dst_ring_next = 0; @@ -7114,6 +12361,25 @@ static void svc_session_flush(void) memset(svc_ipin_alt, 0, sizeof(svc_ipin_alt)); memset(svc_cnt_der, 0, sizeof(svc_cnt_der)); svc_cnt_der_n = 0; + /* r551: delivered-release marks are per-session (same handles reused). */ + memset(svc_opdef_done, 0, sizeof(svc_opdef_done)); + svc_opdef_done_next = 0; + /* + * r565: the GR exactly-once memories are boot-scoped caches keyed + * (partly) by push VA, and the UVM pool reuses the same launch push + * VAs for every client of the boot. svc_gr_cap_hist matches on the VA + * ALONE (8 entries, no hash), so a previous client's capture could + * classify a later client's fresh launch as "already captured" and + * skip it (`LAUNCH-SKIP (replay; already captured=1)`); the hash-keyed + * svc_gr_seen and the content-keyed svc_inline_once are lower risk but + * share the same lifetime. Reset all three per session - they refill + * within the session from the live client's own pushes. + */ + memset(svc_gr_cap_hist, 0, sizeof(svc_gr_cap_hist)); + svc_gr_cap_hist_pos = 0; + memset(svc_gr_seen, 0, sizeof(svc_gr_seen)); + memset(svc_inline_once, 0, sizeof(svc_inline_once)); + svc_inline_once_next = 0; /* * Re-arm the diagnostic log budgets per session. The first client of * a boot would otherwise spend every budget and the next session's @@ -7122,9 +12388,32 @@ static void svc_session_flush(void) * because the counters were already exhausted by the first client). */ svc.counter_rel_dbg = 0; + svc.crel_entry_dbg = 0; + svc.walk_dbg = 0; + svc.gr_rel_auth_log = 0; svc.counter_noslot_dbg = 0; svc.sem256_log = 0; svc.xfer_raw_log = 0; + svc.pushtrace_all = 0; + svc.opadd_log = 0; + svc.dmatrace_log = 0; + svc.copyent_log = 0; + svc_wait_log = 0; + svc_wait_pass_log = 0; + svc_remapconst_log = 0; + svc_pitch_log = 0; + svc_fill_raw_log = 0; + svc_lowfill_log = 0; + svc_tabcopy_log = 0; + svc_fill_noremap = 0; + svc_wait_n = 0; + svc_wait_rel_have = false; + memset(svc_wait_holds, 0, sizeof(svc_wait_holds)); + svc.bigrun_log = 0; + svc.lid_hdr_log = 0; + svc.skiprun_log = 0; + svc.mystery_log = 0; + svc.next_hdr_log = 0; /* * The push decode budget (`sema-push` lines) is per-session too: the * first client of a boot otherwise spends it and the next session's @@ -7135,6 +12424,14 @@ static void svc_session_flush(void) svc.entry_log = 0; svc_pushalt_log = 0; /* + * m169: the op-trace budget is session state too. Left un-reset, the + * first client of a boot spends it and the failing client's copy ops + * are invisible in exactly the runs that need them (r172: streams2's + * H2D evidence was blacked out because the shared budget was spent at + * 29704, before the client ran). + */ + svc.op_trace_log = 0; + /* * Per-entry/stream decode diagnostics are per-session: the first * client otherwise spends them and the second client's CE->GR * transition is invisible (r24-r30: c2's follow-up pushes never @@ -7153,6 +12450,8 @@ static void svc_session_flush(void) memset(svc_retry_q, 0, sizeof(svc_retry_q)); memset(svc_pool_pa, 0, sizeof(svc_pool_pa)); memset(svc_fwd_rel, 0, sizeof(svc_fwd_rel)); + memset(svc_inline_once, 0, sizeof(svc_inline_once)); + svc_inline_once_next = 0; memset(svc_grce_dst_ring, 0, sizeof(svc_grce_dst_ring)); for (i = 0; i < SVC_VAS_SLOTS; i++) { if (svc_vas[i].data) @@ -7162,11 +12461,117 @@ static void svc_session_flush(void) } svc_vas_n = 0; memset(svc.lat, 0, sizeof(svc.lat)); + /* m171 (AGENT-F R2): the write/read ledger is per-session. */ + svc_ledger_clear(); spin_lock_irqsave(&svc.vas_lock, flags); for (i = 0; i < SVC_MAX_VASPACES; i++) svc.vas[i].valid = false; spin_unlock_irqrestore(&svc.vas_lock, flags); - pr_info("sema-svc: session flush (all channels closed): discovery state reset for the next client\n"); + /* + * m176: carryover state that previously survived the flush and could + * poison the next client: + * - user-verified scan failures (tr_failed) held the pool VA page for + * 60 s even after the new client's page was reachable by targeted + * scan - clear them so the next session starts unblocked; + * - a forwarding hold whose channel is gone is resolved against a + * recycled handle with stale entry indices - drop it (no channels + * are live by definition of this flush); + * - the captured GR candidate belongs to the dead session's QMD/push; + * offering it to a recycled channel would replay a dead launch. + */ + memset(tr_failed, 0, sizeof(tr_failed)); + tr_fail_next = 0; + svc_fwd_hold_clear(); + memset(&svc_gr_cand, 0, sizeof(svc_gr_cand)); + /* m176: re-arm the opdef diagnostic budget (it was never per-session, + * so a late client's strand could be silent). */ + svc_opdef_log = 0; + /* One-cycle diag budgets (C2/C3/C7/C8/C9/C10) are per-session too. */ + svc_relx_log = 0; + svc_opdef_clr_log = 0; + svc_busy_defer_log = 0; + svc_hold_fb_log = 0; + svc_cand_ovw_log = 0; + svc_inline_recs_log = 0; + /* + * r401: the exactly-once capture memory is session-scoped. A new + * client's pushes restart their release counters, so their content + * (and hence hash) can collide with the previous client's - keeping + * the table would wrongly classify a fresh launch of the new + * session as already captured (LAUNCH-SKIP, silent drop). + */ + memset(svc_gr_seen, 0, sizeof(svc_gr_seen)); + svc_gr_seen_stale = 0; + /* m176: new session generation - channels and scrubbers re-stamp on + * registration and never inherit per-session learned state. */ + svc_sess_gen++; + pr_info("sema-svc: session flush (all channels closed): discovery state reset for the next client (gen %u)\n", + svc_sess_gen); +} + +/* + * m177: boot-scoped wipe for a GUEST boot (RM reload). Unlike the + * per-client session flush this clears the state that is valid for one + * guest boot only: the direct-map base and every content-scanned table. + * Runs on the svc thread (svc_guest_reset_wipe is invoked from the thread + * loop below), the only writer of this state. + */ +static void svc_guest_reset_wipe(void) +{ + svc.direct_base = 0; + svc.map_n = 0; + svc_pd_found = 0; + svc_pd_attempts = 0; + svc_pd_best = 0; + svc_pd_cands = 0; + svc_pd_ev_hash = 0; + svc_pd_r_found = 0; + svc_pd_r_attempts = 0; + svc_pd_r_base = 0; + svc_pd_r_ev_hash = 0; + svc_pd_r_n = 0; + svc_pb_n = 0; + svc_pool_boot_n = 0; + svc_pool_boot_next = 0; + svc_pool_pa_n = 0; + svc_pool_pa_next = 0; + memset(svc_pool_pa, 0, sizeof(svc_pool_pa)); + memset(tr_failed, 0, sizeof(tr_failed)); + tr_fail_next = 0; + memset(svc.lat, 0, sizeof(svc.lat)); + pr_info("sema-svc: guest reset: boot-scoped discovery state wiped (direct_base, map, caches)\n"); +} + +/* + * m177: called by gsp_rm_emu_reset() when the guest's GSP-RM is (re)loaded + * - a new guest boot, where every VA->PA relation can have changed. The + * request is latched and executed on the svc thread, which owns the + * discovery state (a cross-thread wipe here would race svc_rebuild). + */ +void gsp_sema_service_guest_reset(void) +{ + WRITE_ONCE(svc_guest_reset_req, 1); + if (svc.task) + wake_up_process(svc.task); +} + +/* + * r317: queue a pass's recorded forward releases through the pending + * machinery instead of applying them at a hold break / fallback while an + * op of that pass is still owed. Applying the fence early let the driver + * refill the ping-pong staging before the copy read it (the cold-run + * capture gap: vector_add run 1 of every cold boot, last ~2 chunks). The + * pending store applies each release once the word resolves (the copy has + * run by then, since the op execution is what clears the pending state on + * the retry). + */ +static void svc_fwd_rel_queue_pending(struct svc_fwd_rel *rel, u8 n) +{ + u8 i; + + for (i = 0; i < n; i++) + svc_pending_add(rel[i].va, rel[i].payload, + rel[i].inc_claimed ? 1u : 0u); } /* @@ -7200,8 +12605,11 @@ static void svc_apply_pending(struct nv_vgpu_device *vgpu) * resolver: the learned map can answer with a stale * page whose value makes this release look already * delivered and drops it (see svc_sem_write). + * r316: resolve with the channel the release was + * decoded for (h=0 never matches the per-channel + * pool entries - matmul r322 starved/hung). */ - if (svc_pool_pa_resolve(vgpu, 0, p->va, &ppa, &pap, + if (svc_pool_pa_resolve(vgpu, p->h, p->va, &ppa, &pap, NULL)) { /* not translatable yet: keep (ISSUE-0001) */ p->tries++; @@ -7229,7 +12637,7 @@ static void svc_apply_pending(struct nv_vgpu_device *vgpu) * a pool VA the base may not have landed: applying while * the word is still 0 is absorbed, so hold and retry. */ - if (svc_pool_sem_va(p->va) && svc_pool_hold_active()) { + if (svc_pool_sem_va(p->va) && svc_pool_hold_active_va(p->va)) { p->tries++; svc_pending_note_starved(p); i++; @@ -7255,7 +12663,7 @@ static void svc_apply_pending(struct nv_vgpu_device *vgpu) if (!svc_sem_write(vgpu, 0, p->va, p->op ? cur + p->payload : p->payload)) { svc.releases_applied++; - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); if (gsp_sema_dbg) pr_info("sema-svc: deferred release va=0x%llx payload=%u (was cur=%u)\n", p->va, p->payload, cur); @@ -7387,6 +12795,25 @@ static u32 __maybe_unused __svc_apply_page_data_disabled(struct nv_vgpu_device * } /* + * F1-b: while the forwarded GR launch this release belongs to + * is still in flight, the sweep must not author its QMD + * completion word - the backend poll would break on a + * host-authored ack (r198: qmd_rel satisfied ~6 ms after the + * doorbell, zero engine writes). Defer: nothing is written or + * consumed here; the release stays in the guest page and later + * sweep passes re-see it, while the hold/fallback path owns a + * failed hardware copy. + */ + if (gsp_sema_gr_swrel_gate && svc_gr_rel_inflight(va, NULL)) { + if (svc.gr_rel_auth_log < 256) { + svc.gr_rel_auth_log++; + pr_info("sema-diag: GR-REL-AUTHOR src=sweep va=0x%llx pay=%u h=0x%x DEFER\n", + va, payload, 0u); + } + continue; + } + + /* * A release at page offset 0x10 with payload 1 in front of a * channel's first push identifies the page as that channel's * push-buffer base (see svc_pb_add). @@ -7395,9 +12822,21 @@ static u32 __maybe_unused __svc_apply_page_data_disabled(struct nv_vgpu_device * (lo & 0xfff) < 0x20 && svc_pb_first_push(buf)) svc_pb_add(pa_page, lo & 0xfff); + /* + * F1-a: tag software authorship of the forwarded launch's + * release word (gate off, or the request already resolved and + * the forward progress rule still allows the write). + */ + if (svc_gr_rel_match(va, NULL) && + svc.gr_rel_auth_log < 256) { + svc.gr_rel_auth_log++; + pr_info("sema-diag: GR-REL-AUTHOR src=sweep va=0x%llx pay=%u h=0x%x\n", + va, payload, 0u); + } + if (svc_sem_write(vgpu, 0, va, payload) == 0) { svc.releases_applied++; - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); applied++; if (payload != cur + 1) { svc.releases_jumped++; @@ -7428,7 +12867,6 @@ static u32 svc_apply_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page) * *semaphore* address inside a release needs to resolve through the map. */ #define SVC_SCAN_WINDOW (32ULL << 20) /* 32 MB per pass */ -#define SVC_SCAN_CHUNK (64 * 1024) /* 16 pages per guest read */ #define SVC_MBUF_MAX 6 /* rescue pin candidates */ /* @@ -7616,7 +13054,8 @@ static void svc_pb_discover(struct nv_vgpu_device *vgpu) u32 slot, k; bool have = false; - if (!svc.chan[i].valid || !svc.chan[i].engine_type) + if (!svc.chan[i].valid || svc.chan[i].vgpu != vgpu || + !svc.chan[i].engine_type) continue; ce = true; if (svc.chan[i].ce_index >= 32) @@ -8397,6 +13836,119 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, return 0; } } + + /* + * r496b: no candidate matched. For a GR channel whose ring is known + * (translated and pinned - the sysmem content scan or the VRAM + * conventions), the USERD may still be elsewhere: the X/GL driver + * allocates its USERD in VRAM and the CPU RM maps it through a BAR1 + * window, so the driver's GP_PUT write lands at a BAR1 offset that is + * neither the registered memdesc base nor ring-derived. Scan the + * emulated VRAM for the USERD shape and validate each hit against the + * ring: the put it claims must correspond to entries whose pushbuffer + * VAs all sit in the ring's allocation window. A foreign channel's + * USERD cannot pass that test. One scan per channel (the result + * persists through the pump's write-back), logged on latch. + */ + if (!ch->engine_type && ch->userd_scans < 8 && ring_pa) { + u64 off, lim; + u8 *base = vgpu->bars[1].vaddr; + u64 best = 0, best_dist = ~0ULL; + u32 best_put = 0, best_get = 0; + bool best_pending = false; + + ch->userd_scans++; + lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x20000000ULL); + if (base && lim) { + for (off = 0; off + 0xa0 <= lim; off += 0x200) { + u32 hp, hg; + bool ok = true; + u32 k; + u64 dist; + bool pending; + + memcpy(&hp, base + off + SVC_USERD_GP_PUT, 4); + memcpy(&hg, base + off + SVC_USERD_GP_GET, 4); + if (!hp || hp > ch->gpfifo_entries || hg > hp) + continue; + /* Validate: every claimed entry decodes to a + * pushbuffer VA in the ring's window. */ + for (k = 0; k < hp && k < 64; k++) { + u8 e[8]; + u64 eva; + u32 elen, a0, a1; + + if (svc_read_va(vgpu, + ch->gpfifo_va + + (u64)k * 8, + e, sizeof(e))) { + ok = false; + break; + } + memcpy(&a0, e, 4); + memcpy(&a1, e + 4, 4); + eva = ((u64)(a1 & 0xff) << 32) | + (u64)(a0 & 0xfffffffc); + elen = ((a1 >> 10) & 0x1fffff) << 2; + /* + * Every entry the put claims must name a + * VA in the ring's allocation window - + * pushbuffers below the ring, the + * progress tracker above it. + */ + if (!eva || + eva + SVC_GR_RAM_WIN / 2 < + ch->gpfifo_va || + eva >= ch->gpfifo_va + + SVC_GR_RAM_WIN / 2 || + elen < 4 || elen > 0x100000) { + ok = false; + break; + } + } + if (!ok) + continue; + /* + * Several blocks can validate: a previous + * session leaves its old USERD holding the same + * put/entry values while the new channel's block + * sits elsewhere (observed r504: the re- + * registered channel re-scanned, adopted the + * stale block, and its DMA test timed out). + * Prefer a block with work PENDING (get < put); + * among equals prefer the one nearest the + * registered memdesc offset (the driver's own + * hint). The pump's GP_GET write-back goes to + * the adopted block, so a stale adoption strands + * the client's poll. + */ + dist = off > ch->userd_pa ? off - ch->userd_pa + : ch->userd_pa - off; + pending = hg < hp; + if (best && + (best_pending > pending || + (best_pending == pending && dist >= best_dist))) + continue; + best = off; + best_dist = dist; + best_put = hp; + best_get = hg; + best_pending = pending; + } + if (best) { + ch->userd_pa = best; + ch->userd_as = SVC_APERTURE_FB; + ch->userd_ok = true; + *put = best_put; + *get = best_get; + pr_info("sema-chan: GR USERD h=0x%x rank=%u found by VRAM shape scan at fb 0x%llx (put=%u get=%u%s, ring-validated)\n", + ch->h_object, ch->gr_index, best, + best_put, best_get, + best_pending ? "" : ", idle"); + return 0; + } + } + } return -ENOENT; } @@ -8494,7 +14046,7 @@ static void svc_notify_complete(struct nv_vgpu_device *vgpu, /* Keep the completion visible to clients that start waiting * after the event fired. */ - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); } } @@ -8791,6 +14343,9 @@ static bool svc_gr_content_pin(struct nv_vgpu_device *vgpu, u32 rank, svc_pin_store(push_va, push_pa, SVC_APERTURE_SYS); svc_lat_store(push_va, push_pa, SVC_APERTURE_SYS); svc_mark_hot(push_pa); + /* m177c: a content-verified UVM-space pair - feed the ring-space + * directory discovery (counter pages resolve through it). */ + svc_pd_ring_note(push_va, push_pa); if (ring_ap == SVC_APERTURE_FB) pr_info("sema-chan: content-discovered GR ring rank=%u va=0x%llx -> fb 0x%llx (entry0 va=0x%llx len=%u; push pa=0x%llx; userd rc=%d put=%u get=%u)\n", rank, ring_va, ring_pa, push_va, len, push_pa, prc, @@ -8897,7 +14452,8 @@ static void svc_gr_content_discover(struct nv_vgpu_device *vgpu) int pap = 0; u32 r = c->gr_index; - if (!c->valid || c->engine_type || r >= SVC_GR_CONTENT_RANKS) + if (!c->valid || c->vgpu != vgpu || c->engine_type || + r >= SVC_GR_CONTENT_RANKS) continue; if (test_bit(r, &svc_gr_ring_done)) continue; @@ -9126,6 +14682,202 @@ bool gsp_sema_counter_derive = true; unsigned gsp_sema_gr_grace_ms = 8000; /* + * GRFIX FIX1b (r213): absolute per-LAUNCH bound (ms) for the data-stalled + * hold of svc_gr_data_fill. The identity is the launch push VA, not the + * GPFIFO slot: pin-batch rewinds/replays re-decode the same launch at + * different slot indices (r213: entry 107 -> 107 -> 99 -> ... on one + * channel), so an idx-keyed deadline re-armed on every pass and never + * fired - 410+ holds over 216 s, gr_hold_exp=0, client killed at its 240 s + * timeout. The deadline is fixed at the first hold of a launch; when it + * passes, svc_gr_request proceeds with the launch exactly as the + * pre-8b058884 path did (forward best-effort - never block the guest). + */ +unsigned gsp_phys_gr_hold_max_ms = 3000; + +/* + * GRFIX FIX1e (r215): master gate for the GR fill hold on dropped operand + * pages. Default 0 = LOG-ONLY: svc_gr_data_fill never holds on dropped + * pages - the drops are logged (GRFILL-DROP) and the launch takes the + * historical forward path immediately (exactly the pre-8b058884 behaviour); + * the GRFILL-HOLD/GRFILL-EXPIRED machinery and the per-launch absolute + * bound are skipped entirely (the pump's own m150/m156 hold and its + * absolute bound are NOT - they run regardless of this gate; the record + * fields are now in the snapshot write-back list, see the GRFIX NOTE). + * When set to 1 the page-count hold + absolute bound apply on the fill + * path too. + */ +unsigned gsp_phys_gr_fill_hold; /* default 0: log-only, never hold */ + +/* + * r346: trace GMMU walks for unresolved high-VA operands (the arg-miss + * diagnostic re-walks each published vaspace for a failing base). When + * nonzero, those walks print the per-level entries (L0..L2, PD0 halves) + * so a failing VA shows exactly where the walk dies. + */ +unsigned gsp_sema_walk_trace; + +/* + * F1-b: GR release-word authorship gate (default 1). While a forwarded GR + * launch is in flight, the RAM-content sweep must not author the very QMD + * completion release the backend poll is waiting for: a sweep write could + * satisfy the poll before the engine retires the QMD (false completion, + * silent corruption). With the gate on, such a write is deferred - the + * release stays in RAM and later sweep passes re-see it; when the hardware + * copy resolves, the hold/fallback path applies it. 0 restores the + * unconditional sweep apply for an A/B. + */ +bool gsp_sema_gr_swrel_gate = true; + +/* + * m164 CLASS-2: hold window (ms) for a readable GR launch push whose decode + * saw QMD words but built no launch candidate for its channel (a replay- + * suppressed capture, a partial/split QMD, or a push page the pin stream has + * not reached yet). The entry is left unconsumed for this window so the next + * pass re-decodes it with a forced fresh capture (gr_rehold_idx); on expiry + * the historical consume applies. Without this, such entries were consumed + * silently and the client's launch never ran (streams2 second vadd: 56 + * consumes / 2 captures on one channel; the client read zeros). + */ +unsigned gsp_sema_gr_nocap_hold_ms = 6000; + +/* + * m165 (D-F2a): when set (default), a pool release is only deferred by the + * post-registration hold while NO host-verified injected pin exists for its + * VA; a fresh pin is this session's ground truth and lets the apply land at + * pin time (~0.5-2 s) instead of registration+5 s. Set 0 to restore the + * legacy blanket 5 s hold. + */ +bool gsp_sema_pool_hold_fresh = 1; + +/* + * m169 (AGENT-F R1): hold device-destination copies whose only + * translation is a page-directory walk or RAM scan (no pin). Default on; + * 0 restores the historical immediate-write behavior. + */ +bool gsp_sema_copy_dst_pin_guard = 1; +/* + * r336/r34x wait gate: honor the C56F host-semaphore acquire ops captured + * for a ring entry before running its work. DEFAULT ON since r408: the + * r408 streams2d forensics reproduced the class the gate was built for + * with the gate off - the c2 readback (C56F ACQ_CIRC_GEQ pay=4 on the + * scale channel's progress counter 0x20400ffd0) ran at 106617.279 with + * cur=2, i.e. BEFORE the c2-add launch was even submitted (106617.411), + * read the not-yet-produced buffer (zeros) and the client failed with + * 'c2 all zeros'. With the gate on the same readback holds (sema-wait: + * hold ... cur=2 waited_ms=353) until the counter reaches 4 (pass ... + * cur=4 at 106812.259) and the ordering is correct. The semantics are + * the class contract's: op 0 = equal, 2 = strict GEQ, 3 = circular GEQ, + * 4 = AND, 5 = NOR; the wait set is captured PER ENTRY (the old + * single-last-wait global leaked a later entry's acquire into an earlier + * entry - the r340 over-hold); the entry's own release counts only when + * its method order precedes the acquire; pending releases our machinery + * is merely late about are drained first; and every hold is bounded at + * gsp_sema_wait_hold_ms (default 4000 ms, r401) then fails open loudly + * ('give-up') so a value that can only be installed behind an upstream + * opdef hold cannot park a channel. Diagnostics: 'sema-wait: + * pass/hold/give-up' rows carry op, payload, observed value and decision. + * sema_wait_gate=0 restores the pre-r336 pass-through for an A/B. + */ +bool gsp_sema_wait_gate = 1; + +/* + * m169b: the guard above must never deadlock the boot's first client. The + * first device-destination H2D of a client runs before any launch, and the + * pins for device VAs are (re)discovered by the launch/fill paths - so at + * that instant NO pin can exist. r173: mail's H2D was held forever + * (opdef_held ~6000 passes, six expiry cycles, the entry re-held on every + * re-decode because the retry keeps resolving prov>=3). Bound the wait per + * destination page: after this many ms the guard releases and the copy runs + * with the walk translation (the historical behavior; the walk of a + * just-built client table is fresh), logged loudly. + */ +unsigned gsp_sema_copy_dst_guard_ms = 3000; + +/* + * r231 READ-ORDER guard (default 0: witness only). A device-src copy whose + * source VA has a PENDING device-dst write hold (the m169 guard window is + * still open) reads the PRE-write content: the writer's bytes land only + * when the guard resolves, while the reader can execute immediately. The + * ctaid_probe RAWA round trip is the reproducer - upload a fresh 64 KB + * buffer, read it straight back with no kernel - and the readback came + * back as foreign deterministic content. With this on, such a read is + * deferred through the existing bounded-hold machinery (-ENOENT) so it + * re-decodes after the write resolves; with it off the condition is + * logged (READ-ORDER-WIT) so the next capture proves the coincidence. + * Opt-in until an A/B proves no green regression. + */ +bool gsp_sema_read_order_guard; + +/* + * COPYWIT fail4: per-boot budget for the sema-bigcopy resolution witnesses + * (svc_copy's large-copy proof: both operands' VA -> pa/aperture/provenance + * plus the first 16 bytes each side). The shipped default kept the boot's + * head only; a failing-client run happens minutes later, so the budget is a + * module parameter now - raise it for a discrimination cycle and correlate + * each D2H's resolved source PA with the booter's OUT-PAGE-WIT copy-out PA + * for the same device VA. 0 disables the lines. + */ +unsigned gsp_sema_bigcopy_log_max = 24; +#define SVC_DST_GUARD_SLOTS 16 +struct svc_dst_guard { + u64 va_page; + unsigned long first; +}; +static struct svc_dst_guard svc_dst_guard[SVC_DST_GUARD_SLOTS]; +static u32 svc_dst_guard_next; + +/* true = the guard may hold this destination; false = release it. */ +static bool svc_dst_guard_hold(u64 va) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + unsigned long now = jiffies; + u32 i, slot = SVC_DST_GUARD_SLOTS; + + for (i = 0; i < SVC_DST_GUARD_SLOTS; i++) { + if (svc_dst_guard[i].va_page == page) { + if (time_before(now, svc_dst_guard[i].first + + msecs_to_jiffies( + gsp_sema_copy_dst_guard_ms))) + return true; + if (svc_dst_guard[i].first) { + svc_dst_guard[i].first = 0; + svc.copy_dst_relaxed++; + pr_info("sema-svc: device-dst guard released without a pin va=0x%llx (using the walk translation)\n", + page); + } + return false; + } + if (!svc_dst_guard[i].va_page) + slot = i; + } + if (slot == SVC_DST_GUARD_SLOTS) + slot = svc_dst_guard_next++ % SVC_DST_GUARD_SLOTS; + svc_dst_guard[slot].va_page = page; + svc_dst_guard[slot].first = now; + return true; +} + +/* + * r231 READ-ORDER guard: true while a held device-dst write for this VA + * page is still inside its bounded window (the writer's bytes have not + * landed yet). See the gsp_sema_read_order_guard declaration. + */ +static bool svc_dst_guard_pending(u64 va) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + unsigned long now = jiffies; + u32 i; + + for (i = 0; i < SVC_DST_GUARD_SLOTS; i++) + if (svc_dst_guard[i].va_page == page && + svc_dst_guard[i].first && + time_before(now, svc_dst_guard[i].first + + msecs_to_jiffies(gsp_sema_copy_dst_guard_ms))) + return true; + return false; +} + +/* * CE release fallback: when a push carrying a tracking-semaphore release * cannot be read (and the content rescue failed), advance the channel's * tracking semaphore to the expected payload directly. The push's data @@ -9156,6 +14908,24 @@ bool gsp_sema_rel_recover; bool gsp_sema_dbg; /* + * r365: execute fill-shaped CE transfers whose destination is a VIDMEM + * WINDOW VA (>= SVC_VIDMEM_VA_LO, below the 1 TiB VA floor). The guest's + * page-table writes arrive exactly this way (dma=0x586, REMAP+PITCH, a + * count=1 SET_REMAP_CONST_A, line=4, out=0x2000c0xxx) and previously + * decoded no op, so the shadow never saw them and the first client of a + * fresh boot dropped operand pages (GRFILL-DROP -> launch held/fail-open + * with 8 dropped pages -> wrong results or a client wedge). + * + * The op is built with the VA unchanged (phys=0): svc_fill's virtual path + * resolves it through the guest's own published page tables + * (svc_op_translate) and only falls back to the shadow for floor-range + * VAs - never through the window convention (r362 proved a forced + * `addr - SVC_VIDMEM_VA_BASE` writes the wrong page and corrupts results: + * reduce sum=0 + mail hang). Default off for A/B. + */ +bool gsp_sema_winfill; + +/* * Skip executing a push's collected work (CE memsets/copies into the vidmem * shadow) while still applying its semaphore release. The client only ever * observes the release; the shadow writes exist so the emulation can walk @@ -9214,6 +14984,139 @@ unsigned gsp_sema_push_hold_ms = 2500; static unsigned long svc_gr_ring_tried; /* ranks that got a diagnostic line */ +static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 *pa_out) +{ + static u8 *buf; + u64 start = ch->gr_ram_cursor; + u64 end = start + SVC_GR_RAM_SLICE; + u64 gpa; + int i; + unsigned long flags; + + if (end > SEMA_PA_MAX) + end = SEMA_PA_MAX; + if (!buf) { + buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); + if (!buf) + return false; + } + for (gpa = start; gpa + SEMA_PAGE_SIZE <= end; + gpa += SVC_SCAN_CHUNK) { + u64 chunk = min_t(u64, SVC_SCAN_CHUNK, end - gpa); + u32 o; + + if (svc_read(vgpu, gpa, buf, chunk)) + continue; + for (o = 0; o + 16 <= chunk; o += SEMA_PAGE_SIZE) { + u32 e0, e1, f0, f1; + u64 va0, va1; + u32 len0, len1; + + memcpy(&e0, buf + o, 4); + memcpy(&e1, buf + o + 4, 4); + va0 = ((u64)(e1 & 0xff) << 32) | + (u64)(e0 & 0xfffffffc); + len0 = ((e1 >> 10) & 0x1fffff) << 2; + /* entry0: main pushbuffer, just below the ring */ + if (va0 >= ch->gpfifo_va || + va0 + SVC_GR_RAM_WIN / 2 < ch->gpfifo_va) + continue; + if (va0 < SEMA_VA_LO || va0 >= SEMA_VA_HI) + continue; + if (len0 < 4 || len0 > 0x100000) + continue; + memcpy(&f0, buf + o + 8, 4); + memcpy(&f1, buf + o + 12, 4); + va1 = ((u64)(f1 & 0xff) << 32) | + (u64)(f0 & 0xfffffffc); + len1 = ((f1 >> 10) & 0x1fffff) << 2; + /* entry1: progress tracker, above entry0 */ + if (va1 <= va0 || + va1 > ch->gpfifo_va + SVC_GR_RAM_WIN / 2) + continue; + if (len1 < 4 || len1 > 0x100000) + continue; + *pa_out = gpa + o; + return true; + } + } + ch->gr_ram_cursor = (end >= SEMA_PA_MAX) ? 0 : end; + ch->gr_ram_passes++; + if (ch->gr_ram_cursor == 0 && !ch->gr_ram_logged) { + ch->gr_ram_logged = true; + pr_info("sema-chan: GR ring h=0x%x rank=%u RAM content scan found no candidate in %u slice(s) (va=0x%llx)\n", + ch->h_object, ch->gr_index, ch->gr_ram_passes, + ch->gpfifo_va); + } + /* + * Persist the cursor immediately: the pump's stalled-userd path + * `continue`s past the end-of-pass write-back, so without this the + * scan would restart from slice 0 every pass and never reach the + * ring's page. + */ + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].h_object == ch->h_object) { + svc.chan[i].gr_ram_cursor = ch->gr_ram_cursor; + svc.chan[i].gr_ram_passes = ch->gr_ram_passes; + svc.chan[i].gr_ram_logged = ch->gr_ram_logged; + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + return false; +} + +/* + * r496f: discover a kernel-RM client VAS's page directory in the emulated + * VRAM by verification. The CPU RM builds internal-VAS page tables in + * vidmem and never publishes them (SET_PAGE_DIRECTORY is an externally + * owned / UVM-only RPC), so the pushbuffer pages and the progress + * semaphore of an X/GL channel cannot be translated from any existing + * source. Try each 4 KB page as the walk root and accept the first whose + * walk reproduces the ring VA -> ring PA pair we already validated by + * content; registering it with the VAS table makes every other VA of the + * same client resolvable through the existing walker. Bounded to the + * vidmem shadow; verified, so a wrong candidate cannot be registered. + */ +static void svc_xvas_pdb_discover(struct nv_vgpu_device *vgpu, u64 ring_va, + u64 ring_pa) +{ + static const u32 ent_tab[] = { 4, 8, 16, 32, 64, 2, 128, 256, 512 }; + u64 lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x20000000ULL); + u64 off; + unsigned int ei; + + if (!vgpu->bars[1].vaddr || !lim) + return; + for (off = 0; off + 0x1000 <= lim; off += 0x1000) { + for (ei = 0; ei < ARRAY_SIZE(ent_tab); ei++) { + u64 pa = 0; + int ap = 0; + + if (!svc_gmmu_walk(vgpu, off, ent_tab[ei], + ring_va & ~0xfffULL, + SVC_APERTURE_FB, &pa, &ap, false) && + (pa & ~0xfffULL) == (ring_pa & ~0xfffULL)) { + struct sema_vaspace_desc d; + + memset(&d, 0, sizeof(d)); + d.h_vaspace = 0x5a5a0000u | + (u32)(ring_va >> 20); + d.pdb = off; + d.num_entries = ent_tab[ei]; + d.b_vidmem = true; + gsp_sema_service_vaspace_add(&d); + pr_info("sema-chan: client VAS page directory discovered pdb=0x%llx entries=%u (ring VA 0x%llx -> pa 0x%llx verified; the client's other VAs are now walkable)\n", + off, ent_tab[ei], ring_va, ring_pa); + return; + } + } + } +} + static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { @@ -9238,7 +15141,8 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, */ spin_lock_irqsave(&svc.chan_lock, flags); for (k = 0; k < SVC_MAX_CHANNELS; k++) - if (svc.chan[k].valid && svc.chan[k].engine_type && + if (svc.chan[k].valid && svc.chan[k].vgpu == vgpu && + svc.chan[k].engine_type && svc.chan[k].ce_index + 1 > max_ce) max_ce = svc.chan[k].ce_index + 1; spin_unlock_irqrestore(&svc.chan_lock, flags); @@ -9489,6 +15393,124 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, pr_info("sema-chan: fbs scan done h=0x%x hits=%d (lim 0x%llx)\n", ch->h_object, hits, lim); } + + /* + * r495 diagnostic (X/GL channel): the X driver's GR channel + * registers its USERD at a memdesc base that may be an FB + * offset or a BAR1 window. The driver's kickoff writes GP_PUT + * through its CPU mapping of that object; if the write landed + * in the emulated FB shadow, scanning for the USERD shape + * (GP_PUT in [1..entries] at +0x8c, GP_GET <= GP_PUT at +0x90, + * both nonzero somewhere in the 0x200 block) names the real + * offset. One-shot per channel, diagnostic only. + */ + if (gsp_sema_dbg && vgpu->bars[1].vaddr && + svc_fb_shadow_size(vgpu) && ch->gpfifo_entries <= 0x20000) { + u64 lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x20000000ULL); + u64 off; + int hits = 0; + + for (off = 0; off + 0xa0 <= lim && hits < 8; + off += 0x200) { + u32 hp, hg; + + memcpy(&hp, vgpu->bars[1].vaddr + off + 0x8c, 4); + memcpy(&hg, vgpu->bars[1].vaddr + off + 0x90, 4); + if (!hp || hp > ch->gpfifo_entries || hg > hp) + continue; + pr_info("sema-chan: userd scan hit off=0x%llx put=%u get=%u (blk %08x %08x %08x %08x)\n", + off, hp, hg, + *(u32 *)(vgpu->bars[1].vaddr + off + 0x80), + *(u32 *)(vgpu->bars[1].vaddr + off + 0x84), + *(u32 *)(vgpu->bars[1].vaddr + off + 0x88), + *(u32 *)(vgpu->bars[1].vaddr + off + 0x94)); + hits++; + } + pr_info("sema-chan: userd scan done h=0x%x hits=%d (lim 0x%llx)\n", + ch->h_object, hits, lim); + } + } + + /* + * r496b: guest-RAM content scan for the ring (see the gr_ram_cursor + * field comment). The driver's kickoff writes GPFIFO entries whose + * pushbuffer VAs sit just BELOW the ring VA in the same allocation + * (main pushbuffer first, GPFIFO after - nvidia-push-priv.h): entry0 + * in (ring_va - 0x100000, ring_va), entry1 above entry0 and up to + * ring_va + 0x100000 (the progress tracker). A page whose first two + * 8-byte words both name such VAs with sane lengths is the ring; pin + * it (validated) and its allocation window (guesses) so the ring + * entry and pushbuffer reads resolve. Sliced so the service thread + * stays responsive; gives up per channel after + * SVC_GR_RAM_PASS_MAX slices. + */ + if (ch->gr_ram_passes < SVC_GR_RAM_PASS_MAX && + !test_bit(ch->gr_index, &svc_gr_ring_done)) { + u64 hit_pa = 0; + + if (svc_gr_ram_ring_scan(vgpu, ch, &hit_pa)) { + /* + * Pin ONLY the content-validated ring page (validated, + * not a window guess). The pushbuffer pages and the + * tracker push of this client resolve through the VAS + * page directory registered from COPY_SERVER_PDES + * (r496g) - the surface is NONCONTIGUOUS, so a linear + * window guess is wrong for them and, being a + * pin_lookup_any hit, would shadow the correct walk in + * svc_lookup_trusted (observed r502: the tracker push + * read zeros through the wrong window page and the + * driver's DMA poll never completed). + */ + svc_pin_store(ch->gpfifo_va, hit_pa, + SVC_APERTURE_SYS); + svc_lat_store(ch->gpfifo_va, hit_pa, + SVC_APERTURE_SYS); + ch->gr_ram_logged = true; + ch->gr_ram_ring = true; + set_bit(ch->gr_index, &svc_gr_ring_done); + svc_ring_n++; + /* + * r496j: with the ring content-validated, verify the + * parked COPY_SERVER_PDES candidates against it and + * register the directory that reproduces the ring's + * PA; its walk then resolves the client's other VAs + * (pushbuffer pages, the progress semaphore the + * driver polls). r496f's blind vidmem sweep stays as + * a fallback for directories the control never + * published. + */ + svc_vas_candidates_verify(vgpu, ch->gpfifo_va, + hit_pa); + svc_xvas_pdb_discover(vgpu, ch->gpfifo_va, hit_pa); + /* + * Persist the latch immediately: a later stalled + * path in this pass may `continue` past the + * end-of-pass write-back, and the next pass's + * snapshot must carry gr_ram_ring (the zero-NOP + * exception and the no-capture-hold skip key on + * it). + */ + { + unsigned long flags2; + int i2; + + spin_lock_irqsave(&svc.chan_lock, flags2); + for (i2 = 0; i2 < SVC_MAX_CHANNELS; i2++) + if (svc.chan[i2].valid && + svc.chan[i2].vgpu == vgpu && + svc.chan[i2].h_object == + ch->h_object) { + svc.chan[i2].gr_ram_ring = true; + svc.chan[i2].gr_ram_logged = true; + break; + } + spin_unlock_irqrestore(&svc.chan_lock, flags2); + } + pr_info("sema-chan: GR ring h=0x%x rank=%u found by RAM content scan va=0x%llx -> pa=0x%llx (allocation window pinned; entry/push reads resolve through it)\n", + ch->h_object, ch->gr_index, ch->gpfifo_va, + hit_pa); + return; + } } } @@ -9496,8 +15518,7 @@ static void svc_ring_autopin(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { u64 pa = 0, va = 0; - int ap = SVC_APERTURE_SYS; - u8 e[SVC_GPFIFO_ENTRY_SIZE]; + int ap = SVC_APERTURE_SYS; u8 e[SVC_GPFIFO_ENTRY_SIZE]; u32 e0, e1, len; if (!ch->engine_type) { @@ -9580,6 +15601,9 @@ static void svc_pb_pair(struct nv_vgpu_device *vgpu, struct svc_channel *ch) } svc_pb[hit_i].va = va; set_bit(ch->ce_index, &svc_pb_done); + /* m177c: the base page was content-read (svc_chan_entry0) - a + * verified UVM-space sample for the ring-space directory. */ + svc_pd_ring_note(va, hit_pa); /* * The pinned pairs are verified samples of UVM's own page directory; @@ -9946,6 +15970,19 @@ static int svc_rescue_run_try(struct nv_vgpu_device *vgpu, } /* + * STALLFREE: wall-clock cap for one content-rescue scan range. The scan is + * bounded by the guest-RAM range, not by time: the full fallback walks up to + * 6 GB in 4 KB vfio reads, and under a pin storm / leftover crashed client's + * churn that walk can run for tens of seconds on the service thread, + * delaying every channel (GR launches included). Green scans answer from the + * hot set or the near window in milliseconds, so a 3 s cap never fires on a + * healthy boot; when it does fire the entry falls through its existing grace + * path (hold, then consume with the software release) instead of pinning the + * whole pump. + */ +#define SVC_RESCUE_SCAN_MAX_MS 3000u + +/* * Scan a guest-RAM range descending for a page that validates as this entry's * push (method stream + expected release payload), pinning it and returning 0 * on the first hit. `hi` and `lo` are inclusive chunk start addresses. @@ -9958,10 +15995,20 @@ static int svc_rescue_scan_range(struct nv_vgpu_device *vgpu, { u64 gpa = hi; unsigned long t0 = jiffies; + unsigned long deadline = + t0 + msecs_to_jiffies(SVC_RESCUE_SCAN_MAX_MS); u32 total = 0; u64 bytes = 0; for (;;) { + if (time_after_eq(jiffies, deadline)) { + svc_rescue_scan_capped++; + if (svc_rescue_scan_capped <= 8) + pr_warn("sema-svc: RESCUE-SCAN-CAP h=0x%x va=0x%llx [0x%llx..0x%llx] %llu MB in %u ms (capped; entry keeps its grace path)\n", + ch->h_object, entry_va, hi, lo, + bytes >> 20, SVC_RESCUE_SCAN_MAX_MS); + break; + } if (!svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK)) { u32 ncand, n; @@ -10311,20 +16358,96 @@ static void svc_counter_publish(struct nv_vgpu_device *vgpu, { u32 val; u32 cur = 0; + /* + * STALL FIX (rank-3 completion-word pollution): the publish must + * target the channel's OWN completion word - the descending slot its + * pushes release, which the autoreg above registers and which the + * decoded releases write (release_pa). Writing the ring PUT into the + * ascending rank slot (F1/eea5ea93) made every channel's publish land + * in the paired rank (19-r)'s waited word: the busy ctx-init channel + * rank 16 (h=0x5c000059, PUT~94) overwrote rank 3's word 0x20400ffc0 + * from 1 to 94 (`pub-adv h=0x5c000059 pa=...fc0 val=94 cur=1`), the + * rank-3 stream's own release (payload 1) was then forward-only- + * skipped, and its exact-value sync wait never completed (evtest3 + * cudaStreamSynchronize(st[2]); victory_streams_events). release_pa + * is the push-named truth and wins when known; counter_pa stays the + * fallback for a channel whose pushes are unreadable. + */ + u64 pa = ch->release_pa ? ch->release_pa : ch->counter_pa; - if (!ch->counter_pa) + if (!pa) + return; + /* + * COMPLETION GATE (r205 mail/matmul regression): GP_PUT is the + * client's SUBMISSION position; the pump's last_get is its next + * unconsumed entry. While any submitted entry is outstanding + * (last_get != put) the pushes up to put have NOT all completed, so + * the put must never be published into the completion word: doing so + * short-circuits every client wait (r205 mail: rank-0's word jumped + * to the submitted 108 at last_get=104 - `pub-adv h=0x5c000019 + * pa=...ff0 val=108 cur=104` - ~1 s before the engine ran + * `output retirement wait: 1/968 ... settled`, so the host's + * sub-put release payloads were then forward-only-skipped and the + * client read a stale buffer and failed; matmul's stage ordering + * broke the same way). Only the release applies deliver completions; + * this publish is the catch-up for entries the pump consumed WITHOUT + * a decoded release, so it may run solely when nothing is + * outstanding - the same equality the entry loop consumes to + * (wrap-safe for the 1024-entry rings). + */ + if (ch->last_get != put) return; svc.counter_pub_calls++; if (gsp_sema_dbg && !ch->engine_type && svc.counter_pub_dbg < 96) { svc.counter_pub_dbg++; pr_info("sema-chan: pub-info h=0x%x rank=%u pa=0x%llx put=%u last=%u\n", - ch->h_object, ch->gr_index, ch->counter_pa, put, + ch->h_object, ch->gr_index, pa, put, ch->counter_last); } svc_counter_flush_pending(vgpu, ch); + /* + * mm2 (r207/r208 second-client hang): suppress the GR publish until + * the channel is proven to belong to the current session. During a + * fresh session's first pass the pump consumes the PREVIOUS + * session's residual ring; the completion gate above passes exactly + * then (the stale consumption satisfies last_get == put) and the + * residual put would land in the client's own completion word + * (release_pa). The fresh client's UVM then sees a far-forward + * payload followed by its real release sequence - a downward + * observation larger than UVM_GPU_SEMAPHORE_MAX_JUMP that corrupts + * the tracking counter and wedges the session's first waits (r207 + * victory_matmul: 1 GR push per channel, zero forwards, 242 s). + * gr_pub_armed is set at the fresh-put-drop rewind (session switch + * proven) or when a GR channel is seen with put <= 1 (pristine + * ring, nothing stale to mispublish); a live mid-stream discovery + * (put high, never dropping) loses only this catch-up publish, + * which the release applies already cover. + */ + if (!ch->engine_type && !ch->gr_pub_armed) + return; + val = put; + /* + * r471: never advance the guest's completion word past a push whose + * content was still unwritten when its entry was consumed. The + * clamp position is the lowest all-zero GR slot (gr_bad_put = slot + * + 1; 0 = none). Without it the publish can satisfy a D2H wait + * before its copy ran and the client drains a reused staging buffer + * holding the previous copy's bytes (streams_events h_c[2] = c0). + * The clamp heals when the rewinds re-read the slot with real + * content; the real completions still arrive through the decoded + * releases. + */ + if (!ch->engine_type && ch->gr_bad_put && val >= ch->gr_bad_put) { + val = ch->gr_bad_put - 1; + if (gsp_sema_dbg && svc.counter_pub_dbg < 96) { + svc.counter_pub_dbg++; + pr_info("sema-diag: pub-clamp h=0x%x pa=0x%llx val=%u (bad=%u) cur=%u\n", + ch->h_object, pa, val, ch->gr_bad_put, cur); + } + } if (val == ch->counter_last) return; /* @@ -10332,22 +16455,40 @@ static void svc_counter_publish(struct nv_vgpu_device *vgpu, * advanced the slot past the put position the client last submitted; * the client polls this u32, so never write it backwards. */ - if (svc_read_ap(vgpu, ch->counter_pa, SVC_APERTURE_SYS, &cur, - sizeof(cur))) + if (svc_read_ap(vgpu, pa, SVC_APERTURE_SYS, &cur, sizeof(cur))) return; if (val <= cur) { if (cur > ch->counter_last) ch->counter_last = cur; return; } - if (svc_write_ap(vgpu, ch->counter_pa, SVC_APERTURE_SYS, - &val, sizeof(val))) - return; + /* + * C5 (GAP-1 H4 secondary): the put-based publish is about to advance + * the same word the decoded releases target; a pre-advance here can + * make a later absolute release a silent skip. Witness every advance. + */ + if (gsp_sema_dbg) + pr_info("sema-diag: pub-adv h=0x%x pa=0x%llx val=%u cur=%u last=%u\n", + ch->h_object, pa, val, cur, + ch->counter_last); + { + int wrc = svc_write_ap(vgpu, pa, SVC_APERTURE_SYS, + &val, sizeof(val)); + if (wrc) { + if (gsp_sema_dbg && svc.pend_dbg_log < 1024) { + svc.pend_dbg_log++; + pr_info("sema-diag: pub-adv-wrc h=0x%x pa=0x%llx val=%u wrc=%d (write did not land)\n", + ch->h_object, pa, val, wrc); + } + return; + } + } ch->counter_last = val; + ch->counter_gen = svc_sess_gen; if (gsp_sema_dbg && svc.counter_pub_dbg < 48) { svc.counter_pub_dbg++; pr_info("sema-chan: counter publish h=0x%x pa=0x%llx val=%u (was %u)\n", - ch->h_object, ch->counter_pa, val, cur); + ch->h_object, pa, val, cur); } } @@ -10435,13 +16576,19 @@ static bool svc_fake_release(struct nv_vgpu_device *vgpu, struct svc_channel *ch sizeof(back)) && back == nv) { delivered = true; - if (svc_pool_sem_va(pool_va)) + if (svc_pool_sem_va(pool_va)) { svc_pool_pa_store(pool_va, spa, svc_ap_of(sap)); + /* r539: reach the secondary candidate + * page too (pin/walk disagreement). */ + svc_pool_boot_fanout(vgpu, + ch->h_object, + pool_va, nv); + } svc.releases_applied++; if (!gsp_sema_fake_rel) svc.releases_recovered++; - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); } } } @@ -10454,6 +16601,22 @@ static bool svc_fake_release(struct nv_vgpu_device *vgpu, struct svc_channel *ch svc_pending_add(pool_va, deliver, 0); svc.releases_forwarded++; } + /* + * r576b DIAG: witness every authored/queued recovery release and the + * target it chose. The r575 atomics wedge (a UVM tracker wait for + * payload 34 that stalled at 25 while the ring was consumed) needs + * this to prove which VA the fakes targeted when the client's pool + * stopped advancing - last_rel_va is learned from readable pushes and + * a foreign release VA there would send every later recovery to the + * wrong semaphore. + */ + if (gsp_sema_dbg && svc_fake_rel_log < 256) { + svc_fake_rel_log++; + pr_info("sema-fake: h=0x%x pool=0x%llx deliver=%u delivered=%d have_pa=%d last_rel=0x%llx exp=%u\n", + ch->h_object, pool_va, deliver, delivered ? 1 : 0, + have_pa ? 1 : 0, ch->last_rel_va, + ch->expected_sem_payload); + } svc.releases_faked++; svc_rel_count++; ch->expected_sem_payload++; @@ -10495,7 +16658,33 @@ static bool svc_entry_want_reached(struct nv_vgpu_device *vgpu, /* Mark a consumed GPFIFO slot for the current put epoch. */ static void svc_replay_note(struct svc_channel *ch, u32 idx, u32 put) { + static u32 consume_log; + + /* + * fail4 r303: name every consumed slot (all channels). Paired with + * the sema-entry decode rows this shows exactly which entries were + * consumed without ops and when (the vector_add chunk-drop chase: + * a consumed slot whose decode row carried a device-dst copy but + * whose op never ran is the silent skip under diagnosis). + */ + if (gsp_sema_dbg && consume_log < 16384) { + consume_log++; + pr_info("sema-consume: h=0x%x idx=%u put=%u\n", + ch->h_object, idx, put); + } (void)put; + /* + * STALL FIX (entry=82/pay=84 class, r186): the entry leaving the + * decode path must never strand a release set that svc_opdef_release() + * withheld for it while an operand was pending. svc_opdef_flush() is + * only reachable from the entry decode path; once the entry is + * consumed (e.g. via the entry_want shortcut) the withheld set sat + * until a reset/drop cleared it WITHOUT applying, and the guest's + * tracker word stopped short of the waiter's value (evtest3 + * cudaStreamSynchronize(st[2]) hung with starved=0). Deliver the set + * here, forward-only, before the slot is marked done. + */ + svc_opdef_finish_ch(ch, idx); if (idx < SVC_GPFIFO_MAX_BITS) set_bit(idx, ch->slot_done_epoch); } @@ -10622,7 +16811,37 @@ struct svc_fwd_req { bool gr; bool gr_bridge_done; bool gr_page_overflow; + /* + * STALLFREE: bounded-completion state for the forwarded launch. The + * request's backend outcome is the only host edge that can complete a + * launch whose release records engage no hold (svc_fwd_rel_n == 0); + * gr_swrel_done is the exactly-once latch for the software authorship + * of the QMD completion word, and gr_fill_jiffies stamps the request + * so the GR-STALL line reports the real waited time. + */ + bool gr_swrel_done; + /* + * STALLFREE: set by the work item when release0 could not target the + * guest page (unresolved release PA / HPA), so a successful launch's + * engine completion lands only on the module semaphore and the guest + * QMD word stays behind. The pump's bounded outcome check then + * authors it in software (the same write the hold's success edge + * performs), so the client always completes. + */ + bool gr_rel_unobserved; + unsigned long gr_fill_jiffies; u32 gr_h_object; + u32 gr_dropped; /* GRFIX: pages the fill's last pass could not + * resolve (0 = launch fully resolved); read + * by the absolute-bound expiry log */ + bool gr_arg_partial; /* r543: an ARGUMENT pointer resolved 0 + * pages while other arguments resolved - the + * launch would run with a stale/zero operand + * (the r542 matmul NOWRITE wedge: fe_esr= + * 0x80000000, engine fetch OK, wrote 0, + * client read zeros). The request path holds + * such a launch once per push VA through the + * bounded busy-defer retry. */ u64 gr_push_va; u32 gr_push_len; u32 gr_push_n; @@ -10670,6 +16889,8 @@ static struct workqueue_struct *svc_fwd_wq; static unsigned long svc_fwd_state; static u64 svc_fwd_requests; static u64 svc_fwd_dropped; +static u32 svc_grvaudit_launches; /* GRVAUDIT: per-launch audit counter */ +static u32 svc_readdiv_log; /* r233 READ-DIVERGE-WIT budget */ /* * Pause gate shared by the pump's request path and the backend release path: * quiesce takes this lock, sets paused and waits for the in-flight work item; @@ -10681,6 +16902,34 @@ static DEFINE_MUTEX(svc_fwd_lock); static bool svc_fwd_paused; /* + * F1 helpers: does a software release write target the release word of the + * forwarded GR launch the request slot holds? The slot is written only by the + * service thread (this code also runs there) and is never reused while the + * hold is active, so no lock is needed - same convention as the existing + * svc_fwd_hold_gr_release()/svc_fwd_hold_expired() reads. svc_gr_rel_match() + * covers the just-resolved hold (done may already be true); the inflight + * variant additionally requires the SVC_FWD_INFLIGHT bit and an unresolved + * work item, i.e. the hardware copy the poll is waiting on is still running. + */ +static bool svc_gr_rel_match(u64 va, u32 *payload) +{ + if (!va || !svc_fwd_req.gr || svc_fwd_req.gr_rel_va != va) + return false; + if (payload) + *payload = svc_fwd_req.gr_rel_payload; + return true; +} + +static bool svc_gr_rel_inflight(u64 va, u32 *payload) +{ + if (!svc_gr_rel_match(va, payload)) + return false; + if (!test_bit(SVC_FWD_INFLIGHT, &svc_fwd_state)) + return false; + return !smp_load_acquire(&svc_fwd_req.done); +} + +/* * Resolve one captured operand to an HPA. SYS (guest RAM) operands resolve * the page-aligned GPA through the guest-RAM backing file; VID operands * resolve the shadow byte offset through the vmalloc backing of BAR1 (the @@ -10710,11 +16959,29 @@ static int svc_fwd_operand_hpa(struct nv_vgpu_device *vgpu, u8 kind, return -EFAULT; *hpa = (u64)page_to_phys(pg); *out_off = page_off; + /* + * r372 two-path identity check: the copy-out writes through + * __va(hpa) while every reader reads base+pa - if those are + * not the same memory the engine's output silently vanishes + * (the sum=0 class). Prove it empirically, bounded. + */ + if (svc_fbhpa_log < 16) { + u64 at_base = 0, at_va = 0; + + svc_fbhpa_log++; + memcpy(&at_base, base + page, sizeof(at_base)); + memcpy(&at_va, (u8 *)__va(*hpa), sizeof(at_va)); + pr_info("sema-fbhpa: pa=0x%llx base=%px pg=%px hpa=0x%llx shadow=0x%llx at_base=%016llx at___va=%016llx %s\n", + page, base, pg, *hpa, + (unsigned long long)svc_fb_shadow_size(vgpu), + at_base, at_va, + at_base == at_va ? "SAME" : "DIFFER"); + } return 0; } if (kind != SVC_APERTURE_SYS) return -EINVAL; - ret = gsp_phys_gpa_to_hpa(page, hpa); + ret = gsp_phys_gpa_to_hpa_vgpu(vgpu, page, hpa); if (ret) return ret; *out_off = page_off; @@ -10829,6 +17096,19 @@ static void svc_fwd_soft_copy_inline(struct nv_vgpu_device *vgpu) * hardware has not produced yet. */ #define SVC_FWD_GR_HOLD_TIMEOUT_MS 12000u +/* + * r420: hard cap on the WEDGE-QUIET deferral of the GR hold deadline. The + * backend's quiet hold is a bounded msleep (its own caps: <=240 s per hold, + * two holds per launch), so the pause can only last a few minutes; the cap is + * a defence-in-depth bound so a stuck quiet-hold flag can never defer the + * client's completion indefinitely. + */ +#define SVC_FWD_GR_HOLD_PAUSE_MAX_MS 600000u +/* + * r420: same defence for the per-launch consumption bound pause in + * svc_gr_hold_bound_expired(). + */ +#define SVC_GR_QUIET_PAUSE_MAX_MS 600000u struct svc_fwd_hold { bool active; @@ -10864,9 +17144,9 @@ struct svc_fwd_hold { /* * P6c completion relay: the request selected record relay_index and * the hardware copy was asked to author it into the guest-visible - * semaphore page. On the success edge the hold re-reads the guest - * word and, when it carries the payload, skips that one record's - * software apply. Cleared with the hold. + * semaphore page. On the success edge the hold re-reads the guest word + * and, when it carries the payload, skips that one record's software + * apply. Cleared with the hold. */ bool relay_armed; u8 relay_index; @@ -10879,13 +17159,26 @@ struct svc_fwd_hold { struct svc_fwd_rel rel[SVC_FWD_REL_MAX]; u64 last_rel_va; unsigned long since; + /* + * r420: WEDGE-QUIET deferral state. While the backend deliberately + * holds the in-flight launch in its bounded quiet window + * (gsp_phys_gr_quiet_hold_active), the GR deadline is pushed forward + * (paused_until) so the hold cannot author the launch's release word + * under the hold - the client would then verify data the post-hold + * copy-out has not written yet (the r419 mid-boot escape). The + * deferral is itself bounded by SVC_FWD_GR_HOLD_PAUSE_MAX_MS from + * `since` and is announced once per hold. + */ + unsigned long paused_until; + bool pause_logged; }; static struct svc_fwd_hold svc_fwd_hold; -static void svc_fwd_hold_abort(u32 h_object) +static void svc_fwd_hold_abort(struct nv_vgpu_device *vgpu, u32 h_object) { if (READ_ONCE(svc_fwd_hold.active) && + svc_fwd_hold.vgpu == vgpu && svc_fwd_hold.ch_handle == h_object) WRITE_ONCE(svc_fwd_hold.aborted, true); } @@ -10929,8 +17222,19 @@ static void svc_fwd_rel_apply_launch_deferred(struct nv_vgpu_device *vgpu, bool pending = r->op != 0; bool inc = DMA_SEM_REDUCTION_EN(r->dma) && DMA_SEM_REDUCTION(r->dma) == SEM_REDUCTION_INC; + int rd = svc_sem_read(vgpu, r->va, &cur); - if (svc_sem_read(vgpu, r->va, &cur)) { + /* + * C2 (GAP-1 H2/H3): prove the opdef apply handed the record to the + * helper and what the software read of the release word returned. + */ + if (gsp_sema_dbg && svc_relx_log < 128) { + svc_relx_log++; + pr_info("sema-diag: relx kind=%u op=%u inc_claim=%u va=0x%llx pay=%u rd=%d\n", + (unsigned)r->kind, (unsigned)r->op, + (unsigned)r->inc_claimed, r->va, r->payload, rd); + } + if (rd) { svc_pending_add(r->va, r->payload, inc ? 1 : 0); svc_rel_apply_launch(vgpu, ch, r->va, r->payload, 0, pending, inc, r->inc_claimed); @@ -11015,8 +17319,17 @@ static void svc_fwd_rel_apply_all(struct nv_vgpu_device *vgpu, * Bounded hold: comfortably above the observed ~1 s pin latency and well * inside the 25 s CE grace, so a genuinely dead operand delays its * channel by this much and then falls back to the old skip-and-release. + * + * r549: 10000 -> 3000. r548 evidence: a reused channel handle whose new + * client submits ~65 init entries with device-dst ops no resolver covers + * walked one entry per 10 s (the journal shows OPDEF-EXPIRED exactly + * every 10.5 s, idx 7 -> 18) while the client spun at 100 % CPU - the + * release set the client waits on advances one entry per hold, so the + * bound is the stall multiplier. Ops that do resolve land in 0.5-2 s; + * 3 s keeps the m163 guarantee (a failed device copy is still retried + * once after its pins arrive) and caps the dead-op walk 3.3x shorter. */ -#define SVC_OPDEF_HOLD_MS 10000u +#define SVC_OPDEF_HOLD_MS 3000u struct svc_opdef_ent { bool active; @@ -11067,11 +17380,110 @@ static struct svc_opdef_ent *svc_opdef_alloc(struct nv_vgpu_device *vgpu, return NULL; } -static void svc_opdef_clear(struct svc_opdef_ent *e) +/* C3 caller tags for the opdef-clear witness (H2: which path cleared). */ +#define SVC_OPDEF_CLR_APPLY 0u +#define SVC_OPDEF_CLR_FINISH 1u +#define SVC_OPDEF_CLR_FLUSH 2u +#define SVC_OPDEF_CLR_DROP 3u +#define SVC_OPDEF_CLR_STALE 4u +#define SVC_OPDEF_CLR_COUNTED 5u +#define SVC_OPDEF_CLR_SWEEP 6u +#define SVC_OPDEF_CLR_STOP 7u + +static void svc_opdef_clear(struct svc_opdef_ent *e, u32 caller) { + /* + * C3 (GAP-1 H2): every clear is witnessed with its caller and the + * slot contents, so a "silent" clear (n=0, or a kind the apply + * switch ignores) is distinguishable from a delivery. + */ + if (gsp_sema_dbg && e->active && svc_opdef_clr_log < 128) { + svc_opdef_clr_log++; + pr_info("sema-diag: opdef-clear caller=%u h=0x%x entry=%u n=%u kind0=%u va0=0x%llx pay0=%u\n", + caller, e->h_object, e->entry_idx, (unsigned)e->n, + e->n ? (unsigned)e->rel[0].kind : 0u, + e->n ? e->rel[0].va : 0ull, + e->n ? e->rel[0].payload : 0u); + } + /* + * r456/r457: the snapshot lifecycle is now "capture once, consume + * on the copy's success, evict on ring-slot reuse, TTL". Every + * opdef_clear caller can still re-decode the entry: + * - APPLY (0): the pass's own op execution (which consumes the + * snapshot on success) follows the apply; + * - FINISH (1): the entry can be REPLAYED after a consume (the + * pin-batch rewind replays entries while mappings stream in), + * and an owed copy still needs the pipeline-order bytes; + * - FLUSH (2): the op stays skipped this pass but a later replay + * may run it. + * The r457 mode-C evidence: the b1 H2D's snapshot was captured at + * 191607.816388 with b1's bytes (first4=00000000 3e800000 ...), + * FINISH freed it on the entry's consume, the rewind replay + * re-captured at 191608.798838 from the REFILLED staging + * (first4=00000000 3d800000 ... = b2) and the deferred execution + * wrote b2's bytes into db1 (dst16_after=000000000000803d...). + * The r444 leak the old free guarded against is already covered + * by the owner key (a stale snapshot can only ever serve its own + * (ch, idx, hash)), the slot-reuse eviction in + * svc_copy_snap_capture, the ops_done skip free in svc_run_ops + * and the 30 s TTL. + */ memset(e, 0, sizeof(*e)); } +/* + * r551: remember releases DELIVERED by the hold-expiry paths (flush and + * sweep) by (channel, entry, va, payload) for a short window. The pump + * re-decodes a parked entry on every pass; without this mark each re-decode + * recorded a fresh opdef slot and held the entry again - r550 + * victory_strided: SWEEP-EXPIRED every ~3 s for the SAME entry six times + * (entries 41..43, ~18-21 s per entry, OPDEF-EXPIRED only after six + * cycles), so a ~40-entry init sequence walked for 12+ minutes with the + * client at 100 % CPU. The release is a monotonic completion signal and + * every delivery path is forward-only, so re-applying it is a no-op: a + * re-decode of an already-delivered entry must neither withhold the + * release again nor re-arm the hold. + */ +static void svc_opdef_done_note(u32 h, u32 entry_idx, const struct svc_fwd_rel *r) +{ + u32 i = svc_opdef_done_next++ % SVC_OPDEF_DONE_SLOTS; + + svc_opdef_done[i].h_object = h; + svc_opdef_done[i].entry_idx = entry_idx; + svc_opdef_done[i].va = r->va; + svc_opdef_done[i].payload = r->payload; + svc_opdef_done[i].j = jiffies; +} + +static bool svc_opdef_done_seen(u32 h, u32 entry_idx, const struct svc_fwd_rel *r) +{ + u32 i; + + for (i = 0; i < SVC_OPDEF_DONE_SLOTS; i++) + if (svc_opdef_done[i].h_object == h && + svc_opdef_done[i].entry_idx == entry_idx && + svc_opdef_done[i].va == r->va && + svc_opdef_done[i].payload == r->payload && + time_before(jiffies, svc_opdef_done[i].j + + msecs_to_jiffies(SVC_OPDEF_DONE_TTL_MS))) + return true; + return false; +} + +/* Any release of this entry delivered recently? (hold gate) */ +static bool svc_opdef_done_entry(u32 h, u32 entry_idx) +{ + u32 i; + + for (i = 0; i < SVC_OPDEF_DONE_SLOTS; i++) + if (svc_opdef_done[i].h_object == h && + svc_opdef_done[i].entry_idx == entry_idx && + time_before(jiffies, svc_opdef_done[i].j + + msecs_to_jiffies(SVC_OPDEF_DONE_TTL_MS))) + return true; + return false; +} + static bool svc_opdef_contains(const struct svc_opdef_ent *e, const struct svc_fwd_rel *r) { @@ -11100,13 +17512,78 @@ static void svc_opdef_apply(struct nv_vgpu_device *vgpu, if (e->n) { svc_fwd_rel_apply_all(vgpu, ch, e->rel, e->n); svc.opdef_applied++; - if (gsp_sema_dbg && svc_opdef_log < 32) { + if (gsp_sema_dbg && svc_opdef_log < (gsp_sema_dbg ? 512 : 32)) { svc_opdef_log++; pr_info("sema-opdef: apply h=0x%x entry=%u n=%u (ops complete)\n", ch->h_object, entry_idx, e->n); } } - svc_opdef_clear(e); + svc_opdef_clear(e, SVC_OPDEF_CLR_APPLY); +} + +/* + * r34x: is a release to this VA currently withheld by our own machinery? + * True when the pending-translation ring holds one (translation late) or + * an opdef slot has one recorded (operand hold). The acquire gate calls + * this to classify an unsatisfied wait in its diagnostics: pend=1 names + * the 'value sits behind our own withhold' case - the bounded 100 ms + * give-up is exactly for it (waiting longer cannot install the value + * until the upstream entry's ops complete). Never forces an opdef release + * here: those releases are withheld because their ops have NOT completed, + * and delivering early re-opens the work-before-fence hole they close. + */ +static bool svc_wait_pend_release(u64 va) +{ + u32 i; + + for (i = 0; i < svc_pending_n; i++) + if (svc_pending[i].va == va) + return true; + for (i = 0; i < SVC_OPDEF_SLOTS; i++) { + struct svc_opdef_ent *e = &svc_opdef[i]; + u8 k; + + if (!e->active) + continue; + for (k = 0; k < e->n; k++) + if (e->rel[k].va == va) + return true; + } + return false; +} + +/* + * STALL FIX (r186 entry=82/pay=84): called from svc_replay_note() on every + * entry-consume path. If a release set was withheld for this entry while an + * operand was pending and the entry is now leaving the decode path (so the + * 10 s flush can never run for it), deliver the set forward-only and clear + * the slot. A drop without delivery is the stall class: the guest word + * stops short of the value its tracker waits for. + */ +static void svc_opdef_finish_ch(struct svc_channel *ch, u32 idx) +{ + u32 i; + + if (!ch || idx == SVC_ENTRY_NONE) + return; + for (i = 0; i < SVC_OPDEF_SLOTS; i++) { + struct svc_opdef_ent *e = &svc_opdef[i]; + + if (!e->active || e->h_object != ch->h_object || + e->vgpu != ch->vgpu || e->entry_idx != idx) + continue; + if (e->n) { + svc_fwd_rel_apply_all(e->vgpu, ch, e->rel, e->n); + svc.opdef_delivered_late++; + if (svc_opdef_log < 64) { + svc_opdef_log++; + pr_info("sema-opdef: LATE-FLUSH h=0x%x entry=%u n=%u (entry consumed before deadline; releases delivered)\n", + ch->h_object, idx, e->n); + } + } + svc_opdef_clear(e, SVC_OPDEF_CLR_FINISH); + return; + } } /* @@ -11135,6 +17612,16 @@ static bool svc_opdef_release(struct nv_vgpu_device *vgpu, if (svc.op_tr_pending) { if (rescue) return e != NULL; + /* + * r551: this release was already delivered forward-only by the + * hold-expiry flush/sweep; the pump's re-decode of the parked + * entry must not withhold it (or re-arm the hold) again. + */ + if (svc_opdef_done_seen(ch->h_object, entry_idx, r)) { + svc_rel_va_seen = r->va; + svc_rel_count++; + return true; + } if (!e) { e = svc_opdef_alloc(vgpu, ch->h_object, entry_idx); if (!e) @@ -11143,7 +17630,7 @@ static bool svc_opdef_release(struct nv_vgpu_device *vgpu, if (!svc_opdef_contains(e, r)) { if (r->payload && e->n < SVC_FWD_REL_MAX) { e->rel[e->n++] = *r; - if (gsp_sema_dbg && svc_opdef_log < 32) { + if (gsp_sema_dbg && svc_opdef_log < (gsp_sema_dbg ? 512 : 32)) { svc_opdef_log++; pr_info("sema-opdef: hold h=0x%x entry=%u kind=%u va=0x%llx pay=%u (operand pending)\n", ch->h_object, entry_idx, @@ -11185,20 +17672,58 @@ static bool svc_opdef_release(struct nv_vgpu_device *vgpu, * flushes the slot and consumes the entry. */ static bool svc_opdef_hold(struct nv_vgpu_device *vgpu, struct svc_channel *ch, - u32 entry_idx) + u32 entry_idx, u64 launch_va) { struct svc_opdef_ent *e = svc_opdef_find(vgpu, ch->h_object, entry_idx); + /* r551: a delivered entry is consumed, never held again. */ + if (svc_opdef_done_entry(ch->h_object, entry_idx)) + return false; if (!e) { e = svc_opdef_alloc(vgpu, ch->h_object, entry_idx); if (!e) return false; } + /* + * r361: absolute per-launch bound keyed to the LAUNCH PUSH VA + * (the GRFIX lesson, r213). The slot's `since` restarts whenever + * a pin-batch rewind replays the entry at a different GPFIFO + * index, so a slot-keyed-only bound can be re-armed forever - + * observed as opdef=3301/32/0 (held climbing, expired 0) with the + * guest wedged. The push VA never changes across replays, so the + * first hold of a launch arms an ABSOLUTE deadline nothing moves. + */ + if (!launch_va) + launch_va = 1; /* anonymous launch: still absolutely bound */ + /* + * r368: an ACTIVE bound is never re-armed. The pump walks up to + * SVC_OPDEF_SLOTS held entries, each with its OWN push VA; the r361 + * arm-on-VA-change rule restarted the deadline on every entry + * change, so with several held launches the deadline moved forever + * (streams2 wedge: opdef=955/13/0, held climbing, expired 0). The + * first hold arms the absolute deadline; entries held while it is + * active ride it and flush together when it fires. A fresh bound is + * armed once the previous one has passed. + */ + if (!ch->opdef_hold_dead_va || + time_after_eq(jiffies, ch->opdef_hold_deadline)) { + ch->opdef_hold_dead_va = launch_va; + ch->opdef_hold_deadline = jiffies + + msecs_to_jiffies(SVC_OPDEF_HOLD_MS); + ch->opdef_hold_exp_logged = false; + } if (time_before(jiffies, - e->since + msecs_to_jiffies(SVC_OPDEF_HOLD_MS))) { + e->since + msecs_to_jiffies(SVC_OPDEF_HOLD_MS)) && + time_before(jiffies, ch->opdef_hold_deadline)) { svc.opdef_held++; return true; } + if (!ch->opdef_hold_exp_logged) { + ch->opdef_hold_exp_logged = true; + pr_err_ratelimited("vgsp-sema: OPDEF-EXPIRED h=0x%x idx=%u va=0x%llx (absolute per-launch bound %u ms; rewind replays no longer re-arm)\n", + ch->h_object, entry_idx, launch_va, + SVC_OPDEF_HOLD_MS); + } return false; } @@ -11211,12 +17736,16 @@ static void svc_opdef_flush(struct nv_vgpu_device *vgpu, struct svc_channel *ch, if (!e) return; if (e->n) { + u8 i; + svc_fwd_rel_apply_all(vgpu, ch, e->rel, e->n); + for (i = 0; i < e->n; i++) + svc_opdef_done_note(ch->h_object, entry_idx, &e->rel[i]); pr_info("sema-opdef: EXPIRED h=0x%x entry=%u n=%u applied (operand never translated; op stays skipped)\n", ch->h_object, entry_idx, e->n); } svc.opdef_expired++; - svc_opdef_clear(e); + svc_opdef_clear(e, SVC_OPDEF_CLR_FLUSH); } /* @@ -11230,17 +17759,67 @@ static void svc_opdef_drop(struct nv_vgpu_device *vgpu, struct svc_channel *ch, { struct svc_opdef_ent *e = svc_opdef_find(vgpu, ch->h_object, entry_idx); - if (e) - svc_opdef_clear(e); + if (!e) + return; + /* + * STALL FIX (r186/r187 entry=82, va=0x20400fef0 pay=84): when the + * forwarding machinery takes ownership of an entry, the withheld set + * may NOT ride the forwarded candidate's rel[] (svc_opdef_release() + * returned before the release was appended), so a blind clear strands + * the guest's tracker word with NO LATE-FLUSH/LOSS/EXPIRED ever + * logged (crel freezes right after the launches; evtest3 + * cudaStreamSynchronize(st[2]) hangs). Counter releases are + * monotonic completion signals: apply them forward-only here - a + * record at/ahead of the word is a no-op, so the fwd edge cannot + * double-apply and an additive reduction is skipped. + */ + { + u8 i; + u32 delivered = 0; + + for (i = 0; i < e->n; i++) { + if (!svc_counter_va(e->rel[i].va)) + continue; + if (e->rel[i].kind == SVC_FWD_REL_906F && + e->rel[i].op == HOST_SEM_OP_REDUCTION) + continue; /* additive: not idempotent */ + svc_fwd_rel_apply_all(vgpu, ch, &e->rel[i], 1); + svc.opdef_delivered_late++; + delivered++; + pr_info_ratelimited("sema-opdef: DROP-APPLY h=0x%x entry=%u va=0x%llx pay=%u (counter release delivered at fwd-ownership handoff)\n", + ch->h_object, entry_idx, + e->rel[i].va, e->rel[i].payload); + } + if (delivered < e->n) { + svc.opdef_drop_lost += (u32)(e->n - delivered); + pr_info_ratelimited("sema-opdef: LOSS h=0x%x entry=%u %u of %u withheld release(s) dropped at fwd-ownership handoff\n", + ch->h_object, entry_idx, + (u32)(e->n - delivered), e->n); + } + } + svc_opdef_clear(e, SVC_OPDEF_CLR_DROP); } /* * The ring advanced past a slot's entry (a slot for another entry of the * same channel is being processed): its set belongs to a consumed entry. * Deliver late (forward-only, so it cannot double-apply) and drop it. + * + * r406: an entry is only "passed" if the ring actually CONSUMED it + * (slot_done_epoch). Without that proof the pin-batch rewind breaks the + * invariant: it re-opens lower indices (e.g. 83) while a held entry + * (e.g. 91) sits AHEAD in the same epoch, and the old unconditional drop + * treated the held entry as consumed and delivered its withheld releases + * early. Observed (r406 inplace_scale): the H2D copy's operand was + * unresolved (rc=-2), the entry held with its release withheld; a rewind + * pass at idx=83 fired drop-stale caller=4 for idx=91 and the release + * went out at 104955.751 - while the copy only executed at 104957.418. + * The client raced past its blocking memcpy, launched, the launch read + * the stale buffer (previous run's data), and the copy landed after the + * launch's copy-out: wrong results with 32/1032 "changed". */ static void svc_opdef_drop_stale(struct nv_vgpu_device *vgpu, - struct svc_channel *ch, u32 keep_idx) + struct svc_channel *ch, u32 keep_idx, u32 put) { int i; @@ -11250,36 +17829,168 @@ static void svc_opdef_drop_stale(struct nv_vgpu_device *vgpu, if (!e->active || e->vgpu != vgpu || e->h_object != ch->h_object || e->entry_idx == keep_idx) continue; + if (!svc_replay_seen(ch, e->entry_idx, put)) + continue; /* never consumed: not stale (rewind) */ if (e->n) svc_fwd_rel_apply_all(vgpu, ch, e->rel, e->n); - svc_opdef_clear(e); + svc_opdef_clear(e, SVC_OPDEF_CLR_STALE); + } +} + +/* A slot of THIS pump's guest whose channel is gone from the pass snapshot: + * drop it. Slots of other guests are never touched (their channels are not in + * this snapshot by construction). */ +static void svc_opdef_drop_counted(struct svc_opdef_ent *e) +{ + if (e->n) { + svc.opdef_drop_lost += e->n; + pr_info_ratelimited("sema-opdef: LOSS entry h=0x%x idx=%u dropped with %u undelivered release(s)\n", + e->h_object, e->entry_idx, e->n); } + svc_opdef_clear(e, SVC_OPDEF_CLR_COUNTED); } -/* A slot whose channel is gone from this pass's snapshot: drop it. */ -static void svc_opdef_drop_gone(const struct svc_channel *local, int n) +static void svc_opdef_drop_gone(struct nv_vgpu_device *vgpu, + const struct svc_channel *local, int n) { int i, k; for (i = 0; i < SVC_OPDEF_SLOTS; i++) { struct svc_opdef_ent *e = &svc_opdef[i]; - if (!e->active) + if (!e->active || e->vgpu != vgpu) continue; for (k = 0; k < n; k++) - if (local[k].h_object == e->h_object) + if (local[k].vgpu == e->vgpu && + local[k].h_object == e->h_object) break; if (k == n) - svc_opdef_clear(e); + svc_opdef_drop_counted(e); } } static void svc_opdef_reset_all(void) { + int i; + + /* STALL FIX: a non-applying clear is a silent release loss - count it. */ + for (i = 0; i < SVC_OPDEF_SLOTS; i++) + if (svc_opdef[i].active && svc_opdef[i].n) + svc_opdef_drop_counted(&svc_opdef[i]); memset(svc_opdef, 0, sizeof(svc_opdef)); } /* + * STALL FIX (r189 entry=82/83 va=0x20400fef0 pay=84/85): the 10 s deadline + * flush (svc_opdef_flush) is only reachable while the pump keeps RE-DECODING + * the entry. When a withheld entry stops being revisited (ring advanced, the + * channel went quiet, the entry was parked while the pump moved on), the + * deadline never runs and the withheld counter release strands forever - + * the guest's tracker word stops short (evtest3 cudaStreamSynchronize(st[2]) + * hangs; opdef=531/8/0 static, no EXPIRED/LOSS lines). Sweep expired slots + * once per service iteration: deliver forward-only to the still-live owner + * channel, count a LOSS when the owner is gone. + */ +/* + * r563b: forced variant of the expiry sweep for the stall self-heal. The + * regular sweep skips slots younger than SVC_OPDEF_HOLD_MS; when the pump + * has already been stuck for SVC_STALL_WATCH_MS, age no longer matters - + * deliver every withheld release forward-only (recording the delivered + * marks so a re-decode cannot re-hold) and clear the slots. This is the + * same delivery the slot's own 3 s bound would perform, just executed at + * the watchdog's discretion. + */ +static void svc_opdef_force_flush(void) +{ + int i; + + for (i = 0; i < SVC_OPDEF_SLOTS; i++) { + struct svc_opdef_ent *e = &svc_opdef[i]; + u8 k; + + if (!e->active || !e->n) + continue; + for (k = 0; k < e->n; k++) + svc_opdef_done_note(e->h_object, e->entry_idx, + &e->rel[k]); + { + static struct svc_channel ch_scratch; + unsigned long flags; + struct svc_channel *local = NULL; + int c; + + spin_lock_irqsave(&svc.chan_lock, flags); + for (c = 0; c < SVC_MAX_CHANNELS; c++) + if (svc.chan[c].valid && + svc.chan[c].vgpu == e->vgpu && + svc.chan[c].h_object == e->h_object) { + ch_scratch = svc.chan[c]; + local = &ch_scratch; + break; + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + if (local) + svc_fwd_rel_apply_all(e->vgpu, local, + e->rel, e->n); + } + svc.opdef_expired++; + svc_opdef_clear(e, SVC_OPDEF_CLR_SWEEP); + } +} + +static void svc_opdef_expire_sweep(void) +{ + /* + * struct svc_channel is >25 KB (per-slot op bitmaps); a stack copy + * overflows the 16 KB kernel stack. Only the single service thread + * calls this, so a static scratch is safe (same pattern as + * svc_handle_chan_dump / svc_pump_channels). + */ + static struct svc_channel ch_scratch; + int i; + + for (i = 0; i < SVC_OPDEF_SLOTS; i++) { + struct svc_opdef_ent *e = &svc_opdef[i]; + struct svc_channel *local = NULL; + unsigned long flags; + int c; + + if (!e->active || !e->n) + continue; + if (time_before(jiffies, e->since + + msecs_to_jiffies(SVC_OPDEF_HOLD_MS))) + continue; + spin_lock_irqsave(&svc.chan_lock, flags); + for (c = 0; c < SVC_MAX_CHANNELS; c++) + if (svc.chan[c].valid && + svc.chan[c].vgpu == e->vgpu && + svc.chan[c].h_object == e->h_object) { + ch_scratch = svc.chan[c]; + local = &ch_scratch; + break; + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + if (local) { + u8 i; + + svc_fwd_rel_apply_all(e->vgpu, local, e->rel, e->n); + for (i = 0; i < e->n; i++) + svc_opdef_done_note(e->h_object, + (u32)e->entry_idx, + &e->rel[i]); + svc.opdef_expired++; + pr_info_ratelimited("sema-opdef: SWEEP-EXPIRED h=0x%x entry=%u n=%u applied forward-only (entry parked past the hold window)\n", + e->h_object, e->entry_idx, e->n); + } else { + svc.opdef_drop_lost += e->n; + pr_info_ratelimited("sema-opdef: SWEEP-LOSS h=0x%x entry=%u n=%u owner channel gone (releases dropped)\n", + e->h_object, e->entry_idx, e->n); + } + svc_opdef_clear(e, SVC_OPDEF_CLR_SWEEP); + } +} + +/* * Proof line for a held record set resolved without a successful forwarded * copy (failure/timeout/abort): the releases below are applied directly, so * none is lost to a hardware copy that never ran. Called with svc_fwd_hold @@ -11316,6 +18027,16 @@ static void svc_gr_sw_release(struct nv_vgpu_device *vgpu, u32 h, u64 va, return; if (!svc_sem_read(vgpu, va, &cur) && cur >= payload) return; + /* + * F1-a: this is the hold/fallback authorship of the forwarded launch's + * release word (src="hold"), budgeted with the sweep tag so one + * 256-line session window covers both authors. + */ + if (svc_gr_rel_match(va, NULL) && svc.gr_rel_auth_log < 256) { + svc.gr_rel_auth_log++; + pr_info("sema-diag: GR-REL-AUTHOR src=hold va=0x%llx pay=%u h=0x%x\n", + va, payload, h); + } if (!svc_sem_write(vgpu, h, va, payload)) { svc.fwd_gr_sw_rel++; pr_info("vgsp-work: GR launch release software-applied h=0x%x va=0x%llx pay=0x%x (no successful hardware copy; forward-only)\n", @@ -11338,11 +18059,107 @@ static void svc_fwd_hold_gr_release(struct nv_vgpu_device *vgpu, svc_fwd_req.gr_rel_payload); } +/* + * STALLFREE: bounded completion for a failed forwarded GR launch that no hold + * completes. A GR push whose decode recorded no push-level release records + * (svc_fwd_rel_n == 0) engages no release hold, so when the backend fails + * (ladder timeout, fault abort, refusal) nothing authors the QMD completion + * word the guest client polls after the kick: the client's launch wait then + * has no host edge and lasts until the guest's own timeout (the r219/r222 + * stall class, qmd_rel frozen at the previous launch's value). + * + * This runs on the service thread at the top of every pump pass, before any + * request fill of that pass (svc_fwd_request requires SVC_FWD_INFLIGHT free, + * which the work item only releases after publishing done, and the fill's + * first write is done=false), so a pass that sees done also sees this + * request's fields, and every finished request is observed before its slot + * can be reused. Authors the release once (forward-only, full resolver - the + * same write the hold's failure edge performs), latches gr_swrel_done, and + * logs the distinguishable GR-STALL line. + * + * While a hold is active the hold owns the completion (its failure edge + * applies the recorded releases AND the QMD word, bounded by the existing + * hold timeout), so this check stands down for it. + */ +static void svc_fwd_outcome_check(struct nv_vgpu_device *vgpu) +{ + bool done = smp_load_acquire(&svc_fwd_req.done); + unsigned long waited; + + if (!svc_fwd_req.gr || svc_fwd_req.vgpu != vgpu || + svc_fwd_req.gr_swrel_done || svc_fwd_hold.active || !done) + return; + /* + * Complete when the backend failed (the never-completing r219/r222 + * class) OR when a successful launch's release0 could not target the + * guest page, so the engine's completion landed only on the module + * semaphore and the client's QMD word is still behind (the same edge + * the hold's success path covers for held requests). + */ + if (svc_fwd_req.hw_rc == 0 && !svc_fwd_req.gr_rel_unobserved) + return; + /* + * The outcome is final; latch it first so no later pass, hold edge or + * consume tail accounts it again (the release write itself is + * forward-only, so even a duplicate could not corrupt the word). + */ + svc_fwd_req.gr_swrel_done = true; + waited = jiffies_to_msecs(jiffies - svc_fwd_req.gr_fill_jiffies); + if (!svc_fwd_req.gr_rel_va || !svc_fwd_req.gr_rel_payload) { + pr_warn("vgsp-gr: GR-STALL-backend va=0x%llx waited_ms=%lu action=proceed (launch carries no release record; no word to author)\n", + svc_fwd_req.gr_push_va, waited); + return; + } + svc_gr_sw_release(vgpu, svc_fwd_req.gr_h_object, + svc_fwd_req.gr_rel_va, + svc_fwd_req.gr_rel_payload); + svc.fwd_gr_stall_swrel++; + if (svc_fwd_req.hw_rc != 0) + pr_warn("vgsp-gr: GR-STALL-backend va=0x%llx waited_ms=%lu action=proceed (failed forwarded launch, rc=%d; QMD release authored in software)\n", + svc_fwd_req.gr_push_va, waited, svc_fwd_req.hw_rc); + else + pr_warn("vgsp-gr: GR-STALL-backend va=0x%llx waited_ms=%lu action=proceed (successful launch whose release0 could not target the guest page; QMD release authored in software)\n", + svc_fwd_req.gr_push_va, waited); + pr_info("vgsp-gr: GR-COMPLETE-SW h=0x%x push=0x%llx rel=0x%llx pay=0x%x waited_ms=%lu n=%llu (backend outcome without a hold)\n", + svc_fwd_req.gr_h_object, svc_fwd_req.gr_push_va, + svc_fwd_req.gr_rel_va, svc_fwd_req.gr_rel_payload, waited, + svc.fwd_gr_stall_swrel); +} + static bool svc_fwd_hold_expired(void) { - if (svc_fwd_req.gr && !smp_load_acquire(&svc_fwd_req.done)) + if (svc_fwd_req.gr && !smp_load_acquire(&svc_fwd_req.done)) { + /* + * r420: while the backend deliberately holds the in-flight + * launch in its bounded WEDGE-QUIET window, the GR deadline + * must not fire - expiring here authors the launch's release + * word (svc_fwd_hold_gr_release) and applies the recorded + * releases before the post-hold copy-out, so the client + * verifies stale data (the r419 soak's mid-boot + * "matmul fails after time"). Pause instead, bounded by + * SVC_FWD_GR_HOLD_PAUSE_MAX_MS from the hold's start; the + * pause extends the deadline a full GR window past the last + * quiet-hold pass, so the backend's retry/accept can finish + * before the fallback would author. Announced once per hold. + */ + if (gsp_phys_gr_quiet_hold_active() && + time_before(jiffies, svc_fwd_hold.since + + msecs_to_jiffies(SVC_FWD_GR_HOLD_PAUSE_MAX_MS))) { + if (!svc_fwd_hold.pause_logged) { + svc_fwd_hold.pause_logged = true; + pr_warn("vgsp-gr: GR-HOLD-PAUSE va=0x%llx (backend wedge-quiet hold active; release authorship deferred under the bounded hold)\n", + svc_fwd_req.gr_push_va); + } + svc_fwd_hold.paused_until = jiffies + + msecs_to_jiffies(SVC_FWD_GR_HOLD_TIMEOUT_MS); + return false; + } + if (svc_fwd_hold.paused_until && + time_before(jiffies, svc_fwd_hold.paused_until)) + return false; return time_after(jiffies, svc_fwd_hold.since + msecs_to_jiffies(SVC_FWD_GR_HOLD_TIMEOUT_MS)); + } return time_after(jiffies, svc_fwd_hold.since + msecs_to_jiffies(SVC_FWD_HOLD_TIMEOUT_MS)); } @@ -11376,6 +18193,9 @@ static void svc_fwd_hold_engage(struct nv_vgpu_device *vgpu, svc_fwd_hold.relay_armed = svc_fwd_req.relay; svc_fwd_hold.relay_index = svc_fwd_req.relay_index; svc_fwd_hold.fault_delivered = false; + /* r420: a fresh hold starts with no WEDGE-QUIET deferral. */ + svc_fwd_hold.paused_until = 0; + svc_fwd_hold.pause_logged = false; svc.fwd_hold_engaged++; svc.fwd_rel_pending = n; } @@ -11402,6 +18222,8 @@ static void svc_fwd_hold_clear(void) svc_fwd_hold.relay_armed = false; svc_fwd_hold.relay_index = 0; svc_fwd_hold.fault_delivered = false; + svc_fwd_hold.paused_until = 0; + svc_fwd_hold.pause_logged = false; svc.fwd_rel_pending = 0; } @@ -11555,6 +18377,15 @@ static void svc_fwd_hold_expire_check(struct nv_vgpu_device *vgpu, svc_fwd_hold.relay_armed) svc_fwd_fault_deliver(hv, ch); /* + * STALLFREE: the unpumpable-channel expiry edge (the entry loop never + * ran for this hold). The releases and the QMD completion word are + * software-applied below; exactly one line per hold. + */ + if (svc_fwd_req.gr) + pr_warn("vgsp-gr: GR-STALL-fwd-hold va=0x%llx waited_ms=%u action=proceed (hold expired on an unpumpable channel; releases + QMD word software-applied)\n", + svc_fwd_req.gr_push_va, + jiffies_to_msecs(jiffies - svc_fwd_hold.since)); + /* * P6b wave 2: a skipped candidate whose hardware copy did not complete * successfully is authored by one software copy here, BEFORE the * recorded releases below. A successful hardware copy needs none. @@ -11665,9 +18496,36 @@ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu, * done && hw_rc == -ETIMEDOUT; abort/cancel paths never write * guest memory). */ + /* + * C7 (GAP-2 A1): the false-completion edge - a hold whose + * hardware copy failed/timed out still software-applies the + * release set here. + */ + if (gsp_sema_dbg && svc_hold_fb_log < 64) { + svc_hold_fb_log++; + pr_info("sema-diag: hold-fallback h=0x%x gr=%d done=%d hw_rc=%d nrel=%u rel0=0x%llx/%u -> sw apply\n", + ch->h_object, ch->engine_type == 0 ? 1 : 0, + done ? 1 : 0, svc_fwd_req.hw_rc, + (unsigned)svc_fwd_hold.nrel, + svc_fwd_hold.nrel ? + svc_fwd_hold.rel[0].va : 0ull, + svc_fwd_hold.nrel ? + svc_fwd_hold.rel[0].payload : 0u); + } if (done && svc_fwd_req.hw_rc == -ETIMEDOUT && svc_fwd_hold.relay_armed) svc_fwd_fault_deliver(hv, ch); + /* + * STALLFREE: the bounded failure edge of a held forwarded GR + * launch - the releases AND the QMD completion word are + * software-applied below, so the client always completes. + * Exactly one line per hold (the hold is cleared at the end). + */ + if (svc_fwd_req.gr) + pr_warn("vgsp-gr: GR-STALL-fwd-fail va=0x%llx waited_ms=%u action=proceed (hold resolved on backend failure/expiry; releases + QMD word software-applied)\n", + svc_fwd_req.gr_push_va, + jiffies_to_msecs(jiffies - + svc_fwd_hold.since)); svc_fwd_hold_soft_copy(hv); if (svc_fwd_hold.relay_armed) svc.fwd_relay_fallback++; @@ -11693,6 +18551,14 @@ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu, ch->last_get = (idx + 1) % ch->gpfifo_entries; ch->rescue_fail = 0; ch->defer_since = 0; + /* + * GRFIX FIX1c (r214): do NOT reset the per-launch bound record + * here. This consume can belong to a different entry than the held + * launch, and resetting on every consume re-armed the bound before + * it could pass (r214: 420 re-holds, GRFILL-EXPIRED=0 over 220 s). + * The record is keyed by the launch push VA; it resets only on + * channel (re)registration or the session diag reset. + */ ch->submissions++; svc.gpfifo_entries_done++; if (ch->engine_type && ch->expected_sem_payload) @@ -11709,6 +18575,31 @@ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu, } /* + * Is this mdev still on the driver list with an open VFIO device? Decides + * whether a hold that names it can still be resolved through it: a removed + * device must never be dereferenced (its struct can be freed before the next + * pump pass would see the hold). Pointer comparisons only - safe to call + * with a possibly-stale pointer. + */ +static bool svc_vgpu_open(const struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_device *d; + bool open = false; + + if (!vgpu) + return false; + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(d, &nv_vgpu_driver.devices, list) { + if (d == vgpu && atomic_read(&d->vfio_dev_open_count) > 0) { + open = true; + break; + } + } + mutex_unlock(&nv_vgpu_driver.lock); + return open; +} + +/* * Channel vanished while a hold was active. Any resolved hold (completed, * aborted, failed or expired) applies the recorded releases through a shim * channel - holds exist only for CE channels and the vanished channel object @@ -11728,6 +18619,18 @@ static void svc_fwd_hold_gone_finish(struct nv_vgpu_device *vgpu) bool done = smp_load_acquire(&svc_fwd_req.done); bool aborted = READ_ONCE(svc_fwd_hold.aborted); + if (svc_fwd_hold.vgpu && !svc_vgpu_open(svc_fwd_hold.vgpu)) { + /* + * The owning mdev was removed mid-hold (the mdev cleanup path + * already released its session state): its guest can never + * observe the releases and dereferencing the device would be a + * use-after-free. Drop the hold; the request slot frees for the + * next guest. + */ + svc.fwd_hold_canceled++; + svc_fwd_hold_clear(); + return; + } if (!done && !aborted && !svc_fwd_hold_expired()) return; /* @@ -11780,6 +18683,15 @@ static void svc_fwd_hold_stop_flush(struct nv_vgpu_device *vgpu) if (!svc_fwd_hold.active) return; + /* + * A hold whose owning mdev is already gone must never be dereferenced: + * drop it (the guest cannot observe the releases anyway). + */ + if (svc_fwd_hold.vgpu && !svc_vgpu_open(svc_fwd_hold.vgpu)) { + svc.fwd_hold_canceled++; + svc_fwd_hold_clear(); + return; + } done = smp_load_acquire(&svc_fwd_req.done); /* * P6b wave 2: the backend is going away; a skipped candidate whose @@ -11836,6 +18748,10 @@ static void svc_fwd_gr_work(void) struct gsp_phys_backend_copy_result cp = { 0 }; u64 rel_hpa = 0; u32 off = 0, i; + /* GRVAUDIT: aggregated from the data[]-build loop below (first + * problem page in ga_va/ga_used/ga_led; 0 while clean). */ + u32 ga_unlanded = 0, ga_ptebad = 0; + u64 ga_va = 0, ga_used = 0, ga_led = 0; int hw_rc; memset(&gs, 0, sizeof(gs)); @@ -11868,6 +18784,7 @@ static void svc_fwd_gr_work(void) (SEMA_PAGE_SIZE - 1)); u8 rel_kind = (svc_fwd_req.gr_rel_ap == SVC_APERTURE_FB) ? SVC_APERTURE_FB : SVC_APERTURE_SYS; + bool rel_unobserved = false; if (svc_fwd_req.gr_rel_pa && (svc_fwd_operand_hpa(svc_fwd_req.vgpu, rel_kind, @@ -11880,11 +18797,24 @@ static void svc_fwd_gr_work(void) * means the engine would author nothing this launch; point * release0 at the module semaphore instead so the host still * observes completion. + * + * STALLFREE: when the guest page could not be targeted at all + * (unresolved release PA or HPA), the engine's completion will + * land only on the module semaphore and the guest QMD word + * stays behind - a launch whose client would otherwise wait + * forever even on a SUCCESSFUL submit. Record it so the pump's + * bounded outcome check authors the word in software (the + * same write the hold's success edge performs). A resolved + * page whose word is already ahead needs nothing. */ - if (rel_hpa && - READ_ONCE(*(u32 *)((u8 *)__va(rel_hpa) + roff)) >= + if (rel_hpa) { + if (READ_ONCE(*(u32 *)((u8 *)__va(rel_hpa) + roff)) >= svc_fwd_req.gr_rel_payload) - rel_hpa = 0; + rel_hpa = 0; + } else { + rel_unobserved = true; + } + svc_fwd_req.gr_rel_unobserved = rel_unobserved; } gs.qmd = svc_fwd_req.gr_qmd; @@ -11920,6 +18850,96 @@ static void svc_fwd_gr_work(void) data[gs.n_data].va = dg->va; data[gs.n_data].hpa = dhpa; data[gs.n_data].reloc_va = dg->reloc_va; + /* + * r348 diagnostic: the last 32 high-VA data pages (the output + * operands live there) with the exact (gpa, ap) -> hpa the + * writeback will use - comparable against the D2H read's + * resolution line (sema-bigcopy) for the same VA. + */ + if (dg->va >= SVC_VAS_FLOOR && + i + 32 >= svc_fwd_req.gr_n_data) + pr_info("svc-fwd-hpa: va=0x%llx gpa=0x%llx ap=%u hpa=0x%llx bi=%u\n", + dg->va, dg->pa, dg->ap, dhpa, i); + /* + * GRVAUDIT (diagnostic only): audit the m180 landing + * authority against the PA this submit will put on the wire + * for every tracked data page - the data[]/n_data set the + * backend's NOWRITE analysis later enumerates. The lookup + * runs BEFORE the store underneath re-records this launch's + * own landing, so it reads the pre-existing authority: + * unlanded = no landing recorded for this VA at submit; + * pte_bad = a landing exists on a different page/aperture + * than the PA the submit path computed/used. Pages below + * SVC_VAS_FLOOR are outside the ledger's domain (all store + * sites gate on the floor) and are not classified. + */ + if (dg->va >= SVC_VAS_FLOOR) { + u64 lpa = 0; + u32 lap = 0; + + if (svc_ledger_lookup(dg->va, &lpa, &lap)) { + ga_unlanded++; + if (!ga_va) { + ga_va = dg->va; + ga_used = dg->pa; + ga_led = 0; + } + } else if ((lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (dg->pa & ~(u64)(SEMA_PAGE_SIZE - 1)) || + (u32)lap != (u32)dkind) { + ga_ptebad++; + if (!ga_va) { + ga_va = dg->va; + ga_used = dg->pa; + ga_led = lpa; + } + } + } + /* + * r233 READ-DIVERGE-WIT: the engine's read of this VA uses + * exactly what the fill resolved (dg->pa: the page whose + * content was copied into the bounce). The copy paths (H2D + * writes, D2H reads) resolve independently through + * svc_op_translate/ledger. For small launches (the probe + * class) compare the three for the first recs and flag any + * page-level split - the direct witness for a read/write + * permutation. Bounded: first 8 recs, 4 lines per launch, + * boot budget 128. + */ + if (svc_fwd_req.gr_n_data <= 1024u && i < 8u && + svc_readdiv_log < 128u) { + u64 opa = 0, lpa2 = 0; + u32 oap = 0, lap2 = 0; + bool op_miss, have_led; + + op_miss = svc_op_translate(svc_fwd_req.vgpu, dg->va, + &opa, &oap) != 0; + have_led = !svc_ledger_lookup(dg->va, &lpa2, &lap2); + if (op_miss || (!op_miss && + (opa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (dg->pa & ~(u64)(SEMA_PAGE_SIZE - 1))) || + (have_led && + (lpa2 & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (dg->pa & ~(u64)(SEMA_PAGE_SIZE - 1)))) { + svc_readdiv_log++; + pr_info("vgsp-gr: READ-DIVERGE-WIT va=0x%llx fill_pa=0x%llx op_pa=0x%llx op_src=%u op_rc=%d ledger_pa=0x%llx%s\n", + dg->va, dg->pa, opa, + op_miss ? 0xffffffffu : svc.op_tr_src, + op_miss ? -ENOENT : 0, + have_led ? lpa2 : 0, + op_miss ? " (op-miss)" : ""); + } + } + /* + * C3 (data class): after the run the backend copies this + * page's results back to dg->pa (gsp_phys_booter.c copy-out). + * Record that page as the truth for this VA so the guest's + * D2H read of an engine-authored output follows the page the + * results actually landed on, whatever a later resolver says + * (atomics: engine ran, readback read a different page). + */ + if (dg->va >= SVC_VAS_FLOOR) + svc_ledger_store(dg->va, dg->pa, dkind); gs.n_data++; } gs.data = data; @@ -11939,6 +18959,12 @@ static void svc_fwd_gr_work(void) gs.rel_hpa = rel_hpa; gs.rel_off = (u32)(svc_fwd_req.gr_rel_va & (SEMA_PAGE_SIZE - 1)); gs.rel_payload = svc_fwd_req.gr_rel_payload; + /* + * r378: carry the release VA so the backend can re-validate the + * release-word write targets (zero-on-retry, restore-on-accept) + * through gsp_sema_gr_write_guard before they land. + */ + gs.rel_va = svc_fwd_req.gr_rel_va; gs.sem_hpa = 0; gs.sem_off = 0; gs.sem_payload = 0; @@ -11950,7 +18976,29 @@ static void svc_fwd_gr_work(void) gs.raw_qmd_end = svc_fwd_req.gr_push_qmd_end; gs.h_object = svc_fwd_req.gr_h_object; + /* + * GRVAUDIT: one line per launch (n_data matches the backend + * NOWRITE line's req->n_data); nonzero counts always log, an + * all-clean audit logs once per 50 launches. + */ + svc_grvaudit_launches++; + if (ga_unlanded || ga_ptebad || !(svc_grvaudit_launches % 50u)) + pr_info("vgsp-gr: GRVAUDIT n_data=%u unlanded=%u pte_bad=%u sample_va=0x%llx pa_used=0x%llx ledger_pa=0x%llx\n", + gs.n_data, ga_unlanded, ga_ptebad, + ga_va, ga_used, ga_led); + hw_rc = gsp_phys_backend_guest_gr_submit(&gs, &cp); + /* + * r420b: the backend redirected the launch's QMD release0 to the + * module semaphore (rel_mod=1 or no targetable guest release page), so + * the ENGINE never writes the guest's own QMD release word. Treat the + * word as unobserved so the bounded outcome check (and the hold's + * edges) author it in software on every outcome; without this a + * rel_mod=1 launch with no deferred release records is never completed + * on the guest word and the client hangs (the r414 saxpy stall). + */ + if (cp.qmd_rel_redirected) + svc_fwd_req.gr_rel_unobserved = true; if (!hw_rc) { svc.fwd_gr_ok++; } else { @@ -12221,12 +19269,16 @@ static void svc_fwd_relay_select(struct svc_fwd_req *req) * svc_gr_req_fill from its QMD pointer association and the captured inline * bytes. Overflow is a failure (fail-closed), not a silent drop. */ +static const char *svc_gr_src_probe(struct nv_vgpu_device *vgpu, u64 va, + u64 *pa_out, u32 *ap_out); static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word) { u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); u64 pa = 0; u32 ap = 0; u32 i; + /* r372 diagnostic: which branch resolved this launch page. */ + const char *src = "?"; if (!page) return; @@ -12245,10 +19297,132 @@ static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word) i = svc_fwd_req.gr_n_pages++; svc_fwd_req.gr_pages[i].va = page; svc_fwd_req.gr_pages[i].qmd_word = qmd_word; + /* + * m181 (relocation/ledger unification): the unified landing table is + * the authority for a page a host write already landed on. The CE + * inline writes that author these launch pages (param, program, + * constant banks, inline destinations) go through + * svc_grce_inline_write(), which records its landing. The backend's + * QMD relocation reads the page content through pg->pa (win0 shadow + * copy) and the m129 mirror patches that same page, so resolving + * independently here could hand the engine a stale page whose + * pointer values are not this launch's VAs: the relocation then + * matches nothing, the engine executes against unrelocated guest VAs + * it cannot walk, and the grid aborts silently (zero stores, ack + * still delivered) - the cross-session zero-write class - or runs on + * the previous client's parameters (the multi-page wrongness). + * Resolve once, from the table, and record the independent answer as + * stale when it differs. + */ + { + u64 lpa = 0; + u32 lap = 0; + + /* + * r372 pin-wins: a live injected pin is the guest's newest + * statement about this VA. If it disagrees with the landing + * table, the pin wins and the table is repaired (see the + * PIN-REPAIR note in svc_handle_map_add): the old landing was + * a freed page, and resolving the launch there wrote engine + * output onto memory the guest kernel had reused. + */ + { + u64 ipa = 0; + int iap = SVC_APERTURE_SYS; + + if (!svc_ipin_lookup(page, &ipa, &iap)) { + u64 lpa2 = 0; + u32 lap2 = 0; + + if (svc_ledger_lookup(page, &lpa2, &lap2) || + ((lpa2 & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (ipa & ~(u64)(SEMA_PAGE_SIZE - 1))) || + (u32)lap2 != (u32)iap) { + svc_ledger_pin_repairs++; + if (svc_ledger_pin_repairs <= 32) + pr_info("sema-ledger: PIN-WINS note va=0x%llx ledger=0x%llx/%u pin=0x%llx/%d\n", + page, lpa2, lap2, ipa, iap); + svc_ledger_store(page, + ipa & ~(u64)(SEMA_PAGE_SIZE - 1), + (u32)iap); + } + svc_fwd_req.gr_pages[i].pa = + ipa & ~(u64)(SEMA_PAGE_SIZE - 1); + svc_fwd_req.gr_pages[i].ap = (u8)iap; + svc_pin_store(page, + ipa & ~(u64)(SEMA_PAGE_SIZE - 1), + iap); + src = "ipin"; + goto note_done; + } + } + if (!svc_ledger_lookup(page, &lpa, &lap)) { + u64 rpa = 0; + u32 rap = 0; + const char *rsrc = svc_gr_src_probe(vgpu, page, &rpa, + &rap); + + if (rpa && + ((lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (rpa & ~(u64)(SEMA_PAGE_SIZE - 1)) || + (u32)lap != (u32)rap)) { + svc_ledger_reloc_diverges++; + if (svc_ledger_reloc_diverges <= 16) + pr_info("sema-ledger: RELOC-DIVERGE va=0x%llx ledger_pa=0x%llx %s_pa=0x%llx (ledger wins)\n", + page, + lpa & ~(u64)(SEMA_PAGE_SIZE - 1), + rsrc, rpa & + ~(u64)(SEMA_PAGE_SIZE - 1)); + } + svc_fwd_req.gr_pages[i].pa = + lpa & ~(u64)(SEMA_PAGE_SIZE - 1); + svc_fwd_req.gr_pages[i].ap = (u8)lap; + /* + * Keep the pin table consistent with the authority so + * the win0 copy path and later resolves agree (and no + * stale self-pin is planted for this VA). + */ + svc_pin_store(page, lpa & ~(u64)(SEMA_PAGE_SIZE - 1), + (int)lap); + pa = lpa; + ap = lap; + src = "ledger"; + goto note_done; + } + } if (svc_resolve_va(vgpu, page, &pa, &ap)) return; svc_fwd_req.gr_pages[i].pa = pa & ~(u64)(SEMA_PAGE_SIZE - 1); svc_fwd_req.gr_pages[i].ap = (u8)svc_ap_of(ap); + /* + * r370: store the landing table for EVERY launch page, not just the + * data-fill spans. An indirect/bank operand (the kernel's output + * buffer, qmd_word NONE) was resolved HERE, the engine wrote it via + * this hpa, the copy-out wrote it back to this hpa - and the client's + * D2H readback then resolved the same VA through the ledger/pin to a + * DIFFERENT page (an older landing), read stale zeros and the kernel + * looked wrong (victory_reduce sum=0 with the correct sum proven in + * the bounce page by OUT-PAGE-WIT). Recording this resolution makes + * the readback read exactly the page the engine wrote. + */ + svc_ledger_store(page, pa & ~(u64)(SEMA_PAGE_SIZE - 1), + (u32)svc_ap_of(ap)); + svc_pin_store(page, pa & ~(u64)(SEMA_PAGE_SIZE - 1), + (int)svc_ap_of(ap)); + src = "walk"; +note_done: + /* + * r372 diagnostic: the exact (pa, ap) and branch the launch page + * resolved through - the value the engine's hpa and the copy-out use. + * A D2H readback that lands elsewhere (the sum=0 class) names its + * disagreement against these lines. + */ + if (svc_gr_pagenote_log < 512) { + svc_gr_pagenote_log++; + pr_info("sema-gr: page-note va=0x%llx pa=0x%llx ap=%u src=%s qmd=%u\n", + page, svc_fwd_req.gr_pages[i].pa, + svc_fwd_req.gr_pages[i].ap, src, qmd_word); + } } /* Does any captured inline write land in this page? */ @@ -12282,35 +19456,65 @@ static bool svc_gr_page_inline_covers(u64 page) * Resolution only happens on the service thread; the work item turns the * guest PAs into HPAs and the backend plants the identity PTEs. */ -#define SVC_GR_DATA_RELOC_BASE 0x110000ULL +/* + * B-2 (large-operand data path): the SVC assigns each resolved operand page + * a reloc VA in the grown boot-planted region (the layout constants live in + * gsp_phys_booter.h so the two sides cannot drift; gsp_phys_gr_data_region + * selects grown vs legacy, default grown). The legacy 0x110000 window below + * the mail page (205 data pages max, 68 per pointer) remains selectable with + * gsp_phys_gr_data_region=0 for A/B trialing. + */ +#define SVC_GR_DATA_RELOC_BASE GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 +#define SVC_GR_DATA_RELOC_BASE_OLD 0x110000ULL /* * Per candidate pointer, map at most this many pages from the pointer's - * page onward (the engine-visible main-PT window has ~220 free 4 KB slots; - * the demo buffers are small on the native path). - * m120: lowered 64 -> 32 so every pointer page fits under the 100-page - * bounce cap (mail 2x32=64, vadd 3x32=96); at 64/ptr the second/third - * candidate run was truncated by the cap and its pointer word never - * rewritten (m119: first LDG base c[0x0][0x168] possibly untouched). - * m140: 32 -> 33. A cudaMalloc'd 128 KB buffer whose pointer is 0x400 - * into its page spans 33 pages (32 full + the 1 KB tail), and the kernel - * accesses the tail: with 32 pages the tail read the NEXT candidate's - * relocated page (wrong values; vadd tails). 3x33=99 <= the 100-page cap. - */ -#define SVC_GR_DATA_PAGES_PER_PTR 33u -/* m114: total pages the pre-commit bounce window can carry (slot 100 is - * the mail message). */ -#define SVC_GR_DATA_BOUNCE_MAX 100u + * page onward. With the grown region this is a runaway guard, not a budget: + * a cold page still stops the run (stop-on-miss below), so a real operand's + * span is bounded by its extent. History: 64 -> 32 (m120) -> 33 (m140) under + * the old 100-page cap -> 68 (m150, 256 KB operands under the 205-page + * window); the old value is kept for the legacy path. + */ +#define SVC_GR_DATA_PAGES_PER_PTR 4096u +#define SVC_GR_DATA_PAGES_PER_PTR_OLD 68u +/* + * Total operand pages the engine-visible data region can carry; derived from + * the booter's shared constant (no drift). A cap hit is FAIL-CLOSED: the + * fill marks the launch stalled for the hold/retry path and returns instead + * of silently truncating later pointers - a truncated operand must never + * reach the engine. The legacy value is the pre-B-2 window below the mail + * page (GSP_PHYS_RMCALL_GR_MAIL_SLOT = 206). + */ +/* + * fail4: each launch may relocate up to half of the grown region - the + * other half is the alternate base of a concurrently in-flight launch + * (see the per-capture base flip in svc_gr_data_fill). + */ +#define SVC_GR_DATA_BOUNCE_MAX (GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES / 2u) +#define SVC_GR_DATA_BOUNCE_MAX_OLD 205u static u32 svc_gr_data_log; /* m142: bounded per-page attribution log */ static u32 svc_gr_src_log; /* m150: bounded per-candidate source log */ static bool svc_gr_data_stalled; /* m150: high data args unresolved */ +/* + * m156 (R1): the GR request was refused because a forward was already + * in flight (or a release hold was active) - the launch COULD run later, so + * the entry must be deferred, not consumed with a fabricated software + * release. r157 streams2: kernels 2 and 3 were refused-while-busy (kernel 1 + * in flight) and soft-released 36 us later; the client read all zeros. + */ +static bool svc_gr_busy_defer; static u64 svc_gr_hold_log; /* m150: bounded GR-hold proof lines */ +static u64 svc_gr_drop_log_va; /* GRFIX: launch whose dropped-page detail + * (GRFILL-DROP) was already logged - the + * detail prints once per launch episode, + * not once per ~100 ms re-decode */ /* m150: re-arm both fill diagnostics for each client session. */ static void svc_gr_diag_reset(void) { svc_gr_data_log = 0; svc_gr_src_log = 0; + svc_gr_drop_log_va = 0; } /* @@ -12376,6 +19580,255 @@ static const char *svc_gr_src_probe(struct nv_vgpu_device *vgpu, u64 va, } /* + * relfix PART 3: append one attempt to the resolver's diagnostic chain + * ("name:rc,"). Bounded; the tail is simply dropped once full. + */ +static void svc_gr_reloc_stage_note(struct gsp_sema_gr_reloc *out, + const char *name, int rc) +{ + size_t n = strlen(out->stage); + + if (n + 16 >= sizeof(out->stage)) + return; + scnprintf(out->stage + n, sizeof(out->stage) - n, "%s:%d,", name, rc); +} + +/* + * relfix (agent-A FIX A): rewrite-time GR relocation resolver (see + * gsp_sema_service.h). Mirrors the per-page source order of the fill + * (svc_gr_data_fill: svc_ipin_lookup -> svc_resolve_va's trusted chain -> + * ledger) and of the read-only attribution probe svc_gr_src_probe above, + * then applies the m180 ledger in the fill's two roles: fallback when + * nothing above answered (the C1 landing path), and the m171 landing + * preference for VAs at/above the ledger floor when the ledger disagrees + * with the resolver (the skip-ahead class where a walk result is stale but + * the landing is authoritative). + * + * Deliberately excluded versus svc_resolve_va: pin_store/lat_store writes, + * svc_pd_discover and the sliced targeted RAM scan - those mutate + * service-thread state or can block for seconds, and this runs on the GR + * work item (process context; the page-table walks take svc.vas_lock only + * for their publication snapshot and may sleep in vfio_dma_rw, exactly the + * discipline the walk comment above svc_walk_vaspaces mandates). + * + * Every attempt is recorded in out->stage ("name:rc," chain; the resolving + * source ends in ":0,") for the booter's RELOC-GAP-STAGE diagnostic. + * + * Returns 0 with out filled on a successful translation, -ENOENT on a miss + * (caller must leave the word untouched), or a conversion errno. + */ +int gsp_sema_gr_reloc_resolve(u64 va, struct gsp_sema_gr_reloc *out) +{ + struct nv_vgpu_device *vgpu = svc_fwd_req.vgpu; + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 pa = 0, lpa = 0, hpa = 0; + int ap = SVC_APERTURE_SYS; + u32 lap = 0, off = 0; + bool resolved = false; + int rc, ret; + + if (!vgpu || !out) + return -EINVAL; + + out->stage[0] = '\0'; + out->stage_rc = 0; + + rc = svc_ipin_lookup(page, &pa, &ap); + svc_gr_reloc_stage_note(out, "ipin", rc); + if (!rc) + resolved = true; + if (!resolved) { + rc = svc_pin_lookup(page, &pa, &ap); + svc_gr_reloc_stage_note(out, "pin", rc); + if (!rc) + resolved = true; + } + if (!resolved) { + rc = svc_lat_lookup(page, &pa, &ap); + svc_gr_reloc_stage_note(out, "lat", rc); + if (!rc) + resolved = true; + } + if (!resolved) { + rc = svc_map_lookup(page, &pa); + svc_gr_reloc_stage_note(out, "map", rc); + if (!rc) + resolved = true; + } + if (!resolved) { + rc = svc_walk_vaspaces(vgpu, page, &pa, &ap); + svc_gr_reloc_stage_note(out, "vas", rc); + if (!rc) + resolved = true; + } + if (!resolved) { + rc = svc_walk_discovered(vgpu, page, &pa, &ap); + svc_gr_reloc_stage_note(out, "disc", rc); + if (!rc) + resolved = true; + } + + if (!resolved) { + /* C1 fallback: a landing is a page the client already backed + * with content even when every resolver missed (r174). */ + rc = svc_ledger_lookup(page, &lpa, &lap); + svc_gr_reloc_stage_note(out, "ledger", rc); + if (rc) { + out->stage_rc = -ENOENT; + return -ENOENT; + } + pa = lpa; + ap = (int)lap; + } else if (page >= SVC_VAS_FLOOR) { + /* m171: the landing table wins over a disagreeing resolver. */ + if (!svc_ledger_lookup(page, &lpa, &lap) && + ((lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1)) || + (u32)lap != svc_ap_of(ap))) { + pa = lpa; + ap = (int)lap; + } + } + + ret = svc_fwd_operand_hpa(vgpu, + svc_ap_of(ap) == SVC_APERTURE_FB ? + SVC_APERTURE_FB : SVC_APERTURE_SYS, + pa, 0, SEMA_PAGE_SIZE, &hpa, &off); + if (ret || off) { + svc_gr_reloc_stage_note(out, "hpa", ret ? ret : -EFAULT); + out->stage_rc = ret ? ret : -EFAULT; + return ret ? ret : -EFAULT; + } + + out->pa = pa & ~(u64)(SEMA_PAGE_SIZE - 1); + out->ap = svc_ap_of(ap); + out->hpa = hpa; + out->stage_rc = 0; + return 0; +} + +/* + * r378 write guard, GR backend half (see gsp_sema_service.h). The GR submit + * resolves every operand page once and the copy-out/release writes run + * seconds later (the bounded poll, the retire wait, the outcome retries); a + * guest buffer freed and re-homed inside that window would receive a full + * page of another buffer's content through the stale HPA - the guest + * boot-corruption class (slab freelist clobbered by a foreign page write). + * + * Authority order mirrors the fill's own choice (r372/r376 pin-wins): a + * FRESH injected pin is the guest's newest statement, so the guard compares + * the write target against the pin's page first; only a pin-less VA falls + * back to the trusted resolver chain. The landing table is deliberately NOT + * the authority here - our own launches feed it, so it would confirm the + * target by construction. A miss on every source keeps the historical + * behaviour (the recorded landing is the write authority), so the guard can + * only remove provably stale writes. Bounded diagnostic budget. + */ +static u32 svc_wguard_log; /* r378 WGUARD-SKIP budget (256/boot) */ +bool gsp_sema_gr_write_guard(u64 va, u64 hpa, const char *path) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 ipa = 0, cur_hpa = 0; + int iap = SVC_APERTURE_SYS; + u32 off = 0; + int ret; + + if (!va || !hpa) + return true; + if (!svc_ipin_lookup(page, &ipa, &iap)) { + u32 kind = svc_ap_of(iap) == SVC_APERTURE_FB ? + SVC_APERTURE_FB : SVC_APERTURE_SYS; + + ret = svc_fwd_operand_hpa(svc_fwd_req.vgpu, kind, + ipa & ~(u64)(SEMA_PAGE_SIZE - 1), + 0, SEMA_PAGE_SIZE, &cur_hpa, &off); + if (ret || off) + return true; /* pin page unresolvable: keep */ + } else { + struct gsp_sema_gr_reloc r; + + if (gsp_sema_gr_reloc_resolve(va, &r)) + return true; /* no current answer: keep */ + cur_hpa = r.hpa; + } + if ((cur_hpa & ~(u64)(SEMA_PAGE_SIZE - 1)) == + (hpa & ~(u64)(SEMA_PAGE_SIZE - 1))) + return true; + if (svc_wguard_log < 256) { + svc_wguard_log++; + pr_warn("vgsp-sema: WGUARD-SKIP path=%s va=0x%llx used=0x%llx now=0x%llx (stale write target)\n", + path, va, hpa, cur_hpa); + } + return false; +} + +/* + * r378 write guard, service half (see gsp_sema_service.h). A fresh injected + * pin is the guest's newest statement about a VA; a pin that disagrees with + * the recorded write target means the page was freed/re-homed (the r372/r376 + * pin-wins rule applied at the write instead of only at the resolution). A + * missing pin keeps the historical behaviour - the recorded landing is then + * the write authority - so no green path loses its write. + */ +bool gsp_sema_write_guard(u64 va, u64 pa, u32 ap, const char *path) +{ + u64 ipa = 0; + int iap = SVC_APERTURE_SYS; + + if (!va || !pa || va < SVC_VAS_FLOOR) + return true; + if (svc_ipin_lookup(va, &ipa, &iap)) + return true; /* no fresh pin: nothing newer to obey */ + if ((ipa & ~(u64)(SEMA_PAGE_SIZE - 1)) == + (pa & ~(u64)(SEMA_PAGE_SIZE - 1)) && + svc_ap_of(iap) == svc_ap_of((int)ap)) + return true; + if (svc_wguard_log < 256) { + svc_wguard_log++; + pr_warn("vgsp-sema: WGUARD-SKIP path=%s va=0x%llx used=0x%llx/%u pin=0x%llx/%d (stale write target)\n", + path, va, pa, ap, ipa, iap); + } + return false; +} + +/* + * r391 diagnostic (sum=0 class): the landing-table resolution of a GR data VA + * plus the first 8 bytes read through it - the exact page the D2H reader's + * ledger-first branch will use. Printed by the GR copy-out next to the write + * target, so a launch/readback disagreement is visible in one line. + */ +bool gsp_sema_gr_va_snap(u64 va, u64 *pa, u32 *ap, u64 *val8) +{ + u64 lpa = 0; + u32 lap = 0; + + if (svc_ledger_lookup(va, &lpa, &lap)) + return false; + *pa = lpa & ~(u64)(SEMA_PAGE_SIZE - 1); + *ap = lap; + *val8 = 0; + if (!svc_fwd_req.vgpu) + return true; + if (lap == SVC_APERTURE_FB) { + u8 *base = svc_fwd_req.vgpu->bars[1].vaddr; + size_t size = svc_fb_shadow_size(svc_fwd_req.vgpu); + + if (base && *pa + 8 <= size) + memcpy(val8, base + *pa, 8); + } else { + svc_read(svc_fwd_req.vgpu, *pa, val8, 8); + } + return true; +} + +/* r391 diagnostic: the shadow base this service resolves through (compare + * against the CE copy path's own vgpu->bars[1].vaddr in the same run). */ +u8 *gsp_sema_gr_shadow_base(void) +{ + return svc_fwd_req.vgpu ? svc_fwd_req.vgpu->bars[1].vaddr : NULL; +} + +/* * m144 (second-client launch-state divergence): collect device-VA-shaped * candidate pointers from the bytes the ENGINE will actually execute. * @@ -12407,7 +19860,7 @@ static void svc_gr_scan_cand(u64 *wins, u32 *n_cand, const u8 *data, u32 bytes) for (w = 0; w < *n_cand; w++) if (wins[w] == wv) break; - if (w == *n_cand && *n_cand < 16u) + if (w == *n_cand && *n_cand < SVC_GR_CAND_MAX) wins[(*n_cand)++] = wv; } } @@ -12425,14 +19878,171 @@ static void svc_gr_scan_inline(u64 *wins, u32 *n_cand, u64 page_va) } } -static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) +/* + * STALLFREE: identity of the launch currently held/retried on this channel - + * the capture's push VA when it belongs to this channel (the busy-refusal + * path never reaches the fill, so the request slot can still hold a previous + * launch's VA), else the request slot's when it names this channel. The + * absolute hold bound is keyed to this identity so a pin-batch rewind that + * replays the entry at a different slot index can never re-arm it. + */ +static u64 svc_gr_launch_va(const struct svc_channel *ch) +{ + if (svc_gr_cand.valid && svc_gr_cand.h_object == ch->h_object) + return svc_gr_cand.push_va; + if (svc_fwd_req.gr && svc_fwd_req.gr_h_object == ch->h_object) + return svc_fwd_req.gr_push_va; + return 0; +} + +/* + * GRFIX FIX1b (r213): absolute per-launch hold bound. The m150/m156 data + * hold had no working bound: the 8b058884 deadline was keyed by the GPFIFO + * slot index, and a pin-batch rewind/replay re-decodes the same launch push + * at different slots (r213: entry 107 -> 107 -> 99 on one channel, other + * passes at yet other indices), so "first arm" re-armed on every pass; the + * launch held for 216+ s (410+ GRFILL-HOLD lines) until the client's 240 s + * timeout killed it (gr_hold_exp=0). The identity is now the launch push VA: + * the first hold of a VA arms an ABSOLUTE deadline that nothing moves while + * the launch is held. Returns true when the bound has passed (caller must + * let the launch proceed); false leaves the record armed. + */ +static bool svc_gr_hold_bound_expired(struct svc_channel *ch, u64 launch_va) +{ + u32 cap_ms = gsp_phys_gr_hold_max_ms; + + /* + * STALLFREE (r213-r215 follow-up): the absolute per-launch bound is + * now effective on the default path. Before the write-back fix the + * bound's record fields were excluded from the pump snapshot and + * zeroed every pass, so this returned false forever and the legacy + * m150/m156 hold window (gr_hold_until) was the only expiry - and + * that window is keyed to the GPFIFO slot index, which a pin-batch + * rewind changes on every replay (the r213 identity bug). Under pin + * churn the same launch was therefore re-held indefinitely: r220 + * held one launch from 155221 to 155417 (196 s, 1631 re-fills of + * entry 106) until the guest's 240 s kill. The bound is keyed to the + * LAUNCH PUSH VA and nothing re-arms it, so a replayed slot no + * longer extends the hold; at the cap the pump falls through to the + * historical consume, where STALLFREE authors the QMD completion + * release and logs GR-STALL-hold action=proceed. Default cap 3000 ms + * (gsp_phys_gr_hold_max_ms) sits above the observed 0.5-2 s pin + * latency, so the legitimate pin-late retry still wins. + */ + if (!launch_va) + launch_va = 1; /* anonymous launch: still absolutely bound */ + if (!cap_ms) + cap_ms = 3000; + /* + * r368: mirror of the opdef fix - an ACTIVE bound is never re-armed + * by a different launch VA. The pump can hold several launches whose + * push VAs differ; arming on every VA change moved the deadline + * forever (the same identity bug class as the opdef=955/13/0 + * streams2 wedge). The first hold arms; others ride it; a fresh + * bound arms once the previous one has passed. + * + * r509: the bound must actually REPORT expiry. The historical arming + * rule re-armed the deadline on the very call that detected expiry + * and then compared against the NEW deadline - so the function could + * never return true, and a data-stalled launch whose arguments never + * resolved held its entry forever (r508 witness: matmul plus ten + * sibling channels in "GR hold ... data args unresolved" with zero + * GR-STALL-hold-bound lines for 10+ minutes; the designed + * retry-forward + consume never ran). The expiry is now one-shot: + * push the deadline, latch gr_hold_bound_hit for the r351 block of + * the same pass, return true. A fresh arm clears the latch (and the + * once-per-launch retry log). + */ + if (!ch->gr_hold_dead_va || + time_after_eq(jiffies, ch->gr_hold_deadline)) { + ch->gr_hold_dead_va = launch_va; + ch->gr_hold_dead_since = jiffies; + ch->gr_hold_deadline = jiffies + msecs_to_jiffies(cap_ms); + ch->gr_hold_exp_logged = false; + ch->gr_hold_bound_hit = false; + return false; + } + /* + * r420: pause the per-launch consumption bound while the backend + * deliberately holds the in-flight launch in its bounded WEDGE-QUIET + * window. At the bound the pump falls through to the historical + * consume, which authors the capture's QMD completion release in + * software (GR-STALL-hold) - but the backend is still going to + * submit/accept this very launch after the hold and copy its data + * out; authoring now releases the client onto stale data (the r419 + * mid-boot escape). The quiet hold is a bounded msleep (the + * backend's own caps), and the deferral here is bounded by + * SVC_GR_QUIET_PAUSE_MAX_MS from the bound's arm time as + * defence-in-depth. Rate-limited announcement. + */ + if (ch->gr_hold_dead_va && gsp_phys_gr_quiet_hold_active() && + time_before(jiffies, ch->gr_hold_dead_since + + msecs_to_jiffies(SVC_GR_QUIET_PAUSE_MAX_MS))) { + ch->gr_hold_deadline = jiffies + msecs_to_jiffies(cap_ms); + pr_info_ratelimited("vgsp-gr: GR-HOLD-BOUND-PAUSE launch=0x%llx (backend wedge-quiet hold active; consumption bound deferred)\n", + launch_va); + return false; + } + if (time_after_eq(jiffies, ch->gr_hold_deadline)) { + /* + * r509: one-shot expiry. Push the deadline so the hold window + * restarts for the caller's retry, latch the verdict for the + * second call site of this pass (the r351 retry-forward + * block), and report expired. + */ + ch->gr_hold_deadline = jiffies + msecs_to_jiffies(cap_ms); + ch->gr_hold_bound_hit = true; + return true; + } + return false; +} + +static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, + struct svc_channel *ch) { - u64 wins[16]; - bool is_arg[16]; /* m150: candidate came from the argument scan */ - u32 n_win = 0, n_cand; + u64 wins[SVC_GR_CAND_MAX]; + bool is_arg[SVC_GR_CAND_MAX]; /* m150: candidate came from the arg scan */ + u32 n_cand; u32 off = 0, i, w; u8 kind; /* + * B-2: active data-region geometry. gsp_phys_gr_data_region=1 (the + * default) selects the grown 16384-page region planted by the booter + * at GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0; 0 keeps the legacy 205-page + * window below the mail page for A/B trialing. + */ + bool region_on = gsp_phys_gr_data_region_value() != 0; + /* + * fail4: two concurrent forwarded launches (streams2's vadd pair) + * both relocated their operands from the same base, so the second + * launch's copy-in overwrote the first's bounce windows and each + * engine read the other's inputs (c1 came out as a2+b2). Alternate + * the per-launch base across the grown region's two halves (32 MB + * each, above any single launch's span), keyed to the capture push + * so a retried launch keeps its half, so concurrent launches + * cannot collide. + */ + static u64 svc_gr_reloc_half_key; + static u8 svc_gr_reloc_half; + u64 reloc_off = 0; + + if (region_on) { + if (svc_gr_cand.push_va != svc_gr_reloc_half_key) { + svc_gr_reloc_half_key = svc_gr_cand.push_va; + svc_gr_reloc_half ^= 1; + } + if (svc_gr_reloc_half) + reloc_off = (u64)(GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES / + 2u) * SEMA_PAGE_SIZE; + } + u64 reloc_base = region_on ? + SVC_GR_DATA_RELOC_BASE + reloc_off : + SVC_GR_DATA_RELOC_BASE_OLD; + u32 per_ptr = region_on ? SVC_GR_DATA_PAGES_PER_PTR : + SVC_GR_DATA_PAGES_PER_PTR_OLD; + u32 bounce_max = region_on ? SVC_GR_DATA_BOUNCE_MAX : + SVC_GR_DATA_BOUNCE_MAX_OLD; + /* * m150: high (user/device-space) argument candidates and how many of * them produced at least one resolved page. None resolving is the * second-client zeros signature: the engine would run the kernel @@ -12440,8 +20050,36 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) * retry instead of forwarded (svc_gr_data_stalled). */ u32 n_arg_hi = 0, n_arg_hi_res = 0; + /* + * GRFIX FIX1 (full-page fail-closed): every page of every tracked + * span whose resolution chain (ipin > resolve > ledger) misses is + * counted here, first page or mid-span alike. Any drop holds the + * launch through the same bounded m150/m156 retry path as the + * first-page guard below, so a QMD is never forwarded with a data + * pointer left as an unresolvable guest VA (the zero-store class) + * and the bounce window is never served stale for this launch. + * FIX1b: the hold carries an absolute per-launch bound + * (gsp_phys_gr_hold_max_ms) - when it passes the launch proceeds. + */ + u32 dropped_pages = 0; + bool bound_hit = false; /* GRFIX FIX1c: the absolute bound fired this + * pass - the launch proceeds, no hold */ + /* + * GRFIX FIX1c (r213): detail for the first (up to 4) dropped pages, + * so the cycle records WHICH VAs and which candidate class dropped. + * There is no read/write direction anywhere in the candidate or + * data[] structures (both are va/pa/reloc_va only), so kind is the + * truthful candidate origin: 'a' = argument page (c[0x0] param / + * inline), 'b' = const-bank page; idx is the candidate index (the + * same index the per-candidate probe lines use). + */ + u64 drop_va[4]; + char drop_kind[4]; + u32 drop_idx[4]; svc_fwd_req.gr_n_data = 0; + svc_fwd_req.gr_dropped = 0; + svc_fwd_req.gr_arg_partial = false; svc_fwd_req.gr_n_data_wins = 0; svc_gr_data_stalled = false; memset(is_arg, 0, sizeof(is_arg)); @@ -12502,6 +20140,52 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) * actually dereferences. */ for (w = 0; w < n_cand; w++) is_arg[w] = true; + /* + * relfix PART 3 (fill candidate superset): the rewrite scans the WHOLE + * authored param page (0x0..0xfff) and re-resolves every + * device-VA-shaped word it finds; any word this fill does not collect + * resurfaces as a RELOC-GAP (r217: the two words the late path fixed + * were param/program descriptor fields below 0x150 / past 0x800 that + * the argument scan never looked at). Collect the page remainder here, + * AFTER the argument marking, so these page words keep is_arg=false - + * the per-argument hold and diagnostic semantics stay exactly as + * before. Budget priority is also unchanged: the argument candidates + * above are resolved first. Launch-internal pages remain filtered at + * resolution time (R3 hygiene below). + */ + for (i = 0; i < svc_fwd_req.gr_n_pages; i++) { + const struct svc_gr_page *pp = &svc_fwd_req.gr_pages[i]; + u64 hpa = 0; + const u8 *page; + u32 k; + + if (pp->qmd_word != 32u || !pp->pa) + continue; + kind = (pp->ap == SVC_APERTURE_FB) ? SVC_APERTURE_FB : + SVC_APERTURE_SYS; + if (svc_fwd_operand_hpa(vgpu, kind, pp->pa, 0, + SEMA_PAGE_SIZE, &hpa, &off) || off) + continue; + page = (const u8 *)__va(hpa); + if (!page) + continue; + for (k = 0; k < SEMA_PAGE_SIZE / 8; k++) { + u64 v, wv; + + if (k >= 0x150u / 8u && k < 256u) + continue; /* arg region: scanned above */ + v = get_unaligned_le64(page + k * 8); + if (v < 0x10000000ULL || v >= (1ULL << 47) || + (v & 3)) + continue; + wv = v & ~(u64)(SEMA_PAGE_SIZE - 1); + for (w = 0; w < n_cand; w++) + if (wins[w] == wv) + break; + if (w == n_cand && n_cand < ARRAY_SIZE(wins)) + wins[n_cand++] = wv; + } + } for (i = 0; i < svc_fwd_req.gr_n_pages; i++) { const struct svc_gr_page *pp = &svc_fwd_req.gr_pages[i]; u64 hpa = 0; @@ -12519,7 +20203,12 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) page = (const u8 *)__va(hpa); if (!page) continue; - for (k = 0; k < 256; k++) { + /* + * relfix PART 3: whole page, matching the rewrite's scan + * range (it walks 0x0..0xfff qword-pairs); the old 0..0x7ff + * bound left the second half as permanent RELOC-GAPs. + */ + for (k = 0; k < SEMA_PAGE_SIZE / 8; k++) { u64 v = get_unaligned_le64(page + k * 8); u64 wv; @@ -12537,17 +20226,69 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) * inline-captured bytes as well (see svc_gr_scan_inline). */ svc_gr_scan_inline(wins, &n_cand, pp->va); } - /* m110: one contiguous run of pages per candidate pointer, relocated - * into the engine-walkable main-PT window (0x110000 onward, fresh - * PTE lines; the commit-time snapshot only preserves that window). */ + /* m110/B-2: one contiguous run of pages per candidate pointer, + * relocated into the active engine-visible data region (grown region + * 0x0a000000+ with gsp_phys_gr_data_region=1 - the default - or the + * legacy 0x110000 window; both are boot-planted, never-rewritten PTE + * ranges the commit-time snapshot preserves). */ for (w = 0; w < n_cand; w++) { u64 base = wins[w]; u32 p; u32 before = svc_fwd_req.gr_n_data; + bool first_unresolved = false; /* m170: span's first page failed */ u64 pba = 0; u32 pba_ap = 0; + u32 span; const char *pba_src = "?"; + /* + * Window budget discipline (large buffers): only real device + * operands (high, cudaMalloc-shaped VAs) get the full per-pointer + * span. Low-VA candidates (the launch's own QMD/release/reloc + * pointers) need a single page at most, and the launch's QMD VA + * is not a kernel operand at all - relocating a full span for it + * starved the operands that follow and blew the old window cap + * (r141: a 256 KB operand resolved 0 pages because the QMD and + * param candidates consumed the window). + */ + if ((base & ~(u64)(SEMA_PAGE_SIZE - 1)) == + (svc_fwd_req.gr_qmd_dev_va & ~(u64)(SEMA_PAGE_SIZE - 1))) + continue; + /* + * R3 hygiene (large-buffer budget discipline): the launch's own + * param/bank pages (the qmd_word 32..47 gr_pages entries) and + * the param VA are driver bookkeeping, not kernel operands - the + * observed fills resolved a full 68-page span for the param page + * in every launch and starved the real operands under the old + * window. Skip them as data candidates; they are + * shadowed/rewritten through gr_pages/Tier 2, not this path. + * P6: the QMD program pair (word 48/49) is launch-internal in + * exactly the same way and is skipped with them. + */ + if (svc_fwd_req.gr_param_va && + (base & ~(u64)(SEMA_PAGE_SIZE - 1)) == + (svc_fwd_req.gr_param_va & ~(u64)(SEMA_PAGE_SIZE - 1))) + continue; + { + bool launch_internal = false; + + for (i = 0; i < svc_fwd_req.gr_n_pages; i++) { + const struct svc_gr_page *pp = + &svc_fwd_req.gr_pages[i]; + + if (pp->qmd_word >= 32u && + pp->qmd_word <= 49u && + (base & ~(u64)(SEMA_PAGE_SIZE - 1)) == + (pp->va & ~(u64)(SEMA_PAGE_SIZE - 1))) { + launch_internal = true; + break; + } + } + if (launch_internal) + continue; + } + span = (base >= 0x40000000000ULL) ? per_ptr : 1u; + if (is_arg[w] && base >= 0x40000000000ULL) n_arg_hi++; @@ -12562,7 +20303,7 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) svc_pd_found, svc.ipin_n); } - for (p = 0; p < SVC_GR_DATA_PAGES_PER_PTR; p++) { + for (p = 0; p < span; p++) { u64 va = wins[w] + (u64)p * SEMA_PAGE_SIZE; u64 pa = 0; u32 ap = 0, x; @@ -12592,29 +20333,193 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) pa = ipa; ap = (u32)iap; } else if (svc_resolve_va(vgpu, va, &pa, &ap)) { - break; + /* + * C1 (stall/data class): the page may + * still be readable - the ledger holds + * the landing page of the H2D that + * wrote it even when every resolver + * misses (r174 cand[] probe=none, 0 + * pages, bad=65536). Use it before + * declaring the page dead; only a page + * with no ledger entry can truncate + * the span. + */ + u64 lpa = 0; + u32 lap = 0; + + if (!svc_ledger_lookup(va, &lpa, &lap)) { + pa = lpa; + ap = lap; + } else { + /* + * m170 (R4a): the very first + * page of this span failed to + * resolve - the fail-closed + * trigger below distinguishes + * it from the benign "0 new + * pages" of a span whose + * pages were already mapped. + * + * GRFIX FIX1: every failed + * page counts (first or mid- + * span) for the launch-level + * hold; the span still stops + * here exactly as before. + * FIX1c: capture the first 4 + * (VA + candidate origin + + * candidate index) for the + * GRFILL-DROP detail. + */ + if (p == 0 && + svc_fwd_req.gr_n_data == before) + first_unresolved = true; + /* + * COPYAUDIT (2026-10-18): a + * span cut after at least one + * authored page is the + * completeness hazard - the + * rewritten base pointer is + * still handed to the engine, + * whose base+offset reads then + * run past the authored pages + * into later operands' bounce + * pages. Make the truncation + * loud (want is a lower bound: + * the guest asked for >= p+1 + * pages; done is the authored + * page count before the cut). + */ + if (p > 0) + pr_info_ratelimited("vgsp: COPY-TRUNC va=0x%llx want>=0x%llx done=0x%llx path=gr-fill-span\n", + wins[w], + ((u64)p + 1) << 12, + (u64)p << 12); + /* + * r377 pin-demand: name every + * unresolved operand page as a + * stalled VA. The userspace pin + * stream watches dmesg for this + * exact line and pushes the VA + * to the in-guest scanner's + * demand file, so the page gets + * pinned and the bounded launch + * hold (gsp_phys_gr_fill_hold) + * resumes on real data instead + * of waiting out its bound (the + * r345 wedge: held forever + * because nobody demanded the + * page). + */ + svc_log_stalled_va(va); + if (is_arg[w]) + svc_fwd_req.gr_arg_partial = true; + if (dropped_pages < 4) { + drop_va[dropped_pages] = va; + drop_kind[dropped_pages] = + is_arg[w] ? 'a' : 'b'; + drop_idx[dropped_pages] = w; + } + dropped_pages++; + break; + } } } - if (svc_fwd_req.gr_n_data >= SVC_GR_DATA_BOUNCE_MAX) { - pr_info("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved (reloc 0x%llx+) CAP-HIT\n", + /* + * m171 (AGENT-F R2): prefer the write/read ledger's + * landing page for this operand VA when it disagrees + * with the resolution - the kernel then reads exactly + * the page a large H2D wrote (r174: write landed on + * pa X while the fill read a stale page Y). Compare + * aperture-aware (a SYS page and an FB page with the + * same numeric address are not the same page). + */ + if (va >= SVC_VAS_FLOOR) { + u64 lpa = 0; + u32 lap = 0; + u64 ipa = 0; + int iap = SVC_APERTURE_SYS; + + /* + * r372 pin-wins: when a live pin exists it is the + * newest truth; the landing table may name a page + * the guest has since freed (VA reuse across + * clients). Keep the pin and repair the table. + * Only a pin-less VA still follows the table. + */ + if (!svc_ipin_lookup(va, &ipa, &iap)) { + pa = ipa; + ap = (u32)iap; + if (!svc_ledger_lookup(va, &lpa, &lap) && + (((lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (ipa & ~(u64)(SEMA_PAGE_SIZE - 1))) || + (u32)lap != (u32)iap)) { + svc_ledger_pin_repairs++; + if (svc_ledger_pin_repairs <= 32) + pr_info("sema-ledger: PIN-WINS fill va=0x%llx ledger=0x%llx/%u pin=0x%llx/%d\n", + va, + lpa & ~(u64)(SEMA_PAGE_SIZE - 1), + lap, ipa, iap); + svc_ledger_store(va, + ipa & ~(u64)(SEMA_PAGE_SIZE - 1), + (u32)iap); + } + } else if (!svc_ledger_lookup(va, &lpa, &lap) && + ((lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1)) || + (u32)lap != (u32)ap)) { + svc_ledger_diverges++; + if (svc_ledger_diverges <= 32) + pr_info("sema-ledger: DIVERGE fill va=0x%llx wrote=0x%llx read=0x%llx\n", + va, lpa, pa); + pa = lpa; + ap = lap; + } + } + if (svc_fwd_req.gr_n_data >= bounce_max) { + /* + * B-2 fail-closed cap: the active region is + * full and later pointers would be silently + * dropped (the old CAP-HIT return forwarded + * a truncated operand set). Mark the launch + * stalled so svc_gr_request holds/retries it + * via the existing bounded hold instead of + * forwarding incomplete data. + */ + svc_gr_data_stalled = true; + pr_err("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved (reloc 0x%llx+) CAP-HIT: region full, launch held fail-closed (no silent truncation)\n", n_cand, svc_fwd_req.gr_n_data, - (u64)SVC_GR_DATA_RELOC_BASE); + reloc_base); return; } rec = &svc_fwd_req.gr_data[svc_fwd_req.gr_n_data]; rec->va = va; rec->pa = pa & ~(u64)(SEMA_PAGE_SIZE - 1); - rec->reloc_va = SVC_GR_DATA_RELOC_BASE + + rec->reloc_va = reloc_base + ((u64)svc_fwd_req.gr_n_data << 12); rec->ap = (u8)svc_ap_of(ap); svc_fwd_req.gr_n_data++; /* + * r349: record the launch's own resolution as the + * ledger's landing authority for this VA. The read + * path's m171 read-after-write rule follows the + * ledger when it disagrees with the live translation, + * and the engine writeback below writes exactly this + * (pa, ap). Without the store the ledger kept a stale + * page for engine-written outputs (victory_reduce: + * the launch wrote FB 0x7c00000 while the ledger + * said SYS 0x7c00000; the D2H read followed the + * ledger and returned zeros while the correct sum + * sat on the written page). + */ + if (va >= SVC_VAS_FLOOR) + svc_ledger_store(va, rec->pa, rec->ap); + /* * m142: bounded per-page attribution log. A wrong data * page is invisible in the aggregate counters; these * lines let a post-mortem compare the chosen PA against * the guest's own mapping. */ - if (svc_gr_data_log < 16) { + if (svc_gr_data_log < 192) { svc_gr_data_log++; pr_info("vgsp-sema: svc-gr data page va=0x%llx -> pa=0x%llx ap=%u\n", va, rec->pa, rec->ap); @@ -12627,6 +20532,24 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) pr_info("vgsp-sema: svc-gr candidate[%u] base=0x%llx -> %u new pages (total %u)\n", w, base, svc_fwd_req.gr_n_data - before, svc_fwd_req.gr_n_data); + if (is_arg[w] && svc_fwd_req.gr_n_data == before && + first_unresolved) { + /* + * m170 (AGENT-F R4a): a high argument pointer whose + * FIRST page resolved nothing at fill time is the + * partial-operand signature - the launch would forward + * with stale/zero data for that operand (r174: matmul + * cand[2]/[3] probe=none, 0 pages, bad=65536; the + * other operands resolved via pins). Fail closed: mark + * the launch stalled so the m150 hold keeps the entry + * and retries the fill while the pins land (bounded, + * then forwards anyway - never a hang, never silent + * truncation). + */ + svc_gr_data_stalled = true; + pr_err_ratelimited("vgsp-sema: svc-gr data fill: arg base=0x%llx resolved 0 pages; launch held fail-closed\n", + base); + } /* * m150: an argument pointer that resolves zero pages is the * second-client zeros signature; record the walk result per @@ -12651,7 +20574,7 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) wrc = svc_gmmu_walk(vgpu, svc.vas[vi].pdb, svc.vas[vi].entries, base, svc.vas[vi].ap, &wa, &wap, - false); + gsp_sema_walk_trace != 0); o += snprintf(vb + o, sizeof(vb) - o, " vas[%d] pdb=0x%llx ent=%u ap=%d rc=%d pa=0x%llx", vi, svc.vas[vi].pdb, @@ -12663,14 +20586,104 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) svc.ipin_n, svc_pd_base, svc_pd_ap, vb); } } - svc_gr_data_stalled = (n_arg_hi > 0 && n_arg_hi_res == 0); + /* + * m170 revert: the per-argument fail-closed signal is kept as + * DIAGNOSTIC only (the held fail-closed line above identifies the + * exact unresolved operand); the historical aggregate rule held only + * a launch with NO resolvable high argument, because a fresh client's + * launch stream carries BENIGN 0-page candidates (junk pointer words + * that never translate) and persisting the per-arg flag held mail's + * first launch forever (r176). + * + * GRFIX FIX1: the stall is now page-count based - ANY page in a + * tracked span that failed resolution (or any tracked data VA whose + * resolution returned nothing) holds the launch, so a QMD can never + * be forwarded with a data pointer left as an unresolvable guest VA + * and no bounce page is ever served stale for this launch. The + * response is the unchanged bounded m150/m156 hold (leave the GPFIFO + * entry unconsumed, retry the fill while the pins land); the pump + * side now carries a hard wall-clock bound (gr_hold_deadline) so a + * word that never resolves can no longer re-arm the window forever: + * at the bound the entry is consumed with the historical software + * release - bounded, never a hang, never a silent stale-operand + * submission. + */ + svc_gr_data_stalled = (dropped_pages > 0) || + (n_arg_hi > 0 && n_arg_hi_res == 0); + svc_fwd_req.gr_dropped = dropped_pages; + /* + * GRFIX FIX1e (r215): gsp_phys_gr_fill_hold is the master gate. + * Default 0 = LOG-ONLY: the fill NEVER holds on dropped pages - the + * stalled verdict keeps only the historical pre-8b058884 aggregate + * rule (all high argument pointers unresolved) and the HOLD/EXPIRED + * machinery below is skipped entirely; the launch takes the forward + * path immediately. When 1 the page-count hold applies, bounded by + * the per-launch absolute deadline below. + * + * GRFIX FIX1c/FIX1d (live only with fill_hold=1): the decision point + * is self-enforcing - the absolute per-launch bound is consulted at + * the only place the stalled verdict is produced, before the + * GRFILL-HOLD line can be emitted. When the bound passes the launch + * PROCEEDS via the historical forward path. The benign class + * (probe=none / walk rc=-2) is indistinguishable at fill time from a + * page whose pin has not landed yet, so a "never hold" predicate + * would also disable the pin-late retry; the bound is the only + * guard. NOTE (r213-r215): the bound never fired in the live cycles + * because these per-channel record fields are not in the + * svc_pump_channels snapshot write-back list (see the GRFIX NOTE + * there) and were zeroed on every pass - which is why the gate now + * defaults to 0. + */ + if (!gsp_phys_gr_fill_hold) { + svc_gr_data_stalled = (n_arg_hi > 0 && n_arg_hi_res == 0); + } else if (svc_gr_data_stalled && ch && + svc_gr_hold_bound_expired(ch, svc_fwd_req.gr_push_va)) { + svc.fwd_gr_hold_exp++; + bound_hit = true; + if (!ch->gr_hold_exp_logged) { + ch->gr_hold_exp_logged = true; + pr_info("vgsp-gr: GRFILL-EXPIRED launch=0x%llx dropped=%u waited_ms=%u\n", + svc_fwd_req.gr_push_va, dropped_pages, + jiffies_to_msecs(jiffies - + ch->gr_hold_dead_since)); + } + svc_gr_data_stalled = false; /* proceed: forward */ + } + if (dropped_pages > 0) { + if (gsp_phys_gr_fill_hold && !bound_hit) + pr_info_ratelimited("vgsp-gr: GRFILL-HOLD launch=0x%llx dropped=%u kept=%u\n", + svc_fwd_req.gr_push_va, dropped_pages, + svc_fwd_req.gr_n_data); + /* + * GRFIX FIX1c: name the dropped pages once per launch + * episode (the fill re-runs every pass for a held launch; + * the marker makes the detail a first-pass-only record). + * kind 'a' = argument-page candidate, 'b' = bank candidate; + * idx = candidate index (same indexing as the probe lines). + */ + if (svc_gr_drop_log_va != svc_fwd_req.gr_push_va) { + u32 dl; + + svc_gr_drop_log_va = svc_fwd_req.gr_push_va; + for (dl = 0; dl < 4 && dl < dropped_pages; dl++) + pr_info("vgsp-gr: GRFILL-DROP va=0x%llx kind=%c idx=%u\n", + drop_va[dl], drop_kind[dl], + drop_idx[dl]); + } + } if (svc_gr_data_stalled) - pr_info("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved; %u high argument ptr(s), 0 resolved -> launch held for a bounded retry\n", - n_cand, svc_fwd_req.gr_n_data, n_arg_hi); + pr_info("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved; %u high argument ptr(s), %u resolved; %u dropped page(s) -> launch held for a bounded retry\n", + n_cand, svc_fwd_req.gr_n_data, n_arg_hi, n_arg_hi_res, + dropped_pages); + else if (bound_hit) + pr_info_ratelimited("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved; %u dropped page(s); hold bound expired -> launch proceeds\n", + n_cand, svc_fwd_req.gr_n_data, dropped_pages); + else if (dropped_pages > 0 && !gsp_phys_gr_fill_hold) + pr_info_ratelimited("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved; %u dropped page(s); fill hold disabled (log-only) -> launch proceeds\n", + n_cand, svc_fwd_req.gr_n_data, dropped_pages); else pr_info("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved (reloc 0x%llx+)\n", - n_cand, svc_fwd_req.gr_n_data, - (u64)SVC_GR_DATA_RELOC_BASE); + n_cand, svc_fwd_req.gr_n_data, reloc_base); } static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, @@ -12684,6 +20697,11 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, svc_fwd_req.guest = false; svc_fwd_req.gr = true; svc_fwd_req.gr_bridge_done = false; + /* STALLFREE: fresh request - re-arm the software-completion latch and + * stamp the fill time for the GR-STALL waited_ms accounting. */ + svc_fwd_req.gr_swrel_done = false; + svc_fwd_req.gr_rel_unobserved = false; + svc_fwd_req.gr_fill_jiffies = jiffies; svc_fwd_req.defer = true; svc_fwd_req.relay = false; svc_fwd_req.relay_index = 0; @@ -12786,7 +20804,14 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, if (!svc_gr_page_inline_covers(pg->va)) return -ENOENT; } - svc_gr_data_fill(vgpu); + /* + * GRFIX FIX1c: the fill hosts the self-enforcing decision point and + * therefore needs the channel's per-launch bound record. The const + * view exists for the read-only gates; the channel is the pump's + * mutable live bookkeeping object and this runs on the same + * service thread. + */ + svc_gr_data_fill(vgpu, (struct svc_channel *)ch); return 0; } @@ -12825,17 +20850,298 @@ static void svc_gr_req_bridge_fallback(struct nv_vgpu_device *vgpu) */ static void svc_handle_map_add(void); +/* + * r543: one bounded deferral per push VA for a partially-resolved argument + * set (see svc_fwd_req.gr_arg_partial). The latch keeps a genuine junk + * pointer word from adding a retry window to every launch: the first time a + * push is seen with a partial argument set it is deferred (pins land within + * ~1-2 s in the healthy case); the same push VA seen again inside the + * window proceeds through the historical forward. A re-used ring slot after + * the window gets its own deferral. + * + * r548: the latch is a bounded per-VA table, not a single slot. A client + * with TWO in-flight launches (victory_saxpy r547: push VAs A and B + * alternating re-decodes) made the single slot ping-pong: every call saw + * "a different VA than last time" and deferred again - both launches + * deferred forever, the client's cudaDeviceSynchronize never completed + * (held launches, zero GR LAUNCH lines, zero GR-COMPLETE). The table gives + * each VA exactly one deferral inside the window; the second sighting of + * any VA proceeds (bounded, cannot hang), interleaved launches included. + */ +#define SVC_GR_ARG_DEFER_MS 30000u +/* + * r554: 64, not 8. The latch must remember every launch push VA of the + * current client (the UVM pool allocates ~8-10 channels and each command + * buffer reuses two launch push slots, so >8 distinct VAs are in rotation); + * with 8 slots each new VA evicted another and the "one deferral per VA" + * guarantee thrashed - r553 soak: victory_reduce re-deferred the same push + * VA (0x20043be64) every pass for 430 s ("partial argument set: 7-8 pages + * unresolved") and never forwarded. 64 covers a full client with margin; + * the table is per-boot tiny (64 * 16 B). + */ +#define SVC_GR_ARG_DEFER_SLOTS 64 +static u64 svc_gr_arg_defer_va[SVC_GR_ARG_DEFER_SLOTS]; +static unsigned long svc_gr_arg_defer_j[SVC_GR_ARG_DEFER_SLOTS]; +static u32 svc_gr_arg_defer_pos; + +static bool svc_gr_arg_defer_ok(u64 push_va) +{ + u32 i; + + for (i = 0; i < SVC_GR_ARG_DEFER_SLOTS; i++) { + if (svc_gr_arg_defer_va[i] == push_va && + time_before(jiffies, svc_gr_arg_defer_j[i] + + msecs_to_jiffies(SVC_GR_ARG_DEFER_MS))) + return false; /* deferred once already: forward */ + } + i = svc_gr_arg_defer_pos++ % SVC_GR_ARG_DEFER_SLOTS; + svc_gr_arg_defer_va[i] = push_va; + svc_gr_arg_defer_j[i] = jiffies; + return true; +} + +/* + * r556: the ABSOLUTE per-launch deferral cap. Every defer path (r541 + * dstpend, r543 partial args, m150 data-stalled hold, busy-defer re-holds) + * is individually bounded, but the bounds interact and re-arm across + * re-decode passes; the r553/r555 boots show clients still walking these + * loops for 400+ s (victory_reduce 430 s, victory_saxpy 254 s at write + * time, work_guard deferred climbing by the thousand). This is the single + * backstop that makes "a launch can never be deferred for more than + * SVC_GR_DEFER_CAP_MS" true regardless of which guard keeps winning: the + * first sighting of a push VA arms an absolute deadline; once it passes, + * svc_gr_defer_expired() returns true and every defer site forwards + * best-effort. 15 s is well above the 0.5-2 s pin latency the deferrals + * exist to cover and well below every client-side timeout (>= 60 s). + */ +#define SVC_GR_DEFER_CAP_MS 15000u +static u64 svc_gr_defer_cap_va[SVC_GR_ARG_DEFER_SLOTS]; +static unsigned long svc_gr_defer_cap_j[SVC_GR_ARG_DEFER_SLOTS]; +static u32 svc_gr_defer_cap_pos; + +static bool svc_gr_defer_expired(u64 push_va) +{ + u32 i; + + if (!push_va) + return false; + for (i = 0; i < SVC_GR_ARG_DEFER_SLOTS; i++) { + if (svc_gr_defer_cap_va[i] == push_va) + return !time_before(jiffies, svc_gr_defer_cap_j[i] + + msecs_to_jiffies(SVC_GR_DEFER_CAP_MS)); + } + i = svc_gr_defer_cap_pos++ % SVC_GR_ARG_DEFER_SLOTS; + svc_gr_defer_cap_va[i] = push_va; + svc_gr_defer_cap_j[i] = jiffies; + return false; +} + +/* + * r548: is a forward still OWED for this push VA - a captured, never + * offered candidate sitting in the candidate ring? The r547 saxpy stall: + * both launches were deferred by the r541/r543 guards, the candidate ring + * held them as unoffered, and the later passes re-decoded the entries as + * REPLAYS (LAUNCH-SKIP; already captured=1). The replay consume then + * dropped the entries without forwarding them and without authoring any + * completion (no GR LAUNCH, no GR-COMPLETE-SW) - the client's + * cudaDeviceSynchronize waited forever while the pump re-walked the ring. + * The pump's no-capture hold must therefore also fire for a replay whose + * forward is owed, forcing one fresh capture + request per bounded window. + */ +static bool svc_gr_cand_push_owed(u64 push_va) +{ + u32 i; + + if (!push_va) + return false; + for (i = 0; i < SVC_GR_CAND_RING; i++) + if (svc_gr_cand_ring[i].valid && + !svc_gr_cand_ring[i].offered && + svc_gr_cand_ring[i].push_va == push_va) + return true; + return false; +} + +/* + * r548: last-resort resolution of an owed forward when the bounded + * no-capture hold expires without the launch ever being submitted - the + * client's completion word must not be left to a launch that never ran. + * Author the candidate's QMD release in software (forward-only) and mark + * the candidate consumed so a later pass can never forward it late onto + * refilled buffers (the r59 double-run class). Returns true when a pending + * candidate for this channel+push VA was resolved. + */ +static bool svc_gr_owed_resolve(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch, u64 push_va) +{ + u32 i; + + if (!push_va) + return false; + for (i = 0; i < SVC_GR_CAND_RING; i++) { + struct svc_gr_cand_rec *c = &svc_gr_cand_ring[i]; + + if (!c->valid || c->offered || + c->push_va != push_va || + c->h_object != ch->h_object) + continue; + svc_gr_sw_release(vgpu, ch->h_object, c->rel_va, + c->rel_payload); + c->offered = true; + svc.gr_owed_resolved++; + return true; + } + return false; +} + +/* + * r541: does any of the filled launch's data pages carry an unexecuted + * device write (see svc_dstpend_note)? Runs on the service thread right + * after the fill, so svc_fwd_req.gr_data holds this launch's records. + */ +static bool svc_gr_dstpend_overlap(void) +{ + u32 i; + + for (i = 0; i < svc_fwd_req.gr_n_data; i++) + if (svc_dstpend_hit(svc_fwd_req.gr_data[i].va)) + return true; + return false; +} + +/* + * r568: a CE channel entry with a withheld release is a device write that + * has NOT executed yet. A GR launch must not read any of the client's + * device pages while such a write is pending. The page-level dstpend guard + * cannot cover this when the launch's argument base was dropped + * (GRFILL-DROP): the launch's data set then does not contain the buffer's + * pages, so the overlap check is blind to them - the r571 mail VADD read b + * 1.5 s before the b H2D executed (copy-in 319818.68, b copy 319820.16) and + * computed c = a + 0. Gate the launch on the coarser but complete + * condition: any active opdef slot whose channel is a CE channel means at + * least one copy/fill is unexecuted. Bounded by the same busy-defer + + * absolute 15 s cap machinery as every other guard, so a genuinely stuck + * copy delays a launch by the cap and no more. GR channels are excluded + * (a launch's own withheld release must not defer itself). + */ +static bool svc_copy_pending(void) +{ + unsigned long flags; + bool pending = false; + u32 i, c; + + for (i = 0; i < SVC_OPDEF_SLOTS && !pending; i++) { + struct svc_opdef_ent *e = &svc_opdef[i]; + + if (!e->active || !e->n) + continue; + spin_lock_irqsave(&svc.chan_lock, flags); + for (c = 0; c < SVC_MAX_CHANNELS; c++) + if (svc.chan[c].valid && + svc.chan[c].vgpu == e->vgpu && + svc.chan[c].h_object == e->h_object) { + if (svc.chan[c].engine_type) + pending = true; + break; + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + } + return pending; +} + +/* + * r576 DIAG: the GR launch-decision witness (gsp_phys_gr_dec_log=N budget + * per boot). Logged from svc_gr_request right after the fill, for launches + * whose fill saw dropped pages / a partial argument set or whose decision + * point carries a dstpend/copy-pending verdict - i.e. exactly the draws the + * r566/r567/r568 guards can defer or (on their blind spots) forward. The + * data-page list is the operand evidence: each VA -> guest PA the engine + * will read; a wrong-but-valid PA here (vs the page the client's H2D wrote + * for the same VA) is the failure mechanism the readback lottery draws. + */ +unsigned gsp_phys_gr_dec_log_max; +static unsigned svc_gr_dec_log; + +static void svc_gr_dec_dump(const struct svc_channel *ch) +{ + bool dstpend, copypend; + u32 i, n = 0; + + if (svc_gr_dec_log >= gsp_phys_gr_dec_log_max) + return; + svc_gr_dec_log++; + + dstpend = svc_gr_dstpend_overlap(); + copypend = svc_copy_pending(); + + pr_info("vgsp-gr: GR-DEC h=0x%x push=0x%llx n_data=%u dropped=%u partial=%d stalled=%d dstpend=%d copypend=%d defer_exp=%d\n", + ch->h_object, svc_gr_cand.push_va, svc_fwd_req.gr_n_data, + svc_fwd_req.gr_dropped, + svc_fwd_req.gr_arg_partial ? 1 : 0, + svc_gr_data_stalled ? 1 : 0, + dstpend ? 1 : 0, copypend ? 1 : 0, + svc_gr_defer_expired(svc_gr_cand.push_va) ? 1 : 0); + + /* + * r577: raised from 32 to 256 pages - the mail VADD's fill carried + * n_data=969 pages and the red-draw divergence (a wrong-but-similar + * input page) can sit beyond the first 32 pages. + */ + for (i = 0; i < svc_fwd_req.gr_n_data && i < 256; i++) + pr_info("vgsp-gr: GR-DEC data[%u] va=0x%llx pa=0x%llx ap=%u reloc=0x%llx\n", + i, svc_fwd_req.gr_data[i].va, + svc_fwd_req.gr_data[i].pa, svc_fwd_req.gr_data[i].ap, + svc_fwd_req.gr_data[i].reloc_va); + + for (i = 0, n = 0; i < svc_dstpend_n && n < 16; i++) { + if (!svc_dstpend[i].va_page) + continue; + n++; + pr_info("vgsp-gr: GR-DEC dstpend va=0x%llx age_ms=%u\n", + svc_dstpend[i].va_page, + jiffies_to_msecs(jiffies - svc_dstpend[i].since)); + } + + for (i = 0, n = 0; i < SVC_OPDEF_SLOTS && n < 8; i++) { + if (!svc_opdef[i].active || !svc_opdef[i].n) + continue; + n++; + pr_info("vgsp-gr: GR-DEC opdef h=0x%x entry=%u n=%u age_ms=%u\n", + svc_opdef[i].h_object, svc_opdef[i].entry_idx, + svc_opdef[i].n, + jiffies_to_msecs(jiffies - svc_opdef[i].since)); + } +} + static bool svc_gr_request(struct nv_vgpu_device *vgpu, const struct svc_channel *ch) { int rc; + /* m156: never let a previous call's busy verdict leak into an early + * (candidate-mismatch) return; only the busy paths below set it. */ + svc_gr_busy_defer = false; /* * The candidate is per-channel: a capture left unconsumed by an * earlier pass (e.g. the entry was held before its request) must never * be queued against a different channel's handle by that channel's - * own end-of-pass request call. + * own end-of-pass request call. fail4: with several reserved ring + * slots, select THIS channel's oldest unconsumed candidate first. */ + if (gsp_phys_gr_forward_enabled()) { + u32 ri; + + for (ri = 0; ri < SVC_GR_CAND_RING; ri++) { + u32 rp = (svc_gr_cand_pos + ri) % SVC_GR_CAND_RING; + + if (svc_gr_cand_ring[rp].valid && + !svc_gr_cand_ring[rp].offered && + svc_gr_cand_ring[rp].h_object == ch->h_object) { + svc_gr_cand_pos = rp; + break; + } + } + } if (!gsp_phys_gr_forward_enabled() || !svc_gr_cand.valid || svc_gr_cand.h_object != ch->h_object) return false; @@ -12854,6 +21160,17 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, if (svc_fwd_hold.active) { svc.fwd_gr_refused++; svc.fwd_gr_fallback++; + /* + * m159: with the bridge disabled the capture is not consumed: + * the candidate survives for the caller's bounded + * wait+retry (the launch can still run once the active + * forward resolves). With the bridge enabled the fallback + * owns the work and consumes the candidate as before. + */ + if (!gsp_cuda_bridge_enabled()) { + svc_gr_busy_defer = true; + return false; + } svc_gr_bridge_fallback(vgpu); return false; } @@ -12863,6 +21180,10 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, mutex_unlock(&svc_fwd_lock); svc.fwd_gr_refused++; svc.fwd_gr_fallback++; + if (!gsp_cuda_bridge_enabled()) { + svc_gr_busy_defer = true; + return false; + } svc_gr_bridge_fallback(vgpu); return false; } @@ -12890,6 +21211,90 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, return false; } /* + * r576 DIAG: witness the decision inputs of the draws the guards + * care about (see svc_gr_dec_dump). + */ + if (gsp_phys_gr_dec_log_max && + (svc_fwd_req.gr_dropped > 0 || svc_fwd_req.gr_arg_partial || + svc_gr_data_stalled || svc_gr_dstpend_overlap() || + svc_copy_pending())) + svc_gr_dec_dump(ch); + /* + * r541: launch-ordering guard. The fill's data pages are what the + * engine will read; if any of them is the target of a device copy + * that decoded but has NOT executed yet (svc_dstpend_note - the + * smiley/streams2 class: the client's memcpy completed on its + * release while the H2D op was still held, and the launch read the + * PREVIOUS page), do not forward: return the busy-defer verdict so + * the pump holds the entry through its bounded retry window and + * re-captures once the copy has run (or the dstpend window expired, + * which restores the historical forward). Bounded by the same + * busy-defer + gr_hold machinery as every other busy refusal, so it + * can never hang the launch. + */ + if (svc_gr_dstpend_overlap() && + !svc_gr_defer_expired(svc_gr_cand.push_va)) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + mutex_unlock(&svc_fwd_lock); + svc_gr_busy_defer = true; + svc.fwd_gr_refused++; + svc.fwd_gr_dstpend++; + if (svc.fwd_gr_dstpend_log < 32) { + svc.fwd_gr_dstpend_log++; + pr_info("vgsp-work: GR forward deferred h=0x%x push=0x%llx (data page has an unexecuted device write; bounded retry)\n", + ch->h_object, svc_gr_cand.push_va); + } + return false; + } + /* + * r568: the coarse companion gate - any unexecuted CE write at all + * (see svc_copy_pending). Covers the dropped-argument-base case the + * page-level guard cannot see (the r571 mail b-copy). + */ + if (svc_copy_pending() && + (svc_fwd_req.gr_arg_partial || svc_fwd_req.gr_dropped > 0) && + !svc_gr_defer_expired(svc_gr_cand.push_va)) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + mutex_unlock(&svc_fwd_lock); + svc_gr_busy_defer = true; + svc.fwd_gr_refused++; + svc.fwd_gr_dstpend++; + if (svc.fwd_gr_dstpend_log < 32) { + svc.fwd_gr_dstpend_log++; + pr_info("vgsp-work: GR forward deferred h=0x%x push=0x%llx (a CE write is unexecuted; bounded retry)\n", + ch->h_object, svc_gr_cand.push_va); + } + return false; + } + /* + * r543: a partially-resolved ARGUMENT set must not forward either - + * the engine would run with a stale/zero operand (the r542 matmul + * wedge: 2 of 13 high arg bases resolved 0 pages, the fill proceeded + * "fill hold disabled (log-only)", the engine fetched the QMD and + * wrote nothing: NOWRITE-AMBIGUOUS fe_esr=0x80000000, client read + * zeros). Defer once per push VA through the bounded busy-defer + * retry so the pins can land; the latch (svc_gr_arg_defer_ok) keeps + * a genuinely junk pointer word from paying the window on every + * launch. After the retry the historical forward stands - the guard + * delays, it cannot hang. + */ + if (svc_fwd_req.gr_arg_partial && !svc_gr_data_stalled && + !svc_gr_defer_expired(svc_gr_cand.push_va) && + svc_gr_arg_defer_ok(svc_gr_cand.push_va)) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + mutex_unlock(&svc_fwd_lock); + svc_gr_busy_defer = true; + svc.fwd_gr_refused++; + svc.fwd_gr_argdefer++; + if (svc.fwd_gr_argdefer_log < 32) { + svc.fwd_gr_argdefer_log++; + pr_info("vgsp-work: GR forward deferred h=0x%x push=0x%llx (partial argument set: %u page(s) unresolved; bounded retry)\n", + ch->h_object, svc_gr_cand.push_va, + svc_fwd_req.gr_dropped); + } + return false; + } + /* * m150: the fill found high argument pointers that no source can * translate (the second-client zeros class). Do NOT forward a launch * whose kernel would dereference unresolved pointers: leave the @@ -12898,7 +21303,50 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, * the channel's gr_hold window at the call site. Once the window * expires the entry is consumed exactly as before. */ - if (svc_gr_data_stalled) { + if (svc_gr_data_stalled && gsp_phys_gr_fill_hold) { + u64 lva = svc_fwd_req.gr_push_va; + /* + * The channel is the pump's mutable live bookkeeping object + * (the const view exists for the read-only gates); these + * record fields are the same ones the pump's hold writes, + * on this same service thread. + */ + struct svc_channel *mch = (struct svc_channel *)ch; + + /* + * GRFIX FIX1b (r213): the ABSOLUTE per-launch bound. The + * first hold of this push VA arms a fixed deadline + * (svc_gr_hold_bound_expired) that nothing re-arms; once it + * passes, the launch PROCEEDS exactly as the pre-8b058884 + * path - the candidate is consumed below and forwarded + * best-effort - so the guest is never blocked by a page that + * never resolves. r213: a slot-keyed deadline was re-armed + * on every re-decode (the same launch is replayed at + * different GPFIFO slots after rewinds) and held one launch + * for 216+ s until the client's 240 s timeout killed it. + * GRFIX FIX1e: gated by gsp_phys_gr_fill_hold (default 0) - + * with the gate off the fill never stalls on drops, so this + * block only ever sees the historical aggregate stall. + */ + if (svc_gr_hold_bound_expired(mch, lva)) { + svc.fwd_gr_hold_exp++; + if (!mch->gr_hold_exp_logged) { + mch->gr_hold_exp_logged = true; + pr_info("vgsp-gr: GRFILL-EXPIRED launch=0x%llx dropped=%u waited_ms=%u\n", + lva, svc_fwd_req.gr_dropped, + jiffies_to_msecs(jiffies - + mch->gr_hold_dead_since)); + } + svc_gr_data_stalled = false; /* proceed: forward */ + if (svc_gr_retry_log < 64) { + svc_gr_retry_log++; + pr_info("vgsp-gr: GRFILL-PROCEED launch=0x%llx (bound expired; forwarding best-effort)\n", + lva); + } + } + } + if (svc_gr_data_stalled && + !svc_gr_defer_expired(svc_gr_cand.push_va)) { clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); mutex_unlock(&svc_fwd_lock); svc.fwd_gr_refused++; @@ -13283,7 +21731,7 @@ static bool svc_push_read_displaced(struct nv_vgpu_device *vgpu, u32 h, static void svc_entry_outcome(const char *what, const struct svc_channel *ch, u64 idx, u64 va, u32 len) { - if (!gsp_sema_dbg || !ch->engine_type || svc.gpfifo_entry_log >= 512) + if (!gsp_sema_dbg || svc.gpfifo_entry_log >= 16384) return; svc.gpfifo_entry_log++; pr_info("sema-entry: %s h=0x%x idx=%llu va=0x%llx len=%u want=%u exp=%u unread=%d\n", @@ -13351,6 +21799,172 @@ static void svc_userd_sweep(struct nv_vgpu_device *vgpu, struct svc_channel *ch, ch->h_object, hi, lo); } +/* + * STALLFREE r225: stale-epoch / stale-replay wedge watchdog. + * + * r225 evidence (witness-r225b.log): after a client was killed mid-run, a + * fresh client's GR channel (0x5c000059) was registered with a RESIDUAL + * USERD put=101 left by the dead session. Its fresh session wrote no put + * (it was stuck at ctx-init, polling MC_SERVICE_INTERRUPTS once a second), + * so the m181 fresh-put-drop recovery - which requires put < put_prev - + * never fired, and the host-side completion-counter publish stayed + * withheld (gr_pub_armed false). Meanwhile the pin-batch rewind + * (svc_handle_map_add, SVC_PIN_REWIND=8) kept yanking last_get from 101 + * back to 93 on every injection batch, so the entry loop re-walked slots + * 93..100 forever: 8 stale entries per batch, all decoding to nothing + * (rel=0, no capture) and consuming with zero delivery. The whole stall + * window shows zero captures, zero GR requests and therefore zero + * GR-STALL lines - the launch never reached the bounded submit path + * because the session epoch itself was wedged. + * + * Two signatures are detected, both keyed to NO FORWARD PROGRESS for + * SVC_GR_STALE_WEDGE_MS (the (last_get, put) pair unchanged across the + * window): + * + * 1. stale-epoch: a GR channel that still carries a residual put + * (put > 1 at this pass, publish withheld) and whose ring the host + * believes is fully consumed (last_get == put) - the fresh session + * never proved its switch with a put drop. The host applies the + * m181 recovery on a TIMEOUT instead: arm the counter publish and + * void the withheld publish state (no last_get rewind - a timeout is + * not proof the ring is the dead session's; the proven m181 rewind + * stays reserved for its put-drop signature). This is the recovery + * the killed-client rerun needs. + * + * 2. stale-replay: last_get pinned behind put while put == get (the + * producer has nothing pending) AND the loop consumed entries during + * the window (submissions moved) - i.e. the rewind is re-opening an + * already-consumed window faster than the loop closes it. The window + * is dropped (last_get = put) and the pin-batch rewind is put on a + * cooldown (wd_rewind_off_until) so the futile retry cannot pin the + * pump; the rewind budget returns later and a translation that lands + * can still retry (bounded duty cycle). + * + * Nothing here can fire on a green channel: a fresh channel has put <= 1 + * (signature 1) or consumes its first entries within milliseconds + * (submissions > 0); signature 2 additionally requires BOTH a stalled + * (last_get, put) pair for 3 s with no pending producer work AND consumed + * entries during that window, which only the rewind re-opening of a + * consumed window produces. + */ +#define SVC_GR_STALE_WEDGE_MS 3000u +static void svc_gr_stale_wedge_check(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u32 put, u32 get) +{ + bool residual, replay; + const char *reason = ""; + unsigned long waited; + + if (ch->engine_type || !ch->gpfifo_entries) + return; + /* New work (put advanced) clears the replay cooldown immediately. */ + if (ch->wd_rewind_off_until && ch->wd_put != put) + ch->wd_rewind_off_until = 0; + if (ch->wd_since && + (ch->wd_put != put || ch->wd_last_get != ch->last_get)) + ch->wd_since = 0; /* progress (or a new position): re-arm */ + residual = (!ch->gr_pub_armed && put > 1 && ch->last_get == put); + replay = (ch->last_get != put && put == get && + ch->wd_since && ch->submissions != ch->wd_sub); + if (!residual && !replay) { + ch->wd_since = 0; + ch->wd_put = put; + ch->wd_last_get = ch->last_get; + return; + } + if (!ch->wd_since) { + ch->wd_since = jiffies; + ch->wd_put = put; + ch->wd_last_get = ch->last_get; + ch->wd_sub = ch->submissions; + return; + } + if (time_before(jiffies, ch->wd_since + + msecs_to_jiffies(SVC_GR_STALE_WEDGE_MS))) + return; + waited = jiffies_to_msecs(jiffies - ch->wd_since); + if (residual) { + /* + * m181 recovery on timeout: the fresh session never dropped + * the residual put, so prove the epoch switch here. Arm the + * publish (the withheld completion counter was the plausible + * ctx-init wait - r225's client polled service-interrupts 1/s + * with no host edge) and void any publish state the stale + * pass advanced, exactly like m181; the guest word itself is + * only zeroed when the host authored it. No last_get rewind: + * a timeout is not proof that the ring is the dead session's + * (re-walking it could re-capture a stale launch), so the + * proven m181 rewind stays reserved for its put-drop + * signature; the client's own first put write will trigger it. + */ + ch->gr_pub_armed = true; + if (ch->counter_last) { + u64 upa = ch->release_pa ? ch->release_pa : + ch->counter_pa; + u32 zero = 0; + + /* + * r311: zero only a word authored by a DEAD + * generation. Zeroing a word this live session + * authored makes the client's UVM observe a + * downward jump (its tracker then never completes) + * while the publish re-advances it - the r311 + * vector_add wedge (pub-adv/zero cycles, client + * polling cur=0 forever). + */ + if (ch->counter_gen == svc_sess_gen) { + if (gsp_sema_dbg && svc.pend_dbg_log < 1024) { + svc.pend_dbg_log++; + pr_info("sema-diag: stale-recover-zero-skip h=0x%x upa=0x%llx last=%u (authored by live gen)\n", + ch->h_object, upa, + ch->counter_last); + } + ch->counter_last = 0; + } else if (upa) { + int zrc; + + /* + * r378: the zero is authored from a PA + * learned earlier (release_va -> release_pa); + * re-validate it before the 4-byte write + * lands so a freed/re-homed slot is never + * zeroed through the stale page. + */ + if (!gsp_sema_write_guard(ch->release_va, + upa, + SVC_APERTURE_SYS, + "stale-recover-zero")) + zrc = -ESTALE; + else + zrc = svc_write_ap(vgpu, upa, + SVC_APERTURE_SYS, + &zero, sizeof(zero)); + if (gsp_sema_dbg && svc.pend_dbg_log < 1024) { + svc.pend_dbg_log++; + pr_info("sema-diag: stale-recover-zero h=0x%x upa=0x%llx last=%u zrc=%d\n", + ch->h_object, upa, + ch->counter_last, zrc); + } + } + ch->counter_last = 0; + } + reason = "stale-epoch"; + } else { + /* Drop the rewind-reopened consumed window; cool the rewind. */ + ch->last_get = put; + ch->wd_rewind_off_until = jiffies + + msecs_to_jiffies(8u * SVC_GR_STALE_WEDGE_MS); + reason = "stale-replay"; + } + ch->wd_since = 0; + ch->wd_put = put; + ch->wd_last_get = ch->last_get; + svc.gr_stale_wedge++; + pr_warn_ratelimited("vgsp-gr: GR-STALL-stale-wedge h=0x%x va=0x%llx last=%u put=%u get=%u sub=%llu waited_ms=%u action=recover reason=%s\n", + ch->h_object, ch->gpfifo_va, (u32)ch->last_get, + put, get, ch->submissions, waited, reason); +} + static void svc_pump_channels(struct nv_vgpu_device *vgpu) { /* Only ever called from the service thread. */ @@ -13366,8 +21980,28 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) bool stalled = false; spin_lock_irqsave(&svc.chan_lock, flags); + /* + * r507o stream-order: walk the CE channels BEFORE the GR channels. + * The client's stream order places its H2D copies before the kernel + * that consumes them, but the channels register in handle order (the + * RM's GR channel early, the copy-ce channels later), and a pass that + * decoded the launch before the CE channel's entries ran let the + * forward workqueue execute the launch's copy-in against pre-copy + * data (r480-sep: copy-in 115 ms before its stream's H2D soft + * copies; r481 proved no host semaphore links the two, so the + * emulation must enforce stream order itself). The CE copies execute + * on this same service thread, so consuming the CE entries first + * fixes the inversion directly - no cross-channel hold, no latency + * tax. Stable partition; relative order within each class preserved. + */ for (i = 0; i < SVC_MAX_CHANNELS; i++) { - if (svc.chan[i].valid) + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].engine_type) + local[n++] = svc.chan[i]; + } + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + !svc.chan[i].engine_type) local[n++] = svc.chan[i]; } spin_unlock_irqrestore(&svc.chan_lock, flags); @@ -13375,6 +22009,14 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) n_gr = SVC_GR_COUNTER_TOTAL; /* + * STALLFREE: complete a failed forwarded GR launch no hold owns (see + * svc_fwd_outcome_check). Runs before this pass's channel loop, so a + * finished request is always observed before any fill can reuse its + * slot. + */ + svc_fwd_outcome_check(vgpu); + + /* * H1 last resort: one self-gated content-discovery pass for the GR * ctx-init rings before this pass's channel loop. The pins it * installs (ring VA + push VA pages, both validated from content and @@ -13403,7 +22045,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * pass's snapshot belongs to a vanished channel; its waiter is gone, * so drop the set instead of resolving it later. */ - svc_opdef_drop_gone(local, n); + svc_opdef_drop_gone(vgpu, local, n); for (i = 0; i < n; i++) { struct svc_channel *ch = &local[i]; @@ -13412,6 +22054,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) int rrc; u32 put = 0, get = 0, guard = 0; u32 get_before; + u32 put_prev_wit; u64 applied_before; bool held = false; @@ -13458,7 +22101,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * svc_counter_autoreg). Skip while a relay hold owns * the channel, exactly like the end-of-pass call. */ - if (ch->counter_pa && ch->userd_pa && + if ((ch->release_pa || ch->counter_pa) && + ch->userd_pa && !(svc_fwd_hold.active && svc_fwd_hold.vgpu == vgpu && svc_fwd_hold.ch_handle == ch->h_object) && @@ -13576,6 +22220,20 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->logged = true; } + /* + * mm2: a GR channel whose USERD already reads PUT <= 1 cannot + * hold a stale consumed prefix, so its publishes (the catch-up + * for releases missing from consumed entries) are safe to + * enable immediately - this is the pristine first-boot ring. + * Channels discovered with a residual put stay unarmed until + * the fresh-put-drop rewind proves the session switch, so the + * residual put can never reach the fresh client's completion + * word (see gr_pub_armed and svc_counter_publish). + */ + if (!ch->engine_type && !ch->gr_pub_armed && put <= 1 && + !ch->userd_wait_fresh) + ch->gr_pub_armed = true; + /* Initialize expected semaphore payload from the pool (if mapped). */ svc_init_expected_payload(vgpu, ch); @@ -13629,12 +22287,103 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->h_object, ch->ce_index, put, (u32)ch->last_get); ch->last_get = put; + } else if (!ch->engine_type && ch->gpfifo_entries && + ch->last_get > put && + ch->submissions < ch->gpfifo_entries / 2) { + /* + * m181 (r203 GR ctx-init stall): a GR channel's + * ring/USERD live in fixed VRAM slots + * (fb 0x140000 + rank*0x3000) that keep the + * PREVIOUS session's final state across the + * session flush (m176 reset learned counter + * state but not the ring-consumption + * bookkeeping). A new session's first pump + * pass therefore sees the residual put (e.g. + * 107 = the last session's final GP_PUT) and + * consumes the stale ring up to it, advancing + * last_get before the client has written its + * own entries. The client's fresh put then + * drops below put_prev, and without this + * rewind last_get stays ahead of the producer: + * the live ctx-init entries are never decoded, + * their REPORT_SEM releases are never applied + * (the completion word freezes at the first + * stale-decoded payload), and the client's + * ctx-init wait polls forever - r203 rank 0 + * hung exactly so in multikernel / streams2 / + * victory_mail / smiley whenever the prior + * session had reached a high put (the sessions + * after a hung one, whose final put is 1, + * proceeded - a perfect alternation). Rewind + * to slot 0 like the r48 resync: a fresh + * session restarts its push sequence at slot + * 0, and re-decoding slot 0 is idempotent + * (the stale pass could only have authored the + * same payload, and the forward-only apply + * drops a duplicate). A genuine wrap cannot be + * this fresh (submissions guard), mirroring + * the r73 CE rewind. + */ + pr_info_ratelimited("sema-chan: fresh put drop (GR) h=0x%x rank=%u put=%u last=%u -> rewind to 0\n", + ch->h_object, ch->gr_index, put, + (u32)ch->last_get); + ch->last_get = 0; + /* + * mm2: the session switch is now proven - arm + * the publish for the live entries that follow, + * and void any publish state the stale pass + * advanced. Everything consumed to reach + * last_get was the dead session's ring; a + * publish issued on it wrote the residual put + * into the very word the fresh client waits on + * (its tracking semaphore; pub target is + * release_pa since 8c634144) and the + * forward-only apply then skips the real + * release payloads that follow. Re-arm the + * word to 0 and reset the catch-up bookkeeping + * so the re-decoded live releases re-advance + * it from the client's own sequence (1, 2, + * ...). Idempotent when no stale publish + * happened (counter_last == 0). + */ + ch->gr_pub_armed = true; + if (ch->counter_last) { + u64 upa = ch->release_pa ? + ch->release_pa : ch->counter_pa; + u32 zero = 0; + + if (upa) + (void)svc_write_ap(vgpu, upa, + SVC_APERTURE_SYS, + &zero, sizeof(zero)); + ch->counter_last = 0; + } } bitmap_zero(ch->slot_done_epoch, SVC_GPFIFO_MAX_BITS); + memset(ch->slot_pend_tries, 0, + sizeof(ch->slot_pend_tries)); + /* GRFIX FIX1b: a recycled slot in the next put epoch + * gets a fresh absolute-bound record; without this a + * wrapped slot would inherit a stale expired launch + * record. */ + ch->gr_hold_dead_va = 0; + ch->gr_hold_exp_logged = false; + /* r361: same for the opdef absolute bound. */ + ch->opdef_hold_dead_va = 0; + ch->opdef_hold_exp_logged = false; } + put_prev_wit = ch->put_prev; ch->put_prev = put; /* + * STALLFREE r225: stale-epoch / stale-replay wedge watchdog. + * Runs before the entry loop so a wedged session epoch is + * recovered (and logged) instead of spinning: see + * svc_gr_stale_wedge_check. + */ + svc_gr_stale_wedge_check(vgpu, ch, put, get); + + /* * m136 (SA-STALL M2): liveness watchdog for a held CE entry. * When this channel's entry loop is not re-entered (early * continue paths above, or no fresh pushes), a deferred entry @@ -13669,6 +22418,20 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) get_before = ch->last_get; applied_before = svc.releases_applied; + /* + * C4 (GAP-1 H1): the GR walk witness - does put advance, and + * does last_get reach it? Budget 512/session; fires on a put + * change or while last_get is in the stalled 80..100 band. + */ + if (gsp_sema_dbg && svc.walk_dbg < 512 && + (put != put_prev_wit || + ((u32)ch->last_get >= 80 && (u32)ch->last_get <= 100))) { + svc.walk_dbg++; + pr_info("sema-diag: walk h=0x%x put=%u get=%u last=%u prev=%u\n", + ch->h_object, put, get, (u32)ch->last_get, + put_prev_wit); + } + while (ch->last_get != put && guard++ < ch->gpfifo_entries) { u8 entry[SVC_GPFIFO_ENTRY_SIZE]; u64 va, idx = ch->last_get; @@ -13701,7 +22464,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * (forward-only, so it cannot double-apply) and drop * it before this entry is examined. */ - svc_opdef_drop_stale(vgpu, ch, (u32)idx); + svc_opdef_drop_stale(vgpu, ch, (u32)idx, put); /* * A pin-batch rewind replays recently consumed @@ -13771,10 +22534,17 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ { u64 ident = (u64)e0 | ((u64)e1 << 32); - bool same = ident && + /* + * r505b: the per-slot op bookkeeping arrays are + * SVC_GPFIFO_MAX_BITS (1024) entries; the X/GL + * channel's ring is 8192 slots. Bound every slot + * index here or a large ring writes out of + * bounds. + */ + bool same = ident && idx < SVC_GPFIFO_MAX_BITS && ch->slot_ops_ident[idx] == ident; - entry_ident = ident; + entry_ident = (idx < SVC_GPFIFO_MAX_BITS) ? ident : 0; /* * A complete prior pass suppresses every op. * A partial pass (an operand's targeted scan @@ -13782,6 +22552,20 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * leaves the per-op done bitmap to decide: * work that already ran must not run again, * the pending op must get its retry. + * + * r354: the content witness decides (the hash + * override sits below, after the push bytes + * are read): entry words alone cannot tell a + * true rewind replay (identical bytes - must + * not run twice) from a REWRITTEN push that + * reuses the same push-buffer VA and length + * with new methods (the driver's repeated + * page-table/fill pushes at the same ring + * slots - suppressing those left the guest's + * GPU page tables stale, the GMMU walk + * missing operand pages, and the first CUDA + * client of every fresh boot failing with + * dropped operands). */ entry_replay = same && ch->slot_ops_complete[idx]; @@ -13873,7 +22657,20 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) injected = (svc_ipin_lookup(va, &ipa, &iap) == 0); - if (!svc_method_stream_plausible(pb, len)) { + /* + * r496e: an all-zero push on a + * content-scanned-ring channel is the + * driver's NOP sanity push (one zero + * dword - NVA06F_DMA_NOP), not a bad + * translation: accept it and let the + * decode find no ops. + */ + bool zero_nop = ch->gr_ram_ring && + len >= 4 && + !memchr_inv(pb, 0, len); + + if (!svc_method_stream_plausible(pb, len) && + !zero_nop) { svc.false_trans++; if (injected) { u8 *fb = kmalloc(len, @@ -14024,22 +22821,106 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) { u32 rel_n_before = svc_rel_count; bool rel_seen; + /* + * m156: this entry is inside a live + * GR hold window and its decode is a + * replay that would suppress the + * launch capture. Force exactly one + * fresh capture pass (ops stay + * suppressed: they already ran; only + * the candidate build is re-done) so + * the deferred launch is forwarded + * for real once the engine's + * in-flight work completes. + */ + bool rehold = + !ch->engine_type && + ch->gr_rehold_idx == + (u32)idx; + /* + * m160: the capture gate used to + * require a host-injected pin. A + * fresh ring entry whose push read + * succeeded and whose bytes parse + * as a plausible method stream is + * the client's actual work - the + * same evidence the untrusted ops + * gate already accepts - and a GR + * capture built from it is exactly + * what the m156 rehold forces for + * deferred launches. Without this, + * a channel whose pages the pin + * stream had not reached yet (the + * streams2 second-channel case: + * entry consumed, no capture, the + * client read zeros) never gets + * its launch forwarded at all. + * Replays stay suppressed: a + * consumed entry must not run its + * work twice. + */ + bool cap_trust = injected || rehold; + + /* + * r354: the content witness for + * the once-only op gate. Entry + * words alone cannot tell a true + * rewind replay (identical bytes) + * from a rewritten push that + * reuses the same push VA and + * length with new methods (the + * driver's repeated fill / + * page-table pushes at the same + * ring slots). Suppressing the + * rewritten push left the GPU + * page tables stale and the first + * client's operands unresolved. + * slot_ops_hash has been stored + * since r62 for this comparison. + */ + if (entry_replay && + ch->slot_ops_hash[idx] != + svc_push_hash(pb, len)) { + entry_replay = false; + if (svc_gr_retry_log < 64) { + svc_gr_retry_log++; + pr_info("vgsp-sema: rewritten push runs (idx=%llu va=0x%llx len=%u)\n", + (unsigned long long)idx, va, len); + } + } + if (rehold && !injected && + svc_gr_retry_log < 64) { + svc_gr_retry_log++; + pr_info("vgsp-gr: GR rehold capture idx=%llu va=0x%llx\n", + (unsigned long long)idx, va); + } + if (!cap_trust && !entry_replay && + !ch->engine_type && + gsp_phys_gr_forward_enabled() && + !unreadable && + svc_method_stream_plausible(pb, + len)) + cap_trust = true; svc_entry_outcome("decode", ch, idx, va, len); svc.op_tr_pending = 0; + svc.op_owed = 0; svc_decode_stream(vgpu, pb, len, 0, (allow_ops || injected) && - !entry_replay, + !entry_replay && + !rehold, ch, idx, va, - injected, - entry_replay, - entry_replay ? + cap_trust, + entry_replay && !rehold, + (entry_replay && !rehold) ? NULL : &entry_done); + if (rehold) + ch->gr_rehold_idx = ~0u; /* * Record what ran for this entry * identity before any hold path can @@ -14059,8 +22940,20 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) entry_ident; ch->slot_ops_done[idx] = entry_done; + /* + * r540: an owed + * denied op keeps + * the slot + * incomplete too - + * otherwise every + * later pass is a + * replay and the + * op can never run + * (see op_owed). + */ ch->slot_ops_complete[idx] = - svc.op_tr_pending ? + (svc.op_tr_pending || + svc.op_owed) ? 0 : 1; ch->slot_ops_hash[idx] = svc_push_hash(pb, @@ -14085,10 +22978,37 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * data), so the hold and the * deferral move together. */ + /* + * r558: the opdef hold is the one + * pump hold path without the + * absolute per-launch bound. r557 + * STALL-DUMP evidence: the pump + * consumed NOTHING for 20+ s while + * held/opdef counters climbed + * (opdef=3880/16/64, held=155) - + * a held entry whose operand never + * resolves was re-held every pass + * and the whole pump froze on it. + * Consult the same absolute + * per-launch-VA bound the GR hold + * paths use (svc_gr_hold_bound_ + * expired, one-shot r509): once it + * fires the entry flushes and is + * consumed instead of held again. + */ if (svc.op_tr_pending && + !svc_gr_hold_bound_expired(ch, + svc_gr_launch_va(ch)) && + !svc_hold_bypass() && svc_opdef_hold(vgpu, ch, - (u32)idx)) { + (u32)idx, + va)) { svc.gpfifo_nomap++; + if (gsp_sema_dbg && svc.pend_dbg_log < 1024) { + svc.pend_dbg_log++; + pr_info("sema-pend: hold h=0x%x idx=%u va=0x%llx\n", + ch->h_object, (u32)idx, va); + } /* * P6b wave 1.5: * releases recorded @@ -14112,7 +23032,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) kfree(pb); break; } - if (svc.op_tr_pending) + if (svc.op_tr_pending) { /* * Deadline hit: the * operand never @@ -14124,6 +23044,34 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ svc_opdef_flush(vgpu, ch, (u32)idx); + /* + * r505b: mark the slot's + * ops complete. The op was + * permanently skipped, but + * the decode above stored + * complete=0 (op_tr_pending + * was set), so every pin- + * batch rewind replay of + * this slot re-decoded it, + * re-set op_tr_pending and + * re-held the entry in a + * livelock (multikernel on + * r505: opdef held=18359, + * expired=61, the client + * waiting forever while the + * pump looped on slot 1). + * Nothing is owed after the + * flush, so a replay must + * skip the ops. + */ + if ((u32)idx < + SVC_GPFIFO_MAX_BITS) { + ch->slot_ops_complete[idx] = + 1; + ch->slot_ops_done[idx] = + entry_done; + } + } rel_seen = (svc_rel_count != rel_n_before); { bool fwd_guest_queued; @@ -14152,42 +23100,260 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) !ch->engine_type) ch->gr_hold_idx = ~0u; /* + * m159: a busy refusal (forward + * in flight / release hold + * active) means this launch CAN + * still run - only later. The + * candidate survives the refusal + * (the busy paths do not consume + * it), so wait bounded for the + * slot to free - resolving the + * active hold through its own + * channel, since this pump thread + * is the hold resolver - then + * retry the request inline. This + * serializes multi-launch clients + * (streams2: kernels 2/3 queued + * while kernel 1 is in flight) + * without losing their releases. + * On timeout the historical + * consume + soft-release path + * stands (bounded, never a hang). + */ + if (!fwd_guest_queued && + !ch->engine_type && + svc_gr_busy_defer) { + unsigned long wend = jiffies + + msecs_to_jiffies(4000); + + while (time_before(jiffies, + wend)) { + if (svc_fwd_hold.active && + svc_fwd_hold.vgpu == + vgpu) { + int hk; + + for (hk = 0; hk < n; hk++) { + if (local[hk].h_object != + svc_fwd_hold.ch_handle) + continue; + svc_fwd_hold_poll(vgpu, + &local[hk], + 0); + break; + } + } + if (!svc_fwd_hold.active && + !test_bit(SVC_FWD_INFLIGHT, + &svc_fwd_state) && + smp_load_acquire( + &svc_fwd_req.done)) + break; + msleep(5); + } + if (!svc_fwd_hold.active && + !test_bit(SVC_FWD_INFLIGHT, + &svc_fwd_state)) + fwd_guest_queued = + svc_fwd_request(vgpu, + ch); + if (fwd_guest_queued) { + svc.fwd_gr_defer++; + } else if (!svc_hold_bypass() && + !svc_gr_hold_bound_expired(ch, + svc_gr_launch_va(ch)) && + (ch->gr_hold_idx != (u32)idx || + time_before(jiffies, + ch->gr_hold_until) || + /* + * r420: the backend's + * bounded WEDGE-QUIET + * hold outlives this + * one-shot window; keep + * re-holding (never + * consume-with-sw-release) + * while it is active, so + * the launch is still + * forwarded and copied out + * after the hold instead of + * being released onto stale + * data. + */ + gsp_phys_gr_quiet_hold_active())) { + u32 hold_ms = gsp_sema_gr_grace_ms ? + gsp_sema_gr_grace_ms : 8000; + + /* + * P2 (GAP-2 A3): the wait + * expired without queueing. + * Do NOT fall through to the + * software-release consume - + * the launch was never + * submitted, so authoring + * its release would be a + * false completion. Hold + * exactly like the + * data-stalled path + * (m150/m156); the one-shot + * window lets the historical + * consume resume after it + * expires. + */ + ch->gr_rehold_idx = (u32)idx; + ch->gr_hold_idx = (u32)idx; + ch->gr_hold_until = jiffies + + msecs_to_jiffies(hold_ms); + svc.fwd_gr_defer++; + if (svc_busy_defer_log < 64) { + svc_busy_defer_log++; + pr_info("vgsp-work: GR busy-defer timeout h=0x%x idx=%llu -> hold %u ms\n", + ch->h_object, + (unsigned long long)idx, + hold_ms); + } + kfree(pb); + break; + } else if (svc_busy_defer_log < 64) { + /* + * C8 (GAP-2 A3): the bounded + * hold window is over; the + * historical consume with the + * software release resumes. + */ + svc_busy_defer_log++; + pr_info("vgsp-work: GR busy-defer timeout h=0x%x idx=%llu -> consume w/ sw release\n", + ch->h_object, + (unsigned long long)idx); + } + } + /* * m150: the GR fill held this - * launch because its high - * argument pointers do not - * translate yet. Leave the - * entry unconsumed (do not - * write GP_GET) so the pump - * re-captures and retries it; - * one bounded window per entry, - * then fall through and consume - * exactly as before. The slot's - * identity makes the hold - * one-shot per entry: after - * expiry the same entry is - * never re-held. + * launch because it carries + * data pages that do not + * resolve yet. Leave the entry + * unconsumed (do not write + * GP_GET) so the pump + * re-captures and retries it. + * GRFIX FIX1b (r213): the hold + * is bounded ABSOLUTELY per + * launch push VA + * (svc_gr_hold_bound_expired + * fixes the deadline on the + * first hold and nothing + * re-arms it; svc_gr_request + * enforces it and forwards the + * launch once it passes). The + * extra check here is the + * defence for a stale stalled + * verdict when the request + * path did not run this pass - + * past the bound the entry + * falls through to the + * historical consume. */ if (!fwd_guest_queued && !ch->engine_type && svc_gr_data_stalled && - ch->gr_hold_idx != (u32)idx) { + !svc_hold_bypass() && + !svc_gr_hold_bound_expired(ch, + svc_gr_launch_va(ch)) && + (ch->gr_hold_idx != (u32)idx || + time_before(jiffies, + ch->gr_hold_until))) { u32 hold_ms = gsp_sema_gr_grace_ms ? gsp_sema_gr_grace_ms : 8000; + /* + * m156: arm the forced + * re-capture so the next + * decode of this entry + * rebuilds the GR candidate + * (a replay decode would + * suppress it) and the + * deferred launch is + * forwarded for real once + * the in-flight work has + * completed. + */ + ch->gr_rehold_idx = + (u32)idx; ch->gr_hold_idx = (u32)idx; ch->gr_hold_until = jiffies + msecs_to_jiffies(hold_ms); svc.fwd_gr_defer++; if (svc_gr_hold_log < 16) { svc_gr_hold_log++; - pr_info("sema-svc: GR hold h=0x%x entry=%u (data args unresolved; retry window %u ms)\n", + pr_info("sema-svc: GR hold h=0x%x entry=%u (%s; retry window %u ms)\n", ch->h_object, - (u32)idx, hold_ms); + (u32)idx, + svc_gr_data_stalled ? + "data args unresolved" : + "forward busy (m156)", + hold_ms); } kfree(pb); break; } /* + * STALLFREE: the absolute per-launch + * hold bound just fired - the entry + * falls through to the historical + * consume, where the tail authors the + * QMD completion release and logs + * GR-STALL-hold. This line pins the + * bound expiry itself (the r220 hold + * storm's fix: one line per launch, + * nothing re-arms it across rewind + * replays). + */ + if (!fwd_guest_queued && + !ch->engine_type && + svc_gr_data_stalled && + ch->gr_hold_bound_hit && + !ch->gr_hold_exp_logged) { + ch->gr_hold_exp_logged = true; + /* r509: the latch is this pass's only + * report of the expiry; clear it so the + * NEXT expiry cycle (gr_hold_exp_logged + * now set) falls through to the + * historical consume. */ + ch->gr_hold_bound_hit = false; + svc.fwd_gr_hold_exp++; + /* + * r351: the bound firing used to + * fall through to the historical + * consume (entry consumed, QMD + * release authored) WITHOUT ever + * forwarding the launch - the + * client's kernel never ran and + * the client hung (victory_reduce + * with gsp_phys_gr_fill_hold=1: + * bound fired every 3 s, zero GR + * LAUNCH lines, 110 s+ hang). + * Instead, force ONE more fresh + * capture (rehold) with the stall + * verdict cleared: the request's + * own bound is expired too, so it + * proceeds and forwards the launch + * best-effort with whatever + * resolved. If that retry cannot + * run, the next pass (flag set) + * falls through to the historical + * consume as before. + */ + pr_warn("vgsp-gr: GR-STALL-hold-bound va=0x%llx waited_ms=%u action=retry-forward (bound expired; one best-effort forward pass)\n", + svc_gr_launch_va(ch), + jiffies_to_msecs(jiffies - + ch->gr_hold_dead_since)); + svc_gr_data_stalled = false; + ch->gr_rehold_idx = (u32)idx; + ch->gr_hold_idx = (u32)idx; + ch->gr_hold_until = jiffies + + msecs_to_jiffies(200); + kfree(pb); + break; + } + /* * P6b wave 1.5: when * this pass recorded * releases and the @@ -14263,14 +23429,183 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * bridge job owns the release. */ if (!ch->engine_type && - !gsp_cuda_bridge_enabled()) + !gsp_cuda_bridge_enabled()) { svc_gr_sw_release(vgpu, ch->h_object, svc_gr_cand.rel_va, svc_gr_cand.rel_payload); + /* STALLFREE: latch so the + * consume tail cannot author + * the same release twice. */ + svc_gr_cand.rel_done = true; + } } } /* + * m164 CLASS-2 no-capture hold: a + * readable GR launch push whose + * decode saw QMD words but built + * no launch candidate for this + * channel (replay-suppressed + * capture, partial/split QMD, or + * a page the pin stream has not + * reached) must not be consumed + * at the tail - consuming drops + * the client's launch and only + * the RAM sweep's release reaches + * it (r161/r167 streams2 second + * vadd: 56 consumes / 2 captures + * on channel 0x5c000023). Hold + * the entry and force a fresh + * capture on the next pass + * (gr_rehold_idx makes the replay + * decode rebuild the candidate); + * one bounded window per + * (slot, push VA), then fall + * through to the historical + * consume so the channel can + * never livelock. + * + * r470: an ALL-ZERO push page is + * the same "not the client's work + * yet" condition - the ring entry + * is written (the previous + * session's or the client's own + * slot reuse) but the push + * CONTENT has not landed, so the + * decode sees no QMD and the + * entry would be consumed as a + * no-op; the end-of-pass counter + * publish then advances the + * guest's completion word while + * the real push was never read, + * and the client drains a + * staging buffer that still holds + * the PREVIOUS copy's bytes + * (streams_events: h_c[2] = c0 + * because the c2 D2H's + * completion was published 4.5 s + * before its copy ran, the ring + * slots having been decoded as + * all-zero pushes). Deliberately + * NARROWER than the CE-shaped + * plausibility verdict: compute + * pushes legitimately fail that + * matcher, so only an entirely + * zero page counts as unwritten. + */ + { + bool all_zero = false; + + if (!ch->engine_type && + va && len) { + u32 zi; + + all_zero = true; + for (zi = 0; zi < len; + zi++) + if (pb[zi]) { + all_zero = + false; + break; + } + } + /* + * r471: publish clamp bookkeeping. + * A zero-content GR entry that is + * held or consumed must never let + * the end-of-pass counter publish + * advance the guest's completion + * word past it (the client would + * satisfy its readback wait before + * the real push/copy ran - the + * staging buffer still holds the + * previous copy's bytes). Record + * the lowest zero slot; a decode + * that reads non-zero content + * clears the clamp. + */ + if (!ch->engine_type) { + if (all_zero) { + if (!ch->gr_bad_put || + (u32)idx + 1 < + ch->gr_bad_put) + ch->gr_bad_put = + (u32)idx + 1; + } else if (va && len) { + ch->gr_bad_put = 0; + } + } + if (!ch->engine_type && va && len && + gsp_phys_gr_forward_enabled() && + (svc_gr_qmd_seen > 0 || + all_zero) && + (!entry_replay || + svc_gr_cand_push_owed(va)) && + !(svc_gr_cand.valid && + svc_gr_cand.h_object == + ch->h_object && + svc_gr_cand.push_va == va)) { + if (ch->gr_nocap_idx != + (u32)idx || + ch->gr_nocap_va != va) { + ch->gr_nocap_idx = + (u32)idx; + ch->gr_nocap_va = va; + ch->gr_nocap_replay = + entry_replay ? + 1 : 0; + ch->gr_nocap_since = + jiffies; + } + if (gsp_sema_gr_nocap_hold_ms && + !ch->gr_ram_ring && + time_before(jiffies, + ch->gr_nocap_since + + msecs_to_jiffies( + gsp_sema_gr_nocap_hold_ms)) && + !svc_hold_bypass()) { + /* Force a fresh capture on + * the retry: a replay decode + * would suppress it. */ + ch->gr_rehold_idx = + (u32)idx; + svc.gpfifo_held++; + svc.gr_nocap_held++; + if (svc_gr_nocap_log < 16) { + svc_gr_nocap_log++; + pr_info("sema-chan: GR no-capture hold h=0x%x entry=%u va=0x%llx len=%u replay=%d owed=%d qmd_seen=%u (forcing capture; window %u ms)\n", + ch->h_object, + (u32)idx, va, + len, + ch->gr_nocap_replay, + svc_gr_cand_push_owed(va) ? + 1 : 0, + svc_gr_qmd_seen, + gsp_sema_gr_nocap_hold_ms); + } + kfree(pb); + break; + } + /* Window expired: consume + * exactly as the pre-change + * tail does, with a proof + * line. r548: a forward + * still owed at expiry is + * software-completed first so + * the entry is never consumed + * without a completion. */ + int owed_done = + svc_gr_owed_resolve(vgpu, ch, va); + + svc.gr_nocap_consumed++; + pr_info_ratelimited("sema-chan: GR no-capture hold expired h=0x%x entry=%u va=0x%llx len=%u owed_resolved=%d (consuming without a launch)\n", + ch->h_object, + (u32)idx, va, len, + owed_done); + } + } + /* * A CE push that should carry this * channel's next tracking release but * decoded none read from a @@ -14326,8 +23661,21 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch, idx, va, true, true, NULL); + /* + * r577: for a CE channel only a + * pool-shaped release target is + * this channel's tracking slot. A + * rescued stale page can decode a + * release to a foreign VA (r575: + * 0x12006c004 pay=60); counting + * that as the tracking release + * advanced the expectation without + * delivering to the pool. + */ rel_seen = - (svc_rel_count != rel_n2); + (svc_rel_count != rel_n2) && + (!ch->engine_type || + svc_pool_sem_va(svc_rel_va_seen)); /* * The rescue re-synced the * expectation down to the RAM @@ -14354,16 +23702,66 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) true, NULL); rel_seen = - (svc_rel_count != rel_n2); + (svc_rel_count != rel_n2) && + (!ch->engine_type || + svc_pool_sem_va(svc_rel_va_seen)); } + /* + * r581: the rescue decoded a + * release but the r577 gate + * refused it as a tracking slot + * (rel_seen false while the + * counter moved) - the content + * is provably stale/foreign, no + * work may run from it, and + * waiting the 8 s translation + * grace per entry crawls the + * client's init for minutes + * (r577/r580 boots: 'skipping + * stale entry ... after 8000 ms + * grace' every ~8 s). Backdate + * the deferral so the consume + * path delivers the pool step + * through the exact-slot + * fallback immediately. + */ + if (!rel_seen && + svc_rel_count != rel_n2 && + ch->engine_type && + ch->expected_sem_payload) + ch->defer_since = jiffies - + msecs_to_jiffies(gsp_sema_ce_grace_ms + 100); } } if (rel_seen) { - ch->last_rel_va = - svc_rel_va_seen; + /* + * r577: the exact slot every + * recovery release targets; only + * a pool-shaped VA may claim it + * for a CE channel. The r575 + * wedge adopted a foreign rescued + * release (0x12006c004) here and + * every later recovery went away + * from the client's pool + * (0x121000000 stayed at 25 while + * the client waited for 34; the + * exact-slot poke recovered it). + */ + if (!ch->engine_type || + svc_pool_sem_va(svc_rel_va_seen)) + ch->last_rel_va = + svc_rel_va_seen; + else if (svc_ce_foreign_log < 16) { + svc_ce_foreign_log++; + pr_info("sema-chan: foreign release VA 0x%llx refused as tracking slot h=0x%x eng=%u (recovery stays on the pool slot)\n", + svc_rel_va_seen, + ch->h_object, + ch->engine_type); + } ch->unread_va = 0; ch->unread_until = 0; } + ch->submissions++; svc.gpfifo_entries_done++; if (ch->engine_type && ch->expected_sem_payload) { @@ -14597,6 +23995,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) /* Rescue succeeded, retry the read. */ if (!svc_read_va_push(vgpu, ch, va, pb, len)) { + bool track_rel = false; + /* * Never run work decoded from a * rescued page: the rescue matches @@ -14621,14 +24021,43 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) false, false, NULL); - if (svc_rel_count != rel_n_before) + /* + * r577: only a pool-shaped + * release counts as this CE + * channel's tracking release + * (see the norel-path comment). + * A foreign target must not + * claim the exact slot, and the + * pool step must still be + * delivered. + */ + track_rel = + (svc_rel_count != rel_n_before) && + (!ch->engine_type || + svc_pool_sem_va(svc_rel_va_seen)); + if (track_rel) ch->last_rel_va = svc_rel_va_seen; + else if (svc_rel_count != rel_n_before && + svc_ce_foreign_log < 16) { + svc_ce_foreign_log++; + pr_info("sema-chan: foreign release VA 0x%llx refused (rescue) h=0x%x eng=%u (pool step via fallback)\n", + svc_rel_va_seen, + ch->h_object, + ch->engine_type); + } } ch->submissions++; svc.gpfifo_entries_done++; - if (ch->engine_type && ch->expected_sem_payload) - ch->expected_sem_payload++; + if (ch->engine_type && + ch->expected_sem_payload) { + if (track_rel) + ch->expected_sem_payload++; + else if (!gsp_sema_fake_rel) + svc_fake_release(vgpu, ch); + else + ch->expected_sem_payload++; + } } else { svc.gpfifo_nomap++; kfree(pb); @@ -14806,10 +24235,20 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->defer_since = 0; ch->unread_va = 0; ch->unread_until = 0; - if (gsp_sema_fake_rel || - gsp_sema_rel_recover) - svc_replay_note(ch, idx, - put); + /* + * m177: deliver any + * withheld set even + * on the GR stale-skip + * path. GR channels + * have no rescue and + * their consume-time + * delivery gate is + * below; without this + * the slot strands + * until the sweep. + */ + svc_replay_note(ch, idx, + put); ch->last_get = (idx + 1) % ch->gpfifo_entries; svc.gpfifo_skipped++; @@ -14845,13 +24284,61 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) } } - if (ch->engine_type && svc_rel_count != rel_n_iter) + /* + * m177: GR channels (engine_type == 0) have no rescue and + * no CE-shaped delivery branches above; deliver any + * withheld set for this entry whenever the entry is + * consumed so a held release can never strand until the + * 10 s sweep (evtest3's st[2] sync hang). + */ + if (svc_rel_count != rel_n_iter || !ch->engine_type) svc_replay_note(ch, idx, put); + /* + * STALLFREE: bounded completion for a GR capture that was + * consumed without ever being forwarded - a data-stall, + * busy-defer or no-capture hold window expired and the + * entry fell through to the historical consume. No + * hardware copy is coming and no release hold owns the + * launch, so without this the QMD completion word the + * client polls is never authored by the host and the + * launch wait runs to the guest's own timeout. Author + * it now, forward-only, latched once per capture. + * svc_gr_cand.push_va is the entry's push VA, so a + * candidate captured for a later entry of the same + * channel can never be completed by this entry's + * consume. + */ + if (!ch->engine_type && svc_gr_cand.valid && + svc_gr_cand.h_object == ch->h_object && + !svc_gr_cand.offered && !svc_gr_cand.rel_done && + svc_gr_cand.rel_va && + svc_gr_cand.push_va == va) { + svc_gr_cand.rel_done = true; + svc_gr_sw_release(vgpu, ch->h_object, + svc_gr_cand.rel_va, + svc_gr_cand.rel_payload); + svc.fwd_gr_stall_swrel++; + pr_warn("vgsp-gr: GR-STALL-hold va=0x%llx waited_ms=%u action=proceed (unforwarded GR capture consumed; QMD release authored in software)\n", + va, + jiffies_to_msecs(jiffies - + svc_gr_cand.captured_jiffies)); + } ch->last_get = (idx + 1) % ch->gpfifo_entries; ch->rescue_fail = 0; ch->defer_since = 0; + /* m164: the entry was consumed (or fell through the + * no-capture window); clear the hold identity so a + * future push on this slot opens a fresh window. */ + ch->gr_nocap_idx = ~0u; + ch->gr_nocap_since = 0; + ch->gr_nocap_va = 0; ch->unread_va = 0; ch->unread_until = 0; + /* GRFIX FIX1c (r214): do NOT reset the per-launch + * bound record here - a consume of an unrelated entry + * would re-arm the held launch's bound every pass + * (see svc_fwd_hold_poll). Reset happens on channel + * (re)registration and the session diag reset. */ /* Publish progress the way the GPU would. */ g = ch->last_get; @@ -14916,11 +24403,19 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) spin_lock_irqsave(&svc.chan_lock, flags); for (k = 0; k < SVC_MAX_CHANNELS; k++) { - if (svc.chan[k].valid && + if (svc.chan[k].valid && svc.chan[k].vgpu == vgpu && svc.chan[k].h_object == ch->h_object) { svc.chan[k].userd_pa = ch->userd_pa; svc.chan[k].userd_as = ch->userd_as; svc.chan[k].userd_ok = ch->userd_ok; + /* r496b: persist the RAM-scan cursor and the + * one-shot USERD-scan latch so they survive + * the per-pass snapshot copy. */ + svc.chan[k].gr_ram_cursor = ch->gr_ram_cursor; + svc.chan[k].gr_ram_passes = ch->gr_ram_passes; + svc.chan[k].gr_ram_logged = ch->gr_ram_logged; + svc.chan[k].gr_ram_ring = ch->gr_ram_ring; + svc.chan[k].userd_scans = ch->userd_scans; svc.chan[k].logged = ch->logged; svc.chan[k].logged_expected = ch->logged_expected; svc.chan[k].notified = ch->notified; @@ -14947,6 +24442,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.chan[k].push_hold_until = ch->push_hold_until; svc.chan[k].defer_since = ch->defer_since; svc.chan[k].put_prev = ch->put_prev; + /* r471: the publish clamp watermark is set + * and cleared on the pump thread; persist it + * so the next pass still clamps. */ + svc.chan[k].gr_bad_put = ch->gr_bad_put; /* * Persist the cheap-wait window and the content * watermark: the pump works on a snapshot, so @@ -14967,6 +24466,80 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ svc.chan[k].gr_hold_until = ch->gr_hold_until; svc.chan[k].gr_hold_idx = ch->gr_hold_idx; + /* + * GRFIX NOTE (r213-r215): ANY new per-channel + * record field MUST be added to this snapshot + * write-back list or it is zeroed on every pass + * (the pump works on the local[] snapshot copied + * at the top of svc_pump_channels). + * + * STALLFREE (r213-r215 follow-up): the GRFIX + * absolute-bound fields were the proven record + * destroyer - they were intentionally left out, + * which is why gsp_phys_gr_fill_hold defaulted + * to 0 (the bound could never fire). They are now + * written back, so the per-launch bound survives + * the snapshot and is effective on the default + * path: a data-stalled launch is held for at most + * gsp_phys_gr_hold_max_ms (default 3000) and then + * is consumed with a software completion, instead + * of being re-held forever by a rewind replay at + * a different slot index (the r220 196 s hold). + */ + svc.chan[k].gr_hold_dead_va = ch->gr_hold_dead_va; + svc.chan[k].gr_hold_dead_since = + ch->gr_hold_dead_since; + svc.chan[k].gr_hold_deadline = + ch->gr_hold_deadline; + svc.chan[k].gr_hold_exp_logged = + ch->gr_hold_exp_logged; + /* r509: the one-shot expiry latch is snapshot + * state too. */ + svc.chan[k].gr_hold_bound_hit = + ch->gr_hold_bound_hit; + /* r361: the opdef absolute-bound record is + * snapshot state too (the GRFIX lesson) - + * without the write-back the deadline would + * be zeroed every pass and the bound could + * never fire. */ + svc.chan[k].opdef_hold_dead_va = + ch->opdef_hold_dead_va; + svc.chan[k].opdef_hold_deadline = + ch->opdef_hold_deadline; + svc.chan[k].opdef_hold_exp_logged = + ch->opdef_hold_exp_logged; + /* m164: the CLASS-2 no-capture window is snapshot + * state too - without the write-back every pass + * would re-arm it and rehold the entry forever + * (the m150 lesson). */ + svc.chan[k].gr_nocap_idx = ch->gr_nocap_idx; + svc.chan[k].gr_nocap_replay = + ch->gr_nocap_replay; + svc.chan[k].gr_nocap_va = ch->gr_nocap_va; + svc.chan[k].gr_nocap_since = ch->gr_nocap_since; + /* + * STALLFREE r225: the stale-wedge watchdog + * state is snapshot state too - without the + * write-back the window would restart every + * pass and never expire. + */ + svc.chan[k].wd_last_get = ch->wd_last_get; + svc.chan[k].wd_put = ch->wd_put; + svc.chan[k].wd_sub = ch->wd_sub; + svc.chan[k].wd_since = ch->wd_since; + /* + * r312: gr_pub_armed / counter_gen live only on + * the snapshot copy if not persisted. Losing the + * armed bit re-arms the residual watchdog every + * pass; its recovery then keeps zeroing the + * completion word and re-publishing - the r311/ + * r312 vector_add wedge loop (66 stale-wedge + * rows with the client polling forever). + */ + svc.chan[k].gr_pub_armed = ch->gr_pub_armed; + svc.chan[k].counter_gen = ch->counter_gen; + svc.chan[k].wd_rewind_off_until = + ch->wd_rewind_off_until; svc.chan[k].pushpg_va = ch->pushpg_va; svc.chan[k].pushpg_n = ch->pushpg_n; svc.chan[k].content_max = ch->content_max; @@ -14992,6 +24565,9 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) memcpy(svc.chan[k].slot_ops_ident, ch->slot_ops_ident, sizeof(ch->slot_ops_ident)); + memcpy(svc.chan[k].slot_pend_tries, + ch->slot_pend_tries, + sizeof(ch->slot_pend_tries)); memcpy(svc.chan[k].slot_ops_done, ch->slot_ops_done, sizeof(ch->slot_ops_done)); @@ -15230,7 +24806,7 @@ static void svc_handle_chan_dump(struct nv_vgpu_device *vgpu) spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) - if (svc.chan[i].valid) + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu) local[n++] = svc.chan[i]; spin_unlock_irqrestore(&svc.chan_lock, flags); @@ -15308,17 +24884,67 @@ static u32 svc_map_add_head; /* producer */ static u32 svc_map_add_tail; /* consumer (service thread) */ static DEFINE_SPINLOCK(svc_map_add_lock); +/* + * m154: does this injected-pin VA fall inside any live channel's GPFIFO ring + * range? The ring VA translation must come from the channel's own discovery + * path (channel object / push-buffer scan); a scanner run that happens to + * span the ring VA is enumerating a different mapping and its translation is + * always wrong for the ring (r153: an all-zero read consumed a ring entry + * and wedged the client for the boot). Called on the service thread; takes + * chan_lock like the rewind below. + */ +static bool svc_va_hits_chan_ring(u64 va) +{ + unsigned long flags; + u32 i; + bool hit = false; + + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + struct svc_channel *ch = &svc.chan[i]; + u64 lo, hi; + + if (!ch->valid || !ch->gpfifo_va || !ch->gpfifo_entries) + continue; + lo = ch->gpfifo_va & ~(u64)(SEMA_PAGE_SIZE - 1); + hi = (ch->gpfifo_va + + (u64)ch->gpfifo_entries * 8 + SEMA_PAGE_SIZE - 1) & + ~(u64)(SEMA_PAGE_SIZE - 1); + if (va >= lo && va < hi) { + hit = true; + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + return hit; +} + +/* + * r561: the drain is BOUNDED per call. r559 soak evidence (victory_saxpy, + * 167 s at 100% CPU, pump alive but consuming nothing): the guest pin + * stream flooded the injection queue (svc_handle_map_add: 1867-6101 + * callbacks suppressed per burst) and the old unbounded for(;;) drain + * spent every service pass processing injections - the ring walk never + * ran, no GPFIFO entry was consumed, and the client spun. The queue is + * drained again on every pass (and from svc_gr_request / svc_run_ops), so + * a bounded batch loses nothing; it just guarantees the ring walk gets + * its cycles under an injection flood. + */ +#define SVC_MAP_ADD_BATCH 64u static void svc_handle_map_add(void) { unsigned long flags; bool rewound = false; u32 i; + u32 batch = 0; for (;;) { struct svc_map_add_req req; u32 ap; u64 off; + if (batch++ >= SVC_MAP_ADD_BATCH) + break; spin_lock_irqsave(&svc_map_add_lock, flags); if (svc_map_add_tail == svc_map_add_head) { spin_unlock_irqrestore(&svc_map_add_lock, flags); @@ -15332,8 +24958,66 @@ static void svc_handle_map_add(void) ap = req.ap ? req.ap : SVC_APERTURE_SYS; for (off = 0; off < (req.len ? req.len : 1); off += SEMA_PAGE_SIZE) { - svc_ipin_store(req.va + off, req.pa + off, ap); - svc_lat_store(req.va + off, req.pa + off, ap); + u64 va = req.va + off; + + /* + * m154: never let an injected pin shadow a live + * channel's GPFIFO ring. r153: the scanner's merged + * CE-window run (va=0x121000000 len=0x40000, the + * length varies per session) covered a CE channel's + * ring VA 0x121010000; the injected translation took + * precedence in svc_ipin_lookup/read, the ring then + * read the guest page at 0x102e02000 - an all-zero + * entry was silently consumed, the pool release was + * never authored, and the client spun in userspace + * for the rest of the boot (pushes frozen). The ring + * pages already have their own discovery path (the + * channel object / push-buffer scan); a colliding + * injection is always the wrong translation, so drop + * it and leave the ring to that path. + */ + if (svc_va_hits_chan_ring(va)) { + svc_ring_pin_skips++; + if (svc_ring_pin_skips <= 8) + pr_info("sema-svc: injected pin va=0x%llx DROPPED (live channel ring range; ring reads stay on the channel's own translation)\n", + va); + continue; + } + svc_ipin_store(va, req.pa + off, ap); + svc_lat_store(va, req.pa + off, ap); + /* + * r372 (stale-ledger corruption class): a fresh pin is the + * guest's newest statement of where this VA's content + * lives. The landing table (ledger) otherwise only moves + * on host writes, so a VA re-pinned to a NEW page (the + * guest freed the old buffer and reallocated it - CUDA + * reuses VA ranges across processes) kept resolving to + * the OLD, freed page: every later write through that + * ledger entry landed on memory the guest kernel had + * since reused (proven class: 8 zero bytes into an + * nv_free_system_pages page array -> guest oops at + * client teardown). Repair the entry only when it + * disagrees with the pin (a no-op otherwise, so the + * table is not flooded with pin pages). + */ + { + u64 lpa = 0; + u32 lap = 0; + + if (!svc_ledger_lookup(va, &lpa, &lap) && + (((lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + ((req.pa + off) & ~(u64)(SEMA_PAGE_SIZE - 1))) || + (u32)lap != (u32)ap)) { + svc_ledger_pin_repairs++; + if (svc_ledger_pin_repairs <= 32) + pr_info("sema-ledger: PIN-REPAIR va=0x%llx old=0x%llx ap=%u -> new=0x%llx ap=%u\n", + va, + lpa & ~(u64)(SEMA_PAGE_SIZE - 1), + lap, + req.pa + off, ap); + svc_ledger_store(va, req.pa + off, ap); + } + } } /* * m149: bounded per-session evidence that the CE push-buffer @@ -15376,6 +25060,20 @@ static void svc_handle_map_add(void) if (!ch->valid || !ch->gpfifo_entries) continue; /* + * STALLFREE r225: a channel under the stale-replay + * watchdog's rewind cooldown is NOT rewound. Without + * this the rewind re-opened the same consumed window + * every batch while the producer had nothing pending, + * pinning last_get and re-walking stale entries + * forever (r225: slots 93..100 of 0x5c000059 for + * 42+ s). The cooldown expires on its own (bounded + * duty cycle) and is cleared by new work, so a + * translation landing later can still retry. + */ + if (ch->wd_rewind_off_until && + time_before(jiffies, ch->wd_rewind_off_until)) + continue; + /* * Clamp the rewind at 0: wrapping below zero (get=0 -> * entries - N) makes the pump read ring slots past the * producer that hold stale entries, so replay only @@ -15555,9 +25253,303 @@ int gsp_sema_service_counter_add(u32 h_object, u64 pa) return ret; } +/* + * Register the guest PMA scrubber's channel-buffer words. Called from the + * sema_scrubbuf_add debug parameter with the PAs the guest scanner resolved + * (vmallocinfo vmap area + kcore page-table walk). fin_pa is the finish + * payload the guest's scrub wait polls; hsem_pa is the channel's host + * semaphore (channelWaitForFreeEntry polls it). Plain state update. + * + * Dual-guest: the words are written through a specific guest's address space + * (see svc_scrub_pass), so the registration binds to the FIRST open mdev - + * the same single-guest convention the scanner tooling follows (it tracks + * one client). Refused with no open mdev rather than parked globally. + */ +int gsp_sema_service_scrubbuf_add(u64 fin_pa, u64 hsem_pa) +{ + struct nv_vgpu_device *vgpu = NULL; + struct nv_vgpu_device *d; + struct svc_scrub *s; + + if (!svc.task) + return -ENODEV; + /* Both words are u32: require 4-byte alignment and a RAM-shaped PA. */ + if (!fin_pa || (fin_pa & 0x3) || fin_pa >= SEMA_PA_MAX) + return -EINVAL; + if (!hsem_pa || (hsem_pa & 0x3) || hsem_pa >= SEMA_PA_MAX) + return -EINVAL; + + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(d, &nv_vgpu_driver.devices, list) { + if (atomic_read(&d->vfio_dev_open_count) > 0) { + vgpu = d; + break; + } + } + mutex_unlock(&nv_vgpu_driver.lock); + if (!vgpu) { + pr_warn_ratelimited("sema-svc: scrubber buffer registration ignored: no open mdev to bind it to\n"); + return -ENODEV; + } + s = svc_scrub_get(vgpu, true); + if (!s) + return -ENOSPC; + + s->fin_pa = fin_pa; + s->hsem_pa = hsem_pa; + s->pa_ok = true; + s->pa_gen++; + pr_info("sema-svc: scrubber buffer registered fin_pa=0x%llx hsem_pa=0x%llx (gen %u)\n", + fin_pa, hsem_pa, s->pa_gen); + return 0; +} + +/* + * Author the CeUtils scrubber channel's progress once per service pass. + * + * The physical CE would advance, per completed scrub block, the channel's + * finish payload (channel buffer + 0x6c004) and write the channel's host + * semaphore as it consumes GPFIFO entries. Nothing in this emulation does, + * so the guest PMA's scrub wait (scrubCheckAndWaitForSize) freezes from the + * third CUDA session on. This pass closes that loop from the channel's own + * USERD GPPut (which the guest driver writes before every submit): + * + * submitted = number of scrub blocks submitted (GPPut is end-exclusive + * and counts submissions until the ring wraps; track the wrap ourselves), + * then author USERD GP_GET = put (consumed), host sema = put and finish + * payload = submitted. Forward-only, idempotent, retried every pass so a + * late scanner registration (or a post-flush reset) heals by itself. + * + * The scrub work itself is not executed (no shadow memsets): the guest + * only needs the progress to proceed. Executing the memsets requires + * decoding the channel's pushes, which the scanner can later provide pins + * for; correctness impact is limited to stale data in pages the guest + * believes scrubbed. + */ +static void svc_scrub_pass(struct nv_vgpu_device *vgpu) +{ + struct svc_scrub *s = svc_scrub_get(vgpu, false); + u32 put = 0; + u64 delta; + + if (!s || !s->userd_ok || !s->pa_ok || !s->entries) + return; + if (svc_read_ap(vgpu, s->userd_pa + SVC_USERD_GP_PUT, s->userd_as, + &put, sizeof(put))) { + /* Retry through the alternate aperture (as registration did). */ + if (svc_read_ap(vgpu, s->userd_pa + SVC_USERD_GP_PUT, + s->userd_as == SVC_APERTURE_FB ? + SVC_APERTURE_SYS : SVC_APERTURE_FB, + &put, sizeof(put))) + return; + } + if (put >= s->entries) + return; /* junk translation; retry next pass */ + + if (!s->seeded) { + s->seeded = true; + s->put_last = put; + /* GPPut = (lastSubmittedEntry + 1) % entries: the submission + * count until the ring wraps. Seeding from it adopts any + * pre-host scrub work (the boot session's scrubbers). */ + s->submitted = put; + } else if (put != s->put_last) { + delta = (put + s->entries - s->put_last) % s->entries; + if (!delta) + delta = s->entries; /* exact wrap */ + s->submitted += delta; + s->put_last = put; + } else if (s->submitted == s->authored) { + return; /* nothing new; no guest writes */ + } + + if (s->log_budget < 64) { + s->log_budget++; + pr_info("sema-svc: scrub author put=%u submitted=%llu fin_pa=0x%llx\n", + put, s->submitted, s->fin_pa); + } + + /* + * Author (forward-only). The USERD GP_GET and host-sema writes mark + * the submissions consumed; the finish write releases the scrub + * wait. Written only when GP_PUT advanced (or on the seeding pass), + * so the per-pass cost is zero while the scrubber idles. + */ + svc_write_ap(vgpu, s->userd_pa + SVC_USERD_GP_GET, s->userd_as, + &put, sizeof(put)); + svc_write_ap(vgpu, s->hsem_pa, SVC_APERTURE_SYS, &put, sizeof(put)); + { + u32 fin = (u32)s->submitted; + + svc_write_ap(vgpu, s->fin_pa, SVC_APERTURE_SYS, &fin, + sizeof(fin)); + } + s->authored = s->submitted; +} + +/* + * r539: init-stall watchdog. The r530-r538 stall boots all share one shape: + * the pump keeps looping (rebuilds, pending retries) but the entry loop stops + * consuming anything, a client spins at DEVICE (uvm channel_pool_add -> + * uvm_push_end_and_wait) and every counter the guest waits on is frozen. The + * historical evidence for that state was spread over many bounded logs; this + * dump collects the whole stall state in one place when the entry loop makes + * no progress for SVC_STALL_WATCH_MS while withheld work exists. Bounded: + * SVC_STALL_WATCH_DUMPS per boot, one per re-armed window. + */ +#define SVC_STALL_WATCH_MS 20000u +#define SVC_STALL_WATCH_DUMPS 4u + +static unsigned long svc_stall_watch_last; +static u64 svc_stall_watch_done; +static u32 svc_stall_watch_dumps; + +/* + * r563: stall-watchdog self-heal. The r562 STALL-DUMP shape (pump alive, + * no entry consumed for 20 s, held/opdef counters frozen) has been closed + * one hole at a time (r548..r558), but every remaining interaction hole + * has the same end state: some hold condition keeps winning and the walk + * breaks on the same entry forever. When the watchdog fires, arm a small + * budget of forced consumes: the next hold decisions that would otherwise + * park an entry are bypassed, the entry is consumed with its withheld + * releases applied forward-only (and the GR QMD completion authored by the + * existing tail), and the client's wait advances. Bounded by the dump + * cadence (<= 4 per boot) and by the budget (4 hold bypasses per arming), + * so a healthy boot never sees it and a stalled one cannot livelock. + */ +static u32 svc_force_consume; + +static inline bool svc_hold_bypass(void) +{ + if (!svc_force_consume) + return false; + svc_force_consume--; + pr_info_ratelimited("vgsp-sema: STALL-HEAL: hold bypassed (forced consume, budget left %u)\n", + svc_force_consume); + return true; +} + +static void svc_stall_watch_dump(void) +{ + u32 i; + + pr_err("vgsp-sema: STALL-DUMP #%u: no GPFIFO entry consumed for %us (done=%llu held=%llu opdef=%llu/%llu/%llu pending=%u starved=%llu)\n", + svc_stall_watch_dumps, SVC_STALL_WATCH_MS / 1000u, + svc.gpfifo_entries_done, svc.gpfifo_held, + svc.opdef_held, svc.opdef_applied, svc.opdef_expired, + svc_pending_n, svc.gpfifo_starved); + for (i = 0; i < svc_pending_n && i < 16; i++) + pr_err("vgsp-sema: STALL-DUMP pending[%u] h=0x%x va=0x%llx pay=%u op=%u tries=%u\n", + i, svc_pending[i].h, svc_pending[i].va, + svc_pending[i].payload, svc_pending[i].op, + svc_pending[i].tries); + for (i = 0; i < SVC_OPDEF_SLOTS; i++) { + struct svc_opdef_ent *e = &svc_opdef[i]; + + if (!e->active) + continue; + pr_err("vgsp-sema: STALL-DUMP opdef[%u] h=0x%x entry=%u n=%u age_ms=%u first_va=0x%llx pay=%u\n", + i, e->h_object, e->entry_idx, (unsigned)e->n, + jiffies_to_msecs(jiffies - e->since), + e->n ? e->rel[0].va : 0ull, + e->n ? e->rel[0].payload : 0u); + } + for (i = 0; i < svc_pool_boot_n; i++) { + struct svc_pool_boot_pa *b = &svc_pool_boot[i]; + + if (!b->va_page) + continue; + pr_err("vgsp-sema: STALL-DUMP pool[%u] va=0x%llx pa=0x%llx ap=%d pa2=0x%llx ap2=%d age_ms=%u\n", + i, b->va_page, b->pa_page, b->ap, b->pa2_page, + b->ap2, jiffies_to_msecs(jiffies - b->seen)); + } + pr_err("vgsp-sema: STALL-DUMP pool_dual=%llu alt=%llu conflicts=%u residue=%llu\n", + svc.pool_dual_applied, svc.pool_alt_applied, + svc_pool_boot_conflicts, svc.pool_residue); +} + +static void svc_stall_watch_check(void) +{ + if (svc.gpfifo_entries_done != svc_stall_watch_done) { + svc_stall_watch_done = svc.gpfifo_entries_done; + svc_stall_watch_last = jiffies; + return; + } + if (!svc_stall_watch_last) + svc_stall_watch_last = jiffies; + if (!time_after(jiffies, svc_stall_watch_last + + msecs_to_jiffies(SVC_STALL_WATCH_MS))) + return; + svc_stall_watch_last = jiffies; /* re-arm the window */ + if (svc_stall_watch_dumps >= SVC_STALL_WATCH_DUMPS) + return; + /* + * r563c: gate on ACTIVE withheld work, not on the cumulative counters. + * r566 evidence: the dump fired four times with done=8929 and + * opdef=215/7/5 frozen - but the ring was EMPTY (every channel + * last_get == put: the clients were simply waiting and no entry was + * pending), and the forced flush had nothing to deliver. `opdef_held` + * is a lifetime counter (215 holds applied/expired long ago), not + * proof of a current hold. A real stall has either a pending + * translation (svc_pending_n) or an ACTIVE opdef slot with releases + * recorded (e->active && e->n) - only then is there something the + * pump is withholding. + */ + { + bool work = svc_pending_n != 0; + u32 wi; + + for (wi = 0; wi < SVC_OPDEF_SLOTS && !work; wi++) + if (svc_opdef[wi].active && svc_opdef[wi].n) + work = true; + if (!work) + return; + } + svc_stall_watch_dumps++; + svc_stall_watch_dump(); + /* + * r563: self-heal after the dump. The r564 boot showed the force + * budget alone is not enough: the walk broke BEFORE the instrumented + * hold sites (no STALL-HEAL lines while held=727 climbed), so clear + * the state blockers every break site consults, and force-deliver + * the withheld opdef slots: + * - per channel: the cheap-wait unread window (its break at the + * loop top) and the GR hold/nocap indices; + * - the opdef slots: deliver forward-only and mark delivered, so + * the next decode cannot re-hold (the same edge the 3 s bound + * would run); + * - the force-consume budget stays as the belt for the hold sites. + */ + svc_opdef_force_flush(); + { + unsigned long flags; + u32 c; + + spin_lock_irqsave(&svc.chan_lock, flags); + for (c = 0; c < SVC_MAX_CHANNELS; c++) { + struct svc_channel *ch = &svc.chan[c]; + + if (!ch->valid) + continue; + ch->unread_va = 0; + ch->unread_until = 0; + ch->gr_hold_idx = ~0u; + ch->gr_hold_until = 0; + ch->gr_nocap_idx = ~0u; + ch->gr_nocap_va = 0; + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + } + pr_err("vgsp-sema: STALL-HEAL: forced opdef flush + walk-state clear (dumps=%u)\n", + svc_stall_watch_dumps); + svc_force_consume = 4; +} + static int svc_thread(void *arg) { + struct nv_vgpu_device *vgpus[SVC_MAX_GUESTS]; struct nv_vgpu_device *vgpu; + int n_vgpus, g; pr_info("sema-svc: thread started\n"); @@ -15577,129 +25569,212 @@ static int svc_thread(void *arg) complete_all(&svc_sess_flush_done); } - vgpu = NULL; + /* + * m177: a guest RM (re)load means a new guest boot: wipe the + * boot-scoped discovery state here, on the thread that owns + * it. Raised by gsp_sema_service_guest_reset(). + */ + if (READ_ONCE(svc_guest_reset_req)) { + svc_guest_reset_req = 0; + svc_guest_reset_wipe(); + } + + /* + * STALL FIX (r189): expire withheld release sets whose entry + * stopped being re-decoded (the deadline flush is only + * reachable from the entry path). Forward-only delivery to a + * live owner; counted LOSS otherwise. Cheap: 8 slots. + */ + svc_opdef_expire_sweep(); + + /* + * Service EVERY open mdev each pass, in list order. With one + * mdev this is exactly the historical single-vgpu pass; with + * two, both guests get a pump turn per iteration (hardware + * forwarding stays serialized by the single-inflight slot; see + * the forwarding section). The list walk is the historical + * bounded scan under nv_vgpu_driver.lock. + */ + n_vgpus = 0; mutex_lock(&nv_vgpu_driver.lock); { struct nv_vgpu_device *d; list_for_each_entry(d, &nv_vgpu_driver.devices, list) { - if (atomic_read(&d->vfio_dev_open_count) > 0) { - vgpu = d; - break; - } + if (atomic_read(&d->vfio_dev_open_count) > 0 && + n_vgpus < SVC_MAX_GUESTS) + vgpus[n_vgpus++] = d; } } mutex_unlock(&nv_vgpu_driver.lock); - if (!vgpu) { + if (!n_vgpus) { msleep(500); continue; } - svc_handle_probe(vgpu); - svc_handle_walk_probe(vgpu); + /* Host-injection queue drain: once per pass, not per guest. */ svc_handle_map_add(); - svc_handle_chan_dump(vgpu); - if (svc_sess_flush || !svc.direct_base || - time_after(jiffies, - (unsigned long)(svc.last_build + 10 * HZ))) - svc_rebuild(vgpu); + for (g = 0; g < n_vgpus; g++) { + struct svc_guest_sess *sess; + u64 done_before; - /* - * Channel-driven execution: notice submissions as they are made - * and run the pushed work immediately, so the guest's UVM - * progress waits complete in milliseconds rather than in the - * time it takes the fallback scan to find the push. A doorbell - * token means a channel was just scheduled; run everything now. - */ - { - u64 done_before = svc.gpfifo_entries_done; + vgpu = vgpus[g]; + svc_handle_probe(vgpu); + svc_handle_walk_probe(vgpu); + svc_handle_chan_dump(vgpu); + svc_scrub_pass(vgpu); + + /* + * Per-guest rebuild freshness: the discovery map is + * still shared between guests (staged G4), but each + * guest's RAM must be scanned on its own clock, or the + * first guest's 10 s tick would starve the second + * guest of rebuilds. The freshness stamp is the one + * svc_rebuild() publishes (svc.last_build). + */ + sess = svc_guest_get(vgpu, true); + if (svc_sess_flush || !svc.direct_base || + !sess || + time_after(jiffies, + (unsigned long)(sess->last_build + + 10 * HZ))) { + svc_rebuild(vgpu); + if (sess) + sess->last_build = svc.last_build; + } + /* + * r537: bounded fallback for a parked VAS candidate + * whose content-verified ring walk never anchored + * (see svc_vas_candidates_age_out). + */ + svc_vas_candidates_age_out(vgpu); + + /* + * Channel-driven execution: notice submissions as they + * are made and run the pushed work immediately, so the + * guest's UVM progress waits complete in milliseconds + * rather than in the time it takes the fallback scan + * to find the push. A doorbell token means a channel + * was just scheduled; run everything now. + */ + done_before = svc.gpfifo_entries_done; svc.doorbell_pending = false; svc_pump_channels(vgpu); - pumped = (svc.gpfifo_entries_done != done_before); - } + if (svc.gpfifo_entries_done != done_before) + pumped = true; - /* - * Host CUDA bridge: marshal one queued launch into a daemon - * job and resolve completions. Inert until a launch is seen. - */ - gsp_cuda_bridge_service(vgpu); + /* + * Host CUDA bridge: marshal one queued launch into a + * daemon job and resolve completions. Inert until a + * launch is seen. + */ + gsp_cuda_bridge_service(vgpu); - /* - * Releases whose target VA was not translatable when the scan - * saw them: retry every pass, so a pin or a learned pair applies - * them immediately instead of on the next full sweep. - */ - svc_apply_pending(vgpu); + /* + * Releases whose target VA was not translatable when + * the scan saw them: retry every pass, so a pin or a + * learned pair applies them immediately instead of on + * the next full sweep. + */ + svc_apply_pending(vgpu); - /* - * A fresh VA->PA injection changes which releases are - * applicable: run the fast sliced sweep while the forced window - * is open (the pump's stall path also uses it). - */ - if (time_before(jiffies, (unsigned long)svc.fast_force_until)) - svc_apply_sweep_slice(vgpu); + /* + * A fresh VA->PA injection changes which releases are + * applicable: run the fast sliced sweep while the + * forced window is open (the pump's stall path also + * uses it). + */ + if (time_before(jiffies, + (unsigned long)svc.fast_force_until)) + svc_apply_sweep_slice(vgpu); + } + + /* r539: one stall dump when the entry loop stops progressing. */ + svc_stall_watch_check(); /* * While the pump is making progress the client is streaming * pushes and waiting on their releases: loop tightly and skip - * the discovery/sweep passes, which otherwise dominate the pass - * period (the pump cadence is what the client's context init - * deadline sees). Scans resume the first pass without progress. + * the discovery/sweep passes, which otherwise dominate the + * pass period (the pump cadence is what the client's context + * init deadline sees). Scans resume the first pass without + * progress. (With two guests, any guest's progress keeps the + * loop tight; the next iteration pumps both.) + */ + /* + * r580: the completion hint must also run while the pump is + * making progress - the fast path below skips it, and a + * completion burst saturates the event burst budget and drops + * wake-ups (r579 boot: events_dropped=724; the sentinel + * "no postable event (events_n=8 dropped=724)" repeated every + * 0.5 s). A client that starts waiting mid-burst needs the + * 0.5 s re-post even under load; the per-event dedup keeps the + * extra volume bounded. */ + for (g = 0; g < n_vgpus; g++) { + vgpu = vgpus[g]; + if (time_before(jiffies, + (unsigned long)svc.post_hint_until) && + time_after(jiffies, + (unsigned long)svc.last_hint + HZ / 2)) { + int rc; + + svc.last_hint = jiffies; + rc = gsp_rm_post_all_events(vgpu); + if (gsp_sema_dbg) + pr_info_ratelimited("sema-svc: completion hint rc=%d\n", + rc); + } + } + if (pumped) { msleep(1); continue; } - /* - * Push-buffer base discovery: a chunked walk of guest RAM - * finds the pages whose first push names a channel, so a - * channel that appears with work already waiting is pinned and - * pumped without waiting for the rolling sweep. - */ - svc_pb_discover(vgpu); + for (g = 0; g < n_vgpus; g++) { + vgpu = vgpus[g]; - /* - * Completion hint: re-post the events for a few seconds after a - * completion, so a client that starts waiting after the - * edge-triggered wake-up still observes it. - */ - if (time_before(jiffies, (unsigned long)svc.post_hint_until) && - time_after(jiffies, (unsigned long)svc.last_hint + HZ / 2)) { - int rc; + /* + * Push-buffer base discovery: a chunked walk of guest + * RAM finds the pages whose first push names a + */ + svc_pb_discover(vgpu); - svc.last_hint = jiffies; - rc = gsp_rm_post_all_events(vgpu); - if (gsp_sema_dbg) - pr_info_ratelimited("sema-svc: completion hint rc=%d\n", - rc); - } + /* + * r580: the completion hint was hoisted above the + * pumped fast path (it must run under load too); it is + * not repeated here. + */ - /* - * Fast path for the pushes themselves: every mapped page is a - * candidate pushbuffer page, and there are far fewer of those - * than there are pages of guest RAM. - */ - if (svc.direct_base && svc.map_n) - svc_apply_mapped(vgpu); + /* + * Fast path for the pushes themselves: every mapped + * page is a candidate pushbuffer page, and there are + * far fewer of those than there are pages of guest + * RAM. + */ + if (svc.direct_base && svc.map_n) + svc_apply_mapped(vgpu); - /* - * Rolling RAM scan: the pushbuffers of some clients are not - * covered by the VA map, so sweep guest RAM in bounded windows - * in parallel with the targeted pass. Work per pass is fixed, - * which keeps the emulation responsive while a full sweep - * completes every few seconds. - */ - svc_apply_pushes(vgpu); + /* + * Rolling RAM scan: the pushbuffers of some clients + * are not covered by the VA map, so sweep guest RAM in + * bounded windows in parallel with the targeted pass. + * Work per pass is fixed, which keeps the emulation + * responsive while a full sweep completes every few + * seconds. + */ + svc_apply_pushes(vgpu); - /* - * Deferred readability probes for faked CE pushes (observation - * only; see svc_retry_poll). - */ - svc_retry_poll(vgpu); + /* + * Deferred readability probes for faked CE pushes + * (observation only; see svc_retry_poll). + */ + svc_retry_poll(vgpu); + } msleep(10); } @@ -15822,6 +25897,97 @@ void gsp_sema_service_stop(void) svc_vas_free_all(); } +/* + * Release every piece of service state owned by one mdev (dual-guest goal + * 1.0). Called from the mdev cleanup path after that mdev's emulation thread + * stopped and before its struct can be freed: + * - quiesce forwarding so no in-flight work item can still be reading the + * request's vgpu, and abort a release hold that names it; + * - drop any hold that survives (a request that never queued leaves no work + * item to clear it) BEFORE the device can go away - resolution would + * dereference it; + * - drop the guest's channels; request the global session flush only when no + * guest has live channels left (the discovery tables are still shared; + * staged G4); + * - drop the guest's held-operand release slots (its waiters are gone); + * - free the per-guest session and scrubber records so the vgpu pointer can + * never match a newly allocated device. + */ +void gsp_sema_service_vgpu_cleanup(struct nv_vgpu_device *vgpu) +{ + unsigned long flags; + int i; + + if (!vgpu) + return; + + /* + * The forward work item runs on its own kthread and may be using the + * request's vgpu; quiesce waits for it (bounded by the submit's + * 200 ms poll cap) and aborts an active hold. Resume afterwards so the + * other guests keep forwarding. + */ + gsp_sema_service_work_quiesce(); + if (READ_ONCE(svc_fwd_hold.active) && svc_fwd_hold.vgpu == vgpu) + WRITE_ONCE(svc_fwd_hold.aborted, true); + gsp_sema_service_work_resume(); + + if (READ_ONCE(svc_fwd_hold.active) && svc_fwd_hold.vgpu == vgpu) { + /* + * Pure state clear (never a resolution through the vanishing + * device): the releases belong to a guest that is gone. + */ + svc.fwd_hold_canceled++; + svc_fwd_hold_clear(); + } + + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + struct svc_channel *ch = &svc.chan[i]; + + if (!ch->valid || ch->vgpu != vgpu) + continue; + pr_info("sema-svc: channel 0x%x dropped (mdev cleanup; submissions=%llu)\n", + ch->h_object, ch->submissions); + memset(ch, 0, sizeof(*ch)); + } + if (svc_sess_active) { + struct svc_guest_sess *sess = svc_guest_get(vgpu, false); + bool any = false; + + for (i = 0; i < SVC_MAX_CHANNELS; i++) + if (svc.chan[i].valid) { + any = true; + break; + } + if (sess) + sess->active = false; + if (!any) { + svc_sess_active = 0; + svc_sess_flush = 1; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + + /* Held-operand release slots of this guest cannot be delivered. */ + for (i = 0; i < SVC_OPDEF_SLOTS; i++) + if (svc_opdef[i].active && svc_opdef[i].vgpu == vgpu) + svc_opdef_clear(&svc_opdef[i], SVC_OPDEF_CLR_STOP); + + /* Free the per-guest session and scrubber records. */ + spin_lock_irqsave(&svc_guest_lock, flags); + for (i = 0; i < SVC_MAX_GUESTS; i++) + if (svc_guests[i].vgpu == vgpu) + memset(&svc_guests[i], 0, sizeof(svc_guests[i])); + spin_unlock_irqrestore(&svc_guest_lock, flags); + for (i = 0; i < SVC_MAX_GUESTS; i++) + if (svc_scrubs[i].vgpu == vgpu) + memset(&svc_scrubs[i], 0, sizeof(svc_scrubs[i])); + + if (svc_sess_flush && svc.task) + wake_up_process(svc.task); +} + /* ------------------------------------------------------------------ */ /* host CUDA bridge exports (gsp_cuda_bridge.c) */ /* ------------------------------------------------------------------ */ @@ -16017,6 +26183,6 @@ int gsp_sema_service_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, * after the write still receives the completion event. */ if (!rc) - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); return rc; } diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h index 4b04892..31512ef 100644 --- a/kernel/gsp_sema_service.h +++ b/kernel/gsp_sema_service.h @@ -18,8 +18,16 @@ * * All addresses are guest-physical except gpfifo_va, which is the GPU VA of * the channel's GPFIFO ring (translated through the discovered VA->PA map). + * + * vgpu names the owning mdev (guest session). Two guests' stock allocators + * choose the same object handles, so the channel table is keyed by + * (vgpu, h_object): a registration can never overwrite another guest's slot, + * and ranks/session bookkeeping are computed per guest. */ +struct nv_vgpu_device; + struct sema_channel_desc { + struct nv_vgpu_device *vgpu; /* owning mdev (guest session) */ u32 h_object; u32 engine_type; u32 cid; /* RM channel id (the doorbell token vector) */ @@ -33,7 +41,27 @@ struct sema_channel_desc { }; void gsp_sema_service_channel_add(const struct sema_channel_desc *desc); -void gsp_sema_service_channel_del(u32 h_object); +void gsp_sema_service_channel_del(struct nv_vgpu_device *vgpu, u32 h_object); + +/* + * Release every piece of service state that belongs to one mdev before its + * guest session disappears: drop the mdev's channels (per-vgpu session end), + * clear a release hold that names it (never dereference a removed device), + * and free its per-guest session record. Called from the mdev cleanup path + * after the device's emulation thread stopped and before the mdev struct can + * be freed. No-op for a vgpu that never registered a channel. + */ +void gsp_sema_service_vgpu_cleanup(struct nv_vgpu_device *vgpu); + +/* + * Register the guest PMA scrubber's channel-buffer words (the CeUtils + * finish payload the guest's scrub wait polls, and the channel's host + * semaphore). The PAs come from the guest scanner (vmallocinfo + kcore): + * the guest RM kernel channel lives in a GPU VA space that collides + * numerically with the client's CE window in the global translation map, + * so it cannot be resolved through that map. + */ +int gsp_sema_service_scrubbuf_add(u64 fin_pa, u64 hsem_pa); /* * A GPU VA space whose top-level page directory the driver has published (the @@ -53,6 +81,18 @@ struct sema_vaspace_desc { void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc); void gsp_sema_service_vaspace_del(u32 h_vaspace); +/* + * r496j: park a page-directory candidate (from COPY_SERVER_PDES) until a + * content-validated walk proves it belongs to a live client; only then is + * it registered (a blind registration let stale kernel directories answer + * other clients' VAs in the walker's first-success order). + */ +void gsp_sema_service_vaspace_candidate_add(u32 h_vaspace, u64 pdb, + u32 entries, bool vidmem); +/* m177: guest RM reload = new guest boot; wipe boot-scoped discovery state + * (direct-map base, content-scanned tables) on the svc thread. Client + * session flushes keep that state - see svc_session_flush. */ +void gsp_sema_service_guest_reset(void); int gsp_sema_service_start(void); void gsp_sema_service_stop(void); @@ -235,4 +275,173 @@ extern unsigned gsp_sema_push_hold_ms; */ extern unsigned gsp_sema_gr_grace_ms; +/* + * GRFIX FIX1b (r213): absolute per-launch bound (ms) for the GR + * data-stalled hold. Keyed by the launch push VA (slot indices are not + * stable across pin-batch rewinds); first hold fixes the deadline, nothing + * re-arms it, and when it passes the launch proceeds (the pre-8b058884 + * behaviour) so the guest can never wait unboundedly. + */ +extern unsigned gsp_phys_gr_hold_max_ms; + +/* + * GRFIX FIX1e (r215): master gate for the GR fill hold (default 0 = + * log-only, never hold). At 0 the fill logs dropped operand pages + * (GRFILL-DROP) and takes the historical forward path immediately; the + * GRFILL-HOLD/GRFILL-EXPIRED machinery is skipped. At 1 the page-count + * hold and its absolute bound apply. + */ +extern unsigned gsp_phys_gr_fill_hold; + +/* + * r576 DIAG: per-boot witness budget for the GR launch-decision dump + * (default 0 = off). Nonzero logs, for the first N launches whose fill saw + * dropped pages / a partial argument set or whose decision point had a + * dstpend/copy-pending verdict: the decision verdicts, the launch's + * resolved data pages (VA -> guest PA), the live deferred-write records and + * the pending CE opdef entries - the evidence needed to explain a red draw + * of the readback lottery. + */ +extern unsigned gsp_phys_gr_dec_log_max; +extern unsigned gsp_sema_walk_trace; + +/* + * r365: execute fill-shaped CE transfers to VIDMEM-window VAs (the guest's + * page-table writes). Off by default; see the definition in the .c. + */ +extern bool gsp_sema_winfill; + +/* + * m164 CLASS-2: hold window (ms) for a GR launch push whose decode saw QMD + * words but built no launch candidate; 0 disables the hold. + */ +extern unsigned gsp_sema_gr_nocap_hold_ms; + +/* + * m165 (D-F2a): skip the post-registration pool hold when a verified + * injected pin exists for the VA (default on); 0 restores the blanket 5 s. + */ +extern bool gsp_sema_pool_hold_fresh; + +/* + * m169 (AGENT-F R1): hold device-destination copies translated only by a + * walk/RAM scan until a pin confirms (default on). + */ +extern bool gsp_sema_copy_dst_pin_guard; +extern bool gsp_sema_wait_gate; +/* r401: bounded acquire-wait hold (fail-open) in ms; must exceed the + * emulated launch completion latency for copy-after-launch ordering. */ +extern unsigned gsp_sema_wait_hold_ms; +/* r481: verbatim method-stream dump for the first N large GR pushes + * (default 0 off). Diagnostic only; no behaviour change. */ +extern unsigned gsp_sema_push_raw; + +/* + * r384 (guest boot-corruption lottery): fill-destination guards (default + * on). Large virtual fills whose only authority is the walk/scan resolver + * are held through a short bounded window while their VA is demanded + * through the pin channel (fail-open after gsp_sema_fill_guard_ms); zero + * fills targeting a page-table-shaped guest page are refused loudly. + */ +extern bool gsp_sema_fill_guard; +extern unsigned gsp_sema_fill_guard_ms; + +/* + * r231 READ-ORDER guard (default 0: witness only). Defer a device-src copy + * whose source VA has a pending device-dst write hold (READ-ORDER-WIT) so + * the read re-decodes after the write resolves. Opt-in pending an A/B. + */ +extern bool gsp_sema_read_order_guard; + +/* + * COPYWIT fail4: per-boot line budget for the sema-bigcopy resolution + * witnesses (svc_copy's large-copy VA -> pa proof). Raise it for a + * discrimination cycle so the failing clients' H2D/D2H resolutions are + * visible; 0 disables the lines. + */ +extern unsigned gsp_sema_bigcopy_log_max; + +/* + * F1-b: authorship gate for the GR completion release the forwarded launch's + * poll waits on. When on (default), a RAM-sweep release whose VA equals the + * in-flight forward request's gr_rel_va is deferred (skipped, not applied) + * while the hardware copy is still in flight, so the poll can only break on + * an engine-authored word; the deferred release is re-seen by later sweep + * passes and the hold/fallback paths apply it if the hardware copy fails. + * 0 restores the unconditional sweep apply. + */ +extern bool gsp_sema_gr_swrel_gate; + +/* + * relfix (agent-A FIX A): rewrite-time GR relocation resolver. Resolves one + * device VA through the same trusted source chain the GR fill's per-page + * resolution uses (ipin -> self pin -> learned cache -> discovered map -> + * published vaspaces -> discovered directory), then the m180 ledger in the + * fill's two roles (fallback when nothing else answered; m171 landing + * preference above the ledger floor when it disagrees). The result is + * converted to a host page the caller can copy/map. Deliberately read-only + * w.r.t. the resolver tables - no pin or cache stores, no directory + * discovery, no targeted RAM scan: the caller is the GR work item, not the + * service thread, and must not write service state or block on a full-RAM + * scan. The page-table walks may sleep in vfio_dma_rw exactly like the + * fill's walks; callers run in process context (the GR work item, holding + * gsp_phys_rm_lock) with no spinlocks held. A live forwarded GR request + * (svc_fwd_req.vgpu) is required. A successful return (0) means the VA is a + * translated page of that request's client - the caller may act on it; a + * miss returns -ENOENT and the caller must change nothing. + * + * `stage` carries the per-source diagnostic chain of THIS call + * ("ipin:-2,pin:-2,...,ledger:-2," with the resolving source ending in + * ":0,") and `stage_rc` the overall result (0 on success, else the miss + * errno). relfix PART 3: the booter's RELOC-GAP-STAGE line prints them for + * words that stayed unrelocated, so the failing source is on the record. + */ +struct gsp_sema_gr_reloc { + u64 pa; /* resolved guest physical page base */ + u32 ap; /* aperture: 1 = SYS (guest RAM), 2 = FB (vidmem) */ + u64 hpa; /* host physical address of that guest page */ + char stage[96]; /* per-attempt "name:rc," source chain (diagnostic) */ + int stage_rc; /* 0 on success; else the failure errno */ +}; + +int gsp_sema_gr_reloc_resolve(u64 va, struct gsp_sema_gr_reloc *out); + +/* + * r378 write guards. Both re-validate a guest-memory write target that was + * resolved earlier (ledger/pin/walk/counter registration, or the GR submit's + * per-page resolution) against the CURRENT resolution before the bytes land, + * and return false when a positive disagreement proves the target stale: + * + * gsp_sema_gr_write_guard(va, hpa, path): GR backend pages. A fresh + * injected pin is checked first (the r372/r376 pin-wins rule - the guest's + * newest statement), then the trusted resolver chain for a pin-less VA. + * The submit-to-copy-out window is seconds wide (the GR poll), and a guest + * buffer freed/re-homed in it would otherwise receive a full page of + * another buffer's content (the boot-corruption class). The landing table + * is deliberately not the authority: our own launches feed it, so it would + * confirm the target by construction. + * + * gsp_sema_write_guard(va, pa, ap, path): service-side SYS/FB targets. A + * fresh injected pin is the guest's newest statement about the VA; a pin + * that disagrees with the recorded target means the page moved (the + * r372/r376 pin-wins rule), so the stale write is skipped. + * + * Both fail OPEN when no current answer exists (the recorded landing is then + * the write authority) and log every skip (bounded, 256 lines) with the path + * and the VA/target/current values. + */ +bool gsp_sema_gr_write_guard(u64 va, u64 hpa, const char *path); +bool gsp_sema_write_guard(u64 va, u64 pa, u32 ap, const char *path); + +/* + * r391 diagnostic (sum=0 class): resolve a GR data VA through the landing + * table (the exact source the D2H reader's ledger-first branch uses) and + * return the resolution plus the first 8 bytes read through it. The GR + * copy-out prints it next to the write target it used, so a launch/readback + * disagreement (the write went to another page/aperture than the reader + * resolves) is one line instead of a forensics session. + */ +bool gsp_sema_gr_va_snap(u64 va, u64 *pa, u32 *ap, u64 *val8); +u8 *gsp_sema_gr_shadow_base(void); + #endif /* _GSP_SEMA_SERVICE_H_ */ diff --git a/kernel/main.c b/kernel/main.c index ac970f1..d821762 100644 --- a/kernel/main.c +++ b/kernel/main.c @@ -1740,6 +1740,16 @@ static void nv_vgpu_device_cleanup_work(struct work_struct *work) gsp_emu_cleanup(vgpu); /* + * Dual-guest (goal 1.0): release everything the sema service still holds + * for this mdev (channel entries, session/scrubber records, a release + * hold that names it - quiescing any in-flight forwarding work) and drop + * its per-mdev guest-RAM record, BEFORE the vgpu struct can be freed: a + * stale pointer in the service's tables must never match a later device. + */ + gsp_sema_service_vgpu_cleanup(vgpu); + gsp_phys_guest_ram_vgpu_clear(vgpu); + + /* * Last-reference backend teardown: gsp_emu_cleanup() above has stopped * this mdev's emulation thread, so no forward can be in flight when the * claim is released. The backend session stays up until the last mdev @@ -2544,6 +2554,37 @@ module_param_cb(sema_probe, &nv_vgpu_sema_probe_ops, NULL, 0200); MODULE_PARM_DESC(sema_probe, "Dump sema-service map + slot values for a GPU VA"); /* + * Register the guest PMA scrubber channel buffer words (finish payload PA, + * host-semaphore PA) the guest scanner resolved: + * echo 0x11a8ff004,0x11a8ff000 > .../parameters/sema_scrubbuf_add + */ +static int nv_vgpu_sema_scrubbuf_set(const char *val, + const struct kernel_param *kp) +{ + char buf[64]; + char *comma; + u64 fin_pa = 0, hsem_pa = 0; + + strscpy(buf, val, sizeof(buf)); + comma = strchr(buf, ','); + if (!comma) + return -EINVAL; + *comma = '\0'; + if (kstrtoull(buf, 0, &fin_pa) || kstrtoull(comma + 1, 0, &hsem_pa)) + return -EINVAL; + + return gsp_sema_service_scrubbuf_add(fin_pa, hsem_pa); +} + +static const struct kernel_param_ops nv_vgpu_sema_scrubbuf_ops = { + .set = nv_vgpu_sema_scrubbuf_set, +}; + +module_param_cb(sema_scrubbuf_add, &nv_vgpu_sema_scrubbuf_ops, NULL, 0200); +MODULE_PARM_DESC(sema_scrubbuf_add, + "Register the PMA scrubber finish/host-sema PAs (hex fin,hex hsem)"); + +/* * Debug: run the page-table walks for a GPU VA with per-level tracing, * bypassing the pin/lat/pair map (see gsp_sema_service_walk_probe): * echo 0x20400f000 > .../parameters/walk_probe @@ -2730,6 +2771,141 @@ module_param_named(sema_gr_grace_ms, gsp_sema_gr_grace_ms, uint, 0644); MODULE_PARM_DESC(sema_gr_grace_ms, "GR pending-entry grace window in ms (default 8000)"); /* + * GRFIX FIX1b (r213): absolute per-launch bound in ms for the GR fill + * data-stalled hold (the FIX1 page-drop hold). The deadline is keyed by the + * launch push VA, fixed on the first hold and never re-armed; when it passes + * the launch proceeds (forwarded best-effort) so a never-resolving page can + * never block the guest. 0 keeps the 3000 ms default. + */ +module_param_named(gsp_phys_gr_hold_max_ms, gsp_phys_gr_hold_max_ms, uint, 0644); +MODULE_PARM_DESC(gsp_phys_gr_hold_max_ms, "Absolute GR fill hold bound in ms, keyed by launch push VA (default 3000; launch proceeds when it passes)"); + +/* + * GRFIX FIX1e (r215): master gate for the GR fill hold. At 0 (default) the + * fill logs dropped operand pages and takes the historical forward path + * immediately - it can never hold or delay a launch. At 1 the page-count + * hold and the absolute bound apply. + */ +module_param_named(gsp_phys_gr_fill_hold, gsp_phys_gr_fill_hold, uint, 0644); +MODULE_PARM_DESC(gsp_phys_gr_fill_hold, "GR fill hold on unresolved operand pages (default 0: log-only, never hold)"); + +/* + * r576 DIAG: per-boot witness budget for the GR launch-decision dump (see + * svc_gr_dec_dump in gsp_sema_service.c). Nonzero logs, for the first N + * launches whose fill saw dropped pages / a partial argument set or whose + * decision point carries a dstpend/copy-pending verdict: the decision + * verdicts, the launch's resolved data pages (VA -> guest PA), the live + * deferred-write records and the pending CE opdef entries. 0 (default) = + * off. + */ +module_param_named(gsp_phys_gr_dec_log, gsp_phys_gr_dec_log_max, uint, 0644); +MODULE_PARM_DESC(gsp_phys_gr_dec_log, "GR launch-decision dump budget per boot (default 0: off)"); + +/* + * r576 DIAG: per-boot witness budget for GSP event-post payloads (see + * gsp_rm_evlog_post in gsp_rm_emulation.c). Nonzero logs every + * EVENT_POST_EVENT record actually sent, with its full payload + * (hClient/hEvent/notifyIndex/data) and send result - the evidence needed + * when a client parks waiting for a completion wake-up that never arrives. + * 0 (default) = off. + */ +module_param_named(gsp_rm_ev_log, gsp_rm_ev_log_max, uint, 0644); +MODULE_PARM_DESC(gsp_rm_ev_log, "GSP event-post payload witness budget per boot (default 0: off)"); + +/* + * r577 experiment knobs for the guest-RM event drop (runtime-writable so + * one boot can A/B). See gsp_rm_emulation.c. All default 0 = legacy. + */ +module_param_named(gsp_rm_ev_seq, gsp_rm_ev_seq, uint, 0644); +MODULE_PARM_DESC(gsp_rm_ev_seq, "event records carry the status-queue sequence as RPC sequence (default 0: legacy 0)"); +module_param_named(gsp_rm_ev_nlist, gsp_rm_ev_nlist, uint, 0644); +MODULE_PARM_DESC(gsp_rm_ev_nlist, "set bNotifyList on event records (default 0: specific-hEvent dispatch)"); +module_param_named(gsp_rm_ev_cl, gsp_rm_ev_cl, uint, 0644); +MODULE_PARM_DESC(gsp_rm_ev_cl, "override event record hClient (default 0: the allocating client)"); +module_param_named(gsp_rm_ev_func, gsp_rm_ev_func, uint, 0644); +MODULE_PARM_DESC(gsp_rm_ev_func, "reachability probe: event records carry this RPC function instead of POST_EVENT 0x1003 (default 0)"); + +module_param_named(gsp_sema_walk_trace, gsp_sema_walk_trace, uint, 0644); +MODULE_PARM_DESC(gsp_sema_walk_trace, "Trace GMMU walks for unresolved high-VA operands (default 0: off)"); +module_param_named(gsp_sema_winfill, gsp_sema_winfill, bool, 0644); +MODULE_PARM_DESC(gsp_sema_winfill, "Execute fill-shaped CE transfers to VIDMEM-window VAs (default 0: off)"); + +/* + * F1-b: while a forwarded GR launch is in flight, defer (do not apply) a + * RAM-sweep release whose VA is that launch's QMD completion word, so the + * backend poll can only break on an engine-authored ack. Deferred releases + * are re-seen by later sweep passes; the hold/fallback path applies them when + * the hardware copy fails. 0 restores the unconditional sweep apply. + */ +module_param_named(gsp_sema_gr_swrel_gate, gsp_sema_gr_swrel_gate, bool, 0644); +MODULE_PARM_DESC(gsp_sema_gr_swrel_gate, "Defer a sweep-authored in-flight GR release word (default 1; 0 = unconditional sweep apply)"); + +/* + * m164 CLASS-2: hold window (ms) for a readable GR launch push whose decode + * saw QMD words but built no launch candidate (replay-suppressed capture, + * partial/split QMD, or an unreadable push page). The entry stays unconsumed + * for this window and the next pass forces a fresh capture; on expiry the + * historical consume applies. 0 disables. + */ +module_param_named(sema_gr_nocap_hold_ms, gsp_sema_gr_nocap_hold_ms, uint, 0644); +MODULE_PARM_DESC(sema_gr_nocap_hold_ms, "GR no-capture entry hold in ms (default 6000, 0 disables)"); + +/* + * m165 (D-F2a): skip the post-registration pool-release hold when a + * host-verified injected pin for the VA exists (default on). 0 restores the + * legacy blanket 5 s hold. + */ +module_param_named(sema_pool_hold_fresh, gsp_sema_pool_hold_fresh, bool, 0644); +MODULE_PARM_DESC(sema_pool_hold_fresh, "apply pool releases at fresh-pin time instead of registration+5s (default 1)"); + +/* + * m169 (AGENT-F R1): hold device-destination copies translated only by a + * page-directory walk until a pin confirms (default on). A wrong-but-valid + * stale table entry silently writes the wrong vidmem page; the hold lets + * the m163 retry path re-resolve with pins. + */ +module_param_named(sema_copy_dst_pin_guard, gsp_sema_copy_dst_pin_guard, bool, 0644); +module_param_named(sema_wait_gate, gsp_sema_wait_gate, bool, 0644); +MODULE_PARM_DESC(sema_wait_gate, "honor C56F host-semaphore acquires before running a ring entry's work (default 1 since r408; 0 restores pass-through)"); +module_param_named(sema_wait_hold_ms, gsp_sema_wait_hold_ms, uint, 0644); +MODULE_PARM_DESC(sema_wait_hold_ms, "bounded acquire-wait hold before fail-open, ms (r401, default 4000; must exceed the emulated launch latency)"); +module_param_named(sema_push_raw, gsp_sema_push_raw, uint, 0644); +MODULE_PARM_DESC(sema_push_raw, "r481 diagnostic: dump the decoded method stream of the first N large GR pushes (default 0 off)"); +MODULE_PARM_DESC(sema_copy_dst_pin_guard, "hold device-dst copies without a pin-resolved translation (default 1)"); + +/* + * r384 (guest boot-corruption lottery): fill-destination guards. A large + * virtual fill whose only authority is the walk/scan resolver is held for + * gsp_sema_fill_guard_ms while its VA is demanded through the pin channel + * (fail-open, mirrors sema_copy_dst_pin_guard); a zero fill targeting a + * page-table-shaped guest page is refused loudly (FILL-PT-REFUSED). + */ +module_param_named(sema_fill_guard, gsp_sema_fill_guard, bool, 0644); +MODULE_PARM_DESC(sema_fill_guard, "guard virtual fill destinations (pin-demand hold + page-table-shape refusal, default 1)"); +module_param_named(sema_fill_guard_ms, gsp_sema_fill_guard_ms, uint, 0644); +MODULE_PARM_DESC(sema_fill_guard_ms, "fill pin-demand hold window in ms (default 800)"); + +/* + * r231 READ-ORDER guard: defer a device-src copy whose source VA has a + * pending device-dst write hold so the read re-decodes after the write + * resolves (fixes the ctaid_probe RAWA upload/readback inversion). + * Default 0: witness only (READ-ORDER-WIT). Opt-in pending an A/B proving + * no green regression. + */ +module_param_named(sema_read_order_guard, gsp_sema_read_order_guard, bool, 0644); +MODULE_PARM_DESC(sema_read_order_guard, "defer device-src reads of a VA with a pending held device-dst write (default 0: witness only)"); + +/* + * COPYWIT fail4 (2026-10-18): per-boot budget for the sema-bigcopy resolution + * witnesses. svc_copy's large-copy proof logs both operands' resolved + * VA -> pa/aperture/provenance, which is the read side of the client data + * path; raise this for a failing-client discrimination cycle so the lines + * survive past the boot's head. 0 disables. + */ +module_param_named(sema_bigcopy_log, gsp_sema_bigcopy_log_max, uint, 0644); +MODULE_PARM_DESC(sema_bigcopy_log, "sema-bigcopy resolution witness budget per boot (default 24; 0 disables)"); + +/* * Debug: inject a verified {GPU VA -> guest PA} translation into the * semaphore service (used when the client's mapping cannot be discovered * from guest structures): |