diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 1170 |
1 files changed, 1099 insertions, 71 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 53e2140..52e921c 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -160,6 +160,15 @@ struct sema_map_entry { #define SVC_USERD_GP_GET 0x88 #define SVC_USERD_GP_PUT 0x8c +/* + * r496b: guest-RAM ring content scan (see gr_ram_cursor in struct + * svc_channel) and the VRAM USERD shape scan (svc_chan_userd). One scan + * slice per service pass so the service thread stays responsive. + */ +#define SVC_GR_RAM_SLICE (512ULL << 20) /* guest RAM per pass */ +#define SVC_GR_RAM_PASS_MAX 16 /* slices before giving up */ +#define SVC_GR_RAM_WIN (2ULL << 20) /* allocation window pinned */ + struct svc_channel { bool valid; u32 session_gen; /* m176: session generation this entry was @@ -420,6 +429,49 @@ struct svc_channel { u64 gr_nocap_va; /* push VA the window was opened for */ unsigned long gr_nocap_since; /* absolute first-seen stamp */ /* + * r496b (X/GL kernel-RM sysmem channel): ring discovery by content in + * guest RAM. The CPU RM maps sysmem objects locally (no + * SET_PAGE_DIRECTORY, no rm_mem record carrying the ring VA), so + * svc_lookup_trusted cannot name the ring page; the driver's kickoff + * GPFIFO entries {pushbuffer VA, len} identify it instead. The scan + * is sliced (SVC_GR_RAM_SLICE per pass) and cursor-resumed so the + * service thread stays responsive. + */ + u64 gr_ram_cursor; /* resume point of the content scan */ + u8 gr_ram_passes; /* slices walked so far */ + bool gr_ram_logged; /* one discovery line per channel */ + /* + * r496e: true when the ring was found by the RAM content scan (a + * kernel-RM sysmem channel - the X/GL 3D channel). Such a channel's + * pushes are 2D/3D method streams the emulation does not execute; + * the correct model is consume-and-apply-releases: no QMD capture, + * no launch forward, no no-capture hold. Also allows an all-zero + * NOP push (the driver's pushbuffer sanity test writes exactly one + * zero dword) to be consumed as the no-op it is. + */ + bool gr_ram_ring; + /* + * r496b: the FB-scan USERD (see svc_userd_fb_scan). Latched when the + * registered/ring-derived candidates all fail; the scan validates the + * hit against the ring entries the put claims, so a foreign channel's + * USERD cannot be adopted. Bounded retries: the driver kicks off a + * few passes after channel registration, so a scan before the first + * put must not retire the discovery. + */ + u8 userd_scans; + /* + * r471: the completion-counter publish must never advance the + * guest's word past a push whose content was still unwritten when + * the entry was consumed (the streams_events staging-reuse + * corruption: the publish satisfied the client's D2H wait before + * its copy ran, and the readback drained a staging buffer that + * still held the PREVIOUS copy's bytes). Lowest ring slot consumed + * with all-zero content; 0 = none. Cleared when a later decode of + * a GR push reads non-zero content (the content landed - the rewinds + * re-read entries constantly, so this heals within a pass or two). + */ + u32 gr_bad_put; + /* * STALLFREE r225: stale-epoch / stale-replay wedge watchdog. A * channel that cannot make forward progress (last_get pinned behind * put, or a residual session put that never drops) is detected here @@ -486,6 +538,12 @@ struct svc_vaspace { u64 pdb; u32 entries; /* root entry count */ int ap; /* SVC_APERTURE_* of the directory itself */ + unsigned long stamp; /* r459: last registration/use; LRU evict + * when the table is full (a stale entry from + * a dead client otherwise blocks every new + * client after the 8th registration: the + * mid-boot walk failure that made the 10th+ + * client's copies fail rc=-2) */ }; struct svc_lat { @@ -819,6 +877,7 @@ struct sema_svc { u64 pushtrace_all; /* fail4: de-windowed push decode * trace (every push, bounded) */ u64 opadd_log; /* fail4: every op appended */ + u64 op2d_raw_log; /* r458: raw 2D-shaped copy pushes */ u64 dmatrace_log; /* fail4: denied-transfer trace */ u64 pend_dbg_log; /* fail4 r305: pending/hold witness */ u64 snap_gate_log; /* fail4 r311: capture-gate witness */ @@ -1599,12 +1658,40 @@ void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc) if (!vs && !svc.vas[i].valid) vs = &svc.vas[i]; } + /* + * r459: the table is 8 slots and the guest driver does not always + * send a teardown for a client's VA space (observed: 23 distinct + * registrations, 0 drops in one boot). Once all slots hold a dead + * client's directory, a NEW client's registration found no slot and + * every walk of its VAs failed (copies rc=-2, launches reading + * stale pages - the 10th-client mid-boot degradation). Evict the + * least-recently-registered entry instead of refusing the new one; + * the old client is gone and its directory pointer is dead guest + * memory, so the eviction can only remove stale authority. + */ + if (!vs) { + unsigned long oldest = 0; + int oi = 0; + + for (i = 0; i < SVC_MAX_VASPACES; i++) { + if (!svc.vas[i].valid) + continue; + if (!oldest || time_before(svc.vas[i].stamp, oldest)) { + oldest = svc.vas[i].stamp; + oi = i; + } + } + vs = &svc.vas[oi]; + pr_info("sema-svc: VA space table full; evicting 0x%x (pdb=0x%llx) for 0x%x\n", + vs->h_vaspace, vs->pdb, desc->h_vaspace); + } if (vs) { vs->valid = true; vs->h_vaspace = desc->h_vaspace; vs->pdb = desc->pdb; vs->entries = desc->num_entries; vs->ap = desc->b_vidmem ? SVC_APERTURE_FB : SVC_APERTURE_SYS; + vs->stamp = jiffies; /* A new directory invalidates what we learned from the old one. */ memset(svc.lat, 0, sizeof(svc.lat)); } @@ -1932,6 +2019,26 @@ static int svc_walk_vaspaces(struct nv_vgpu_device *vgpu, u64 va, } spin_unlock_irqrestore(&svc.vas_lock, flags); + /* + * r459: try the NEWEST registration first. A dead client's stale + * directory (the table outlives clients; only 8 slots) can still + * resolve a VA - through guest memory the driver has since reused - + * to a wrong page, and the first success wins. Ordering by the + * registration stamp makes the live client's directory the + * authority and only falls through to older ones when it cannot + * answer. + */ + for (i = 1; i < n; i++) { + struct svc_vaspace k = local[i]; + int j = i - 1; + + while (j >= 0 && time_before(local[j].stamp, k.stamp)) { + local[j + 1] = local[j]; + j--; + } + local[j + 1] = k; + } + for (i = 0; i < n; i++) { int alt; @@ -4069,6 +4176,23 @@ struct svc_wait_op { }; static struct svc_wait_op svc_waits[SVC_WAIT_MAX]; static u32 svc_wait_n; /* acquires captured this entry */ +/* + * r484 THE STAGING-DRAIN COMPLETION EDGE: when the acquire gate holds an + * entry, the op can still have executed on an earlier decode whose gate did + * not trigger (svc_wait_n was 0 on that pass), and the once-only done + * bitmap then suppresses it on the first pass whose gate finally passes - + * the pass at which the dependency just became installed and the op's + * inputs changed. The release then applies and the guest drains a staging + * buffer holding the PRE-dependency bytes (streams_events s=2: the D2H + * copy read dc2 before the vadd_into wrote it, ran early on gate-less + * passes, was once-only-skipped on the passing pass, and the client + * drained a2+b2 instead of a2+b2+c1). Remember the last held (channel, + * entry) and force exactly ONE re-run of its ops on the first passing + * pass. + */ +static u32 svc_wait_hold_h; +static u32 svc_wait_hold_idx = 0xffffffffu; +static bool svc_wait_repass; static u32 svc_wait_ovf; /* acquires dropped past the array */ static u64 svc_wait_held; static u64 svc_wait_pass; @@ -4820,7 +4944,8 @@ static int svc_fill_1d(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u64 *done_out); static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, u32 dst_phys, u32 dst_aperture, - u32 src_phys, u32 src_aperture, u64 *done_out); + u32 src_phys, u32 src_aperture, u64 *done_out, + u32 owner_ch, u32 owner_idx, u64 owner_hash); /* * r402 row-wise wrapper: a 2D (pitched) fill runs the 1D fill once per @@ -5938,12 +6063,29 @@ static void svc_copy_audit_chunk(struct nv_vgpu_device *vgpu, u64 dst_va, struct svc_copy_snap { u64 dst, src; size_t total; + /* + * Owner identity (r444): the channel, ring entry and push-content + * hash of the decode pass that captured this snapshot. A snapshot + * exists to serve ONE entry's retried copy (the held staging read); + * keying it by (dst,src,total) alone let a snapshot captured for an + * earlier entry poison a later copy with the same operand triple - + * the wedge_ab "bad = mutated-element count" class: step N's H2D + * executed from step N-1's snapshot because the re-decode of the + * completed step N-1 entry re-captured its (unchanged) source after + * its copy had already run, and step N's copy then consumed it. + * The pass hash changes whenever the guest rewrites the push, so a + * ring slot reused by a fresh push can never match a stale owner. + */ + u64 owner_hash; + u32 owner_ch; + u32 owner_idx; u8 *buf; unsigned long stamp; }; static struct svc_copy_snap svc_copy_snaps[SVC_COPY_SNAP_N]; -static u8 *svc_copy_snap_lookup(u64 dst, u64 src, size_t total) +static u8 *svc_copy_snap_lookup(u64 dst, u64 src, size_t total, + u32 owner_ch, u32 owner_idx, u64 owner_hash) { u32 i; @@ -5951,12 +6093,16 @@ static u8 *svc_copy_snap_lookup(u64 dst, u64 src, size_t total) if (svc_copy_snaps[i].buf && svc_copy_snaps[i].dst == dst && svc_copy_snaps[i].src == src && - svc_copy_snaps[i].total == total) + svc_copy_snaps[i].total == total && + svc_copy_snaps[i].owner_ch == owner_ch && + svc_copy_snaps[i].owner_idx == owner_idx && + svc_copy_snaps[i].owner_hash == owner_hash) return svc_copy_snaps[i].buf; return NULL; } -static void svc_copy_snap_free(u64 dst, u64 src, size_t total) +static void svc_copy_snap_free(u64 dst, u64 src, size_t total, + u32 owner_ch, u32 owner_idx, u64 owner_hash) { u32 i; @@ -5964,16 +6110,45 @@ static void svc_copy_snap_free(u64 dst, u64 src, size_t total) if (svc_copy_snaps[i].buf && svc_copy_snaps[i].dst == dst && svc_copy_snaps[i].src == src && - svc_copy_snaps[i].total == total) { + svc_copy_snaps[i].total == total && + svc_copy_snaps[i].owner_ch == owner_ch && + svc_copy_snaps[i].owner_idx == owner_idx && + svc_copy_snaps[i].owner_hash == owner_hash) { kfree(svc_copy_snaps[i].buf); svc_copy_snaps[i].buf = NULL; return; } } +/* + * r456/r457: the snapshot lifecycle is capture-once -> free on the copy's + * own successful consumption (svc_copy) -> free on ring-slot reuse with a + * new push hash (svc_copy_snap_capture) -> free when a re-decode skips an + * already-done op (svc_run_ops) -> TTL. The old per-entry owner free was + * removed from every opdef_clear caller: an entry whose release was + * delivered can still be REPLAYED (the pin-batch rewind) with its copy + * still owed, and freeing at that point let the replay re-capture the + * driver's refilled ping-pong staging (the r457 mode-C b1 H2D wrote b2's + * bytes into db1). A stale snapshot can only ever serve its own + * (ch, idx, hash) owner, so the r444 cross-entry poisoning stays closed. + */ +static void __maybe_unused svc_copy_snap_free_owner(u32 owner_ch, u32 owner_idx) +{ + u32 i; + + for (i = 0; i < SVC_COPY_SNAP_N; i++) + if (svc_copy_snaps[i].buf && + svc_copy_snaps[i].owner_ch == owner_ch && + svc_copy_snaps[i].owner_idx == owner_idx) { + kfree(svc_copy_snaps[i].buf); + svc_copy_snaps[i].buf = NULL; + } +} + /* Capture the source of a virtual-operand copy about to be held. */ static void svc_copy_snap_capture(struct nv_vgpu_device *vgpu, u64 dst, - u64 src, size_t total) + u64 src, size_t total, u32 owner_ch, + u32 owner_idx, u64 owner_hash) { struct svc_copy_snap *s = NULL; u8 *buf; @@ -5982,8 +6157,27 @@ static void svc_copy_snap_capture(struct nv_vgpu_device *vgpu, u64 dst, if (!total || total > SVC_COPY_SNAP_MAX) return; - if (svc_copy_snap_lookup(dst, src, total)) + if (svc_copy_snap_lookup(dst, src, total, owner_ch, owner_idx, + owner_hash)) return; /* already captured */ + /* + * r456: evict any snapshot left by the SAME ring slot under a + * DIFFERENT push hash (the slot was reused by a fresh push). The + * surviving-across-retries snapshot (svc_opdef_clear APPLY no + * longer frees it) must never be readable by the next occupant of + * the slot; the pass hash changes whenever the guest rewrites the + * push, so a same-slot different-hash snapshot is by definition + * stale for the new push. + */ + for (i = 0; i < SVC_COPY_SNAP_N; i++) { + if (svc_copy_snaps[i].buf && + svc_copy_snaps[i].owner_ch == owner_ch && + svc_copy_snaps[i].owner_idx == owner_idx && + svc_copy_snaps[i].owner_hash != owner_hash) { + kfree(svc_copy_snaps[i].buf); + svc_copy_snaps[i].buf = NULL; + } + } for (i = 0; i < SVC_COPY_SNAP_N; i++) { if (!svc_copy_snaps[i].buf) { s = &svc_copy_snaps[i]; @@ -6019,10 +6213,14 @@ static void svc_copy_snap_capture(struct nv_vgpu_device *vgpu, u64 dst, s->dst = dst; s->src = src; s->total = total; + s->owner_ch = owner_ch; + s->owner_idx = owner_idx; + s->owner_hash = owner_hash; s->stamp = jiffies; s->buf = buf; - pr_info_ratelimited("vgsp-sema: copy snapshot captured dst=0x%llx src=0x%llx total=%zu first4=%08x %08x %08x %08x\n", - dst, src, total, + pr_info_ratelimited("vgsp-sema: copy snapshot captured dst=0x%llx src=0x%llx total=%zu owner=0x%x/%u/%016llx first4=%08x %08x %08x %08x\n", + dst, src, total, owner_ch, owner_idx, + owner_hash, total >= 4 ? *(const u32 *)buf : 0, total >= 8 ? *(const u32 *)(buf + 4) : 0, total >= 12 ? *(const u32 *)(buf + 8) : 0, @@ -6039,7 +6237,8 @@ static const char *svc_gr_src_probe(struct nv_vgpu_device *vgpu, u64 va, u64 *pa_out, u32 *ap_out); static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, u32 dst_phys, u32 dst_aperture, u32 src_phys, u32 src_aperture, - u64 *done_out, u32 rows, u32 dpitch, u32 spitch) + u64 *done_out, u32 rows, u32 dpitch, u32 spitch, + u32 owner_ch, u32 owner_idx, u64 owner_hash) { if (rows > 1 && dpitch) { u64 done = 0; @@ -6052,7 +6251,8 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, rc = svc_copy_1d(vgpu, dst + (u64)r * dpitch, src + (u64)r * spitch, total, dst_phys, dst_aperture, src_phys, - src_aperture, &row_done); + src_aperture, &row_done, + owner_ch, owner_idx, owner_hash); done += row_done; if (rc) break; @@ -6062,12 +6262,13 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, return rc; } return svc_copy_1d(vgpu, dst, src, total, dst_phys, dst_aperture, - src_phys, src_aperture, done_out); + src_phys, src_aperture, done_out, + owner_ch, owner_idx, owner_hash); } static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, u32 dst_phys, u32 dst_aperture, u32 src_phys, u32 src_aperture, - u64 *done_out) + u64 *done_out, u32 owner_ch, u32 owner_idx, u64 owner_hash) { u8 bounce[256]; size_t o; @@ -6098,7 +6299,8 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot svc.cur_ch, dst, src, total, dst_phys, dst_aperture, src_phys, src_aperture); } - snap = svc_copy_snap_lookup(dst_orig, src_orig, total); + snap = svc_copy_snap_lookup(dst_orig, src_orig, total, owner_ch, + owner_idx, owner_hash); if (!dst_phys) { u64 pa; @@ -6232,7 +6434,8 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot */ if (!src_phys && src_orig >= SVC_VAS_FLOOR) svc_copy_snap_capture(vgpu, dst_orig, src_orig, - total); + total, owner_ch, owner_idx, + owner_hash); return -ENOENT; } /* @@ -6261,7 +6464,9 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot dst_orig, dst_prov, ipa); if (!src_phys && src_orig >= SVC_VAS_FLOOR) svc_copy_snap_capture(vgpu, dst_orig, - src_orig, total); + src_orig, total, + owner_ch, owner_idx, + owner_hash); return -ENOENT; } } @@ -7036,7 +7241,8 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot svc_ledger_store(dst_orig + po, dst + po, cap); } if (snap) - svc_copy_snap_free(dst_orig, src_orig, total); + svc_copy_snap_free(dst_orig, src_orig, total, owner_ch, + owner_idx, owner_hash); if (done_out) *done_out = (u64)o; /* o == total on the success path */ return 0; @@ -7238,12 +7444,15 @@ static struct svc_wait_hold *svc_wait_hold_get(u32 h, bool create) } static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, - u32 n_ops, u32 ch_h, u64 *ops_done, u32 ops_base) + u32 n_ops, u32 ch_h, u64 *ops_done, u32 ops_base, + u32 entry_idx, u64 pass_hash) { u32 i; static u64 bigop_log; svc.cur_ch = ch_h; + /* r484: one-shot per call; set only by the hold->pass transition. */ + svc_wait_repass = false; /* * m163: apply every queued scanner injection before resolving this * pass's copy operands. Without this, a CE copy decoded in the window @@ -7284,7 +7493,6 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, struct svc_wait_hold *gh; bool hold = false; u32 wi; - /* * Kill switch: a wait that never satisfies (a counter our * emulation does not advance the way the driver expects) @@ -7369,9 +7577,22 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, } if (hold) { svc_wait_held++; + svc_wait_hold_h = ch_h; + svc_wait_hold_idx = entry_idx; svc.op_tr_pending = 1; return; } + /* + * r484: the gate just PASSED for an entry it previously held - + * the dependency is installed now, so this pass's ops must + * re-run even if the once-only bitmap says they already did + * (they ran against the pre-dependency state). + */ + if (ch_h == svc_wait_hold_h && + entry_idx == svc_wait_hold_idx) { + svc_wait_repass = true; + svc_wait_hold_idx = 0xffffffffu; + } svc_wait_pass++; if (gh) gh->active = false; @@ -7389,8 +7610,21 @@ svc_wait_done: * ran; re-running a CE copy would read a staging buffer the * client has since refilled and overwrite finished data. */ - if (ops_done && seq < 64 && ((*ops_done >> seq) & 1)) { + if (!svc_wait_repass && ops_done && seq < 64 && + ((*ops_done >> seq) & 1)) { svc.ops_once_skipped++; + /* + * r456: this op already ran in an earlier pass. Any + * staging snapshot a later re-decode pass re-captured + * for it (the capture sites run on every decode) is + * a leak that the r444 APPLY-free used to sweep; with + * that free removed (r456 keeps the snapshot alive + * across retries) drop it here so it can never serve + * a re-push of the ring slot. + */ + if (m->type == 1 && m->bytes <= SVC_COPY_SNAP_MAX) + svc_copy_snap_free(m->dst, m->src, m->bytes, + ch_h, entry_idx, pass_hash); continue; } @@ -7404,7 +7638,8 @@ svc_wait_done: m->phys, m->aperture, m->src_phys, m->src_aperture, &ce_done, m->rows, m->dpitch, - m->spitch); + m->spitch, ch_h, entry_idx, + pass_hash); else rc = svc_fill(vgpu, m->dst, m->value, m->elem, m->count, m->phys, m->aperture, @@ -8822,6 +9057,87 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, static u64 svc_pass_hash; /* r391f: this decode pass's push content hash */ +/* + * r481 RISK-REDUCTION diagnostic (default 0): dump the full decoded method + * stream of the first few large GR-class pushes (len >= 1024) so the launch + * push's acquire/release method groups (or their absence) are visible + * verbatim. The streams_events class evidence shows the st2 vadd's copy-in + * running ~115 ms BEFORE its stream's H2D soft copies; the ordering the + * driver relied on is only provable by reading the push's own methods. + * Bounded: gsp_sema_push_raw dumps, at most once per push VA. + */ +unsigned gsp_sema_push_raw; +static u64 svc_pushraw_last_va; +static u32 svc_pushraw_n; + +static void svc_push_raw_dump(const u8 *buf, u32 len, u64 push_va, + u32 h_object) +{ + u32 off = 0, gi = 0; + + if (!gsp_sema_push_raw || svc_pushraw_n >= gsp_sema_push_raw) + return; + if (len < 512 || len > 8192) + return; + if (push_va == svc_pushraw_last_va) + return; + svc_pushraw_last_va = push_va; + svc_pushraw_n++; + pr_info("sema-pushraw: BEGIN h=0x%x va=0x%llx len=%u n=%u\n", + h_object, push_va, len, svc_pushraw_n); + while (off + 4 <= len && gi < 160) { + u32 hdr, op, cnt, subch, addr, v[4] = {0,0,0,0}, k, n; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 0x7; + cnt = (hdr >> 16) & 0xfff; + subch = (hdr >> 13) & 0x7; + addr = (hdr & 0x1fff) << 2; + n = cnt > 4 ? 4 : cnt; + for (k = 0; k < n && off + 8 + 4 * k <= len; k++) + memcpy(&v[k], buf + off + 4 + 4 * k, 4); + pr_info("sema-pushraw: [%u] off=0x%x op=%u sub=%u addr=0x%x cnt=%u v0=%08x v1=%08x v2=%08x v3=%08x\n", + gi, off, op, subch, addr, cnt, + v[0], v[1], v[2], v[3]); + if (op == 3 && cnt) { + off += 4 + cnt * 4; + gi++; + continue; + } + if (!cnt) { + gi++; + off += 4; + continue; + } + if (op == 1) { + off += 4 + cnt * 4; + gi++; + continue; + } + { + u32 p; + bool found = false; + + for (p = off + 4; p + 4 <= len; p += 4) { + u32 h2; + + memcpy(&h2, buf + p, 4); + if (((h2 >> 29) & 0x7) == 1 && + ((h2 >> 16) & 0xfff)) { + off = p; + found = true; + break; + } + } + if (!found) + off = len; + gi++; + } + } + pr_info("sema-pushraw: END h=0x%x va=0x%llx consumed=0x%x/%u\n", + h_object, push_va, off, len); +} + static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u32 len, u64 page_pa, bool allow_ops, struct svc_channel *ch, u32 entry_idx, @@ -8886,6 +9202,13 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, allow_ops = false; } + /* + * r481: verbatim method-stream dump for large GR-class pushes + * (default 0 = off). See svc_push_raw_dump. + */ + if (gsp_sema_push_raw && ch && !ch->engine_type && len >= 512) + svc_push_raw_dump(buf, len, push_va, ch->h_object); + memset(&st, 0, sizeof(st)); st.push_va = push_va; /* @@ -9385,7 +9708,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * REPORT_SEM comment below. */ if (allow_ops && n_ops && !gsp_sema_skip_ops) { svc_run_ops(vgpu, ops, n_ops, ch->h_object, - ops_done, ops_seq - n_ops); + ops_done, ops_seq - n_ops, + entry_idx, svc_pass_hash); svc.ring_ops++; ops_runs++; } @@ -9585,7 +9909,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, */ if (allow_ops && n_ops && !gsp_sema_skip_ops) { svc_run_ops(vgpu, ops, n_ops, ch->h_object, - ops_done, ops_seq - n_ops); + ops_done, ops_seq - n_ops, + entry_idx, svc_pass_hash); svc.ring_ops++; ops_runs++; } @@ -9809,6 +10134,23 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, break; case CE_M_LINE_LENGTH_IN: st.line_length = v0; + /* + * r458 THE memset2d root cause: the driver encodes + * LINE_LENGTH_IN (0x418) and the adjacent LINE_COUNT + * (0x41c) as ONE count=2 method group - observed raw + * push bytes `20028106 00001000 00000100` = header + * (subch 4, count 2, addr 0x418), line length 4096, + * line count 0x100 = 256. Reading only v0 left + * line_count = 0, so the 2D-copy builder below took + * the single-row path: one 4 KB op per push, and a + * 4 MB cudaMemcpy2D delivered exactly one page per + * 1 MB block (memset2d probe: per-MB ok=4096, + * bad=12.5M/12.58M). The GRCE variant below has + * always decoded the count=2 form; the CE path did + * not. + */ + if (count >= 2) + st.line_count = v1; break; /* r402: 2D geometry (cudaMemset2D / cudaMemcpy2D). */ case CE_M_LINE_COUNT: @@ -10000,18 +10342,45 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * r402: a 2D fill (cudaMemset2D): * LINE_COUNT rows of * LINE_LENGTH_IN bytes at - * PITCH_OUT stride. Only a pitch - * WIDER than the row selects row - * mode; a flat fill carries - * pitch == row length (or no - * pitch method) and keeps the - * historical single-run path. + * PITCH_OUT stride. A pitch WIDER + * than the row selects row mode; a + * flat fill carries pitch == row + * length (or no pitch method) and + * keeps the historical single-run + * path. + * + * r420c: LINE_COUNT is + * authoritative for the flat case + * too - cudaMemset2D with pitch == + * width (victory_memset2d, + * pitch=4096/w=4096) is CONTIGUOUS + * rows, so the single run must + * cover rows*line bytes. The + * historical path filled one row + * and the readback lost the rest + * (bad ~= 12.5M of 12.58M). */ - if (st.have_pitch_out && - st.line_count > 1 && - st.pitch_out > st.line_length) { - m->rows = st.line_count; - m->dpitch = st.pitch_out; + if (st.line_count > 1) { + u32 fpitch = st.have_pitch_out ? + st.pitch_out : + st.line_length; + + if (fpitch > st.line_length) { + m->rows = + st.line_count; + m->dpitch = fpitch; + } else if (st.line_length % + m->elem) { + m->elem = 1; + m->count = + st.line_length * + st.line_count; + } else { + m->count = + (st.line_length / + m->elem) * + st.line_count; + } } if (m->rows > 1 && svc_pitch_log < 32) { svc_pitch_log++; @@ -10053,20 +10422,74 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * LINE_COUNT rows of * LINE_LENGTH_IN bytes with * PITCH_IN/PITCH_OUT strides. - * Row mode only when a pitch is - * wider than the row (a flat 1D - * copy keeps pitch == length). + * Row mode when a pitch is + * wider than the row. + * + * r420c: pitch == row length + * (or no pitch method) makes the + * rows CONTIGUOUS, so the single + * flat run must cover + * rows*line bytes - the + * historical path copied one row + * (victory_memset2d's pitched + * D2H/H2D at pitch 4096 = width + * 4096 lost all but the first + * row). */ - if (st.line_count > 1 && - st.have_pitch_out && - st.pitch_out > st.line_length) { - m->rows = st.line_count; - m->dpitch = st.pitch_out; - m->spitch = st.have_pitch_in ? + if (st.line_count > 1) { + u32 dpitch = st.have_pitch_out ? + st.pitch_out : + st.line_length; + u32 spitch = st.have_pitch_in ? (st.pitch_in ? st.pitch_in : - st.pitch_out) : - st.pitch_out; + dpitch) : + dpitch; + + if (dpitch == st.line_length && + spitch == st.line_length) { + m->bytes = + st.line_length * + st.line_count; + } else { + m->rows = + st.line_count; + m->dpitch = dpitch; + m->spitch = spitch; + } + } + /* + * r458 memset2d forensics: the + * driver's cudaMemcpy2D pushes decode + * LINE_LENGTH=4096 with PITCH_IN/OUT + * present but LINE_COUNT absent + * (lc=0), so only one 4 KB row per + * push executes and a 4 MB copy + * delivers 4 pages. Dump the raw push + * (bounded) for every pitch-shaped + * copy so the method stream that + * carries the row count is on record. + */ + if (gsp_sema_dbg && + (st.have_pitch_out || + st.have_pitch_in) && + svc.op2d_raw_log < 12) { + u32 dl = len < 220 ? len : 220; + + svc.op2d_raw_log++; + pr_info("sema-op2d-raw: h=0x%x push=0x%llx len=%u off=0x%x line=%u lc=%u po=%u/%u pi=%u/%u out=0x%llx in=0x%llx bytes=%u dma=0x%08x\n", + ch->h_object, push_va, + len, off, + st.line_length, + st.line_count, + st.have_pitch_out, + st.pitch_out, + st.have_pitch_in, + st.pitch_in, + m->dst, m->src, + (u32)m->bytes, dma); + pr_info("sema-op2d-raw: raw %*phN\n", + dl, buf); } n_ops++; ops_seq++; @@ -10247,7 +10670,10 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, svc_copy_snap_capture(vgpu, st.offset_out, st.offset_in, - st.line_length); + st.line_length, + ch->h_object, + entry_idx, + svc_pass_hash); } /* * r308: the `!replay` gate above kept this @@ -10338,7 +10764,10 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, svc_copy_snap_capture(vgpu, lm->dst, lm->src, - lm->bytes); + lm->bytes, + ch->h_object, + entry_idx, + svc_pass_hash); if (gsp_sema_dbg && svc.snap_gate_log < 256) { svc.snap_gate_log++; @@ -10376,6 +10805,27 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, lm->elem, lm->count, lm->aperture, lm->src_aperture, lm->phys, lm->src_phys); + /* + * r420d: 2D geometry of every copy op + * (the memset2d readback forensics - + * the driver decomposes cudaMemcpy2D + * into per-push ops whose LINE_COUNT/ + * PITCH decode must be visible to + * tell "one row per op" from "N rows + * in one op"). Bounded. + */ + if (lm->type == 1) + pr_info("sema-op2d: h=0x%x dst=0x%llx src=0x%llx bytes=%u rows=%u dpitch=%u spitch=%u line=%u lc=%u have_po=%u po=%u have_pi=%u pi=%u\n", + ch->h_object, lm->dst, + lm->src, lm->bytes, + lm->rows, lm->dpitch, + lm->spitch, + st.line_length, + st.line_count, + st.have_pitch_out, + st.pitch_out, + st.have_pitch_in, + st.pitch_in); } } @@ -10638,7 +11088,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, gsp_sema_skip_ops); } svc_run_ops(vgpu, ops, n_ops, ch->h_object, - ops_done, ops_seq - n_ops); + ops_done, ops_seq - n_ops, + entry_idx, svc_pass_hash); svc.ring_ops++; ops_runs++; } @@ -12588,6 +13039,84 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, return 0; } } + + /* + * r496b: no candidate matched. For a GR channel whose ring is known + * (translated and pinned - the sysmem content scan or the VRAM + * conventions), the USERD may still be elsewhere: the X/GL driver + * allocates its USERD in VRAM and the CPU RM maps it through a BAR1 + * window, so the driver's GP_PUT write lands at a BAR1 offset that is + * neither the registered memdesc base nor ring-derived. Scan the + * emulated VRAM for the USERD shape and validate each hit against the + * ring: the put it claims must correspond to entries whose pushbuffer + * VAs all sit in the ring's allocation window. A foreign channel's + * USERD cannot pass that test. One scan per channel (the result + * persists through the pump's write-back), logged on latch. + */ + if (!ch->engine_type && ch->userd_scans < 8 && ring_pa) { + u64 off, lim; + u8 *base = vgpu->bars[1].vaddr; + + ch->userd_scans++; + lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x20000000ULL); + if (base && lim) { + for (off = 0; off + 0xa0 <= lim; off += 0x200) { + u32 hp, hg; + bool ok = true; + u32 k; + + memcpy(&hp, base + off + SVC_USERD_GP_PUT, 4); + memcpy(&hg, base + off + SVC_USERD_GP_GET, 4); + if (!hp || hp > ch->gpfifo_entries || hg > hp) + continue; + /* Validate: every claimed entry decodes to a + * pushbuffer VA in the ring's window. */ + for (k = 0; k < hp && k < 64; k++) { + u8 e[8]; + u64 eva; + u32 elen, a0, a1; + + if (svc_read_va(vgpu, + ch->gpfifo_va + + (u64)k * 8, + e, sizeof(e))) { + ok = false; + break; + } + memcpy(&a0, e, 4); + memcpy(&a1, e + 4, 4); + eva = ((u64)(a1 & 0xff) << 32) | + (u64)(a0 & 0xfffffffc); + elen = ((a1 >> 10) & 0x1fffff) << 2; + /* + * Every entry the put claims must name a + * VA in the ring's allocation window - + * pushbuffers below the ring, the + * progress tracker above it. + */ + if (!eva || + eva + SVC_GR_RAM_WIN / 2 < + ch->gpfifo_va || + eva >= ch->gpfifo_va + + SVC_GR_RAM_WIN / 2 || + elen < 4 || elen > 0x100000) { + ok = false; + break; + } + } + if (!ok) + continue; + ch->userd_pa = off; + ch->userd_as = SVC_APERTURE_FB; + ch->userd_ok = true; + *put = hp; + *get = hg; + pr_info("sema-chan: GR USERD h=0x%x rank=%u found by VRAM shape scan at fb 0x%llx (put=%u get=%u, ring-validated)\n", + ch->h_object, ch->gr_index, off, hp, hg); + return 0; + } + } + } return -ENOENT; } @@ -13396,20 +13925,28 @@ bool gsp_sema_pool_hold_fresh = 1; bool gsp_sema_copy_dst_pin_guard = 1; /* * r336/r34x wait gate: honor the C56F host-semaphore acquire ops captured - * for a ring entry before running its work. Still DEFAULT OFF (opt-in - * experiment; the module param sema_wait_gate=1 enables it), but the - * semantics are now the class contract's: op 0 = equal, 2 = strict GEQ, - * 3 = circular GEQ, 4 = AND, 5 = NOR; the wait set is captured PER ENTRY - * (the old single-last-wait global leaked a later entry's acquire into an - * earlier entry - the r340 over-hold); the entry's own release counts only - * when its method order precedes the acquire; pending releases our - * machinery is merely late about are drained first; and every hold is - * bounded at SVC_WAIT_HOLD_MS (100 ms) then fails open loudly ('give-up') - * so a value that can only be installed behind an upstream opdef hold - * cannot park a channel. Diagnostics: 'sema-wait: pass/hold/give-up' rows - * carry op, payload, observed value and decision. - */ -bool gsp_sema_wait_gate; + * for a ring entry before running its work. DEFAULT ON since r408: the + * r408 streams2d forensics reproduced the class the gate was built for + * with the gate off - the c2 readback (C56F ACQ_CIRC_GEQ pay=4 on the + * scale channel's progress counter 0x20400ffd0) ran at 106617.279 with + * cur=2, i.e. BEFORE the c2-add launch was even submitted (106617.411), + * read the not-yet-produced buffer (zeros) and the client failed with + * 'c2 all zeros'. With the gate on the same readback holds (sema-wait: + * hold ... cur=2 waited_ms=353) until the counter reaches 4 (pass ... + * cur=4 at 106812.259) and the ordering is correct. The semantics are + * the class contract's: op 0 = equal, 2 = strict GEQ, 3 = circular GEQ, + * 4 = AND, 5 = NOR; the wait set is captured PER ENTRY (the old + * single-last-wait global leaked a later entry's acquire into an earlier + * entry - the r340 over-hold); the entry's own release counts only when + * its method order precedes the acquire; pending releases our machinery + * is merely late about are drained first; and every hold is bounded at + * gsp_sema_wait_hold_ms (default 4000 ms, r401) then fails open loudly + * ('give-up') so a value that can only be installed behind an upstream + * opdef hold cannot park a channel. Diagnostics: 'sema-wait: + * pass/hold/give-up' rows carry op, payload, observed value and decision. + * sema_wait_gate=0 restores the pre-r336 pass-through for an A/B. + */ +bool gsp_sema_wait_gate = 1; /* * m169b: the guard above must never deadlock the boot's first client. The @@ -13615,6 +14152,139 @@ unsigned gsp_sema_push_hold_ms = 2500; static unsigned long svc_gr_ring_tried; /* ranks that got a diagnostic line */ +static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 *pa_out) +{ + static u8 *buf; + u64 start = ch->gr_ram_cursor; + u64 end = start + SVC_GR_RAM_SLICE; + u64 gpa; + int i; + unsigned long flags; + + if (end > SEMA_PA_MAX) + end = SEMA_PA_MAX; + if (!buf) { + buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); + if (!buf) + return false; + } + for (gpa = start; gpa + SEMA_PAGE_SIZE <= end; + gpa += SVC_SCAN_CHUNK) { + u64 chunk = min_t(u64, SVC_SCAN_CHUNK, end - gpa); + u32 o; + + if (svc_read(vgpu, gpa, buf, chunk)) + continue; + for (o = 0; o + 16 <= chunk; o += SEMA_PAGE_SIZE) { + u32 e0, e1, f0, f1; + u64 va0, va1; + u32 len0, len1; + + memcpy(&e0, buf + o, 4); + memcpy(&e1, buf + o + 4, 4); + va0 = ((u64)(e1 & 0xff) << 32) | + (u64)(e0 & 0xfffffffc); + len0 = ((e1 >> 10) & 0x1fffff) << 2; + /* entry0: main pushbuffer, just below the ring */ + if (va0 >= ch->gpfifo_va || + va0 + SVC_GR_RAM_WIN / 2 < ch->gpfifo_va) + continue; + if (va0 < SEMA_VA_LO || va0 >= SEMA_VA_HI) + continue; + if (len0 < 4 || len0 > 0x100000) + continue; + memcpy(&f0, buf + o + 8, 4); + memcpy(&f1, buf + o + 12, 4); + va1 = ((u64)(f1 & 0xff) << 32) | + (u64)(f0 & 0xfffffffc); + len1 = ((f1 >> 10) & 0x1fffff) << 2; + /* entry1: progress tracker, above entry0 */ + if (va1 <= va0 || + va1 > ch->gpfifo_va + SVC_GR_RAM_WIN / 2) + continue; + if (len1 < 4 || len1 > 0x100000) + continue; + *pa_out = gpa + o; + return true; + } + } + ch->gr_ram_cursor = (end >= SEMA_PA_MAX) ? 0 : end; + ch->gr_ram_passes++; + if (ch->gr_ram_cursor == 0 && !ch->gr_ram_logged) { + ch->gr_ram_logged = true; + pr_info("sema-chan: GR ring h=0x%x rank=%u RAM content scan found no candidate in %u slice(s) (va=0x%llx)\n", + ch->h_object, ch->gr_index, ch->gr_ram_passes, + ch->gpfifo_va); + } + /* + * Persist the cursor immediately: the pump's stalled-userd path + * `continue`s past the end-of-pass write-back, so without this the + * scan would restart from slice 0 every pass and never reach the + * ring's page. + */ + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].h_object == ch->h_object) { + svc.chan[i].gr_ram_cursor = ch->gr_ram_cursor; + svc.chan[i].gr_ram_passes = ch->gr_ram_passes; + svc.chan[i].gr_ram_logged = ch->gr_ram_logged; + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + return false; +} + +/* + * r496f: discover a kernel-RM client VAS's page directory in the emulated + * VRAM by verification. The CPU RM builds internal-VAS page tables in + * vidmem and never publishes them (SET_PAGE_DIRECTORY is an externally + * owned / UVM-only RPC), so the pushbuffer pages and the progress + * semaphore of an X/GL channel cannot be translated from any existing + * source. Try each 4 KB page as the walk root and accept the first whose + * walk reproduces the ring VA -> ring PA pair we already validated by + * content; registering it with the VAS table makes every other VA of the + * same client resolvable through the existing walker. Bounded to the + * vidmem shadow; verified, so a wrong candidate cannot be registered. + */ +static void svc_xvas_pdb_discover(struct nv_vgpu_device *vgpu, u64 ring_va, + u64 ring_pa) +{ + static const u32 ent_tab[] = { 4, 8, 16, 32, 64, 2, 128, 256, 512 }; + u64 lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x20000000ULL); + u64 off; + unsigned int ei; + + if (!vgpu->bars[1].vaddr || !lim) + return; + for (off = 0; off + 0x1000 <= lim; off += 0x1000) { + for (ei = 0; ei < ARRAY_SIZE(ent_tab); ei++) { + u64 pa = 0; + int ap = 0; + + if (!svc_gmmu_walk(vgpu, off, ent_tab[ei], + ring_va & ~0xfffULL, + SVC_APERTURE_FB, &pa, &ap, false) && + (pa & ~0xfffULL) == (ring_pa & ~0xfffULL)) { + struct sema_vaspace_desc d; + + memset(&d, 0, sizeof(d)); + d.h_vaspace = 0x5a5a0000u | + (u32)(ring_va >> 20); + d.pdb = off; + d.num_entries = ent_tab[ei]; + d.b_vidmem = true; + gsp_sema_service_vaspace_add(&d); + pr_info("sema-chan: client VAS page directory discovered pdb=0x%llx entries=%u (ring VA 0x%llx -> pa 0x%llx verified; the client's other VAs are now walkable)\n", + off, ent_tab[ei], ring_va, ring_pa); + return; + } + } + } +} + static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { @@ -13891,6 +14561,119 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, pr_info("sema-chan: fbs scan done h=0x%x hits=%d (lim 0x%llx)\n", ch->h_object, hits, lim); } + + /* + * r495 diagnostic (X/GL channel): the X driver's GR channel + * registers its USERD at a memdesc base that may be an FB + * offset or a BAR1 window. The driver's kickoff writes GP_PUT + * through its CPU mapping of that object; if the write landed + * in the emulated FB shadow, scanning for the USERD shape + * (GP_PUT in [1..entries] at +0x8c, GP_GET <= GP_PUT at +0x90, + * both nonzero somewhere in the 0x200 block) names the real + * offset. One-shot per channel, diagnostic only. + */ + if (gsp_sema_dbg && vgpu->bars[1].vaddr && + svc_fb_shadow_size(vgpu) && ch->gpfifo_entries <= 0x20000) { + u64 lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x20000000ULL); + u64 off; + int hits = 0; + + for (off = 0; off + 0xa0 <= lim && hits < 8; + off += 0x200) { + u32 hp, hg; + + memcpy(&hp, vgpu->bars[1].vaddr + off + 0x8c, 4); + memcpy(&hg, vgpu->bars[1].vaddr + off + 0x90, 4); + if (!hp || hp > ch->gpfifo_entries || hg > hp) + continue; + pr_info("sema-chan: userd scan hit off=0x%llx put=%u get=%u (blk %08x %08x %08x %08x)\n", + off, hp, hg, + *(u32 *)(vgpu->bars[1].vaddr + off + 0x80), + *(u32 *)(vgpu->bars[1].vaddr + off + 0x84), + *(u32 *)(vgpu->bars[1].vaddr + off + 0x88), + *(u32 *)(vgpu->bars[1].vaddr + off + 0x94)); + hits++; + } + pr_info("sema-chan: userd scan done h=0x%x hits=%d (lim 0x%llx)\n", + ch->h_object, hits, lim); + } + } + + /* + * r496b: guest-RAM content scan for the ring (see the gr_ram_cursor + * field comment). The driver's kickoff writes GPFIFO entries whose + * pushbuffer VAs sit just BELOW the ring VA in the same allocation + * (main pushbuffer first, GPFIFO after - nvidia-push-priv.h): entry0 + * in (ring_va - 0x100000, ring_va), entry1 above entry0 and up to + * ring_va + 0x100000 (the progress tracker). A page whose first two + * 8-byte words both name such VAs with sane lengths is the ring; pin + * it (validated) and its allocation window (guesses) so the ring + * entry and pushbuffer reads resolve. Sliced so the service thread + * stays responsive; gives up per channel after + * SVC_GR_RAM_PASS_MAX slices. + */ + if (ch->gr_ram_passes < SVC_GR_RAM_PASS_MAX && + !test_bit(ch->gr_index, &svc_gr_ring_done)) { + u64 hit_pa = 0; + + if (svc_gr_ram_ring_scan(vgpu, ch, &hit_pa)) { + u64 win_va = ch->gpfifo_va - SVC_GR_RAM_WIN / 2; + u64 win_pa = hit_pa - SVC_GR_RAM_WIN / 2; + u64 off; + + svc_pin_store(ch->gpfifo_va, hit_pa, + SVC_APERTURE_SYS); + svc_lat_store(ch->gpfifo_va, hit_pa, + SVC_APERTURE_SYS); + for (off = 0; off < SVC_GR_RAM_WIN; + off += SEMA_PAGE_SIZE) { + svc_pin_store_win(win_va + off, win_pa + off, + SVC_APERTURE_SYS); + svc_lat_store(win_va + off, win_pa + off, + SVC_APERTURE_SYS); + } + ch->gr_ram_logged = true; + ch->gr_ram_ring = true; + set_bit(ch->gr_index, &svc_gr_ring_done); + svc_ring_n++; + /* + * r496f: with the ring content-validated, discover the + * client VAS's page directory in vidmem so the + * channel's other VAs (pushbuffer pages, the progress + * semaphore the driver polls) resolve through the + * walker. Runs before the pins below matter and only + * once per channel (the bitmap is set). + */ + svc_xvas_pdb_discover(vgpu, ch->gpfifo_va, hit_pa); + /* + * Persist the latch immediately: a later stalled + * path in this pass may `continue` past the + * end-of-pass write-back, and the next pass's + * snapshot must carry gr_ram_ring (the zero-NOP + * exception and the no-capture-hold skip key on + * it). + */ + { + unsigned long flags2; + int i2; + + spin_lock_irqsave(&svc.chan_lock, flags2); + for (i2 = 0; i2 < SVC_MAX_CHANNELS; i2++) + if (svc.chan[i2].valid && + svc.chan[i2].vgpu == vgpu && + svc.chan[i2].h_object == + ch->h_object) { + svc.chan[i2].gr_ram_ring = true; + svc.chan[i2].gr_ram_logged = true; + break; + } + spin_unlock_irqrestore(&svc.chan_lock, flags2); + } + pr_info("sema-chan: GR ring h=0x%x rank=%u found by RAM content scan va=0x%llx -> pa=0x%llx (allocation window pinned; entry/push reads resolve through it)\n", + ch->h_object, ch->gr_index, ch->gpfifo_va, + hit_pa); + return; + } } } @@ -13898,8 +14681,7 @@ static void svc_ring_autopin(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { u64 pa = 0, va = 0; - int ap = SVC_APERTURE_SYS; - u8 e[SVC_GPFIFO_ENTRY_SIZE]; + int ap = SVC_APERTURE_SYS; u8 e[SVC_GPFIFO_ENTRY_SIZE]; u32 e0, e1, len; if (!ch->engine_type) { @@ -14810,6 +15592,25 @@ static void svc_counter_publish(struct nv_vgpu_device *vgpu, return; val = put; + /* + * r471: never advance the guest's completion word past a push whose + * content was still unwritten when its entry was consumed. The + * clamp position is the lowest all-zero GR slot (gr_bad_put = slot + * + 1; 0 = none). Without it the publish can satisfy a D2H wait + * before its copy ran and the client drains a reused staging buffer + * holding the previous copy's bytes (streams_events h_c[2] = c0). + * The clamp heals when the rewinds re-read the slot with real + * content; the real completions still arrive through the decoded + * releases. + */ + if (!ch->engine_type && ch->gr_bad_put && val >= ch->gr_bad_put) { + val = ch->gr_bad_put - 1; + if (gsp_sema_dbg && svc.counter_pub_dbg < 96) { + svc.counter_pub_dbg++; + pr_info("sema-diag: pub-clamp h=0x%x pa=0x%llx val=%u (bad=%u) cur=%u\n", + ch->h_object, pa, val, ch->gr_bad_put, cur); + } + } if (val == ch->counter_last) return; /* @@ -15428,6 +16229,19 @@ static void svc_fwd_soft_copy_inline(struct nv_vgpu_device *vgpu) * hardware has not produced yet. */ #define SVC_FWD_GR_HOLD_TIMEOUT_MS 12000u +/* + * r420: hard cap on the WEDGE-QUIET deferral of the GR hold deadline. The + * backend's quiet hold is a bounded msleep (its own caps: <=240 s per hold, + * two holds per launch), so the pause can only last a few minutes; the cap is + * a defence-in-depth bound so a stuck quiet-hold flag can never defer the + * client's completion indefinitely. + */ +#define SVC_FWD_GR_HOLD_PAUSE_MAX_MS 600000u +/* + * r420: same defence for the per-launch consumption bound pause in + * svc_gr_hold_bound_expired(). + */ +#define SVC_GR_QUIET_PAUSE_MAX_MS 600000u struct svc_fwd_hold { bool active; @@ -15463,9 +16277,9 @@ struct svc_fwd_hold { /* * P6c completion relay: the request selected record relay_index and * the hardware copy was asked to author it into the guest-visible - * semaphore page. On the success edge the hold re-reads the guest - * word and, when it carries the payload, skips that one record's - * software apply. Cleared with the hold. + * semaphore page. On the success edge the hold re-reads the guest word + * and, when it carries the payload, skips that one record's software + * apply. Cleared with the hold. */ bool relay_armed; u8 relay_index; @@ -15478,6 +16292,18 @@ struct svc_fwd_hold { struct svc_fwd_rel rel[SVC_FWD_REL_MAX]; u64 last_rel_va; unsigned long since; + /* + * r420: WEDGE-QUIET deferral state. While the backend deliberately + * holds the in-flight launch in its bounded quiet window + * (gsp_phys_gr_quiet_hold_active), the GR deadline is pushed forward + * (paused_until) so the hold cannot author the launch's release word + * under the hold - the client would then verify data the post-hold + * copy-out has not written yet (the r419 mid-boot escape). The + * deferral is itself bounded by SVC_FWD_GR_HOLD_PAUSE_MAX_MS from + * `since` and is announced once per hold. + */ + unsigned long paused_until; + bool pause_logged; }; static struct svc_fwd_hold svc_fwd_hold; @@ -15703,6 +16529,29 @@ static void svc_opdef_clear(struct svc_opdef_ent *e, u32 caller) e->n ? e->rel[0].va : 0ull, e->n ? e->rel[0].payload : 0u); } + /* + * r456/r457: the snapshot lifecycle is now "capture once, consume + * on the copy's success, evict on ring-slot reuse, TTL". Every + * opdef_clear caller can still re-decode the entry: + * - APPLY (0): the pass's own op execution (which consumes the + * snapshot on success) follows the apply; + * - FINISH (1): the entry can be REPLAYED after a consume (the + * pin-batch rewind replays entries while mappings stream in), + * and an owed copy still needs the pipeline-order bytes; + * - FLUSH (2): the op stays skipped this pass but a later replay + * may run it. + * The r457 mode-C evidence: the b1 H2D's snapshot was captured at + * 191607.816388 with b1's bytes (first4=00000000 3e800000 ...), + * FINISH freed it on the entry's consume, the rewind replay + * re-captured at 191608.798838 from the REFILLED staging + * (first4=00000000 3d800000 ... = b2) and the deferred execution + * wrote b2's bytes into db1 (dst16_after=000000000000803d...). + * The r444 leak the old free guarded against is already covered + * by the owner key (a stale snapshot can only ever serve its own + * (ch, idx, hash)), the slot-reuse eviction in + * svc_copy_snap_capture, the ops_done skip free in svc_run_ops + * and the 30 s TTL. + */ memset(e, 0, sizeof(*e)); } @@ -16280,9 +17129,38 @@ static void svc_fwd_outcome_check(struct nv_vgpu_device *vgpu) static bool svc_fwd_hold_expired(void) { - if (svc_fwd_req.gr && !smp_load_acquire(&svc_fwd_req.done)) + if (svc_fwd_req.gr && !smp_load_acquire(&svc_fwd_req.done)) { + /* + * r420: while the backend deliberately holds the in-flight + * launch in its bounded WEDGE-QUIET window, the GR deadline + * must not fire - expiring here authors the launch's release + * word (svc_fwd_hold_gr_release) and applies the recorded + * releases before the post-hold copy-out, so the client + * verifies stale data (the r419 soak's mid-boot + * "matmul fails after time"). Pause instead, bounded by + * SVC_FWD_GR_HOLD_PAUSE_MAX_MS from the hold's start; the + * pause extends the deadline a full GR window past the last + * quiet-hold pass, so the backend's retry/accept can finish + * before the fallback would author. Announced once per hold. + */ + if (gsp_phys_gr_quiet_hold_active() && + time_before(jiffies, svc_fwd_hold.since + + msecs_to_jiffies(SVC_FWD_GR_HOLD_PAUSE_MAX_MS))) { + if (!svc_fwd_hold.pause_logged) { + svc_fwd_hold.pause_logged = true; + pr_warn("vgsp-gr: GR-HOLD-PAUSE va=0x%llx (backend wedge-quiet hold active; release authorship deferred under the bounded hold)\n", + svc_fwd_req.gr_push_va); + } + svc_fwd_hold.paused_until = jiffies + + msecs_to_jiffies(SVC_FWD_GR_HOLD_TIMEOUT_MS); + return false; + } + if (svc_fwd_hold.paused_until && + time_before(jiffies, svc_fwd_hold.paused_until)) + return false; return time_after(jiffies, svc_fwd_hold.since + msecs_to_jiffies(SVC_FWD_GR_HOLD_TIMEOUT_MS)); + } return time_after(jiffies, svc_fwd_hold.since + msecs_to_jiffies(SVC_FWD_HOLD_TIMEOUT_MS)); } @@ -16316,6 +17194,9 @@ static void svc_fwd_hold_engage(struct nv_vgpu_device *vgpu, svc_fwd_hold.relay_armed = svc_fwd_req.relay; svc_fwd_hold.relay_index = svc_fwd_req.relay_index; svc_fwd_hold.fault_delivered = false; + /* r420: a fresh hold starts with no WEDGE-QUIET deferral. */ + svc_fwd_hold.paused_until = 0; + svc_fwd_hold.pause_logged = false; svc.fwd_hold_engaged++; svc.fwd_rel_pending = n; } @@ -16342,6 +17223,8 @@ static void svc_fwd_hold_clear(void) svc_fwd_hold.relay_armed = false; svc_fwd_hold.relay_index = 0; svc_fwd_hold.fault_delivered = false; + svc_fwd_hold.paused_until = 0; + svc_fwd_hold.pause_logged = false; svc.fwd_rel_pending = 0; } @@ -17106,6 +17989,17 @@ static void svc_fwd_gr_work(void) ga_va, ga_used, ga_led); hw_rc = gsp_phys_backend_guest_gr_submit(&gs, &cp); + /* + * r420b: the backend redirected the launch's QMD release0 to the + * module semaphore (rel_mod=1 or no targetable guest release page), so + * the ENGINE never writes the guest's own QMD release word. Treat the + * word as unobserved so the bounded outcome check (and the hold's + * edges) author it in software on every outcome; without this a + * rel_mod=1 launch with no deferred release records is never completed + * on the guest word and the client hangs (the r414 saxpy stall). + */ + if (cp.qmd_rel_redirected) + svc_fwd_req.gr_rel_unobserved = true; if (!hw_rc) { svc.fwd_gr_ok++; } else { @@ -18055,6 +18949,27 @@ static bool svc_gr_hold_bound_expired(struct svc_channel *ch, u64 launch_va) ch->gr_hold_deadline = jiffies + msecs_to_jiffies(cap_ms); ch->gr_hold_exp_logged = false; } + /* + * r420: pause the per-launch consumption bound while the backend + * deliberately holds the in-flight launch in its bounded WEDGE-QUIET + * window. At the bound the pump falls through to the historical + * consume, which authors the capture's QMD completion release in + * software (GR-STALL-hold) - but the backend is still going to + * submit/accept this very launch after the hold and copy its data + * out; authoring now releases the client onto stale data (the r419 + * mid-boot escape). The quiet hold is a bounded msleep (the + * backend's own caps), and the deferral here is bounded by + * SVC_GR_QUIET_PAUSE_MAX_MS from the bound's arm time as + * defence-in-depth. Rate-limited announcement. + */ + if (ch->gr_hold_dead_va && gsp_phys_gr_quiet_hold_active() && + time_before(jiffies, ch->gr_hold_dead_since + + msecs_to_jiffies(SVC_GR_QUIET_PAUSE_MAX_MS))) { + ch->gr_hold_deadline = jiffies + msecs_to_jiffies(cap_ms); + pr_info_ratelimited("vgsp-gr: GR-HOLD-BOUND-PAUSE launch=0x%llx (backend wedge-quiet hold active; consumption bound deferred)\n", + launch_va); + return false; + } return time_after_eq(jiffies, ch->gr_hold_deadline); } @@ -20340,7 +21255,20 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) injected = (svc_ipin_lookup(va, &ipa, &iap) == 0); - if (!svc_method_stream_plausible(pb, len)) { + /* + * r496e: an all-zero push on a + * content-scanned-ring channel is the + * driver's NOP sanity push (one zero + * dword - NVA06F_DMA_NOP), not a bad + * translation: accept it and let the + * decode find no ops. + */ + bool zero_nop = ch->gr_ram_ring && + len >= 4 && + !memchr_inv(pb, 0, len); + + if (!svc_method_stream_plausible(pb, len) && + !zero_nop) { svc.false_trans++; if (injected) { u8 *fb = kmalloc(len, @@ -20771,7 +21699,22 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc_gr_launch_va(ch)) && (ch->gr_hold_idx != (u32)idx || time_before(jiffies, - ch->gr_hold_until))) { + ch->gr_hold_until) || + /* + * r420: the backend's + * bounded WEDGE-QUIET + * hold outlives this + * one-shot window; keep + * re-holding (never + * consume-with-sw-release) + * while it is active, so + * the launch is still + * forwarded and copied out + * after the hold instead of + * being released onto stale + * data. + */ + gsp_phys_gr_quiet_hold_active())) { u32 hold_ms = gsp_sema_gr_grace_ms ? gsp_sema_gr_grace_ms : 8000; @@ -21051,10 +21994,81 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * through to the historical * consume so the channel can * never livelock. + * + * r470: an ALL-ZERO push page is + * the same "not the client's work + * yet" condition - the ring entry + * is written (the previous + * session's or the client's own + * slot reuse) but the push + * CONTENT has not landed, so the + * decode sees no QMD and the + * entry would be consumed as a + * no-op; the end-of-pass counter + * publish then advances the + * guest's completion word while + * the real push was never read, + * and the client drains a + * staging buffer that still holds + * the PREVIOUS copy's bytes + * (streams_events: h_c[2] = c0 + * because the c2 D2H's + * completion was published 4.5 s + * before its copy ran, the ring + * slots having been decoded as + * all-zero pushes). Deliberately + * NARROWER than the CE-shaped + * plausibility verdict: compute + * pushes legitimately fail that + * matcher, so only an entirely + * zero page counts as unwritten. */ + { + bool all_zero = false; + + if (!ch->engine_type && + va && len) { + u32 zi; + + all_zero = true; + for (zi = 0; zi < len; + zi++) + if (pb[zi]) { + all_zero = + false; + break; + } + } + /* + * r471: publish clamp bookkeeping. + * A zero-content GR entry that is + * held or consumed must never let + * the end-of-pass counter publish + * advance the guest's completion + * word past it (the client would + * satisfy its readback wait before + * the real push/copy ran - the + * staging buffer still holds the + * previous copy's bytes). Record + * the lowest zero slot; a decode + * that reads non-zero content + * clears the clamp. + */ + if (!ch->engine_type) { + if (all_zero) { + if (!ch->gr_bad_put || + (u32)idx + 1 < + ch->gr_bad_put) + ch->gr_bad_put = + (u32)idx + 1; + } else if (va && len) { + ch->gr_bad_put = 0; + } + } if (!ch->engine_type && va && len && gsp_phys_gr_forward_enabled() && - svc_gr_qmd_seen > 0 && + (svc_gr_qmd_seen > 0 || + all_zero) && !entry_replay && !(svc_gr_cand.valid && svc_gr_cand.h_object == @@ -21073,6 +22087,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) jiffies; } if (gsp_sema_gr_nocap_hold_ms && + !ch->gr_ram_ring && time_before(jiffies, ch->gr_nocap_since + msecs_to_jiffies( @@ -21106,6 +22121,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->h_object, (u32)idx, va, len); } + } /* * A CE push that should carry this * channel's next tracking release but @@ -21815,6 +22831,14 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.chan[k].userd_pa = ch->userd_pa; svc.chan[k].userd_as = ch->userd_as; svc.chan[k].userd_ok = ch->userd_ok; + /* r496b: persist the RAM-scan cursor and the + * one-shot USERD-scan latch so they survive + * the per-pass snapshot copy. */ + svc.chan[k].gr_ram_cursor = ch->gr_ram_cursor; + svc.chan[k].gr_ram_passes = ch->gr_ram_passes; + svc.chan[k].gr_ram_logged = ch->gr_ram_logged; + svc.chan[k].gr_ram_ring = ch->gr_ram_ring; + svc.chan[k].userd_scans = ch->userd_scans; svc.chan[k].logged = ch->logged; svc.chan[k].logged_expected = ch->logged_expected; svc.chan[k].notified = ch->notified; @@ -21841,6 +22865,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.chan[k].push_hold_until = ch->push_hold_until; svc.chan[k].defer_since = ch->defer_since; svc.chan[k].put_prev = ch->put_prev; + /* r471: the publish clamp watermark is set + * and cleared on the pump thread; persist it + * so the next pass still clamps. */ + svc.chan[k].gr_bad_put = ch->gr_bad_put; /* * Persist the cheap-wait window and the content * watermark: the pump works on a snapshot, so |