diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 74 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.c | 19 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.h | 1 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 749 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.h | 13 | ||||
| -rw-r--r-- | kernel/main.c | 71 |
6 files changed, 878 insertions, 49 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 46060b1..d43430b 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -16744,6 +16744,61 @@ static u32 *gsp_phys_rmcall_build_gr_init(u32 *p) } /* + * r597: extract the SEM_EXECUTE ACQUIRE groups from the captured guest + * prelude and copy them, verbatim, ahead of the rebuilt inline-QMD window. + * Two encodings are recognised: + * - C46F host-semaphore group: INC_METHOD addr 0x5c count>=5, words + * {LO, HI, PAYLOAD_LO, PAYLOAD_HI, EXECUTE}; EXECUTE & 7 != 1 is an + * acquire (the pump's own op decode). + * - C5C0 SET_REPORT_SEMAPHORE A..D: addr 0x1b00 count>=4, word D's low + * two bits select release (0) / acquire (non-zero). + * Only the wait methods are taken, so the proven window (init + inline QMD + * + PCAS + release) is untouched. Returns the number of words emitted. + */ +static u32 gsp_phys_rmcall_gr_emit_acquires(u32 *p, + const struct gsp_phys_gr_submit *req, + u32 budget) +{ + u32 off = 0, words = 0; + + if (!gsp_phys_rmcall_gr_acq_value() || !req || !req->raw) + return 0; + while (off < req->raw_qmd_hdr && words + 8 < budget) { + u32 hdr = req->raw[off]; + u32 op = (hdr >> 29) & 0x7; + u32 count = (hdr >> 16) & 0xfff; + u32 addr = (hdr & 0x1fff) << 2; + u32 n = 1 + count; + + if ((op != 1 && op != 3) || !count || + off + n > req->raw_qmd_hdr) { + off++; + continue; + } + if (addr == 0x5c && count >= 5) { + u32 v4 = req->raw[off + 5]; + u64 va = ((u64)(req->raw[off + 2] & 0xff) << 32) | + (u64)(req->raw[off + 1] & 0xfffffffc); + + if ((v4 & 0x7) != 1 && va) { + memcpy(p + words, &req->raw[off], n * 4); + words += n; + } + } else if (addr >= 0x1b00 && addr <= 0x1b0c && + count >= 4) { + u32 d = req->raw[off + 4] & 0x3; + + if (d != 0) { + memcpy(p + words, &req->raw[off], n * 4); + words += n; + } + } + off += n; + } + return words; +} + +/* * GR launch method block: the inline QMD run at its original shape and the * channel REPORT_SEMAPHORE release. The stock guest push does NOT emit * SET_OBJECT before the inline run (the compute class is bound on the @@ -16776,6 +16831,25 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, GSP_PHYS_RMCALL_GR_SET_QMD_VIRT_CONTROL, GSP_PHYS_RMCALL_GR_QMD_VIRT_IQ2M_SEND_PCAS); /* + * r597: the prelude's acquire groups (if any) go before the QMD run + * so the engine cannot read the launch's data pages until the copy + * channel's semaphore has advanced. + */ + { + u32 w = gsp_phys_rmcall_gr_emit_acquires(p, req, + 128u); + static u32 acq_log; + + if (w) { + p += w; + if (acq_log < 16) { + acq_log++; + pr_info("vgsp-gr: prelude acquire groups emitted words=%u h=0x%x push=0x%llx\n", + w, req->h_object, req->push_va); + } + } + } + /* * Verbatim replay: copy the captured GRCE prelude and the guest's * own inline-QMD run, keeping the method headers, and relocate only * the ADDRESS_A/B data words and the 64 QMD words exactly like the diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c index 74062dd..e9c0c5a 100644 --- a/kernel/gsp_phys_gpu.c +++ b/kernel/gsp_phys_gpu.c @@ -617,6 +617,25 @@ int gsp_phys_rmcall_gr_prelude_value(void) EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_prelude_value); /* + * r597: emit the SEM_EXECUTE ACQUIRE groups found in the captured guest + * prelude ahead of the rebuilt inline-QMD window (default 0). The verbatim + * prelude replay breaks the launch (qmd_rel=0 after an 8.9 s stall, r599), + * so the acquire cannot be taken by replaying the prelude; this gate takes + * only the wait method groups and leaves the proven window in place. + */ +static int gsp_phys_rmcall_gr_acq_param; +module_param_named(gsp_phys_rmcall_gr_acq, gsp_phys_rmcall_gr_acq_param, + int, 0644); +MODULE_PARM_DESC(gsp_phys_rmcall_gr_acq, + "Emit the captured prelude's SEM_EXECUTE ACQUIRE groups ahead of the rebuilt GR window (default 0)"); + +int gsp_phys_rmcall_gr_acq_value(void) +{ + return gsp_phys_rmcall_gr_acq_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_acq_value); + +/* * QMD virtualization control gate (default 0 after m26 falsified the * hypothesis: on Turing the context init value is 0, native CUDA never * writes 0x29c, and the nonzero enable alone does not schedule). When set, diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h index 36c2220..3993c6e 100644 --- a/kernel/gsp_phys_gpu.h +++ b/kernel/gsp_phys_gpu.h @@ -295,6 +295,7 @@ int gsp_phys_rmcall_gr_promote_value(void); * init+window+PCAS+sem shape); 1 replays the verbatim prelude. */ int gsp_phys_rmcall_gr_prelude_value(void); +int gsp_phys_rmcall_gr_acq_value(void); /* * gsp_phys_rmcall_gr_qmdvirt (default 1): emit diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index f1a31df..e76cb7a 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -318,6 +318,21 @@ struct svc_channel { * the end with no operand left pending, so a * replay suppresses every op. When clear, the * done bitmap decides per op (retry path). */ + bool ever_ops; /* r611: this channel has decoded at least one + * transfer op (copy/fill), i.e. it carries the + * client's stream-ordered memory work. The r607 + * stream-order watermark considers only such + * channels: a sibling GR LAUNCH channel's ring + * legitimately holds future launches, and + * deferring on those creates mutual deferral + * between the client's own launch channels + * (streams2: launches on h=0x5c00001f/23 each + * saw the other's unconsumed entry, burned the + * 15 s cap and were refused fail-closed - the + * r610c stream class red). Measured r606/r609/ + * r610c: op-decoding channels are exactly the + * copy ranks (0x5c000059 family) and the CE + * ranks; launch channels decode no ops. */ u64 slot_ops_hash[SVC_GPFIFO_MAX_BITS]; /* Content witness for slot_ops_ident: the * FNV-1a hash of the push bytes the identity @@ -1117,6 +1132,10 @@ struct sema_svc { u64 fwd_gr_argdefer; /* r543: held for a partial * argument set (unresolved arg) */ u32 fwd_gr_argdefer_log; + u64 fwd_gr_argfw; /* r612: non-degenerate fill + * forwarded at the cap (stalled + * candidates are decoys) */ + u32 fwd_gr_argfw_log; u64 fwd_gr_cewm; /* r590: held on the CE ring's * live submission watermark */ u64 fwd_gr_cewm_cap; /* r590: watermark still @@ -3530,6 +3549,9 @@ static inline bool svc_pool_pa_identity(u64 va, u64 pa) static void svc_pool_boot_store(u64 va, u64 pa, int ap); /* m168 */ static bool svc_dst_guard_hold(u64 va); /* m169b */ +static bool svc_dst_guard_should_hold(u64 va); /* r609 */ +static void svc_dst_guard_age(void); /* r609 */ +static void svc_dst_guard_reset(void); /* r609 */ static bool svc_dst_guard_pending(u64 va); /* r231 */ static u32 svc_read_order_log; /* r231 budget */ static u32 svc_atrace_log; /* r234 A-TRACE budget */ @@ -4159,6 +4181,7 @@ static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) int wap, rc; bool verified = false; bool rb_ok = false; + static u32 sem_sw_dev_log; /* r619: non-pool device-VA write budget */ /* * Pool semaphores resolve pin -> verified cache -> targeted rm_mem @@ -4218,10 +4241,23 @@ static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) * sysmem semaphore range (0x792..), with the resolver provenance and * the read-back, so the second session's translations can be * compared with the first session's for the same VA. Bounded. - */ - if (va >= 0x79240000000ULL && va < 0x79260000000ULL && - svc.sem_sw_log < 8192) { - svc.sem_sw_log++; + * + * r619 diag: the same proof for NON-pool device-VA writes (the VAS + * window >= 0x200000000; e.g. 0x20480fef0 push releases, 0x204827f7c + * QMD releases). The r618 hung math client read 0 at 0x20480fef0 + * while the host's deferred releases logged `cur` 97..105 on its own + * target - a write-target divergence that was invisible because this + * range never printed `pa`. r620: the first cut used the 1 TiB + * SVC_VAS_FLOOR and excluded this very window (no lines ever); + * bounded per boot (512). + */ + if (((va >= 0x79240000000ULL && va < 0x79260000000ULL && + svc.sem_sw_log < 8192) || + (va >= 0x200000000ULL && !is_pool && sem_sw_dev_log < 512))) { + if (va >= 0x79240000000ULL && va < 0x79260000000ULL) + svc.sem_sw_log++; + else + sem_sw_dev_log++; pr_info("sema-sw: h=0x%x va=0x%llx pa=0x%llx ap=%d src=%s val=0x%x rc=%d rb_ok=%d back=0x%x pool=%d\n", h, va, addr, wap, src, val, rc, rb_ok, back, is_pool); } @@ -4634,6 +4670,16 @@ static u32 svc_wait_ovf; /* acquires dropped past the array */ static u64 svc_wait_held; static u64 svc_wait_pass; static u64 svc_wait_giveup; +static u64 svc_wait_extended; /* r613: give-up deferred while a + * GR forward is owed */ +static u32 svc_wait_ext_log; +/* + * r613: episode extension bound while a GR forward is owed. Comfortably + * above wait-hold (default 4000 ms) + the absolute launch defer cap + * (SVC_GR_DEFER_CAP_MS, 15000 ms) + margin, and far below every client-side + * timeout (>= 60 s). + */ +#define SVC_WAIT_EXT_CAP_MS 25000u static u32 svc_wait_log; /* per-session held-row print budget */ static u32 svc_wait_pass_log; /* per-session resolved-row budget */ static u32 svc_remapconst_log; /* r343 FILL: SET_REMAP_CONST_A rows */ @@ -5946,6 +5992,29 @@ struct svc_gr_cand_rec { static struct svc_gr_cand_rec svc_gr_cand_ring[SVC_GR_CAND_RING]; static u32 svc_gr_cand_pos; #define svc_gr_cand (svc_gr_cand_ring[svc_gr_cand_pos]) + +/* + * r613: is a GR forward owed for this device - a captured candidate that has + * not been offered (submitted or refused) yet? The wait gate uses this to + * distinguish "the acquire's release can never be installed by firmware + * alone" from "the release is pending behind OUR OWN launch deferral": the + * r612 streams2 capture gave up a D2H readback's acquire (0x20480ffe0 pay=4, + * cur=2) at waited_ms=10333 while the vadd launches that author that + * semaphore sat deferred by the CE watermark for the full 15 s cap - the + * readback then ran BEFORE the kernels and the client read zeros + * (bad=65534). While a forward is owed, the episode extends (bounded by + * wait-hold + defer cap + margin) instead of failing open. + */ +static bool svc_gr_forward_pending(void) +{ + u32 i; + + for (i = 0; i < SVC_GR_CAND_RING; i++) + if (svc_gr_cand_ring[i].valid && + !svc_gr_cand_ring[i].offered) + return true; + return false; +} /* * r383: push-VA history of recent captures, beyond the (four-slot) * candidate ring's lifetime. A replayed decode of a complete QMD asks @@ -6748,6 +6817,7 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot bool audit = total >= SVC_COPY_AUDIT_MIN; u8 *snap = NULL; static u64 bigcopy_log; + static u32 shadow_dst_log; /* r614 diag: shadow-lane dst witness */ bool trace64 = (total == 65536u); const char *dir = "?"; @@ -6864,6 +6934,26 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot } else { dst_shadow = true; svc.vas_shadowed++; + /* + * r614 diag: the r613b red boot's matmul + * wedge began here - the H2D into the + * client's cudaMalloc'd buffer found no + * ledger/pin/walk answer, took the shadow + * silently, and the later GR fill could not + * consume the shadowed bytes (probe=none for + * every arg base; the launch held 15 s, then + * the degenerate refusal consumed it and the + * client spun). Name the first shadow-lane + * device-dst copies of a boot (bounded), so + * the next red boot is greppable where it + * starts instead of only at the launch. + */ + if (shadow_dst_log < 64 && + dst_orig >= SVC_VAS_FLOOR) { + shadow_dst_log++; + pr_info("sema-shadow-dst: h=0x%x dst=0x%llx total=%zu (device VA with no ledger/pin/walk answer; copy runs into the shadow, a GR fill cannot consume it)\n", + owner_ch, dst_orig, total); + } } } else { dst = pa; @@ -6890,7 +6980,7 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot if (dst_virt && dst_aperture == CE_APERTURE_VID && dst_prov >= 3 && dst_orig >= SVC_VAS_FLOOR && gsp_sema_copy_dst_pin_guard && - svc_dst_guard_hold(dst_orig)) { + svc_dst_guard_should_hold(dst_orig)) { svc.copy_dst_hold++; if (svc.copy_dst_hold <= 16) pr_info("sema-svc: device-dst copy held: dst=0x%llx prov=%u (no pin yet; stale-table guard)\n", @@ -8219,26 +8309,57 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, pend = svc_wait_pend_release(w->va); if (waited_ms >= gsp_sema_wait_hold_ms) { /* - * Bounded episode expired: fail open for the - * WHOLE entry (one row, then the ops run). A - * per-wait continue here would ping-pong - * with a second unsatisfied wait: the first - * gave up and reset the shared clock before - * the second was judged, and the pair would - * hold forever 100 ms at a time. + * r613: a deferred GR launch extends the + * episode. The release this acquire waits for + * may be authored by a launch we are holding + * through our own bounded deferral (the r612 + * streams2 capture: the D2H readback's + * acquire on 0x20480ffe0 was given up at + * waited_ms=10333 while the vadd launches + * that release it sat deferred by the CE + * watermark for 15 s; the readback then ran + * before the kernels and the client read + * zeros). While a captured candidate is + * unoffered, "the value cannot be installed + * by firmware alone" is not yet true - keep + * holding, bounded by the defer cap plus + * margin. The client-side timeouts are all + * >= 60 s, so a <= 24 s episode cannot + * strand it. */ - svc_wait_giveup++; - if (svc_wait_log < 2048) { - svc_wait_log++; - pr_info("sema-wait: give-up ch=0x%x va=0x%llx op=%u pay=%u cur=%u pend=%u waited_ms=%u -> fail-open (cannot be satisfied by firmware alone; ops run)\n", - ch_h, w->va, w->op, w->payload, - cur, pend, waited_ms); + if (waited_ms < SVC_WAIT_EXT_CAP_MS && + svc_gr_forward_pending()) { + svc_wait_extended++; + if (svc_wait_ext_log < 32) { + svc_wait_ext_log++; + pr_info("sema-wait: extend ch=0x%x va=0x%llx op=%u pay=%u cur=%u waited_ms=%u -> hold (a GR forward is owed; the release is behind our own defer)\n", + ch_h, w->va, w->op, + w->payload, cur, + waited_ms); + } + } else { + /* + * Bounded episode expired: fail open for the + * WHOLE entry (one row, then the ops run). A + * per-wait continue here would ping-pong + * with a second unsatisfied wait: the first + * gave up and reset the shared clock before + * the second was judged, and the pair would + * hold forever 100 ms at a time. + */ + svc_wait_giveup++; + if (svc_wait_log < 2048) { + svc_wait_log++; + pr_info("sema-wait: give-up ch=0x%x va=0x%llx op=%u pay=%u cur=%u pend=%u waited_ms=%u -> fail-open (cannot be satisfied by firmware alone; ops run)\n", + ch_h, w->va, w->op, w->payload, + cur, pend, waited_ms); + } + if (gh) + gh->active = false; + hold = false; + svc_wait_pass++; + goto svc_wait_done; } - if (gh) - gh->active = false; - hold = false; - svc_wait_pass++; - goto svc_wait_done; } hold = true; if (svc_wait_log < 512) { @@ -11407,8 +11528,34 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * soon as the trust (or the operand translation) * arrives; the bounded SVC_OPDEF_HOLD_MS expiry keeps * the pre-existing skip as the last resort. + * + * r610: a REPLAY pass must be exempt. A replay is the + * pump re-walking an entry whose ops were already + * recorded complete for this identity (entry_replay = + * same && slot_ops_complete): re-marking it as pending + * re-arms a 3 s opdef hold for a copy that HAS run, + * and the fresh pass that follows re-executes the op - + * a replay/fresh/replay cycle (r609 boot, idx=94: the + * d_b H2D re-marked and re-run every ~0.3 s from + * 368841 while the vadd launch waited; the r606 + * captured held chain is the same shape). Measured + * r604-r609: 100% of op_tr_pending marks (27/27, + * 71/71, 72/72) came from replay passes of already- + * executed copies - each one stalled the whole copy + * channel behind it, which is the held-chain class. + * The r308 rationale (pre-r595 the eng==0 copy + * channel's fresh passes never admitted ops, so the + * replay re-mark WAS the retry hatch) no longer + * applies: r595's cap_trust admission runs a plausible + * copy on its fresh pass, and a genuinely failed copy + * keeps its slot incomplete through op_owed / the + * rc!=0 pending rule. The whole accounting block is + * skipped on replay: no dstpend note (the write ran), + * no staging re-snapshot, no hold. A fresh + * (non-replay) denied pass marks pending exactly as + * before. */ - if (!allow_ops && trusted && + if (!allow_ops && trusted && !replay && !ch->engine_type && transfer != 0 && st.line_length && st.have_out && st.have_in && !DMA_REMAP_ENABLE(dma) && @@ -11531,6 +11678,11 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, if (n_ops != n_ops_before) { struct svc_memset_op *lm = &ops[n_ops - 1]; + /* r611: this channel carries transfer work + * (see ever_ops) - the stream-order + * watermark's scope. */ + ch->ever_ops = true; + /* * fail4 CONT.53: H2D-shaped copies (virtual * device destination, virtual host source) @@ -12265,6 +12417,9 @@ static void svc_pending_add(u64 va, u32 payload, u32 op) * exist; drop them instead of resolving them against the next session. */ static void svc_gr_diag_reset(void); /* m150: defined with the fill diag */ +/* r604: defined next to the per-VA defer tables; clears both on session end + * (they are keyed by push VA alone and the UVM pool reuses those VAs). */ +static void svc_gr_defer_reset(void); static void svc_session_flush(void) { @@ -12301,6 +12456,8 @@ static void svc_session_flush(void) } svc_gr_diag_reset(); /* m150: per-session diagnostic budgets */ + svc_gr_defer_reset(); /* r604: per-session defer latches */ + svc_dst_guard_reset(); /* r609: per-session device-dst windows */ svc.map_n = 0; svc_pb_n = 0; svc_pd_found = 0; @@ -14829,6 +14986,88 @@ bool gsp_sema_wait_gate = 1; unsigned gsp_sema_copy_dst_guard_ms = 3000; /* + * r609: the m169 first-touch window is per PAGE, but the stale-table risk it + * covers is per CLIENT TABLE BUILD. The driver's H2D relocation burst on the + * eng==0 copy channel writes N distinct first-write pages back to back, and + * because the pump holds the FIRST held entry and the rest of the chain + * decodes only after it drains, each page paid its own full window, + * serialized (r606 red draw: `sema-pend: hold h=0x5c000059 idx=90` held + * 362167.7->362170.8, idx=92 held 362177.7->362182.9; the b H2D entry ran + * 362182.882, 326 ms AFTER the launch forward -> `VADD bad=32735 + * csum=8139264`, c = a + 0). The guard's bet - a fresh pin lands inside the + * window - is a property of the client's table state, not of one page: once + * ONE device-dst page of a channel has waited the full window with NO pin + * arriving (every observed release logs "without a pin"), the same bet for + * the next page of the same burst is already lost. Age the channel for + * gsp_sema_copy_dst_age_ms after such a release; further first-write + * device-dst pages of that channel inside the window skip the hold and run + * with the walk translation - the exact translation the expired guard would + * have used anyway. The first page of every burst still pays the full + * window, so the anti-stale bet is unchanged; only the N x window + * serialization is removed (r606 class: ~20 s of chain -> one window). + * Keyed by h_object like every channel table; the session flush clears it, + * so a recycled handle can never inherit the previous client's ageing + * (the r604 lifetime lesson). 0 disables (= per-page windows, pre-r609). + */ +unsigned gsp_sema_copy_dst_age_ms = 3000; +#define SVC_DST_AGE_SLOTS 16 +static u32 svc_dst_age_ch[SVC_DST_AGE_SLOTS]; +static unsigned long svc_dst_age_j[SVC_DST_AGE_SLOTS]; +static u32 svc_dst_age_pos; +static u32 svc_dst_age_log; + +/* r609: the current decode channel's guard window expired without a pin. */ +static void svc_dst_guard_age(void) +{ + u32 i; + + if (!gsp_sema_copy_dst_age_ms) + return; + for (i = 0; i < SVC_DST_AGE_SLOTS; i++) + if (svc_dst_age_ch[i] == svc.cur_ch) { + svc_dst_age_j[i] = jiffies; + return; + } + i = svc_dst_age_pos++ % SVC_DST_AGE_SLOTS; + svc_dst_age_ch[i] = svc.cur_ch; + svc_dst_age_j[i] = jiffies; +} + +/* r609: is the current decode channel inside its post-expiry ageing window? */ +static bool svc_dst_guard_aged(void) +{ + unsigned long now = jiffies; + u32 i; + + if (!gsp_sema_copy_dst_age_ms || !svc.cur_ch) + return false; + for (i = 0; i < SVC_DST_AGE_SLOTS; i++) + if (svc_dst_age_ch[i] == svc.cur_ch && + time_before(now, svc_dst_age_j[i] + + msecs_to_jiffies(gsp_sema_copy_dst_age_ms))) + return true; + return false; +} + +/* + * r609: decide whether a first-write device-dst page pays the m169 window. + * An aged channel skips it (the burst's first page already paid and no pin + * arrived); otherwise the normal per-page window runs. + */ +static bool svc_dst_guard_should_hold(u64 va) +{ + if (svc_dst_guard_aged()) { + if (svc_dst_age_log < 16) { + svc_dst_age_log++; + pr_info("sema-svc: device-dst guard age-skipped dst=0x%llx h=0x%x (same-channel burst after an un-pinned release; walk translation)\n", + va, svc.cur_ch); + } + return false; + } + return svc_dst_guard_hold(va); +} + +/* * r231 READ-ORDER guard (default 0: witness only). A device-src copy whose * source VA has a PENDING device-dst write hold (the m169 guard window is * still open) reads the PRE-write content: the writer's bytes land only @@ -14877,6 +15116,7 @@ static bool svc_dst_guard_hold(u64 va) if (svc_dst_guard[i].first) { svc_dst_guard[i].first = 0; svc.copy_dst_relaxed++; + svc_dst_guard_age(); /* r609: burst ageing */ pr_info("sema-svc: device-dst guard released without a pin va=0x%llx (using the walk translation)\n", page); } @@ -14893,6 +15133,24 @@ static bool svc_dst_guard_hold(u64 va) } /* + * r609: clear the m169 page windows and the r609 ageing ring with the + * session. Both are keyed by VA page / h_object alone, and both would + * otherwise pre-age the next client's first touch (a page slot still inside + * its window makes the new client's first device-dst write skip the + * anti-stale wait; an ageing stamp makes the new channel's whole first + * burst skip it). The stale-table risk this guard covers is precisely the + * new-session condition, so the windows must start fresh. + */ +static void svc_dst_guard_reset(void) +{ + memset(svc_dst_guard, 0, sizeof(svc_dst_guard)); + svc_dst_guard_next = 0; + memset(svc_dst_age_ch, 0, sizeof(svc_dst_age_ch)); + memset(svc_dst_age_j, 0, sizeof(svc_dst_age_j)); + svc_dst_age_pos = 0; +} + +/* * r231 READ-ORDER guard: true while a held device-dst write for this VA * page is still inside its bounded window (the writer's bytes have not * landed yet). See the gsp_sema_read_order_guard declaration. @@ -18116,6 +18374,74 @@ static void svc_fwd_hold_gr_release(struct nv_vgpu_device *vgpu, * applies the recorded releases AND the QMD word, bounded by the existing * hold timeout), so this check stands down for it. */ + +/* + * r618: synth-fault for a refused/failed GR launch. The r616 experiment + * (manual grfaultpoke on the live r616b matmul wedge - fill `0/5 high + * argument ptr(s)`, `GR submit refused: degenerate data set n_data=2 + * (min 16)`, sw release authored, client spun in cudaDeviceSynchronize + * forever) proved the edge: the channel error-notifier write + * (status=0xffff + channel event) ends the client's wait with + * `CUDA_ERROR unspecified launch failure` in seconds. Deliver the same + * edge automatically on every FAILED forward so a refused launch is an + * observable error, never an unbounded hang. One-line rollback: + * gsp_sema_refusal_fault=0. + */ +bool gsp_sema_refusal_fault = true; +module_param_named(sema_refusal_fault, gsp_sema_refusal_fault, bool, 0644); +MODULE_PARM_DESC(sema_refusal_fault, + "Deliver a synthetic channel error edge on a failed/refused GR forward (default 1; 0 = legacy silent sw release)"); + +static void svc_gr_fault_synth(struct nv_vgpu_device *vgpu, u32 h_object) +{ + u8 notif[SVC_NOTIFY_SIZE] = {0}; + u16 status = SVC_NOTIFY_ERROR_STATUS; + u32 info32 = 0x1; /* generic engine exception, as proven live */ + unsigned long flags; + struct svc_channel *ch = NULL; + u64 pa; + u32 as, eng; + int i; + + if (!gsp_sema_refusal_fault || !h_object) + return; + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].h_object == h_object && + svc.chan[i].notifier_pa) { + ch = &svc.chan[i]; + break; + } + } + if (!ch) { + spin_unlock_irqrestore(&svc.chan_lock, flags); + return; + } + pa = ch->notifier_pa; + as = ch->notifier_as; + eng = ch->engine_type; + ch->notifier_err = true; + ch->notifier_err_until = jiffies + SVC_NOTIFIER_ERR_HOLD; + ch->notifier_err_put = ch->put_prev; + spin_unlock_irqrestore(&svc.chan_lock, flags); + + memcpy(notif + SVC_NOTIFY_INFO32_OFF, &info32, 4); + { + u16 info16 = (u16)eng; + + memcpy(notif + SVC_NOTIFY_INFO16_OFF, &info16, 2); + } + memcpy(notif + SVC_NOTIFY_STATUS_OFF, &status, 2); + if (svc_write_ap(vgpu, pa, as, notif, sizeof(notif))) { + pr_info_ratelimited("vgsp-fault: SYNTH-FAULT notifier write failed ch=0x%x pa=0x%llx as=%u\n", + h_object, pa, as); + return; + } + pr_warn("vgsp-fault: SYNTH-FAULT ch=0x%x engine=%u pa=0x%llx status=0x%x except=0x%x (refused forwarded launch -> client-visible error)\n", + h_object, eng, pa, status, info32); + gsp_rm_post_channel_event(vgpu, h_object); +} static void svc_fwd_outcome_check(struct nv_vgpu_device *vgpu) { bool done = smp_load_acquire(&svc_fwd_req.done); @@ -18149,10 +18475,18 @@ static void svc_fwd_outcome_check(struct nv_vgpu_device *vgpu) svc_fwd_req.gr_rel_va, svc_fwd_req.gr_rel_payload); svc.fwd_gr_stall_swrel++; - if (svc_fwd_req.hw_rc != 0) + if (svc_fwd_req.hw_rc != 0) { pr_warn("vgsp-gr: GR-STALL-backend va=0x%llx waited_ms=%lu action=proceed (failed forwarded launch, rc=%d; QMD release authored in software)\n", svc_fwd_req.gr_push_va, waited, svc_fwd_req.hw_rc); - else + /* + * r618: the software release alone completes only launches the + * engine actually ran; a refused/failed one left the client in + * an unbounded cudaDeviceSynchronize. Deliver the synthetic + * engine-error edge (proven live by the r616 grfaultpoke test) + * so the failed launch is client-observable. + */ + svc_gr_fault_synth(vgpu, svc_fwd_req.gr_h_object); + } else pr_warn("vgsp-gr: GR-STALL-backend va=0x%llx waited_ms=%lu action=proceed (successful launch whose release0 could not target the guest page; QMD release authored in software)\n", svc_fwd_req.gr_push_va, waited); pr_info("vgsp-gr: GR-COMPLETE-SW h=0x%x push=0x%llx rel=0x%llx pay=0x%x waited_ms=%lu n=%llu (backend outcome without a hold)\n", @@ -20595,7 +20929,14 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, if (is_arg[w] && svc_fwd_req.gr_n_data == before && svc_gr_src_log < 64) { int vi; - char vb[240]; + /* r614 diag: the r613b red-boot post-mortem could not + * tell which registered pdb would have answered an + * unresolved arg base (the 240-byte buffer truncated + * the walk list at three entries, and only the red + * boot's misses were ever printed). A larger buffer + * with a clamped append keeps every registered + * VASpace's walk verdict visible. */ + char vb[512]; int o = 0; svc_gr_src_log++; @@ -20610,11 +20951,12 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, svc.vas[vi].entries, base, svc.vas[vi].ap, &wa, &wap, gsp_sema_walk_trace != 0); - o += snprintf(vb + o, sizeof(vb) - o, - " vas[%d] pdb=0x%llx ent=%u ap=%d rc=%d pa=0x%llx", - vi, svc.vas[vi].pdb, - svc.vas[vi].entries, - svc.vas[vi].ap, wrc, wa); + if (o < (int)sizeof(vb) - 1) + o += snprintf(vb + o, sizeof(vb) - o, + " vas[%d] pdb=0x%llx ent=%u ap=%d rc=%d pa=0x%llx", + vi, svc.vas[vi].pdb, + svc.vas[vi].entries, + svc.vas[vi].ap, wrc, wa); } pr_info("vgsp-sema: svc-gr arg-miss base=0x%llx probe=%s n_cand=%u pd_found=%u ipin_n=%u disc=0x%llx/%d%s\n", base, pba_src, n_cand, svc_pd_found, @@ -20991,6 +21333,40 @@ static bool svc_gr_defer_expired(u64 push_va) } /* + * r604: the r543 arg-defer latch and the r556 absolute defer cap are keyed + * by push VA ALONE, and the UVM pool reuses the same launch push VAs for + * every client of a boot - the exact lifetime bug the session flush already + * fixes for the r565 exactly-once memories (svc_gr_cap_hist, svc_gr_seen, + * svc_inline_once: "matches on the VA ALONE ... a previous client's + * capture could classify a later client's fresh launch"). Carried across + * sessions, the r556 stamp arms once - at the previous client's first + * sighting - so a new client's first decision for a reused VA already reads + * defer_expired()=true: every partial-argument defer fails closed (r586 + * path), the engine never launches, the QMD release is authored in + * software, and the client reads stale/zero buffers. Witnessed r603 family + * boot: the first mail client (fresh VAs 0x20043be94/0x20043c364) deferred + * and launched (`GUEST GR LAUNCH OK`), while every later mail client (VAs + * 0x20043beac/0x20043c37c reused from earlier boot runs) saw defer_exp=1 + * at its FIRST decision and `GR forward REFUSED at defer cap ... argument + * pages still unresolved` for both launches; its "green" was the previous + * client's results still sitting in the reused device pages. + * + * Reset both tables with the session: within a session the original + * bounds stand unchanged (one deferred sighting per VA in the arg window, + * one absolute 15 s cap per VA lifetime); a new session starts with fresh + * push generations and must not inherit either stamp. + */ +static void svc_gr_defer_reset(void) +{ + memset(svc_gr_arg_defer_va, 0, sizeof(svc_gr_arg_defer_va)); + memset(svc_gr_arg_defer_j, 0, sizeof(svc_gr_arg_defer_j)); + svc_gr_arg_defer_pos = 0; + memset(svc_gr_defer_cap_va, 0, sizeof(svc_gr_defer_cap_va)); + memset(svc_gr_defer_cap_j, 0, sizeof(svc_gr_defer_cap_j)); + svc_gr_defer_cap_pos = 0; +} + +/* * r548: is a forward still OWED for this push VA - a captured, never * offered candidate sitting in the candidate ring? The r547 saxpy stall: * both launches were deferred by the r541/r543 guards, the candidate ring @@ -21114,15 +21490,52 @@ static struct svc_channel *svc_pass_local; static u32 svc_pass_local_n; /* - * r590: does any CE ring of this device still hold entries this pass's pump - * has not consumed? Reads the ring's live GP_PUT from its own USERD and - * compares it (modulo the ring size) with the snapshot's last_get. A - * mismatch means the client submitted copies/fills the pump has not decoded - * yet - exactly the stream-order backlog the r541/r568 guards cannot see. + * r590: does any ring of this device still hold entries this pass's pump has + * not consumed? Reads the ring's live GP_PUT from its own USERD and compares + * it (modulo the ring size) with the snapshot's last_get. A mismatch means + * the client submitted copies/fills the pump has not decoded yet - exactly + * the stream-order backlog the r541/r568 guards cannot see. + * + * r607 (r606 red-draw capture, final form of r590b): the engine_type==0 + * filter skipped the channel the guard exists for. The driver's H2D + * relocation copies ride a COMPUTE-class channel (h=0x5c000059, eng=0 - + * the r595 finding), and the captured red draw shows the launch forwarding + * while that channel held three published, unconsumed entries behind an + * opdef operand hold: + * 362177.927 GR-DEC push=0x20043c394 dropped=9 partial=1 dstpend=0 + * copypend=0 ceback=0 + * 362178.001 peek-backlog h=0x5c000059 eng=0 put=95 last=92 + * 362182.145 peek-backlog h=0x5c000059 eng=0 put=95 last=92 + * 362182.556 GR LAUNCH submitted (forwarded at the one-shot arg-defer + * retry) -> VADD bad=32735 csum=8139264 (c = a + 0) + * 362182.882 ce-soft-copy va=d_b executed, 326 ms AFTER the forward + * The chain was held by `sema-pend: hold h=0x5c000059 idx=90 ... (operand + * pending)`; the wait group (0x20480fff0 pay=84) passed every pass, so the + * only blindness was the class filter - ceback=0 while the copy channel + * was three entries behind. Consider every other channel of the device; + * the launching channel itself is excluded (its own ring legitimately runs + * ahead with future entries). r590's validated part is unchanged: same + * pass snapshot, same bounded busy-defer, historical forward at the + * absolute cap (this guard itself never refuses; the r586 fail-closed + * path is a separate branch that fires only when argument pages never + * resolve). r595's GR-class ops admission has since cut the hold + * resolution to 0.5-4.7 s (the captured 4.7 s), well inside the 15 s cap. + * * Channels whose USERD is unreadable are skipped (their work cannot be * ordered by this mechanism anyway). - */ -static bool svc_ce_stream_backlog(struct nv_vgpu_device *vgpu) + * + * r611: only channels that carry transfer ops are considered (ever_ops). + * The r607 blanket "every other ring" caught sibling GR LAUNCH channels + * whose rings hold the client's own future launches: streams2's two stream + * channels (h=0x5c00001f/23) each deferred on the other's published entry + * for the full 15 s cap, then the r586 fail-closed refused both kernels + * (measured r610c: every streams2 launch REFUSED, bad=65534). A launch + * channel decodes no ops (measured r606/r609/r610c: ops appear only on the + * 0x5c000059 copy family and the CE ranks), so the flag separates them + * cleanly while keeping the r606 red-draw copy channel in scope. + */ +static bool svc_ce_stream_backlog(struct nv_vgpu_device *vgpu, + const struct svc_channel *self) { u32 k; @@ -21132,8 +21545,11 @@ static bool svc_ce_stream_backlog(struct nv_vgpu_device *vgpu) const struct svc_channel *c = &svc_pass_local[k]; u32 put = 0; - if (!c->valid || (vgpu && c->vgpu != vgpu) || - !c->engine_type) + if (!c->valid || (vgpu && c->vgpu != vgpu)) + continue; + if (self && c->h_object == self->h_object) + continue; + if (!c->ever_ops) continue; if (!c->gpfifo_entries || !c->userd_pa) continue; @@ -21149,6 +21565,93 @@ static bool svc_ce_stream_backlog(struct nv_vgpu_device *vgpu) } /* + * r598 diag: the driver's ring WRITE pointer, not the GPFIFO put. The r568 + * draw needs the copy entries that are already written into the ring but + * whose GP_PUT has not been published yet. When the put says the ring is + * drained, peek at the slots immediately after the consumption position and + * log their {va,len} plus whether the slot already executed that exact + * entry. Bounded per launch, gated by gsp_phys_gr_peek. + */ +u32 gsp_phys_gr_peek; + +static void svc_gr_peek_lookahead(struct nv_vgpu_device *vgpu, + const struct svc_channel *self) +{ + u32 k, logged = 0, backlog_logged = 0; + + if (!gsp_phys_gr_peek || !svc_pass_local) + return; + /* + * r604 diag: 8 was too small. svc_pass_local carries the six CE ranks + * (engine_type 11/12/13) FIRST and the GR-class channels after, so the + * cap was always spent before the copy-carrying GR-class channels + * (0x5c000059 family - the r595 finding: "the driver's relocation / + * H2D copies live on a compute channel") could be logged. Session 48 + * concluded "the copy channel's lookahead read empty" from captures + * where that channel was never in the logged set at all. 24 covers + * every drained channel of a live client with margin; the lookahead + * depth 6 covers the copy behind the launch's own small pushes. + * + * r604: also name every channel the drained-only filter SKIPS because + * its live GP_PUT reads ahead of the pass consumption position. Those + * are exactly the channels the r590 CE watermark watches - but that + * guard skips engine_type==0, so a published backlog on a GR-class + * copy channel is invisible to every existing guard; the capture must + * show it so a red decision can be attributed to write-ahead (put == + * last, entries logged below) vs published-backlog (put != last, + * logged here) without guessing. + */ + for (k = 0; k < svc_pass_local_n && logged < 24; k++) { + const struct svc_channel *c = &svc_pass_local[k]; + u32 put = 0, j; + + if (!c->valid || c->h_object == self->h_object || + !c->gpfifo_va || !c->gpfifo_entries || !c->userd_pa) + continue; + if (svc_read_ap(vgpu, c->userd_pa + SVC_USERD_GP_PUT, + c->userd_as, &put, sizeof(put))) + continue; + put %= c->gpfifo_entries; + if (put != ((u32)c->last_get % c->gpfifo_entries)) { + /* real published backlog */ + if (backlog_logged < 12) { + backlog_logged++; + pr_info("vgsp-gr: peek-backlog h=0x%x eng=%u put=%u last=%u\n", + c->h_object, c->engine_type, put, + (u32)c->last_get); + } + continue; + } + pr_info("vgsp-gr: peek h=0x%x eng=%u put=%u last=%u\n", + c->h_object, c->engine_type, put, (u32)c->last_get); + for (j = 1; j <= 6; j++) { + u32 idx = ((u32)c->last_get + j) % c->gpfifo_entries; + u8 ent[SVC_GPFIFO_ENTRY_SIZE]; + u32 e0, e1, len; + u64 va; + + if (svc_read_va(vgpu, + c->gpfifo_va + + (u64)idx * SVC_GPFIFO_ENTRY_SIZE, + ent, sizeof(ent))) + break; + memcpy(&e0, ent, 4); + memcpy(&e1, ent + 4, 4); + va = ((u64)(e1 & 0xff) << 32) | + (u64)(e0 & 0xfffffffc); + len = ((e1 >> 10) & 0x1fffff) << 2; + if (!va || !len || len > SVC_PUSH_MAX) + break; + pr_info("vgsp-gr: peek +%u idx=%u va=0x%llx len=%u ident_match=%d\n", + j, idx, va, len, + c->slot_ops_ident[idx] == + ((u64)e0 | ((u64)e1 << 32))); + } + logged++; + } +} + +/* * r576 DIAG: the GR launch-decision witness (gsp_phys_gr_dec_log=N budget * per boot). Logged from svc_gr_request right after the fill, for launches * whose fill saw dropped pages / a partial argument set or whose decision @@ -21172,7 +21675,7 @@ static void svc_gr_dec_dump(const struct svc_channel *ch) dstpend = svc_gr_dstpend_overlap(); copypend = svc_copy_pending(); - ceback = svc_ce_stream_backlog(NULL); + ceback = svc_ce_stream_backlog(NULL, ch); pr_info("vgsp-gr: GR-DEC h=0x%x push=0x%llx n_data=%u dropped=%u partial=%d stalled=%d dstpend=%d copypend=%d ceback=%d defer_exp=%d\n", ch->h_object, svc_gr_cand.push_va, svc_fwd_req.gr_n_data, @@ -21318,7 +21821,7 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, if (gsp_phys_gr_dec_log_max && (svc_fwd_req.gr_dropped > 0 || svc_fwd_req.gr_arg_partial || svc_gr_data_stalled || svc_gr_dstpend_overlap() || - svc_copy_pending() || svc_ce_stream_backlog(vgpu))) + svc_copy_pending() || svc_ce_stream_backlog(vgpu, ch))) svc_gr_dec_dump(ch); /* * r541: launch-ordering guard. The fill's data pages are what the @@ -21401,7 +21904,8 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, * historical forward stands (witnessed) - the guard delays a launch, * it never refuses one, so it cannot recreate r588's mail refusal. */ - if (svc_ce_stream_backlog(vgpu)) { + svc_gr_peek_lookahead(vgpu, ch); + if (svc_ce_stream_backlog(vgpu, ch)) { if (!svc_gr_defer_expired(svc_gr_cand.push_va)) { clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); mutex_unlock(&svc_fwd_lock); @@ -21480,9 +21984,26 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, * no-write). Refuse the forward, author the completion in software, * consume; the dropped pages were already demanded through the pin * channel (r585) so the retry path had its chance first. + * + * r612: refuse only a DEGENERATE fill - one whose dropped set is at + * least as large as the resolved set (r584's 2 resolved / 18 + * dropped). A launch with a full resolved window (969 resolved / + * 9 dropped) whose unresolved candidates are still unresolved at + * the absolute cap has them stalled in EVERY source for the whole + * window; they are the param page's decoy/foreign pointers (the + * r611 streams2 first-client capture: candidates 0x783299000000 / + * 0x783297000000, `stalled VA` from the first fill, no pin, no + * walk, no ledger, never resolving in the run) - waiting longer + * cannot help, and refusing converts a launch whose real operands + * all resolved into a guaranteed no-write client failure (streams2 + * bad=65534). At the cap the copy-related guards (dstpend, CE + * watermark) have already cleared or the refusal branches above + * win first, so a non-degenerate fill forwards historically, + * loudly witnessed. */ if (svc_fwd_req.gr_arg_partial && !svc_gr_data_stalled && - svc_gr_defer_expired(svc_gr_cand.push_va)) { + svc_gr_defer_expired(svc_gr_cand.push_va) && + svc_fwd_req.gr_n_data < svc_fwd_req.gr_dropped) { svc.fwd_gr_refused++; svc.fwd_gr_closed++; if (svc_gr_cand.rel_va) { @@ -21505,6 +22026,21 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, * candidate is owned and sw-completed */ } /* + * r612: see the refusal note above - a non-degenerate fill at the cap + * forwards historically (witnessed once per budget slot). + */ + if (svc_fwd_req.gr_arg_partial && !svc_gr_data_stalled && + svc_gr_defer_expired(svc_gr_cand.push_va)) { + svc.fwd_gr_argfw++; + if (svc.fwd_gr_argfw_log < 32) { + svc.fwd_gr_argfw_log++; + pr_info("vgsp-work: GR forward at defer cap h=0x%x push=0x%llx (partial argument set: %u dropped of %u resolved; stalled candidates never resolve, forwarding historically)\n", + ch->h_object, svc_gr_cand.push_va, + svc_fwd_req.gr_dropped, + svc_fwd_req.gr_n_data); + } + } + /* * m150: the fill found high argument pointers that no source can * translate (the second-client zeros class). Do NOT forward a launch * whose kernel would dereference unresolved pointers: leave the @@ -22209,10 +22745,60 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.chan[i].engine_type) local[n++] = svc.chan[i]; } - for (i = 0; i < SVC_MAX_CHANNELS; i++) { - if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && - !svc.chan[i].engine_type) - local[n++] = svc.chan[i]; + /* + * r605 stream-order, the eng==0 half of the r507o fix. The same + * argument that made CE channels walk before GR ones applies WITHIN + * the GR class: the driver's relocation/H2D copies ride a + * later-registered compute-class channel (h=0x5c000059 family, + * eng=0 - the r595 finding), while the launch channel registers + * early (0x5c000019). The stable partition walked the launch first, + * so at a launch decision the copy channel's PUBLISHED backlog was + * unconsumed and invisible to every guard: the r590 watermark skips + * engine_type==0, and the r541/r568 gates only see DECODED copies. + * r604 capture (fresh boot, peek armed): at the vadd's second + * decision `peek-backlog h=0x5c000059 eng=0 put=95 last=94` - the + * b-copy entry was already in the ring with GP_PUT ahead of the + * pump, executed 8 ms after the decision and 320 ms before the + * launch's copy-in, and the run was green only because the + * partial-argument defer had stretched the window; pins landing one + * pass earlier would have forwarded against the pre-copy page + * (c = a + 0, the r568 red draw). Newest-registered first + * (descending h_object) walks the copy / push-library channels + * before the compute channels, so every launch decision sees the + * copy ring consumed to its live write pointer (GP_PUT): the + * write-pointer drain, by ordering. No deferral, no cap + * interaction, no speculative execution - the pump still consumes + * each entry exactly once through its normal per-slot identity + * bookkeeping (slot_ops_ident/done/complete/hash), so nothing needs + * mirroring. A client whose compute channel registers last simply + * keeps the historical order (no worse). + */ + { + u32 prev_h = 0xffffffffu; + int picked; + + for (picked = 0; picked < SVC_MAX_CHANNELS; picked++) { + int best = -1; + u32 best_h = 0; + + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + struct svc_channel *c = &svc.chan[i]; + + if (!c->valid || c->vgpu != vgpu || + c->engine_type) + continue; + if (c->h_object >= prev_h) + continue; + if (best < 0 || c->h_object > best_h) { + best = i; + best_h = c->h_object; + } + } + if (best < 0) + break; + local[n++] = svc.chan[best]; + prev_h = best_h; + } } spin_unlock_irqrestore(&svc.chan_lock, flags); @@ -26429,3 +27015,68 @@ int gsp_sema_service_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); return rc; } + +/* + * r616 diag: manual fault poke for the refused-launch hang experiment. A GR + * push that the fill cannot resolve is refused (degenerate data set) and the + * entry is consumed with a software release; the authored release lands in + * the guest page (measured: kcore read 0x6 == pay on r613d) yet the guest + * driver never completes cudaDeviceSynchronize for a launch that never ran. + * This writes the channel's error notifier in the exact shape the relayed- + * fault edge uses (info32=exceptType, info16=engine, status=0xffff, latched + * by notifier_err) and posts the channel event, so a hung client's kernel + * can be poked into observing an engine error and ending its wait. Debug + * only; harmless when no client is hung (the pump's completion clear resumes + * after SVC_NOTIFIER_ERR_HOLD). h_object 0 = every valid channel. + */ +int gsp_sema_channel_fault_poke(struct nv_vgpu_device *vgpu, u32 h_object) +{ + u8 notif[SVC_NOTIFY_SIZE] = {0}; + u16 status = SVC_NOTIFY_ERROR_STATUS; + u32 info32 = 0x1; /* generic engine exception */ + int i, done = 0; + + if (!vgpu) + return -ENODEV; + memcpy(notif + SVC_NOTIFY_INFO32_OFF, &info32, 4); + memcpy(notif + SVC_NOTIFY_STATUS_OFF, &status, 2); + + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + struct svc_channel *ch = &svc.chan[i]; + unsigned long flags; + u64 pa; + u32 as, eng, h; + u16 info16; + + spin_lock_irqsave(&svc.chan_lock, flags); + if (!ch->valid || ch->vgpu != vgpu || !ch->notifier_pa || + (h_object && ch->h_object != h_object)) { + spin_unlock_irqrestore(&svc.chan_lock, flags); + continue; + } + h = ch->h_object; + pa = ch->notifier_pa; + as = ch->notifier_as; + eng = ch->engine_type; + /* Latch exactly like the relayed-fault edge so the pump's + * completion clear cannot wipe the poke before the guest + * reads it. */ + ch->notifier_err = true; + ch->notifier_err_until = jiffies + SVC_NOTIFIER_ERR_HOLD; + ch->notifier_err_put = ch->put_prev; + spin_unlock_irqrestore(&svc.chan_lock, flags); + + info16 = (u16)eng; + memcpy(notif + SVC_NOTIFY_INFO16_OFF, &info16, 2); + if (svc_write_ap(vgpu, pa, as, notif, sizeof(notif))) { + pr_info("vgsp-fault: POKE ch=0x%x notifier write failed pa=0x%llx as=%u\n", + h, pa, as); + continue; + } + pr_info("vgsp-fault: POKE ch=0x%x engine=%u pa=0x%llx as=%u status=0x%x except=0x%x (manual experiment)\n", + h, eng, pa, as, status, info32); + gsp_rm_post_channel_event(vgpu, h); + done++; + } + return done ? done : -ENOENT; +} diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h index 31512ef..fbc7e4d 100644 --- a/kernel/gsp_sema_service.h +++ b/kernel/gsp_sema_service.h @@ -44,6 +44,14 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc); void gsp_sema_service_channel_del(struct nv_vgpu_device *vgpu, u32 h_object); /* + * r616 diag: manually write a channel's error notifier (engine-exception + * shape) and post its channel event, to test whether a refused launch's + * hung client can be ended by a fault-conversion edge. h_object 0 = every + * valid channel of the device. Debug only. + */ +int gsp_sema_channel_fault_poke(struct nv_vgpu_device *vgpu, u32 h_object); + +/* * Release every piece of service state that belongs to one mdev before its * guest session disappears: drop the mdev's channels (per-vgpu session end), * clear a release hold that names it (never dereference a removed device), @@ -259,6 +267,7 @@ extern u64 gsp_sema_copy_dst2; /* CE pending-entry grace window in ms before the release fallback fires. */ extern unsigned gsp_sema_ce_grace_ms; +extern u32 gsp_phys_gr_peek; /* * Bounded work-before-release hold (ms) for an unreadable CE entry whose @@ -328,6 +337,10 @@ extern bool gsp_sema_pool_hold_fresh; * walk/RAM scan until a pin confirms (default on). */ extern bool gsp_sema_copy_dst_pin_guard; +/* r609: after a device-dst guard window expires WITHOUT a pin, skip the + * window for further first-write device-dst pages of the same channel for + * this long (ms). 0 restores the per-page windows (pre-r609). */ +extern unsigned gsp_sema_copy_dst_age_ms; extern bool gsp_sema_wait_gate; /* r401: bounded acquire-wait hold (fail-open) in ms; must exceed the * emulated launch completion latency for copy-after-launch ordering. */ diff --git a/kernel/main.c b/kernel/main.c index d821762..9a5d912 100644 --- a/kernel/main.c +++ b/kernel/main.c @@ -2752,6 +2752,10 @@ MODULE_PARM_DESC(sema_copy_dst2, "Second destination window for sema_copy_dst (d * the release the client synchronizes on; the translation usually lands within * a service pass. */ +module_param_named(gsp_phys_gr_peek, gsp_phys_gr_peek, uint, 0644); +MODULE_PARM_DESC(gsp_phys_gr_peek, + "Peek the ring slots past GP_PUT at each GR launch decision (diagnostic, default 0)"); + module_param_named(sema_ce_grace_ms, gsp_sema_ce_grace_ms, uint, 0644); MODULE_PARM_DESC(sema_ce_grace_ms, "CE pending-entry grace window in ms (default 25000)"); @@ -2872,6 +2876,14 @@ MODULE_PARM_DESC(sema_wait_hold_ms, "bounded acquire-wait hold before fail-open, module_param_named(sema_push_raw, gsp_sema_push_raw, uint, 0644); MODULE_PARM_DESC(sema_push_raw, "r481 diagnostic: dump the decoded method stream of the first N large GR pushes (default 0 off)"); MODULE_PARM_DESC(sema_copy_dst_pin_guard, "hold device-dst copies without a pin-resolved translation (default 1)"); +/* + * r609: after a device-dst guard window expires WITHOUT a pin, skip the + * window for further first-write device-dst pages of the same channel for + * this long (ms) - the r606 held-chain serialization fix. 0 restores the + * per-page windows (pre-r609). + */ +module_param_named(sema_copy_dst_age_ms, gsp_sema_copy_dst_age_ms, uint, 0644); +MODULE_PARM_DESC(sema_copy_dst_age_ms, "device-dst guard burst-ageing window after an un-pinned release, ms (r609, default 3000; 0 off)"); /* * r384 (guest boot-corruption lottery): fill-destination guards. A large @@ -3881,5 +3893,64 @@ static const struct kernel_param_ops nv_vgpu_rmpost_ops = { module_param_cb(rmpost, &nv_vgpu_rmpost_ops, NULL, 0200); MODULE_PARM_DESC(rmpost, "Post a GSP-RM event to the guest by event index (debug)"); +/* + * Debug: poke a channel's error notifier with an engine-exception shape and + * post its channel event (the relayed-fault delivery edge, manually): + * echo 0x5c000019 > .../parameters/grfaultpoke + * echo 0 > .../parameters/grfaultpoke (every valid channel) + * Experiment: a refused/degenerate GR launch ends its client's + * cudaDeviceSynchronize spin if a synthetic engine error completes the wait + * (r616; see gsp_sema_channel_fault_poke). + */ +struct nv_vgpu_faultpoke_req { + u32 h_object; + int rc; + struct completion done; +}; + +static int nv_vgpu_faultpoke_thread(void *arg) +{ + struct nv_vgpu_faultpoke_req *req = arg; + struct nv_vgpu_device *vgpu; + + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) { + if (atomic_read(&vgpu->vfio_dev_open_count) <= 0) + continue; + req->rc = gsp_sema_channel_fault_poke(vgpu, req->h_object); + pr_info("vgsp: grfaultpoke h=0x%x -> %d\n", + req->h_object, req->rc); + } + mutex_unlock(&nv_vgpu_driver.lock); + + complete(&req->done); + return 0; +} + +static int nv_vgpu_faultpoke_set(const char *val, const struct kernel_param *kp) +{ + static struct nv_vgpu_faultpoke_req req; + struct task_struct *task; + + memset(&req, 0, sizeof(req)); + req.rc = -ENODEV; + if (kstrtou32(val, 0, &req.h_object)) + return -EINVAL; + + init_completion(&req.done); + task = kthread_run(nv_vgpu_faultpoke_thread, &req, "vgpu-faultpoke"); + if (IS_ERR(task)) + return PTR_ERR(task); + wait_for_completion(&req.done); + return 0; +} + +static const struct kernel_param_ops nv_vgpu_faultpoke_ops = { + .set = nv_vgpu_faultpoke_set, +}; + +module_param_cb(grfaultpoke, &nv_vgpu_faultpoke_ops, NULL, 0200); +MODULE_PARM_DESC(grfaultpoke, "Write a channel error notifier + post its event (r616 experiment)"); + module_init(nv_vgpu_init); module_exit(nv_vgpu_exit); |