diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 1610 |
1 files changed, 1548 insertions, 62 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index a0cdb2b..b6fa436 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -318,6 +318,21 @@ struct svc_channel { * the end with no operand left pending, so a * replay suppresses every op. When clear, the * done bitmap decides per op (retry path). */ + bool ever_ops; /* r611: this channel has decoded at least one + * transfer op (copy/fill), i.e. it carries the + * client's stream-ordered memory work. The r607 + * stream-order watermark considers only such + * channels: a sibling GR LAUNCH channel's ring + * legitimately holds future launches, and + * deferring on those creates mutual deferral + * between the client's own launch channels + * (streams2: launches on h=0x5c00001f/23 each + * saw the other's unconsumed entry, burned the + * 15 s cap and were refused fail-closed - the + * r610c stream class red). Measured r606/r609/ + * r610c: op-decoding channels are exactly the + * copy ranks (0x5c000059 family) and the CE + * ranks; launch channels decode no ops. */ u64 slot_ops_hash[SVC_GPFIFO_MAX_BITS]; /* Content witness for slot_ops_ident: the * FNV-1a hash of the push bytes the identity @@ -1117,11 +1132,22 @@ struct sema_svc { u64 fwd_gr_argdefer; /* r543: held for a partial * argument set (unresolved arg) */ u32 fwd_gr_argdefer_log; + u64 fwd_gr_argfw; /* r612: non-degenerate fill + * forwarded at the cap (stalled + * candidates are decoys) */ + u32 fwd_gr_argfw_log; u64 fwd_gr_cewm; /* r590: held on the CE ring's * live submission watermark */ u64 fwd_gr_cewm_cap; /* r590: watermark still * backlogged at the defer cap */ u32 fwd_gr_cewm_log; + u64 ce_mutual_skips; /* r638: watermark + * skipped a channel whose + * head entry is wait-gate + * held (a consumer-side + * mutual citation, not a + * producer backlog) */ + u32 ce_mutual_log; u64 fwd_gr_timeout; u64 fwd_gr_fault; u64 fwd_gr_bridge; @@ -1610,6 +1636,8 @@ static void svc_opdef_drop(struct nv_vgpu_device *vgpu, static void svc_opdef_reset_all(void); static void svc_opdef_finish_ch(struct svc_channel *ch, u32 idx); static u32 svc_opdef_log; /* m176: forward decl for the session flush */ +/* r633: per-session synthetic shadow budget reset (defined with the fill). */ +static void svc_gr_synth_ep_reset(void); /* * One-cycle GAP-1/GAP-2 instrumentation budgets (agent-datapath plan C1-C10): * small, reset per session where they are service-thread state, so a later @@ -2635,6 +2663,16 @@ static void svc_lat_store(u64 va, u64 addr, int ap) struct svc_vas_rec { u64 va_page; /* 0 = empty */ u8 *data; + /* + * r635: the page holds bytes a client/host WRITE actually landed + * (an H2D shadow-lane write, an inline write, a bridge write, an + * engine writeback). Synthetic pages minted by the GR fill serve + * zeros and must never be mistaken for the client's own content: + * only real content anchors a span or arms the launch-level anchor + * (without this flag the fill would re-anchor on its own minted + * pages and mint without bound across retry passes). + */ + u8 content; }; static struct svc_vas_rec svc_vas[SVC_VAS_SLOTS]; @@ -2670,6 +2708,14 @@ static bool svc_vas_has(u64 va) return svc_vas_find(va) != NULL; } +/* r635: the page holds REAL written content (not a synthetic zero page). */ +static bool svc_vas_has_content(u64 va) +{ + struct svc_vas_rec *r = svc_vas_find(va); + + return r && r->content; +} + /* * A VA may receive a new shadow record only when it is engine-only device * memory: high in the client VA space, where the guest's own sysmem objects @@ -2677,8 +2723,23 @@ static bool svc_vas_has(u64 va) * 100 TiB). Explicit sources (GRCE inline writes, the bridge's output write) * shadow any VA because their target is device memory by construction. */ +bool gsp_sema_no_shadow; /* r626 A/B: disable the shadow lane */ +module_param_named(sema_no_shadow, gsp_sema_no_shadow, bool, 0644); +MODULE_PARM_DESC(sema_no_shadow, + "Disable the VA shadow lane (copy-dst and fill fallbacks) for an A/B boot; unresolved device VAs hold instead (default 0)"); + static bool svc_vas_can_shadow(u64 va) { + /* + * r626 A/B: with the knob set, every shadow fallback is refused and + * the caller takes its -ENOENT/hold path. The r613c/r616/r623c + * corruption class (nvidia channel structures overwritten with + * PTE-like 0x444../0x4456.. patterns, fatal panic in + * kchannelGetGfid_IMPL via nvGpuOpsStopChannel) follows the + * shadow-lane era; this knob isolates the lane as a variable. + */ + if (gsp_sema_no_shadow) + return false; return va >= SVC_VAS_FLOOR; } @@ -2728,6 +2789,7 @@ static int svc_vas_write(u64 va, const u8 *buf, size_t len) if (!r) return -ENOMEM; memcpy(r->data + (va & (SEMA_PAGE_SIZE - 1)), buf, chunk); + r->content = 1; /* r635: real bytes landed here */ svc.vas_writes += chunk; buf += chunk; va += chunk; @@ -3530,6 +3592,9 @@ static inline bool svc_pool_pa_identity(u64 va, u64 pa) static void svc_pool_boot_store(u64 va, u64 pa, int ap); /* m168 */ static bool svc_dst_guard_hold(u64 va); /* m169b */ +static bool svc_dst_guard_should_hold(u64 va); /* r609 */ +static void svc_dst_guard_age(void); /* r609 */ +static void svc_dst_guard_reset(void); /* r609 */ static bool svc_dst_guard_pending(u64 va); /* r231 */ static u32 svc_read_order_log; /* r231 budget */ static u32 svc_atrace_log; /* r234 A-TRACE budget */ @@ -4159,6 +4224,7 @@ static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) int wap, rc; bool verified = false; bool rb_ok = false; + static u32 sem_sw_dev_log; /* r619: non-pool device-VA write budget */ /* * Pool semaphores resolve pin -> verified cache -> targeted rm_mem @@ -4218,10 +4284,23 @@ static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) * sysmem semaphore range (0x792..), with the resolver provenance and * the read-back, so the second session's translations can be * compared with the first session's for the same VA. Bounded. - */ - if (va >= 0x79240000000ULL && va < 0x79260000000ULL && - svc.sem_sw_log < 8192) { - svc.sem_sw_log++; + * + * r619 diag: the same proof for NON-pool device-VA writes (the VAS + * window >= 0x200000000; e.g. 0x20480fef0 push releases, 0x204827f7c + * QMD releases). The r618 hung math client read 0 at 0x20480fef0 + * while the host's deferred releases logged `cur` 97..105 on its own + * target - a write-target divergence that was invisible because this + * range never printed `pa`. r620: the first cut used the 1 TiB + * SVC_VAS_FLOOR and excluded this very window (no lines ever); + * bounded per boot (512). + */ + if (((va >= 0x79240000000ULL && va < 0x79260000000ULL && + svc.sem_sw_log < 8192) || + (va >= 0x200000000ULL && !is_pool && sem_sw_dev_log < 4096))) { + if (va >= 0x79240000000ULL && va < 0x79260000000ULL) + svc.sem_sw_log++; + else + sem_sw_dev_log++; pr_info("sema-sw: h=0x%x va=0x%llx pa=0x%llx ap=%d src=%s val=0x%x rc=%d rb_ok=%d back=0x%x pool=%d\n", h, va, addr, wap, src, val, rc, rb_ok, back, is_pool); } @@ -4634,6 +4713,26 @@ static u32 svc_wait_ovf; /* acquires dropped past the array */ static u64 svc_wait_held; static u64 svc_wait_pass; static u64 svc_wait_giveup; +static u64 svc_wait_extended; /* r613: give-up deferred while a + * GR forward is owed */ +static u32 svc_wait_ext_log; +/* + * r613: episode extension bound while a GR forward is owed. Comfortably + * above wait-hold (default 4000 ms) + the absolute launch defer cap + * (SVC_GR_DEFER_CAP_MS, 15000 ms) + margin, and far below every client-side + * timeout (>= 60 s). + * + * r640: raised 25 s -> 45 s. The r639 streams2 transcript measured the + * readback's episode give up at waited_ms=28378 while the launch whose + * release it waits for only forwarded (GR-STALL-backend, software release + * authored) at 419322.99 - ~9 s AFTER the give-up - because the launch's + * own path paid the watermark defer (15 s cap) + arg-partial defer (8 s) + + * backend poll before it moved. The readback then ran out of order and the + * client read zeros (bad=65534). 45 s still leaves >15 s of headroom under + * the clients' >= 60 s timeouts, and only an episode with an unoffered + * candidate ever reaches it. + */ +#define SVC_WAIT_EXT_CAP_MS 45000u static u32 svc_wait_log; /* per-session held-row print budget */ static u32 svc_wait_pass_log; /* per-session resolved-row budget */ static u32 svc_remapconst_log; /* r343 FILL: SET_REMAP_CONST_A rows */ @@ -4656,6 +4755,10 @@ static bool svc_wait_rel_have; struct svc_wait_hold { bool active; u32 h; + u32 idx; /* r638: GPFIFO entry index this hold owns + * (0xffffffff when unset); the CE stream-order + * watermark uses it to detect a mutual citation + * (a held consumer entry, not a producer backlog) */ unsigned long since; }; #define SVC_WAIT_HOLD_MAX 16 @@ -5881,6 +5984,13 @@ struct svc_gr_data { u64 pa; /* guest PA, page aligned (0 = unresolved) */ u64 reloc_va; /* m102: server-managed VA the page maps to */ u8 ap; /* SVC_APERTURE_* of pa */ + /* + * r632: the page is served from the VA shadow lane instead of a guest + * page (no resolver and no landing answered for it). The submit build + * hands the shadow page's host address to the bounce machinery - see + * the fill's shadow-serve branch and the r632 note there. + */ + u8 shadow; }; /* @@ -5946,6 +6056,29 @@ struct svc_gr_cand_rec { static struct svc_gr_cand_rec svc_gr_cand_ring[SVC_GR_CAND_RING]; static u32 svc_gr_cand_pos; #define svc_gr_cand (svc_gr_cand_ring[svc_gr_cand_pos]) + +/* + * r613: is a GR forward owed for this device - a captured candidate that has + * not been offered (submitted or refused) yet? The wait gate uses this to + * distinguish "the acquire's release can never be installed by firmware + * alone" from "the release is pending behind OUR OWN launch deferral": the + * r612 streams2 capture gave up a D2H readback's acquire (0x20480ffe0 pay=4, + * cur=2) at waited_ms=10333 while the vadd launches that author that + * semaphore sat deferred by the CE watermark for the full 15 s cap - the + * readback then ran BEFORE the kernels and the client read zeros + * (bad=65534). While a forward is owed, the episode extends (bounded by + * wait-hold + defer cap + margin) instead of failing open. + */ +static bool svc_gr_forward_pending(void) +{ + u32 i; + + for (i = 0; i < SVC_GR_CAND_RING; i++) + if (svc_gr_cand_ring[i].valid && + !svc_gr_cand_ring[i].offered) + return true; + return false; +} /* * r383: push-VA history of recent captures, beyond the (four-slot) * candidate ring's lifetime. A replayed decode of a complete QMD asks @@ -6299,6 +6432,106 @@ static u8 svc_fwd_rel_n; static bool svc_fwd_defer_armed; /* + * r633: PARKED push-level release records for a GR launch whose entry was + * HELD before it could be forwarded. The records above are per-decode state + * (reset at the top of every svc_decode_stream); the two GR hold paths + * (busy-defer hold, m150 data-stalled hold) break out of the entry loop + * BEFORE the release block runs, so the entry's records would be lost at + * the next decode - and the eventual forward (queued from the capture ring + * in a later pass, which does not re-decode the entry) would engage the + * release hold with whatever records that pass happens to carry (often + * none, or another entry's), leaving the push's tracking release unapplied + * and the client spinning in cudaDeviceSynchronize forever (the r632 mail + * MALLOC hang: push release 0x20480fff0 pay=112 never landed while the + * client's word stayed at 111). Park keyed by (vgpu, channel, push VA); + * restored into svc_fwd_rel[] by svc_gr_request when the launch is finally + * queued, so the pump's hold-engage holds exactly the queued push's + * records; delivered inline by the paths that consume a capture without a + * forward. Service-thread only. + */ +#define SVC_GRPARK_SLOTS 8 +static struct svc_grpark { + struct nv_vgpu_device *vgpu; + u32 h_object; + u64 push_va; + u8 n; + struct svc_fwd_rel rel[SVC_FWD_REL_MAX]; +} svc_grpark[SVC_GRPARK_SLOTS]; +static u32 svc_grpark_pos; + +static void svc_grpark_put(struct nv_vgpu_device *vgpu, u32 h_object, + u64 push_va, const struct svc_fwd_rel *rel, u8 n) +{ + u32 i, slot = ~0u; + + if (!vgpu || !push_va || !rel || !n) + return; + if (n > SVC_FWD_REL_MAX) + n = SVC_FWD_REL_MAX; + for (i = 0; i < SVC_GRPARK_SLOTS; i++) { + if (svc_grpark[i].vgpu == vgpu && + svc_grpark[i].h_object == h_object && + svc_grpark[i].push_va == push_va) { + slot = i; + break; + } + if (slot == ~0u && !svc_grpark[i].push_va) + slot = i; + } + if (slot == ~0u) + slot = svc_grpark_pos++ % SVC_GRPARK_SLOTS; + svc_grpark[slot].vgpu = vgpu; + svc_grpark[slot].h_object = h_object; + svc_grpark[slot].push_va = push_va; + svc_grpark[slot].n = n; + memcpy(svc_grpark[slot].rel, rel, + sizeof(svc_grpark[slot].rel)); +} + +static bool svc_grpark_get(struct nv_vgpu_device *vgpu, u32 h_object, + u64 push_va, struct svc_fwd_rel *out, u8 *n) +{ + u32 i; + + for (i = 0; i < SVC_GRPARK_SLOTS; i++) { + if (svc_grpark[i].vgpu == vgpu && + svc_grpark[i].h_object == h_object && + svc_grpark[i].push_va == push_va) { + if (out) + memcpy(out, svc_grpark[i].rel, + sizeof(svc_grpark[i].rel)); + if (n) + *n = svc_grpark[i].n; + return true; + } + } + return false; +} + +static void svc_grpark_drop(struct nv_vgpu_device *vgpu, u32 h_object, + u64 push_va) +{ + u32 i; + + for (i = 0; i < SVC_GRPARK_SLOTS; i++) + if (svc_grpark[i].vgpu == vgpu && + svc_grpark[i].h_object == h_object && + svc_grpark[i].push_va == push_va) { + memset(&svc_grpark[i], 0, sizeof(svc_grpark[i])); + return; + } +} + +static void svc_grpark_reset(void) +{ + memset(svc_grpark, 0, sizeof(svc_grpark)); + svc_grpark_pos = 0; +} + +/* r633: bounded evidence that a parked set was restored/delivered. */ +static u32 svc_grpark_log; + +/* * P6b wave 2: positive sentinel returned by svc_copy() when the captured * forwarding candidate's software copy is deliberately withheld (the * hardware-primary path will have the real engine author the data). @@ -6748,6 +6981,7 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot bool audit = total >= SVC_COPY_AUDIT_MIN; u8 *snap = NULL; static u64 bigcopy_log; + static u32 shadow_dst_log; /* r614 diag: shadow-lane dst witness */ bool trace64 = (total == 65536u); const char *dir = "?"; @@ -6864,6 +7098,26 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot } else { dst_shadow = true; svc.vas_shadowed++; + /* + * r614 diag: the r613b red boot's matmul + * wedge began here - the H2D into the + * client's cudaMalloc'd buffer found no + * ledger/pin/walk answer, took the shadow + * silently, and the later GR fill could not + * consume the shadowed bytes (probe=none for + * every arg base; the launch held 15 s, then + * the degenerate refusal consumed it and the + * client spun). Name the first shadow-lane + * device-dst copies of a boot (bounded), so + * the next red boot is greppable where it + * starts instead of only at the launch. + */ + if (shadow_dst_log < 64 && + dst_orig >= SVC_VAS_FLOOR) { + shadow_dst_log++; + pr_info("sema-shadow-dst: h=0x%x dst=0x%llx total=%zu (device VA with no ledger/pin/walk answer; copy runs into the shadow, a GR fill cannot consume it)\n", + owner_ch, dst_orig, total); + } } } else { dst = pa; @@ -6890,7 +7144,7 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot if (dst_virt && dst_aperture == CE_APERTURE_VID && dst_prov >= 3 && dst_orig >= SVC_VAS_FLOOR && gsp_sema_copy_dst_pin_guard && - svc_dst_guard_hold(dst_orig)) { + svc_dst_guard_should_hold(dst_orig)) { svc.copy_dst_hold++; if (svc.copy_dst_hold <= 16) pr_info("sema-svc: device-dst copy held: dst=0x%llx prov=%u (no pin yet; stale-table guard)\n", @@ -7697,6 +7951,22 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot * kept the stale pre-pin pages while the copy landed on the pinned * page (streams2 r254: the b1 rec range carried the a2 pages and * the vadd read a2's pattern as its second input). + * + * r627: NEVER overwrite a per-page landing the loop above already + * stored. That loop stores each page's true resolved PA; this + * pass's `dst + po` is a contiguity guess valid only while the VA + * range maps contiguously. For a scattered UVM host buffer (r623c + * saxpy: page 0 at pa 0x115781000, page 0x7f000 at pa 0x12e700000) + * the guess clobbered the true entries - + * `sema-ledger: rewrite va=0x70e64e77f000 old=0x12e700000 + * new=0x115800000` - and the NEXT copy of the same VA then + * followed the poisoned ledger: 504x `sema-ledger: DIVERGE write + * va=0x70e64e7xxxxx prev=0x1158xxxxx cur=0x12e7xxxxx`; the + * non-pin branch keeps the ledger, so ~2 MiB of client c-data + * landed in guest pages the kernel had since re-used and the + * guest died in kchannelGetGfid_IMPL (r623c panic). The loop's + * per-page entries are the authority; this pass only seeds pages + * the loop did not reach. */ if (!dst_phys && dst_virt && dst_orig >= SVC_VAS_FLOOR && total >= (64u << 10)) { @@ -7704,8 +7974,14 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot SVC_APERTURE_FB : SVC_APERTURE_SYS; size_t po; - for (po = 0; po < total; po += SEMA_PAGE_SIZE) + for (po = 0; po < total; po += SEMA_PAGE_SIZE) { + u64 lpa = 0; + u32 lap = 0; + + if (!svc_ledger_lookup(dst_orig + po, &lpa, &lap)) + continue; svc_ledger_store(dst_orig + po, dst + po, cap); + } } if (snap) svc_copy_snap_free(dst_orig, src_orig, total, owner_ch, @@ -7887,6 +8163,30 @@ static bool svc_wait_satisfied(u32 cur, u8 op, u32 need) } } +/* + * r638: is (h, idx) currently owned by the bounded wait gate? The CE + * stream-order watermark consults this to tell a producer backlog (a CE + * copy the client ordered before the launch - the reason the watermark + * exists) from a consumer-side wait the forward itself will unblock (the + * mutual citation measured on r612/r636: the deferred launch's release is + * the only value that can satisfy the held copy's acquire, and the copy's + * unconsumed entry is the only reason the launch is deferred). + */ +static bool svc_wait_held_entry(u32 h, u32 idx) +{ + int i; + + if (idx == 0xffffffffu) + return false; + for (i = 0; i < SVC_WAIT_HOLD_MAX; i++) { + const struct svc_wait_hold *e = &svc_wait_holds[i]; + + if (e->active && e->h == h && e->idx == idx) + return true; + } + return false; +} + /* r34x: the per-channel first-hold clock for the bounded wait. */ static struct svc_wait_hold *svc_wait_hold_get(u32 h, bool create) { @@ -7906,6 +8206,7 @@ static struct svc_wait_hold *svc_wait_hold_get(u32 h, bool create) memset(free_slot, 0, sizeof(*free_slot)); free_slot->active = true; free_slot->h = h; + free_slot->idx = 0xffffffffu; /* r638: unset until a hold lands */ free_slot->since = jiffies; return free_slot; } @@ -8219,26 +8520,57 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, pend = svc_wait_pend_release(w->va); if (waited_ms >= gsp_sema_wait_hold_ms) { /* - * Bounded episode expired: fail open for the - * WHOLE entry (one row, then the ops run). A - * per-wait continue here would ping-pong - * with a second unsatisfied wait: the first - * gave up and reset the shared clock before - * the second was judged, and the pair would - * hold forever 100 ms at a time. + * r613: a deferred GR launch extends the + * episode. The release this acquire waits for + * may be authored by a launch we are holding + * through our own bounded deferral (the r612 + * streams2 capture: the D2H readback's + * acquire on 0x20480ffe0 was given up at + * waited_ms=10333 while the vadd launches + * that release it sat deferred by the CE + * watermark for 15 s; the readback then ran + * before the kernels and the client read + * zeros). While a captured candidate is + * unoffered, "the value cannot be installed + * by firmware alone" is not yet true - keep + * holding, bounded by the defer cap plus + * margin. The client-side timeouts are all + * >= 60 s, so a <= 24 s episode cannot + * strand it. */ - svc_wait_giveup++; - if (svc_wait_log < 2048) { - svc_wait_log++; - pr_info("sema-wait: give-up ch=0x%x va=0x%llx op=%u pay=%u cur=%u pend=%u waited_ms=%u -> fail-open (cannot be satisfied by firmware alone; ops run)\n", - ch_h, w->va, w->op, w->payload, - cur, pend, waited_ms); + if (waited_ms < SVC_WAIT_EXT_CAP_MS && + svc_gr_forward_pending()) { + svc_wait_extended++; + if (svc_wait_ext_log < 32) { + svc_wait_ext_log++; + pr_info("sema-wait: extend ch=0x%x va=0x%llx op=%u pay=%u cur=%u waited_ms=%u -> hold (a GR forward is owed; the release is behind our own defer)\n", + ch_h, w->va, w->op, + w->payload, cur, + waited_ms); + } + } else { + /* + * Bounded episode expired: fail open for the + * WHOLE entry (one row, then the ops run). A + * per-wait continue here would ping-pong + * with a second unsatisfied wait: the first + * gave up and reset the shared clock before + * the second was judged, and the pair would + * hold forever 100 ms at a time. + */ + svc_wait_giveup++; + if (svc_wait_log < 2048) { + svc_wait_log++; + pr_info("sema-wait: give-up ch=0x%x va=0x%llx op=%u pay=%u cur=%u pend=%u waited_ms=%u -> fail-open (cannot be satisfied by firmware alone; ops run)\n", + ch_h, w->va, w->op, w->payload, + cur, pend, waited_ms); + } + if (gh) + gh->active = false; + hold = false; + svc_wait_pass++; + goto svc_wait_done; } - if (gh) - gh->active = false; - hold = false; - svc_wait_pass++; - goto svc_wait_done; } hold = true; if (svc_wait_log < 512) { @@ -8253,6 +8585,10 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, svc_wait_held++; svc_wait_hold_h = ch_h; svc_wait_hold_idx = entry_idx; + /* r638: record the held entry for the CE watermark's + * mutual-cycle check. */ + if (gh) + gh->idx = entry_idx; svc.op_tr_pending = 1; return; } @@ -11407,8 +11743,34 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * soon as the trust (or the operand translation) * arrives; the bounded SVC_OPDEF_HOLD_MS expiry keeps * the pre-existing skip as the last resort. + * + * r610: a REPLAY pass must be exempt. A replay is the + * pump re-walking an entry whose ops were already + * recorded complete for this identity (entry_replay = + * same && slot_ops_complete): re-marking it as pending + * re-arms a 3 s opdef hold for a copy that HAS run, + * and the fresh pass that follows re-executes the op - + * a replay/fresh/replay cycle (r609 boot, idx=94: the + * d_b H2D re-marked and re-run every ~0.3 s from + * 368841 while the vadd launch waited; the r606 + * captured held chain is the same shape). Measured + * r604-r609: 100% of op_tr_pending marks (27/27, + * 71/71, 72/72) came from replay passes of already- + * executed copies - each one stalled the whole copy + * channel behind it, which is the held-chain class. + * The r308 rationale (pre-r595 the eng==0 copy + * channel's fresh passes never admitted ops, so the + * replay re-mark WAS the retry hatch) no longer + * applies: r595's cap_trust admission runs a plausible + * copy on its fresh pass, and a genuinely failed copy + * keeps its slot incomplete through op_owed / the + * rc!=0 pending rule. The whole accounting block is + * skipped on replay: no dstpend note (the write ran), + * no staging re-snapshot, no hold. A fresh + * (non-replay) denied pass marks pending exactly as + * before. */ - if (!allow_ops && trusted && + if (!allow_ops && trusted && !replay && !ch->engine_type && transfer != 0 && st.line_length && st.have_out && st.have_in && !DMA_REMAP_ENABLE(dma) && @@ -11531,6 +11893,11 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, if (n_ops != n_ops_before) { struct svc_memset_op *lm = &ops[n_ops - 1]; + /* r611: this channel carries transfer work + * (see ever_ops) - the stream-order + * watermark's scope. */ + ch->ever_ops = true; + /* * fail4 CONT.53: H2D-shaped copies (virtual * device destination, virtual host source) @@ -12265,6 +12632,9 @@ static void svc_pending_add(u64 va, u32 payload, u32 op) * exist; drop them instead of resolving them against the next session. */ static void svc_gr_diag_reset(void); /* m150: defined with the fill diag */ +/* r604: defined next to the per-VA defer tables; clears both on session end + * (they are keyed by push VA alone and the UVM pool reuses those VAs). */ +static void svc_gr_defer_reset(void); static void svc_session_flush(void) { @@ -12301,6 +12671,10 @@ static void svc_session_flush(void) } svc_gr_diag_reset(); /* m150: per-session diagnostic budgets */ + svc_gr_defer_reset(); /* r604: per-session defer latches */ + svc_grpark_reset(); /* r633: per-session parked GR releases */ + svc_gr_synth_ep_reset(); /* r633: per-session synthetic budgets */ + svc_dst_guard_reset(); /* r609: per-session device-dst windows */ svc.map_n = 0; svc_pb_n = 0; svc_pd_found = 0; @@ -14829,6 +15203,88 @@ bool gsp_sema_wait_gate = 1; unsigned gsp_sema_copy_dst_guard_ms = 3000; /* + * r609: the m169 first-touch window is per PAGE, but the stale-table risk it + * covers is per CLIENT TABLE BUILD. The driver's H2D relocation burst on the + * eng==0 copy channel writes N distinct first-write pages back to back, and + * because the pump holds the FIRST held entry and the rest of the chain + * decodes only after it drains, each page paid its own full window, + * serialized (r606 red draw: `sema-pend: hold h=0x5c000059 idx=90` held + * 362167.7->362170.8, idx=92 held 362177.7->362182.9; the b H2D entry ran + * 362182.882, 326 ms AFTER the launch forward -> `VADD bad=32735 + * csum=8139264`, c = a + 0). The guard's bet - a fresh pin lands inside the + * window - is a property of the client's table state, not of one page: once + * ONE device-dst page of a channel has waited the full window with NO pin + * arriving (every observed release logs "without a pin"), the same bet for + * the next page of the same burst is already lost. Age the channel for + * gsp_sema_copy_dst_age_ms after such a release; further first-write + * device-dst pages of that channel inside the window skip the hold and run + * with the walk translation - the exact translation the expired guard would + * have used anyway. The first page of every burst still pays the full + * window, so the anti-stale bet is unchanged; only the N x window + * serialization is removed (r606 class: ~20 s of chain -> one window). + * Keyed by h_object like every channel table; the session flush clears it, + * so a recycled handle can never inherit the previous client's ageing + * (the r604 lifetime lesson). 0 disables (= per-page windows, pre-r609). + */ +unsigned gsp_sema_copy_dst_age_ms = 3000; +#define SVC_DST_AGE_SLOTS 16 +static u32 svc_dst_age_ch[SVC_DST_AGE_SLOTS]; +static unsigned long svc_dst_age_j[SVC_DST_AGE_SLOTS]; +static u32 svc_dst_age_pos; +static u32 svc_dst_age_log; + +/* r609: the current decode channel's guard window expired without a pin. */ +static void svc_dst_guard_age(void) +{ + u32 i; + + if (!gsp_sema_copy_dst_age_ms) + return; + for (i = 0; i < SVC_DST_AGE_SLOTS; i++) + if (svc_dst_age_ch[i] == svc.cur_ch) { + svc_dst_age_j[i] = jiffies; + return; + } + i = svc_dst_age_pos++ % SVC_DST_AGE_SLOTS; + svc_dst_age_ch[i] = svc.cur_ch; + svc_dst_age_j[i] = jiffies; +} + +/* r609: is the current decode channel inside its post-expiry ageing window? */ +static bool svc_dst_guard_aged(void) +{ + unsigned long now = jiffies; + u32 i; + + if (!gsp_sema_copy_dst_age_ms || !svc.cur_ch) + return false; + for (i = 0; i < SVC_DST_AGE_SLOTS; i++) + if (svc_dst_age_ch[i] == svc.cur_ch && + time_before(now, svc_dst_age_j[i] + + msecs_to_jiffies(gsp_sema_copy_dst_age_ms))) + return true; + return false; +} + +/* + * r609: decide whether a first-write device-dst page pays the m169 window. + * An aged channel skips it (the burst's first page already paid and no pin + * arrived); otherwise the normal per-page window runs. + */ +static bool svc_dst_guard_should_hold(u64 va) +{ + if (svc_dst_guard_aged()) { + if (svc_dst_age_log < 16) { + svc_dst_age_log++; + pr_info("sema-svc: device-dst guard age-skipped dst=0x%llx h=0x%x (same-channel burst after an un-pinned release; walk translation)\n", + va, svc.cur_ch); + } + return false; + } + return svc_dst_guard_hold(va); +} + +/* * r231 READ-ORDER guard (default 0: witness only). A device-src copy whose * source VA has a PENDING device-dst write hold (the m169 guard window is * still open) reads the PRE-write content: the writer's bytes land only @@ -14877,6 +15333,7 @@ static bool svc_dst_guard_hold(u64 va) if (svc_dst_guard[i].first) { svc_dst_guard[i].first = 0; svc.copy_dst_relaxed++; + svc_dst_guard_age(); /* r609: burst ageing */ pr_info("sema-svc: device-dst guard released without a pin va=0x%llx (using the walk translation)\n", page); } @@ -14893,6 +15350,24 @@ static bool svc_dst_guard_hold(u64 va) } /* + * r609: clear the m169 page windows and the r609 ageing ring with the + * session. Both are keyed by VA page / h_object alone, and both would + * otherwise pre-age the next client's first touch (a page slot still inside + * its window makes the new client's first device-dst write skip the + * anti-stale wait; an ageing stamp makes the new channel's whole first + * burst skip it). The stale-table risk this guard covers is precisely the + * new-session condition, so the windows must start fresh. + */ +static void svc_dst_guard_reset(void) +{ + memset(svc_dst_guard, 0, sizeof(svc_dst_guard)); + svc_dst_guard_next = 0; + memset(svc_dst_age_ch, 0, sizeof(svc_dst_age_ch)); + memset(svc_dst_age_j, 0, sizeof(svc_dst_age_j)); + svc_dst_age_pos = 0; +} + +/* * r231 READ-ORDER guard: true while a held device-dst write for this VA * page is still inside its bounded window (the writer's bytes have not * landed yet). See the gsp_sema_read_order_guard declaration. @@ -18116,6 +18591,87 @@ static void svc_fwd_hold_gr_release(struct nv_vgpu_device *vgpu, * applies the recorded releases AND the QMD word, bounded by the existing * hold timeout), so this check stands down for it. */ + +/* + * r618: synth-fault for a refused/failed GR launch. The r616 experiment + * (manual grfaultpoke on the live r616b matmul wedge - fill `0/5 high + * argument ptr(s)`, `GR submit refused: degenerate data set n_data=2 + * (min 16)`, sw release authored, client spun in cudaDeviceSynchronize + * forever) proved the edge: the channel error-notifier write + * (status=0xffff + channel event) ends the client's wait with + * `CUDA_ERROR unspecified launch failure` in seconds. Deliver the same + * edge automatically on every FAILED forward so a refused launch is an + * observable error, never an unbounded hang. One-line rollback: + * gsp_sema_refusal_fault=0. + */ +bool gsp_sema_refusal_fault = true; +module_param_named(sema_refusal_fault, gsp_sema_refusal_fault, bool, 0644); +MODULE_PARM_DESC(sema_refusal_fault, + "Deliver a synthetic channel error edge on a failed/refused GR forward (default 1; 0 = legacy silent sw release)"); + +/* + * r638: break the CE-watermark/wait-gate mutual citation (see + * svc_wait_held_entry and svc_ce_stream_backlog): a channel whose + * unconsumed head entry is wait-gate held is a consumer-side wait, not a + * producer backlog, and is skipped while the candidate's operands are fully + * resolved and no decoded device write is pending on a data page. Rollback + * to the r636 behaviour: sema_mutual_skip=0. + */ +bool gsp_sema_mutual_skip = true; +module_param_named(sema_mutual_skip, gsp_sema_mutual_skip, bool, 0644); +MODULE_PARM_DESC(sema_mutual_skip, + "Skip a wait-held consumer head in the CE stream-order watermark (default 1; 0 = r636 behaviour)"); + +static void svc_gr_fault_synth(struct nv_vgpu_device *vgpu, u32 h_object) +{ + u8 notif[SVC_NOTIFY_SIZE] = {0}; + u16 status = SVC_NOTIFY_ERROR_STATUS; + u32 info32 = 0x1; /* generic engine exception, as proven live */ + unsigned long flags; + struct svc_channel *ch = NULL; + u64 pa; + u32 as, eng; + int i; + + if (!gsp_sema_refusal_fault || !h_object) + return; + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].h_object == h_object && + svc.chan[i].notifier_pa) { + ch = &svc.chan[i]; + break; + } + } + if (!ch) { + spin_unlock_irqrestore(&svc.chan_lock, flags); + return; + } + pa = ch->notifier_pa; + as = ch->notifier_as; + eng = ch->engine_type; + ch->notifier_err = true; + ch->notifier_err_until = jiffies + SVC_NOTIFIER_ERR_HOLD; + ch->notifier_err_put = ch->put_prev; + spin_unlock_irqrestore(&svc.chan_lock, flags); + + memcpy(notif + SVC_NOTIFY_INFO32_OFF, &info32, 4); + { + u16 info16 = (u16)eng; + + memcpy(notif + SVC_NOTIFY_INFO16_OFF, &info16, 2); + } + memcpy(notif + SVC_NOTIFY_STATUS_OFF, &status, 2); + if (svc_write_ap(vgpu, pa, as, notif, sizeof(notif))) { + pr_info_ratelimited("vgsp-fault: SYNTH-FAULT notifier write failed ch=0x%x pa=0x%llx as=%u\n", + h_object, pa, as); + return; + } + pr_warn("vgsp-fault: SYNTH-FAULT ch=0x%x engine=%u pa=0x%llx status=0x%x except=0x%x (refused forwarded launch -> client-visible error)\n", + h_object, eng, pa, status, info32); + gsp_rm_post_channel_event(vgpu, h_object); +} static void svc_fwd_outcome_check(struct nv_vgpu_device *vgpu) { bool done = smp_load_acquire(&svc_fwd_req.done); @@ -18149,10 +18705,18 @@ static void svc_fwd_outcome_check(struct nv_vgpu_device *vgpu) svc_fwd_req.gr_rel_va, svc_fwd_req.gr_rel_payload); svc.fwd_gr_stall_swrel++; - if (svc_fwd_req.hw_rc != 0) + if (svc_fwd_req.hw_rc != 0) { pr_warn("vgsp-gr: GR-STALL-backend va=0x%llx waited_ms=%lu action=proceed (failed forwarded launch, rc=%d; QMD release authored in software)\n", svc_fwd_req.gr_push_va, waited, svc_fwd_req.hw_rc); - else + /* + * r618: the software release alone completes only launches the + * engine actually ran; a refused/failed one left the client in + * an unbounded cudaDeviceSynchronize. Deliver the synthetic + * engine-error edge (proven live by the r616 grfaultpoke test) + * so the failed launch is client-observable. + */ + svc_gr_fault_synth(vgpu, svc_fwd_req.gr_h_object); + } else pr_warn("vgsp-gr: GR-STALL-backend va=0x%llx waited_ms=%lu action=proceed (successful launch whose release0 could not target the guest page; QMD release authored in software)\n", svc_fwd_req.gr_push_va, waited); pr_info("vgsp-gr: GR-COMPLETE-SW h=0x%x push=0x%llx rel=0x%llx pay=0x%x waited_ms=%lu n=%llu (backend outcome without a hold)\n", @@ -18228,6 +18792,19 @@ static void svc_fwd_hold_engage(struct nv_vgpu_device *vgpu, svc_fwd_hold.relay_armed = svc_fwd_req.relay; svc_fwd_hold.relay_index = svc_fwd_req.relay_index; svc_fwd_hold.fault_delivered = false; + /* + * r633 diag: the held record set, so a later release-delivery gap + * (the streams2 spin class) can be told apart from a hold that + * engaged empty / with another entry's records. Bounded. + */ + if (svc_grpark_log < 64) { + svc_grpark_log++; + pr_info("vgsp-work: hold engage h=0x%x entry=%u nrel=%u va0=0x%llx pay0=%u gr=%d\n", + ch->h_object, entry_idx, n, + n ? svc_fwd_hold.rel[0].va : 0ull, + n ? svc_fwd_hold.rel[0].payload : 0u, + svc_fwd_req.gr ? 1 : 0); + } /* r420: a fresh hold starts with no WEDGE-QUIET deferral. */ svc_fwd_hold.paused_until = 0; svc_fwd_hold.pause_logged = false; @@ -18850,6 +19427,26 @@ static void svc_fwd_gr_work(void) rel_unobserved = true; } svc_fwd_req.gr_rel_unobserved = rel_unobserved; + /* + * r633 diag: the release0 resolution chain, so a redirect + * ("release0 could not target the guest page") names its cause + * (rel_pa unresolved at capture vs HPA unresolved here vs the + * forward-only zeroing). Bounded. + */ + { + static u32 gr_relhpa_log; + + if (gr_relhpa_log < 32) { + gr_relhpa_log++; + pr_info("vgsp-work: gr-rel0 va=0x%llx pa=0x%llx ap=%u hpa=0x%llx off=%u word=%08x pay=%u unobs=%d\n", + svc_fwd_req.gr_rel_va, + svc_fwd_req.gr_rel_pa, + svc_fwd_req.gr_rel_ap, rel_hpa, roff, + rel_hpa ? READ_ONCE(*(u32 *)((u8 *)__va(rel_hpa) + roff)) : 0, + svc_fwd_req.gr_rel_payload, + rel_unobserved ? 1 : 0); + } + } } gs.qmd = svc_fwd_req.gr_qmd; @@ -18875,11 +19472,34 @@ static void svc_fwd_gr_work(void) u64 dhpa = 0; u32 doff = 0; - if (!dg->pa) - continue; - if (svc_fwd_operand_hpa(svc_fwd_req.vgpu, dkind, dg->pa, 0, - SEMA_PAGE_SIZE, &dhpa, &doff) || doff) - continue; + if (dg->shadow) { + /* + * r632: the page was served from the VA shadow lane + * (no resolver and no landing answered for it). The + * shadow page is the page's storage: hand its host + * address to the bounce machinery exactly like a + * resolved guest page. The copy-in stages the client's + * own bytes (H2D'd or zeroed) from it, and the + * copy-out writes the engine's results back into it - + * the same page the D2H copies read through the + * shadow-first rule. Module-owned memory throughout; + * no guest page is involved. + */ + struct svc_vas_rec *vr = svc_vas_find(dg->va); + + if (!vr || !vr->data) + continue; + dhpa = (u64)virt_to_phys(vr->data) & + ~(u64)(SEMA_PAGE_SIZE - 1); + dkind = SVC_APERTURE_SYS; + } else { + if (!dg->pa) + continue; + if (svc_fwd_operand_hpa(svc_fwd_req.vgpu, dkind, + dg->pa, 0, SEMA_PAGE_SIZE, + &dhpa, &doff) || doff) + continue; + } if (gs.n_data >= SVC_GR_DATA_MAX) break; data[gs.n_data].va = dg->va; @@ -19529,6 +20149,19 @@ static bool svc_gr_page_inline_covers(u64 page) static u32 svc_gr_data_log; /* m142: bounded per-page attribution log */ static u32 svc_gr_src_log; /* m150: bounded per-candidate source log */ +static u32 svc_gr_shadow_fill_log; /* r632: bounded shadow-serve log */ +/* + * r632 (the red-draw resolution class): caps on SYNTHETIC shadow pages a + * launch may create for argument candidates whose VAs no resolver answers + * (the output-buffer class). Existing shadow content (the client's own H2D + * writes) serves without a quota. The per-candidate cap bounds how far a + * span may guess past a buffer's true end (the first page past the cap ends + * the span through the historical drop path); the per-launch cap keeps the + * bounce window (16384 pages) safe. The real output operands are argument + * candidates and are collected first. + */ +#define SVC_GR_SHADOW_SYNTH_PER_CAND 1024u +#define SVC_GR_SHADOW_SYNTH_MAX 4096u static bool svc_gr_data_stalled; /* m150: high data args unresolved */ /* * m156 (R1): the GR request was refused because a forward was already @@ -19770,6 +20403,16 @@ bool gsp_sema_gr_write_guard(u64 va, u64 hpa, const char *path) if (!va || !hpa) return true; + /* + * r632: a VA with a shadow page uses the shadow as its storage (the + * copy paths resolve it there before any other source, and the launch + * path serves its pages from it). The engine's writeback targets + * exactly that page, so a later pin or discovery answer must not skip + * the write - skipping it would strand the engine's output in the + * bounce while the D2H copies still read the shadow. + */ + if (svc_vas_has(va)) + return true; if (!svc_ipin_lookup(page, &ipa, &iap)) { u32 kind = svc_ap_of(iap) == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS; @@ -20032,11 +20675,268 @@ static bool svc_gr_hold_bound_expired(struct svc_channel *ch, u64 launch_va) return false; } +/* + * r632 (the red-draw resolution class): serve a data page from the VA shadow + * lane when the resolution chain (ipin > resolver > ledger) misses. + * + * The red draw's chain is: a client session whose device VAs no source + * answers (the FB-aperture directory walk returns rc=-2 for every published + * VA space, the pin stream delivers no pair, the discovery re-validation + * fails) - the client's H2D copies then land in the shadow lane (the copy's + * documented fallback), and the launch fill drops every operand, the hold + * expires, the data set is degenerate and the refusal faults the client. + * + * Serving the page from the shadow instead keeps one story across the whole + * pipeline: the copy paths already read/write a shadowed VA in the shadow + * (svc_vas_has is checked before every resolver there), so the fill hands the + * engine the client's own H2D bytes (or a zeroed page for an output the + * client never wrote), and the submit build passes the shadow page's host + * address to the bounce machinery. The engine's writeback lands back in that + * same page, which is exactly what the D2H copies read. The shadow is + * module-owned memory: no guest page is ever touched on this path, and the + * pages are freed with the session like every other shadow page. + * + * Existing shadow content serves unconditionally (the client wrote it). + * A synthetic page (an output buffer no copy has touched) is created only + * for ANCHORED argument candidates - candidates whose span already has at + * least one page the resolvers answered - under both a per-candidate cap + * (the span has no other way to know where the buffer ends - the cap bounds + * the guess and the first page failing the cap ends the span through the + * historical drop path) and a per-launch cap (the bounce window must + * survive). With sema_no_shadow=1 every fallback is refused and the caller + * keeps its historical -ENOENT/drop path. + * + * r633 (the r632 cap validation): the r632 caps were per-FILL-CALL, and the + * fill re-runs on every retry pass of a held launch - one probe=none junk + * candidate could therefore mint another 1024 synthetic pages per pass and + * the 8192-page shadow table filled within seconds (r632 mail: va-shadow + * 8192/8192 with all pages synthetic, from the two 16 MB guessed spans of + * candidates no resolver ever answered). Two changes bound this: + * - the ANCHOR GATE: synthetic pages exist only to extend a span that + * real evidence has already anchored (>= 1 page resolved through + * ipin/resolve/ledger). A candidate whose every page is answerless is + * the historical drop path - its guessed span is not a buffer; + * - the budget is PERSISTED PER LAUNCH EPISODE, keyed by (push VA, span + * base), so the caps hold across the episode's passes, not per call. + * The anchor gate is what keeps the genuinely useful case working: the + * red-draw chain's H2D-shadowed operand pages serve through the CONTENT + * path (unconditionally), and a real buffer's unresolved tail is anchored + * by its resolved head. + */ +#define SVC_GR_SYNTH_EP_SLOTS 16 +static struct svc_gr_synth_ep { + u64 push_va; + u64 base; /* span base; 0 = the launch-level total record */ + u32 n; + /* + * r635: the cross-mint counter (launch-level anchor, see + * svc_gr_shadow_fill_serve). Tracked separately from n so a cross + * mint can never consume the anchored budgets and vice versa. + */ + u32 cross; +} svc_gr_synth_ep[SVC_GR_SYNTH_EP_SLOTS]; +static u32 svc_gr_synth_ep_pos; + +static struct svc_gr_synth_ep *svc_gr_synth_ep_get(u64 push_va, u64 base, + bool create) +{ + u32 i, slot = ~0u; + + if (!push_va) + return NULL; + for (i = 0; i < SVC_GR_SYNTH_EP_SLOTS; i++) { + if (svc_gr_synth_ep[i].push_va == push_va && + svc_gr_synth_ep[i].base == base) + return &svc_gr_synth_ep[i]; + if (slot == ~0u && !svc_gr_synth_ep[i].push_va) + slot = i; + } + if (!create) + return NULL; + if (slot == ~0u) + slot = svc_gr_synth_ep_pos++ % SVC_GR_SYNTH_EP_SLOTS; + memset(&svc_gr_synth_ep[slot], 0, sizeof(svc_gr_synth_ep[slot])); + svc_gr_synth_ep[slot].push_va = push_va; + svc_gr_synth_ep[slot].base = base; + return &svc_gr_synth_ep[slot]; +} + +static u32 svc_gr_synth_ep_launch_n(u64 push_va) +{ + struct svc_gr_synth_ep *lep = + svc_gr_synth_ep_get(push_va, 0, false); + + return lep ? lep->n : 0; +} + +static void svc_gr_synth_ep_reset(void) +{ + memset(svc_gr_synth_ep, 0, sizeof(svc_gr_synth_ep)); + svc_gr_synth_ep_pos = 0; +} + +/* + * r635 (host-side operand resolution without the scanner): the r634 red + * draws measured the remaining shape - a client whose INPUT buffers are + * shadow-CONTENT served (its own H2D bytes, the copy took the shadow lane) + * while its OUTPUT buffer (an argument candidate the client never wrote) + * still failed closed, because the r633 anchor gate only counted a page + * answered through ipin/resolve/ledger and every page of the output span + * was answerless. Two additions, both host-side (no scanner/pin needed): + * + * - a CONTENT-served page anchors its span exactly like a trusted answer: + * the client's own H2D bytes are first-hand truth, and the output tail + * of a contiguous allocation (matmul d_a|d_b|d_c, streams2's buffers) + * then mints synthetic pages through the unchanged per-candidate / + * per-launch budgets instead of dropping; + * - a LAUNCH-LEVEL anchor (cross): once any argument page of the launch + * is served from content or trust, an answerless ARG candidate whose + * span holds nothing at all (a non-contiguous output allocation) may + * still mint a strictly smaller cross-budget page set. This is what + * makes output buffers resolvable regardless of candidate order or + * allocation placement. + * + * Guards kept from r632/r633: only argument candidates mint, budgets are + * per launch episode, and every synthetic mint additionally respects + * SVC_GR_SHADOW_TABLE_GUARD - once the shared session shadow table is + * three-quarters full, no guess may displace the headroom real client + * content (H2D writes) still needs; content serves unconditionally. + */ +#define SVC_GR_SHADOW_NONE 0 +#define SVC_GR_SHADOW_CONTENT 1 +#define SVC_GR_SHADOW_SYNTH 2 +#define SVC_GR_SHADOW_SYNTH_CROSS 3 +/* + * Cross budgets: the r633 proven caps (1024 per candidate / 4096 per + * launch). They apply to CONTENT-anchored spans; the flood class they + * must never re-open (r632: 8192/8192 synthetic pages from answerless + * junk spans) is excluded by the r635 content gate - a green boot has + * no written shadow content at all, so the cross mint is off there, and + * an answerless junk span on a red boot has no head to anchor it. + */ +#define SVC_GR_SHADOW_CROSS_PER_CAND 1024u +#define SVC_GR_SHADOW_CROSS_MAX 4096u +#define SVC_GR_SHADOW_TABLE_GUARD (SVC_VAS_SLOTS * 3u / 4u) + +static int svc_gr_shadow_fill_serve(u64 va, bool is_arg, bool trusted, + bool span_content, bool launch_content, + u64 push_va, u64 base, bool *real_content) +{ + struct svc_gr_synth_ep *ep = NULL; + bool have; + + if (real_content) + *real_content = false; + if (!svc_vas_can_shadow(va)) + return SVC_GR_SHADOW_NONE; + have = svc_vas_has(va); + if (!have) { + if (!is_arg) + return SVC_GR_SHADOW_NONE; + if (trusted) { + /* + * r633 path: a trusted answer (ipin/resolver/ledger) + * anchors the span; the large r633 budgets apply. + */ + if (svc_vas_n >= SVC_GR_SHADOW_TABLE_GUARD) + return SVC_GR_SHADOW_NONE; + ep = svc_gr_synth_ep_get(push_va, base, true); + if (!ep || ep->n >= SVC_GR_SHADOW_SYNTH_PER_CAND) + return SVC_GR_SHADOW_NONE; + if (svc_gr_synth_ep_launch_n(push_va) >= + SVC_GR_SHADOW_SYNTH_MAX) + return SVC_GR_SHADOW_NONE; + } else if (span_content || launch_content) { + /* + * r635 path: the anchor is the client's own written + * content (its H2D took the shadow lane), not a + * resolver answer. The cross budgets apply - smaller, + * because this serves the output tail of a red-draw + * client, not a trusted span. + */ + struct svc_gr_synth_ep *lep; + + if (svc_vas_n >= SVC_GR_SHADOW_TABLE_GUARD) + return SVC_GR_SHADOW_NONE; + ep = svc_gr_synth_ep_get(push_va, base, true); + if (ep && ep->cross >= SVC_GR_SHADOW_CROSS_PER_CAND) + return SVC_GR_SHADOW_NONE; + lep = svc_gr_synth_ep_get(push_va, 0, false); + if (lep && lep->cross >= SVC_GR_SHADOW_CROSS_MAX) + return SVC_GR_SHADOW_NONE; + } else { + return SVC_GR_SHADOW_NONE; + } + } + { + struct svc_vas_rec *r = svc_vas_get(va, true); + + if (!r) + return SVC_GR_SHADOW_NONE; + if (have) { + /* + * r635: real_content tells the caller whether these + * bytes are the client's own WRITES (anchoring + * material) or a synthetic zero page minted by an + * earlier pass (servable, but never an anchor - the + * guard against the self-feeding mint loop). + */ + if (real_content) + *real_content = r->content != 0; + if (svc_gr_shadow_fill_log < 32) { + svc_gr_shadow_fill_log++; + pr_info("sema-svc: gr data page served from the shadow lane va=0x%llx (content%s)\n", + va, r->content ? "" : ", synthetic"); + } + return SVC_GR_SHADOW_CONTENT; + } + } + if (trusted) { + struct svc_gr_synth_ep *lep = + svc_gr_synth_ep_get(push_va, 0, true); + + ep->n++; + if (lep) + lep->n++; + if (svc_gr_shadow_fill_log < 32) { + svc_gr_shadow_fill_log++; + pr_info("sema-svc: gr data page served from the shadow lane va=0x%llx (synthetic; cand=%u launch=%u)\n", + va, ep->n, + svc_gr_synth_ep_launch_n(push_va)); + } + return SVC_GR_SHADOW_SYNTH; + } + { + struct svc_gr_synth_ep *lep = svc_gr_synth_ep_get(push_va, 0, + true); + + if (ep) + ep->cross++; + if (lep) + lep->cross++; + if (svc_gr_shadow_fill_log < 32) { + svc_gr_shadow_fill_log++; + pr_info("sema-svc: gr data page served from the shadow lane va=0x%llx (synthetic cross; cand=%u launch=%u)\n", + va, ep ? ep->cross : 0, lep ? lep->cross : 0); + } + } + return SVC_GR_SHADOW_SYNTH_CROSS; +} + static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { u64 wins[SVC_GR_CAND_MAX]; bool is_arg[SVC_GR_CAND_MAX]; /* m150: candidate came from the arg scan */ + /* + * r635: launch-level CONTENT anchor for the host-side operand + * fallback (see the pre-pass and svc_gr_shadow_fill_serve). Armed + * only by the client's own WRITTEN shadow content (H2D bytes that + * took the shadow lane) - a resolver/pin/ledger answer is NOT + * enough, so a green boot cannot enable the cross mint; consumed + * by the cross-mint for answerless argument candidates. + */ + bool launch_arg_content = false; u32 n_cand; u32 off = 0, i, w; u8 kind; @@ -20261,6 +21161,38 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, * inline-captured bytes as well (see svc_gr_scan_inline). */ svc_gr_scan_inline(wins, &n_cand, pp->va); } + /* + * r635: the launch-level CONTENT anchor for the host-side fallback, + * computed BEFORE the span loop so candidate order cannot decide. + * An argument candidate whose span already holds the client's own + * WRITTEN shadow bytes (r->content; a synthetic zero page does not + * count) marks the launch's operands as host-visible; an answerless + * ARG candidate (the output buffer the client never wrote, possibly + * not contiguous with the inputs) may then mint a strictly bounded + * cross-budget page set instead of failing the whole launch closed. + * Only the first SVC_GR_CROSS_ANCHOR_PROBE pages of each span are + * consulted - a real H2D'd buffer always has its head written, and + * the probe is a pure hash lookup. Trusted answers (pin/walk/ledger) + * do NOT arm this: on a green boot the cross mint stays off. + */ +#define SVC_GR_CROSS_ANCHOR_PROBE 64u + { + bool found = false; + + for (w = 0; w < n_cand && !found; w++) { + u32 p; + if (!is_arg[w] || wins[w] < 0x40000000000ULL) + continue; + for (p = 0; p < SVC_GR_CROSS_ANCHOR_PROBE; p++) { + if (svc_vas_has_content(wins[w] + + (u64)p * SEMA_PAGE_SIZE)) { + found = true; + break; + } + } + } + launch_arg_content = found; + } /* m110/B-2: one contiguous run of pages per candidate pointer, * relocated into the active engine-visible data region (grown region * 0x0a000000+ with gsp_phys_gr_data_region=1 - the default - or the @@ -20270,6 +21202,15 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, u64 base = wins[w]; u32 p; u32 before = svc_fwd_req.gr_n_data; + bool cand_anchored = false; /* r633: >= 1 page of this span + * resolved through a trusted + * source (ipin/resolve/ledger); + * the r633 large synthetic + * budgets only apply here */ + bool cand_content = false; /* r635: >= 1 page of this span + * holds the client's own + * WRITTEN shadow bytes; the + * cross budgets extend it */ bool first_unresolved = false; /* m170: span's first page failed */ u64 pba = 0; u32 pba_ap = 0; @@ -20343,6 +21284,9 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, u64 pa = 0; u32 ap = 0, x; struct svc_gr_data *rec; + bool shadow_serve = false; + int serve = SVC_GR_SHADOW_NONE; + bool real_content = false; /* r635 */ if (va >= (1ULL << 47)) break; @@ -20367,6 +21311,7 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, if (!svc_ipin_lookup(va, &ipa, &iap)) { pa = ipa; ap = (u32)iap; + cand_anchored = true; } else if (svc_resolve_va(vgpu, va, &pa, &ap)) { /* * C1 (stall/data class): the page may @@ -20385,6 +21330,58 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, if (!svc_ledger_lookup(va, &lpa, &lap)) { pa = lpa; ap = lap; + cand_anchored = true; + } else if ((serve = svc_gr_shadow_fill_serve( + va, is_arg[w], + cand_anchored, + cand_content, + launch_arg_content, + svc_gr_cand.push_va, + wins[w], + &real_content)) != + SVC_GR_SHADOW_NONE) { + /* + * r632/r633: no resolver, no + * landing - serve the page + * from the shadow lane + * instead of dropping it. + * Existing H2D shadow + * content serves always; a + * SYNTHETIC page only when + * the span is anchored by a + * resolved page and the + * per-episode budgets hold. + * The rec is created with + * pa=0 + shadow=1 below; + * the submit build hands + * the shadow page to the + * bounce machinery. Never + * breaks the span: a + * shadow-served page is a + * resolved page. + */ + shadow_serve = true; + /* + * r635: REAL content (the + * client's own written H2D + * bytes, r->content) anchors + * the span for the cross + * budgets and arms the + * launch-level content + * anchor. A synthetic zero + * page served from an + * earlier pass is NOT an + * anchor (the self-feeding + * mint guard). + */ + if (serve == + SVC_GR_SHADOW_CONTENT && + real_content) { + cand_content = true; + if (is_arg[w]) + launch_arg_content = + true; + } } else { /* * m170 (R4a): the very first @@ -20457,6 +21454,13 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, dropped_pages++; break; } + } else { + /* r633: the trusted resolver answered + * this page - the span is anchored + * (the r633 large budgets apply; + * the r635 cross anchor stays + * content-only). */ + cand_anchored = true; } } /* @@ -20528,10 +21532,13 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, } rec = &svc_fwd_req.gr_data[svc_fwd_req.gr_n_data]; rec->va = va; - rec->pa = pa & ~(u64)(SEMA_PAGE_SIZE - 1); + rec->pa = shadow_serve ? 0 : + (pa & ~(u64)(SEMA_PAGE_SIZE - 1)); rec->reloc_va = reloc_base + ((u64)svc_fwd_req.gr_n_data << 12); - rec->ap = (u8)svc_ap_of(ap); + rec->ap = shadow_serve ? SVC_APERTURE_SYS : + (u8)svc_ap_of(ap); + rec->shadow = shadow_serve ? 1 : 0; svc_fwd_req.gr_n_data++; /* * r349: record the launch's own resolution as the @@ -20545,8 +21552,12 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, * said SYS 0x7c00000; the D2H read followed the * ledger and returned zeros while the correct sum * sat on the written page). + * + * r632: a shadow-served rec has no guest page to + * record - the shadow is its storage, and the copy + * paths already prefer the shadow for these VAs. */ - if (va >= SVC_VAS_FLOOR) + if (!shadow_serve && va >= SVC_VAS_FLOOR) svc_ledger_store(va, rec->pa, rec->ap); /* * m142: bounded per-page attribution log. A wrong data @@ -20556,8 +21567,9 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, */ if (svc_gr_data_log < 192) { svc_gr_data_log++; - pr_info("vgsp-sema: svc-gr data page va=0x%llx -> pa=0x%llx ap=%u\n", - va, rec->pa, rec->ap); + pr_info("vgsp-sema: svc-gr data page va=0x%llx -> pa=0x%llx ap=%u%s\n", + va, rec->pa, rec->ap, + shadow_serve ? " (shadow lane)" : ""); } } (void)base; @@ -20595,7 +21607,14 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, if (is_arg[w] && svc_fwd_req.gr_n_data == before && svc_gr_src_log < 64) { int vi; - char vb[240]; + /* r614 diag: the r613b red-boot post-mortem could not + * tell which registered pdb would have answered an + * unresolved arg base (the 240-byte buffer truncated + * the walk list at three entries, and only the red + * boot's misses were ever printed). A larger buffer + * with a clamped append keeps every registered + * VASpace's walk verdict visible. */ + char vb[512]; int o = 0; svc_gr_src_log++; @@ -20610,11 +21629,12 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, svc.vas[vi].entries, base, svc.vas[vi].ap, &wa, &wap, gsp_sema_walk_trace != 0); - o += snprintf(vb + o, sizeof(vb) - o, - " vas[%d] pdb=0x%llx ent=%u ap=%d rc=%d pa=0x%llx", - vi, svc.vas[vi].pdb, - svc.vas[vi].entries, - svc.vas[vi].ap, wrc, wa); + if (o < (int)sizeof(vb) - 1) + o += snprintf(vb + o, sizeof(vb) - o, + " vas[%d] pdb=0x%llx ent=%u ap=%d rc=%d pa=0x%llx", + vi, svc.vas[vi].pdb, + svc.vas[vi].entries, + svc.vas[vi].ap, wrc, wa); } pr_info("vgsp-sema: svc-gr arg-miss base=0x%llx probe=%s n_cand=%u pd_found=%u ipin_n=%u disc=0x%llx/%d%s\n", base, pba_src, n_cand, svc_pd_found, @@ -20991,6 +22011,40 @@ static bool svc_gr_defer_expired(u64 push_va) } /* + * r604: the r543 arg-defer latch and the r556 absolute defer cap are keyed + * by push VA ALONE, and the UVM pool reuses the same launch push VAs for + * every client of a boot - the exact lifetime bug the session flush already + * fixes for the r565 exactly-once memories (svc_gr_cap_hist, svc_gr_seen, + * svc_inline_once: "matches on the VA ALONE ... a previous client's + * capture could classify a later client's fresh launch"). Carried across + * sessions, the r556 stamp arms once - at the previous client's first + * sighting - so a new client's first decision for a reused VA already reads + * defer_expired()=true: every partial-argument defer fails closed (r586 + * path), the engine never launches, the QMD release is authored in + * software, and the client reads stale/zero buffers. Witnessed r603 family + * boot: the first mail client (fresh VAs 0x20043be94/0x20043c364) deferred + * and launched (`GUEST GR LAUNCH OK`), while every later mail client (VAs + * 0x20043beac/0x20043c37c reused from earlier boot runs) saw defer_exp=1 + * at its FIRST decision and `GR forward REFUSED at defer cap ... argument + * pages still unresolved` for both launches; its "green" was the previous + * client's results still sitting in the reused device pages. + * + * Reset both tables with the session: within a session the original + * bounds stand unchanged (one deferred sighting per VA in the arg window, + * one absolute 15 s cap per VA lifetime); a new session starts with fresh + * push generations and must not inherit either stamp. + */ +static void svc_gr_defer_reset(void) +{ + memset(svc_gr_arg_defer_va, 0, sizeof(svc_gr_arg_defer_va)); + memset(svc_gr_arg_defer_j, 0, sizeof(svc_gr_arg_defer_j)); + svc_gr_arg_defer_pos = 0; + memset(svc_gr_defer_cap_va, 0, sizeof(svc_gr_defer_cap_va)); + memset(svc_gr_defer_cap_j, 0, sizeof(svc_gr_defer_cap_j)); + svc_gr_defer_cap_pos = 0; +} + +/* * r548: is a forward still OWED for this push VA - a captured, never * offered candidate sitting in the candidate ring? The r547 saxpy stall: * both launches were deferred by the r541/r543 guards, the candidate ring @@ -21114,15 +22168,52 @@ static struct svc_channel *svc_pass_local; static u32 svc_pass_local_n; /* - * r590: does any CE ring of this device still hold entries this pass's pump - * has not consumed? Reads the ring's live GP_PUT from its own USERD and - * compares it (modulo the ring size) with the snapshot's last_get. A - * mismatch means the client submitted copies/fills the pump has not decoded - * yet - exactly the stream-order backlog the r541/r568 guards cannot see. + * r590: does any ring of this device still hold entries this pass's pump has + * not consumed? Reads the ring's live GP_PUT from its own USERD and compares + * it (modulo the ring size) with the snapshot's last_get. A mismatch means + * the client submitted copies/fills the pump has not decoded yet - exactly + * the stream-order backlog the r541/r568 guards cannot see. + * + * r607 (r606 red-draw capture, final form of r590b): the engine_type==0 + * filter skipped the channel the guard exists for. The driver's H2D + * relocation copies ride a COMPUTE-class channel (h=0x5c000059, eng=0 - + * the r595 finding), and the captured red draw shows the launch forwarding + * while that channel held three published, unconsumed entries behind an + * opdef operand hold: + * 362177.927 GR-DEC push=0x20043c394 dropped=9 partial=1 dstpend=0 + * copypend=0 ceback=0 + * 362178.001 peek-backlog h=0x5c000059 eng=0 put=95 last=92 + * 362182.145 peek-backlog h=0x5c000059 eng=0 put=95 last=92 + * 362182.556 GR LAUNCH submitted (forwarded at the one-shot arg-defer + * retry) -> VADD bad=32735 csum=8139264 (c = a + 0) + * 362182.882 ce-soft-copy va=d_b executed, 326 ms AFTER the forward + * The chain was held by `sema-pend: hold h=0x5c000059 idx=90 ... (operand + * pending)`; the wait group (0x20480fff0 pay=84) passed every pass, so the + * only blindness was the class filter - ceback=0 while the copy channel + * was three entries behind. Consider every other channel of the device; + * the launching channel itself is excluded (its own ring legitimately runs + * ahead with future entries). r590's validated part is unchanged: same + * pass snapshot, same bounded busy-defer, historical forward at the + * absolute cap (this guard itself never refuses; the r586 fail-closed + * path is a separate branch that fires only when argument pages never + * resolve). r595's GR-class ops admission has since cut the hold + * resolution to 0.5-4.7 s (the captured 4.7 s), well inside the 15 s cap. + * * Channels whose USERD is unreadable are skipped (their work cannot be * ordered by this mechanism anyway). - */ -static bool svc_ce_stream_backlog(struct nv_vgpu_device *vgpu) + * + * r611: only channels that carry transfer ops are considered (ever_ops). + * The r607 blanket "every other ring" caught sibling GR LAUNCH channels + * whose rings hold the client's own future launches: streams2's two stream + * channels (h=0x5c00001f/23) each deferred on the other's published entry + * for the full 15 s cap, then the r586 fail-closed refused both kernels + * (measured r610c: every streams2 launch REFUSED, bad=65534). A launch + * channel decodes no ops (measured r606/r609/r610c: ops appear only on the + * 0x5c000059 copy family and the CE ranks), so the flag separates them + * cleanly while keeping the r606 red-draw copy channel in scope. + */ +static bool svc_ce_stream_backlog(struct nv_vgpu_device *vgpu, + const struct svc_channel *self) { u32 k; @@ -21132,8 +22223,11 @@ static bool svc_ce_stream_backlog(struct nv_vgpu_device *vgpu) const struct svc_channel *c = &svc_pass_local[k]; u32 put = 0; - if (!c->valid || (vgpu && c->vgpu != vgpu) || - !c->engine_type) + if (!c->valid || (vgpu && c->vgpu != vgpu)) + continue; + if (self && c->h_object == self->h_object) + continue; + if (!c->ever_ops) continue; if (!c->gpfifo_entries || !c->userd_pa) continue; @@ -21142,13 +22236,138 @@ static bool svc_ce_stream_backlog(struct nv_vgpu_device *vgpu) c->userd_as, &put, sizeof(put))) continue; put %= c->gpfifo_entries; - if (put != ((u32)c->last_get % c->gpfifo_entries)) + if (put != ((u32)c->last_get % c->gpfifo_entries)) { + /* + * r638 mutual-cycle gate. The r612/r636 captures + * showed a provable deadlock: a launch L is deferred + * by this watermark because channel X holds an + * unconsumed entry; X's head entry is held by the + * bounded wait gate because its acquire's release is + * authored by L itself, which cannot forward until + * this watermark clears - both halves cite each + * other, and only the 15 s defer cap / the wait's + * fail-open break it (the copy runs ~20 s late; the + * launch finds its operand pages empty). Such a head + * is a consumer-side wait, not a producer backlog: + * skip it and let the forward install the release. + * Bounded by the same guards the watermark's + * consumers already trust: skip only while the + * candidate's operands are fully resolved (no + * stalled fill pages - a launch that still needs + * producer data stays protected) and no decoded + * device write is pending on a data page. Rollback: + * sema_mutual_skip=0. + */ + if (gsp_sema_mutual_skip && + svc_gr_forward_pending() && + !svc_gr_data_stalled && + !svc_gr_dstpend_overlap() && + svc_wait_held_entry(c->h_object, + (u32)c->last_get % c->gpfifo_entries)) { + svc.ce_mutual_skips++; + if (svc.ce_mutual_log < 32) { + svc.ce_mutual_log++; + pr_info("vgsp-work: watermark skip h=0x%x head=%u (head is wait-gate held; forward unblocks its release)\n", + c->h_object, + (u32)c->last_get % + c->gpfifo_entries); + } + continue; + } return true; + } } return false; } /* + * r598 diag: the driver's ring WRITE pointer, not the GPFIFO put. The r568 + * draw needs the copy entries that are already written into the ring but + * whose GP_PUT has not been published yet. When the put says the ring is + * drained, peek at the slots immediately after the consumption position and + * log their {va,len} plus whether the slot already executed that exact + * entry. Bounded per launch, gated by gsp_phys_gr_peek. + */ +u32 gsp_phys_gr_peek; + +static void svc_gr_peek_lookahead(struct nv_vgpu_device *vgpu, + const struct svc_channel *self) +{ + u32 k, logged = 0, backlog_logged = 0; + + if (!gsp_phys_gr_peek || !svc_pass_local) + return; + /* + * r604 diag: 8 was too small. svc_pass_local carries the six CE ranks + * (engine_type 11/12/13) FIRST and the GR-class channels after, so the + * cap was always spent before the copy-carrying GR-class channels + * (0x5c000059 family - the r595 finding: "the driver's relocation / + * H2D copies live on a compute channel") could be logged. Session 48 + * concluded "the copy channel's lookahead read empty" from captures + * where that channel was never in the logged set at all. 24 covers + * every drained channel of a live client with margin; the lookahead + * depth 6 covers the copy behind the launch's own small pushes. + * + * r604: also name every channel the drained-only filter SKIPS because + * its live GP_PUT reads ahead of the pass consumption position. Those + * are exactly the channels the r590 CE watermark watches - but that + * guard skips engine_type==0, so a published backlog on a GR-class + * copy channel is invisible to every existing guard; the capture must + * show it so a red decision can be attributed to write-ahead (put == + * last, entries logged below) vs published-backlog (put != last, + * logged here) without guessing. + */ + for (k = 0; k < svc_pass_local_n && logged < 24; k++) { + const struct svc_channel *c = &svc_pass_local[k]; + u32 put = 0, j; + + if (!c->valid || c->h_object == self->h_object || + !c->gpfifo_va || !c->gpfifo_entries || !c->userd_pa) + continue; + if (svc_read_ap(vgpu, c->userd_pa + SVC_USERD_GP_PUT, + c->userd_as, &put, sizeof(put))) + continue; + put %= c->gpfifo_entries; + if (put != ((u32)c->last_get % c->gpfifo_entries)) { + /* real published backlog */ + if (backlog_logged < 12) { + backlog_logged++; + pr_info("vgsp-gr: peek-backlog h=0x%x eng=%u put=%u last=%u\n", + c->h_object, c->engine_type, put, + (u32)c->last_get); + } + continue; + } + pr_info("vgsp-gr: peek h=0x%x eng=%u put=%u last=%u\n", + c->h_object, c->engine_type, put, (u32)c->last_get); + for (j = 1; j <= 6; j++) { + u32 idx = ((u32)c->last_get + j) % c->gpfifo_entries; + u8 ent[SVC_GPFIFO_ENTRY_SIZE]; + u32 e0, e1, len; + u64 va; + + if (svc_read_va(vgpu, + c->gpfifo_va + + (u64)idx * SVC_GPFIFO_ENTRY_SIZE, + ent, sizeof(ent))) + break; + memcpy(&e0, ent, 4); + memcpy(&e1, ent + 4, 4); + va = ((u64)(e1 & 0xff) << 32) | + (u64)(e0 & 0xfffffffc); + len = ((e1 >> 10) & 0x1fffff) << 2; + if (!va || !len || len > SVC_PUSH_MAX) + break; + pr_info("vgsp-gr: peek +%u idx=%u va=0x%llx len=%u ident_match=%d\n", + j, idx, va, len, + c->slot_ops_ident[idx] == + ((u64)e0 | ((u64)e1 << 32))); + } + logged++; + } +} + +/* * r576 DIAG: the GR launch-decision witness (gsp_phys_gr_dec_log=N budget * per boot). Logged from svc_gr_request right after the fill, for launches * whose fill saw dropped pages / a partial argument set or whose decision @@ -21172,7 +22391,7 @@ static void svc_gr_dec_dump(const struct svc_channel *ch) dstpend = svc_gr_dstpend_overlap(); copypend = svc_copy_pending(); - ceback = svc_ce_stream_backlog(NULL); + ceback = svc_ce_stream_backlog(NULL, ch); pr_info("vgsp-gr: GR-DEC h=0x%x push=0x%llx n_data=%u dropped=%u partial=%d stalled=%d dstpend=%d copypend=%d ceback=%d defer_exp=%d\n", ch->h_object, svc_gr_cand.push_va, svc_fwd_req.gr_n_data, @@ -21318,7 +22537,7 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, if (gsp_phys_gr_dec_log_max && (svc_fwd_req.gr_dropped > 0 || svc_fwd_req.gr_arg_partial || svc_gr_data_stalled || svc_gr_dstpend_overlap() || - svc_copy_pending() || svc_ce_stream_backlog(vgpu))) + svc_copy_pending() || svc_ce_stream_backlog(vgpu, ch))) svc_gr_dec_dump(ch); /* * r541: launch-ordering guard. The fill's data pages are what the @@ -21366,6 +22585,24 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, svc_gr_cand.rel_payload); svc_gr_cand.rel_done = true; } + /* + * r633: deliver the parked push-level release records too - a + * fail-closed refusal still completes the client's tracking + * word (forward-only; a record already applied is skipped). + */ + { + static struct svc_fwd_rel parked[SVC_FWD_REL_MAX]; + u8 parked_n = 0; + + if (svc_grpark_get(vgpu, ch->h_object, + svc_gr_cand.push_va, parked, + &parked_n)) { + svc_fwd_rel_apply_all(vgpu, ch, parked, + parked_n); + svc_grpark_drop(vgpu, ch->h_object, + svc_gr_cand.push_va); + } + } svc_gr_cand.offered = true; svc_gr_cand.valid = false; if (svc.fwd_gr_closed_log < 32) { @@ -21401,7 +22638,8 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, * historical forward stands (witnessed) - the guard delays a launch, * it never refuses one, so it cannot recreate r588's mail refusal. */ - if (svc_ce_stream_backlog(vgpu)) { + svc_gr_peek_lookahead(vgpu, ch); + if (svc_ce_stream_backlog(vgpu, ch)) { if (!svc_gr_defer_expired(svc_gr_cand.push_va)) { clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); mutex_unlock(&svc_fwd_lock); @@ -21480,9 +22718,26 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, * no-write). Refuse the forward, author the completion in software, * consume; the dropped pages were already demanded through the pin * channel (r585) so the retry path had its chance first. + * + * r612: refuse only a DEGENERATE fill - one whose dropped set is at + * least as large as the resolved set (r584's 2 resolved / 18 + * dropped). A launch with a full resolved window (969 resolved / + * 9 dropped) whose unresolved candidates are still unresolved at + * the absolute cap has them stalled in EVERY source for the whole + * window; they are the param page's decoy/foreign pointers (the + * r611 streams2 first-client capture: candidates 0x783299000000 / + * 0x783297000000, `stalled VA` from the first fill, no pin, no + * walk, no ledger, never resolving in the run) - waiting longer + * cannot help, and refusing converts a launch whose real operands + * all resolved into a guaranteed no-write client failure (streams2 + * bad=65534). At the cap the copy-related guards (dstpend, CE + * watermark) have already cleared or the refusal branches above + * win first, so a non-degenerate fill forwards historically, + * loudly witnessed. */ if (svc_fwd_req.gr_arg_partial && !svc_gr_data_stalled && - svc_gr_defer_expired(svc_gr_cand.push_va)) { + svc_gr_defer_expired(svc_gr_cand.push_va) && + svc_fwd_req.gr_n_data < svc_fwd_req.gr_dropped) { svc.fwd_gr_refused++; svc.fwd_gr_closed++; if (svc_gr_cand.rel_va) { @@ -21491,6 +22746,20 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, svc_gr_cand.rel_payload); svc_gr_cand.rel_done = true; } + /* r633: deliver the parked push-level release records too. */ + { + static struct svc_fwd_rel parked[SVC_FWD_REL_MAX]; + u8 parked_n = 0; + + if (svc_grpark_get(vgpu, ch->h_object, + svc_gr_cand.push_va, parked, + &parked_n)) { + svc_fwd_rel_apply_all(vgpu, ch, parked, + parked_n); + svc_grpark_drop(vgpu, ch->h_object, + svc_gr_cand.push_va); + } + } svc_gr_cand.offered = true; svc_gr_cand.valid = false; if (svc.fwd_gr_closed_log < 32) { @@ -21505,6 +22774,21 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, * candidate is owned and sw-completed */ } /* + * r612: see the refusal note above - a non-degenerate fill at the cap + * forwards historically (witnessed once per budget slot). + */ + if (svc_fwd_req.gr_arg_partial && !svc_gr_data_stalled && + svc_gr_defer_expired(svc_gr_cand.push_va)) { + svc.fwd_gr_argfw++; + if (svc.fwd_gr_argfw_log < 32) { + svc.fwd_gr_argfw_log++; + pr_info("vgsp-work: GR forward at defer cap h=0x%x push=0x%llx (partial argument set: %u dropped of %u resolved; stalled candidates never resolve, forwarding historically)\n", + ch->h_object, svc_gr_cand.push_va, + svc_fwd_req.gr_dropped, + svc_fwd_req.gr_n_data); + } + } + /* * m150: the fill found high argument pointers that no source can * translate (the second-client zeros class). Do NOT forward a launch * whose kernel would dereference unresolved pointers: leave the @@ -21568,6 +22852,38 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, return false; /* candidate kept: the pump retries the entry */ } /* + * r633: this launch may have been held in an earlier pass whose decode + * recorded its push-level release records - records that are per-decode + * state and were parked at the hold's break. Restore them now so the + * caller's hold-engage holds THIS push's records (a queued launch from + * the capture ring is not accompanied by a re-decode, so the current + * svc_fwd_rel[] can be empty or belong to another entry; the r632 mail + * hang: the vadd push's tracking release never landed and the client + * spun in cudaDeviceSynchronize). + */ + { + static struct svc_fwd_rel parked[SVC_FWD_REL_MAX]; + u8 parked_n = 0; + + if (svc_grpark_get(vgpu, ch->h_object, + svc_gr_cand.push_va, parked, &parked_n)) { + if (parked_n > SVC_FWD_REL_MAX) + parked_n = SVC_FWD_REL_MAX; + memcpy(svc_fwd_rel, parked, sizeof(svc_fwd_rel)); + svc_fwd_rel_n = parked_n; + svc_fwd_defer_armed = true; + svc_grpark_drop(vgpu, ch->h_object, + svc_gr_cand.push_va); + if (svc_grpark_log < 32) { + svc_grpark_log++; + pr_info("vgsp-work: GR parked releases restored h=0x%x push=0x%llx n=%u va0=0x%llx pay0=%u\n", + ch->h_object, svc_gr_cand.push_va, + parked_n, parked[0].va, + parked[0].payload); + } + } + } + /* * The candidate is consumed: the request copy owns it from here, so a * later capture/replay cannot re-offer it and the bridge is only * reached through the request's own latch. @@ -22209,10 +23525,60 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.chan[i].engine_type) local[n++] = svc.chan[i]; } - for (i = 0; i < SVC_MAX_CHANNELS; i++) { - if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && - !svc.chan[i].engine_type) - local[n++] = svc.chan[i]; + /* + * r605 stream-order, the eng==0 half of the r507o fix. The same + * argument that made CE channels walk before GR ones applies WITHIN + * the GR class: the driver's relocation/H2D copies ride a + * later-registered compute-class channel (h=0x5c000059 family, + * eng=0 - the r595 finding), while the launch channel registers + * early (0x5c000019). The stable partition walked the launch first, + * so at a launch decision the copy channel's PUBLISHED backlog was + * unconsumed and invisible to every guard: the r590 watermark skips + * engine_type==0, and the r541/r568 gates only see DECODED copies. + * r604 capture (fresh boot, peek armed): at the vadd's second + * decision `peek-backlog h=0x5c000059 eng=0 put=95 last=94` - the + * b-copy entry was already in the ring with GP_PUT ahead of the + * pump, executed 8 ms after the decision and 320 ms before the + * launch's copy-in, and the run was green only because the + * partial-argument defer had stretched the window; pins landing one + * pass earlier would have forwarded against the pre-copy page + * (c = a + 0, the r568 red draw). Newest-registered first + * (descending h_object) walks the copy / push-library channels + * before the compute channels, so every launch decision sees the + * copy ring consumed to its live write pointer (GP_PUT): the + * write-pointer drain, by ordering. No deferral, no cap + * interaction, no speculative execution - the pump still consumes + * each entry exactly once through its normal per-slot identity + * bookkeeping (slot_ops_ident/done/complete/hash), so nothing needs + * mirroring. A client whose compute channel registers last simply + * keeps the historical order (no worse). + */ + { + u32 prev_h = 0xffffffffu; + int picked; + + for (picked = 0; picked < SVC_MAX_CHANNELS; picked++) { + int best = -1; + u32 best_h = 0; + + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + struct svc_channel *c = &svc.chan[i]; + + if (!c->valid || c->vgpu != vgpu || + c->engine_type) + continue; + if (c->h_object >= prev_h) + continue; + if (best < 0 || c->h_object > best_h) { + best = i; + best_h = c->h_object; + } + } + if (best < 0) + break; + local[n++] = svc.chan[best]; + prev_h = best_h; + } } spin_unlock_irqrestore(&svc.chan_lock, flags); @@ -23124,9 +24490,30 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) idx, va, len); svc.op_tr_pending = 0; svc.op_owed = 0; + /* + * r595: a GR-class channel + * (engine_type 0) carries the + * driver's own relocation / + * H2D copies as well as its + * launches, but ops were only + * allowed for CE channels (or + * an injected push page). The + * copies then never executed + * and their releases were + * applied by the hold path - + * exactly the r568 b-copy + * class. Admit a GR push's + * ops when its content is a + * plausible method stream + * (cap_trust is that same + * content arbiter, set just + * above), never on a replay + * or a rehold. + */ svc_decode_stream(vgpu, pb, len, 0, - (allow_ops || injected) && + (allow_ops || injected || + cap_trust) && !entry_replay && !rehold, ch, @@ -23421,6 +24808,21 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->gr_hold_idx = (u32)idx; ch->gr_hold_until = jiffies + msecs_to_jiffies(hold_ms); + /* + * r633: this break leaves + * before the release block - + * park this pass's records + * with the launch so the + * eventual forward's hold + * delivers them. + */ + if (svc_fwd_defer_armed && + svc_fwd_rel_n > 0) + svc_grpark_put(vgpu, + ch->h_object, + svc_gr_cand.push_va, + svc_fwd_rel, + svc_fwd_rel_n); svc.fwd_gr_defer++; if (svc_busy_defer_log < 64) { svc_busy_defer_log++; @@ -23498,6 +24900,25 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->gr_hold_idx = (u32)idx; ch->gr_hold_until = jiffies + msecs_to_jiffies(hold_ms); + /* + * r633: this break leaves + * before the release block - + * park this pass's records + * with the launch so the + * eventual forward's hold + * delivers them (the r632 + * mail hang: the vadd push's + * tracking release was lost + * here and the client spun + * at cudaDeviceSynchronize). + */ + if (svc_fwd_defer_armed && + svc_fwd_rel_n > 0) + svc_grpark_put(vgpu, + ch->h_object, + svc_gr_cand.push_va, + svc_fwd_rel, + svc_fwd_rel_n); svc.fwd_gr_defer++; if (svc_gr_hold_log < 16) { svc_gr_hold_log++; @@ -26408,3 +27829,68 @@ int gsp_sema_service_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); return rc; } + +/* + * r616 diag: manual fault poke for the refused-launch hang experiment. A GR + * push that the fill cannot resolve is refused (degenerate data set) and the + * entry is consumed with a software release; the authored release lands in + * the guest page (measured: kcore read 0x6 == pay on r613d) yet the guest + * driver never completes cudaDeviceSynchronize for a launch that never ran. + * This writes the channel's error notifier in the exact shape the relayed- + * fault edge uses (info32=exceptType, info16=engine, status=0xffff, latched + * by notifier_err) and posts the channel event, so a hung client's kernel + * can be poked into observing an engine error and ending its wait. Debug + * only; harmless when no client is hung (the pump's completion clear resumes + * after SVC_NOTIFIER_ERR_HOLD). h_object 0 = every valid channel. + */ +int gsp_sema_channel_fault_poke(struct nv_vgpu_device *vgpu, u32 h_object) +{ + u8 notif[SVC_NOTIFY_SIZE] = {0}; + u16 status = SVC_NOTIFY_ERROR_STATUS; + u32 info32 = 0x1; /* generic engine exception */ + int i, done = 0; + + if (!vgpu) + return -ENODEV; + memcpy(notif + SVC_NOTIFY_INFO32_OFF, &info32, 4); + memcpy(notif + SVC_NOTIFY_STATUS_OFF, &status, 2); + + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + struct svc_channel *ch = &svc.chan[i]; + unsigned long flags; + u64 pa; + u32 as, eng, h; + u16 info16; + + spin_lock_irqsave(&svc.chan_lock, flags); + if (!ch->valid || ch->vgpu != vgpu || !ch->notifier_pa || + (h_object && ch->h_object != h_object)) { + spin_unlock_irqrestore(&svc.chan_lock, flags); + continue; + } + h = ch->h_object; + pa = ch->notifier_pa; + as = ch->notifier_as; + eng = ch->engine_type; + /* Latch exactly like the relayed-fault edge so the pump's + * completion clear cannot wipe the poke before the guest + * reads it. */ + ch->notifier_err = true; + ch->notifier_err_until = jiffies + SVC_NOTIFIER_ERR_HOLD; + ch->notifier_err_put = ch->put_prev; + spin_unlock_irqrestore(&svc.chan_lock, flags); + + info16 = (u16)eng; + memcpy(notif + SVC_NOTIFY_INFO16_OFF, &info16, 2); + if (svc_write_ap(vgpu, pa, as, notif, sizeof(notif))) { + pr_info("vgsp-fault: POKE ch=0x%x notifier write failed pa=0x%llx as=%u\n", + h, pa, as); + continue; + } + pr_info("vgsp-fault: POKE ch=0x%x engine=%u pa=0x%llx as=%u status=0x%x except=0x%x (manual experiment)\n", + h, eng, pa, as, status, info32); + gsp_rm_post_channel_event(vgpu, h); + done++; + } + return done ? done : -ENOENT; +} |