From b90ced5aa52d202e678918093d1bb497ec1efd2b Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Fri, 16 Oct 2026 20:54:49 -0400 Subject: vgsp: m163 lost-H2D copy integrity (drain injected pins before CE op resolution; hold failed device-copy ops) + m164 CLASS-2 GR no-capture entry hold + m165 pool release applies at fresh-pin time + m166 timeout-ladder warm-sem fail-fast; scanner: first-class pool VA resolution + periodic re-assert + demand routing --- kernel/gsp_sema_service.c | 197 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 193 insertions(+), 4 deletions(-) (limited to 'kernel/gsp_sema_service.c') diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index c9c36b3..d6b9617 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -355,6 +355,12 @@ struct svc_channel { * the capture; forcing it lets a deferred * launch be forwarded once the engine's * in-flight work completes) */ + u32 gr_nocap_idx; /* m164 CLASS-2: slot in the no-capture hold + * (a readable launch push whose decode saw + * QMD words but built no candidate) */ + u32 gr_nocap_replay;/* 1 = window opened on a replay decode */ + u64 gr_nocap_va; /* push VA the window was opened for */ + unsigned long gr_nocap_since; /* absolute first-seen stamp */ }; /* @@ -775,6 +781,11 @@ struct sema_svc { * expectation advanced, write queued */ u64 gpfifo_held; /* passes an entry was left pending * for a translation/grace window */ + u64 gr_nocap_held; /* m164: GR holds opened for a + * launch push whose decode saw QMD + * words but built no candidate */ + u64 gr_nocap_consumed; /* m164: no-capture windows that + * expired and consumed the entry */ u64 gpfifo_starved; /* entries consumed without a * deliverable exact slot */ u64 counter_rel_applied; /* releases to registered counters */ @@ -1194,6 +1205,7 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) memset(ch, 0, sizeof(*ch)); ch->gr_hold_idx = ~0u; /* m156: no hold armed yet */ ch->gr_rehold_idx = ~0u; + ch->gr_nocap_idx = ~0u; /* m164: no no-capture window armed */ if (same) bitmap_copy(ch->rel_done, keep_rel_done, SVC_GPFIFO_MAX_BITS); @@ -2698,6 +2710,29 @@ static inline bool svc_pool_hold_active(void) time_before(jiffies, svc.last_chan_reg + 5 * HZ); } +/* + * m165 (D-F2a): the blanket pool hold exists to stop a release decoded + * before the guest's own injection lands from being applied through a + * stale translation. Once a host-verified injected pin for THIS VA is + * present, that premise is gone: the pin source is the resolver's first + * choice (read-back proven) and the ipin table is wiped at session flush, + * so it is this session's ground truth. r156 measured every pool release + * waiting the full registration+5.00 s with the pin ready all along. + */ +static bool svc_pool_hold_active_va(u64 va) +{ + u64 ipa = 0; + int iap = 0; + + if (!svc_pool_hold_active()) + return false; + if (!gsp_sema_pool_hold_fresh) + return true; + if (svc_ipin_lookup(va, &ipa, &iap) == 0) + return false; + return true; +} + /* * Verified pool-semaphore translations, keyed by page VA. An entry is only * created from an injected pin or from a targeted rm_mem resolve whose page @@ -3591,11 +3626,12 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) svc.false_trans, svc.gpfifo_skipped, svc.gpfifo_rescued, svc.releases_faked, svc.releases_recovered, svc.counter_rel_applied); - pr_info("sema-svc: drain held=%llu starved=%llu forwarded=%llu faked=%llu rec=%llu ops_once=%llu reident=%llu opdef=%llu/%llu/%llu\n", + pr_info("sema-svc: drain held=%llu starved=%llu forwarded=%llu faked=%llu rec=%llu ops_once=%llu reident=%llu opdef=%llu/%llu/%llu gr_nocap=%llu/%llu\n", svc.gpfifo_held, svc.gpfifo_starved, svc.releases_forwarded, svc.releases_faked, svc.releases_recovered, svc.ops_once_skipped, svc.ops_reident, - svc.opdef_held, svc.opdef_applied, svc.opdef_expired); + svc.opdef_held, svc.opdef_applied, svc.opdef_expired, + svc.gr_nocap_held, svc.gr_nocap_consumed); pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu altw=%llu residue=%llu\n", svc.counter_pub_calls, svc.gr_rel_applied, svc.sem_write_reresolve, svc.pushpg_fallback, @@ -4836,6 +4872,11 @@ static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va, return 0; } +/* defined below (svc_handle_map_add); the CE decode path drains the queued + * scanner injections before resolving copy operands, exactly like the GR + * path's m143 drain. */ +static void svc_handle_map_add(void); + /* Execute a decoded operation list (fills and copies). `ops_done` (may be * NULL) is the per-entry bitmap of ops that already ran for the entry's * identity; `ops_base` is the sequence number of ops[0] within the entry. */ @@ -4846,6 +4887,17 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, static u64 bigop_log; svc.cur_ch = ch_h; + /* + * m163: apply every queued scanner injection before resolving this + * pass's copy operands. Without this, a CE copy decoded in the window + * between the client's push and its pins being processed resolves + * through the previous client's stale translation (or fails), while + * its release is still applied - the client's cudaMemcpy "completes" + * with no data and the next GR launch computes from the stale vidmem + * page (streams2 r163: a1 held the previous client's h_a; the vadd + * output was a_stale + b_fresh). + */ + svc_handle_map_add(); for (i = 0; i < n_ops; i++) { struct svc_memset_op *m = &ops[i]; u32 seq = ops_base + i; @@ -4917,9 +4969,18 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, * (observed r72 c2) must not withhold a release that gates a GR * launch with a 5 s poll deadline. On expiry the release is * flushed and the op stays skipped (the pre-r62 fallback). + * m163 extends the hold to copy ops whose destination is a + * device buffer: a lost small H2D that silently "completes" + * corrupts the next launch's input (streams2 r163: the 128 KB + * a1 copy's release was applied with no copy op ever run; the + * vadd computed a_stale + b and the client read the wrong c). + * A small failing device copy now costs a bounded hold and the + * retried decode runs it once the pins land - never silent + * corruption. Small failing memsets keep the old trade. */ if (rc != 0 && rc != SVC_COPY_FORWARDED && - m->bytes >= (256u << 10)) + (m->bytes >= (256u << 10) || + (m->type == 1 && m->dst >= SVC_VAS_FLOOR))) svc.op_tr_pending = 1; if (gsp_sema_dbg && m->bytes == 256 && svc.sem256_log < 64) { svc.sem256_log++; @@ -5906,6 +5967,15 @@ static void svc_fwd_rel_direct_note(u32 h, u64 va, u32 payload) * mapped to its register offset. Each completed QMD resets the accumulator, * so a push may carry more than one launch. */ +/* + * m164: per-decode QMD witness for the CLASS-2 no-capture hold. Set to the + * max number of QMD words any accumulator saw during the current + * svc_decode_stream() call; reset at its top and read by the entry loop's + * no-capture check. Service-thread only. + */ +static u32 svc_gr_qmd_seen; +static u64 svc_gr_nocap_log; + static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, struct svc_push_state *st, u32 addr, u32 count, const u8 *data, const u8 *push, u32 push_len, @@ -5935,6 +6005,8 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, } } } + if (st->qmd_n > svc_gr_qmd_seen) + svc_gr_qmd_seen = st->qmd_n; if (st->qmd_have == 3) st->qmd_dev_va = (((u64)st->qmd_hi << 32) | st->qmd_lo) << 8; if (st->qmd_n == SVC_QMD_WORDS) { @@ -6001,6 +6073,9 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u32 ops_seq = 0; /* ops appended this pass (sequence numbers) */ u32 off = 0; + /* m164: fresh CLASS-2 QMD witness for this decode pass. */ + svc_gr_qmd_seen = 0; + /* * A false VA->PA pair (the discovered map is content-scanned) can * point a ring entry's push VA at an unrelated page - observed: @@ -7440,7 +7515,7 @@ static void svc_apply_pending(struct nv_vgpu_device *vgpu) * a pool VA the base may not have landed: applying while * the word is still 0 is absorbed, so hold and retry. */ - if (svc_pool_sem_va(p->va) && svc_pool_hold_active()) { + if (svc_pool_sem_va(p->va) && svc_pool_hold_active_va(p->va)) { p->tries++; svc_pending_note_starved(p); i++; @@ -9338,6 +9413,27 @@ bool gsp_sema_counter_derive = true; */ unsigned gsp_sema_gr_grace_ms = 8000; +/* + * m164 CLASS-2: hold window (ms) for a readable GR launch push whose decode + * saw QMD words but built no launch candidate for its channel (a replay- + * suppressed capture, a partial/split QMD, or a push page the pin stream has + * not reached yet). The entry is left unconsumed for this window so the next + * pass re-decodes it with a forced fresh capture (gr_rehold_idx); on expiry + * the historical consume applies. Without this, such entries were consumed + * silently and the client's launch never ran (streams2 second vadd: 56 + * consumes / 2 captures on one channel; the client read zeros). + */ +unsigned gsp_sema_gr_nocap_hold_ms = 6000; + +/* + * m165 (D-F2a): when set (default), a pool release is only deferred by the + * post-registration hold while NO host-verified injected pin exists for its + * VA; a fresh pin is this session's ground truth and lets the apply land at + * pin time (~0.5-2 s) instead of registration+5 s. Set 0 to restore the + * legacy blanket 5 s hold. + */ +bool gsp_sema_pool_hold_fresh = 1; + /* * CE release fallback: when a push carrying a tracking-semaphore release * cannot be read (and the content rescue failed), advance the channel's @@ -14775,6 +14871,84 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc_gr_cand.rel_payload); } } + /* + * m164 CLASS-2 no-capture hold: a + * readable GR launch push whose + * decode saw QMD words but built + * no launch candidate for this + * channel (replay-suppressed + * capture, partial/split QMD, or + * a page the pin stream has not + * reached) must not be consumed + * at the tail - consuming drops + * the client's launch and only + * the RAM sweep's release reaches + * it (r161/r167 streams2 second + * vadd: 56 consumes / 2 captures + * on channel 0x5c000023). Hold + * the entry and force a fresh + * capture on the next pass + * (gr_rehold_idx makes the replay + * decode rebuild the candidate); + * one bounded window per + * (slot, push VA), then fall + * through to the historical + * consume so the channel can + * never livelock. + */ + if (!ch->engine_type && va && len && + gsp_phys_gr_forward_enabled() && + svc_gr_qmd_seen > 0 && + !(svc_gr_cand.valid && + svc_gr_cand.h_object == + ch->h_object && + svc_gr_cand.push_va == va)) { + if (ch->gr_nocap_idx != + (u32)idx || + ch->gr_nocap_va != va) { + ch->gr_nocap_idx = + (u32)idx; + ch->gr_nocap_va = va; + ch->gr_nocap_replay = + entry_replay ? + 1 : 0; + ch->gr_nocap_since = + jiffies; + } + if (gsp_sema_gr_nocap_hold_ms && + time_before(jiffies, + ch->gr_nocap_since + + msecs_to_jiffies( + gsp_sema_gr_nocap_hold_ms))) { + /* Force a fresh capture on + * the retry: a replay decode + * would suppress it. */ + ch->gr_rehold_idx = + (u32)idx; + svc.gpfifo_held++; + svc.gr_nocap_held++; + if (svc_gr_nocap_log < 16) { + svc_gr_nocap_log++; + pr_info("sema-chan: GR no-capture hold h=0x%x entry=%u va=0x%llx len=%u replay=%d qmd_seen=%u (forcing capture; window %u ms)\n", + ch->h_object, + (u32)idx, va, + len, + ch->gr_nocap_replay, + svc_gr_qmd_seen, + gsp_sema_gr_nocap_hold_ms); + } + kfree(pb); + break; + } + /* Window expired: consume + * exactly as the pre-change + * tail does, with a proof + * line. */ + svc.gr_nocap_consumed++; + pr_info_ratelimited("sema-chan: GR no-capture hold expired h=0x%x entry=%u va=0x%llx len=%u (consuming without a launch)\n", + ch->h_object, + (u32)idx, va, len); + } /* * A CE push that should carry this * channel's next tracking release but @@ -15355,6 +15529,12 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->last_get = (idx + 1) % ch->gpfifo_entries; ch->rescue_fail = 0; ch->defer_since = 0; + /* m164: the entry was consumed (or fell through the + * no-capture window); clear the hold identity so a + * future push on this slot opens a fresh window. */ + ch->gr_nocap_idx = ~0u; + ch->gr_nocap_since = 0; + ch->gr_nocap_va = 0; ch->unread_va = 0; ch->unread_until = 0; @@ -15472,6 +15652,15 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ svc.chan[k].gr_hold_until = ch->gr_hold_until; svc.chan[k].gr_hold_idx = ch->gr_hold_idx; + /* m164: the CLASS-2 no-capture window is snapshot + * state too - without the write-back every pass + * would re-arm it and rehold the entry forever + * (the m150 lesson). */ + svc.chan[k].gr_nocap_idx = ch->gr_nocap_idx; + svc.chan[k].gr_nocap_replay = + ch->gr_nocap_replay; + svc.chan[k].gr_nocap_va = ch->gr_nocap_va; + svc.chan[k].gr_nocap_since = ch->gr_nocap_since; svc.chan[k].pushpg_va = ch->pushpg_va; svc.chan[k].pushpg_n = ch->pushpg_n; svc.chan[k].content_max = ch->content_max; -- cgit v1.2.3