From 41343c63751daefdd7156d26b817cebc43708c18 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Mon, 26 Oct 2026 21:39:28 -0400 Subject: vgsp: cross-class FB ring-block claims stop the GR autopin stealing a live foreign ring The GR ring autopin fallback (SESSION 133c userd-shape scan) and the CE first-submission capture each guarded block ownership only within their own engine class, so a channel whose own ring was not yet written latched another channel's live block permanently: the GR channels drained the CE rank-0 ring (or an unlatched CE rank's convention block), the real launch entry was never read, and CUDA clients hung after their first device query (victory_mail stalled in UVM channel_pool_add; every post-reboot boot r805-r811 showed the same steal, e.g. 'auto-pinned GR ring h=0x5c000019 rank=0 ... -> fb 0x80000 (userd-shape scan ... entry0 va=0x120000000 len=40 put=35 get=35)' while the CE handle decoded the identical entry[35] raw words from pa=0x10350641c). Changes: - shared FB block claim table (svc_fb_claim_add/_foreign/_reset), session scoped, consulted by both GR scan passes and the CE fallback scan; every live CE rank's convention block is reserved even before that rank latches (r809: CE rank 1's block at fb 0xa0000 was stolen pre-latch); - every GR autopin latch (userd-derived, FB convention, pin, trusted walk, scan pass 1/2) and both CE latch paths record claims; - CE first-submission capture: a rank's own convention block is latched even when its stale content probes incoherent (the previous session's consumed state), instead of falling through to a never-used zero block (r812/r813: client 2's 0x90000 block read put=1/get=146 at the capture instant and the zero 0x82000 block was latched, hanging the client). No module parameters or defaults change; the guards are additive. Validated (plain mdev, script defaults): - victory_mail GREEN repeatedly: 'VADD n=32768 bad=0 checksum=12204516.000000' / 'STATUS OK', all six GR ranks latched at their own fb 0x140000+0x3000*rank rings with 'launch captured h=0x5c000019 push=0x20043be94 len=840'; - a second sequential client on the same boot now progresses (vecadd_loop: 'VERIFY launch=10 bad=0' - the 100-launch loop needs a long budget). --- kernel/gsp_sema_service.c | 205 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 198 insertions(+), 7 deletions(-) (limited to 'kernel/gsp_sema_service.c') diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 55ad4b2..939d1bb 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -9115,6 +9115,107 @@ static unsigned long svc_ce_hold_since[32]; * svc_session_flush. */ static u64 svc_ce_firstsub_off[32]; +/* + * SESSION 144: FB ring-block ownership ACROSS engine classes. The GR + * autopin fallback scan (SESSION 133c) and the CE first-submission + * capture (SESSION 134c) each solved the same problem only within their + * own class; a live-but-foreign ring block passes every shape gate, so a + * channel whose own ring is not written yet latches another channel's + * block permanently and never reads its own entries. Observed on + * r805-r808: every CUDA GR rank's userd-shape scan latch resolved the CE + * rank-0 block (fb 0x80000, userd 0x90000; e.g. + * "auto-pinned GR ring h=0x5c000019 rank=0 ... -> fb 0x80000 (userd-shape + * scan ... entry0 va=0x120000000 len=40 put=35 get=35)" while the CE + * handle decoded the identical entry[35] raw words from pa=0x10350641c), + * so no GR launch was ever captured and victory_mail hung after DEVICE. + * A channel that cannot see its own block must stay unlatched (and retry + * next pass) rather than steal a foreign one. Keyed by the FB ring + * offset; claims are session state, reset by svc_session_flush. + */ +struct svc_fb_claim { + u32 h; /* owning channel handle */ + u64 ring_off; /* FB ring offset */ + u64 userd_off; /* FB USERD offset (0 = unknown) */ +}; +#define SVC_CE_RING_FB_BASE 0x80000ULL +#define SVC_CE_RING_FB_STRIDE 0x20000ULL +static struct svc_fb_claim svc_fb_claim_tab[40]; +static u32 svc_fb_claim_n; + +static void svc_fb_claim_add(u32 h, u64 ring_off, u64 userd_off) +{ + u32 i; + + for (i = 0; i < svc_fb_claim_n; i++) { + if (svc_fb_claim_tab[i].h == h) { + svc_fb_claim_tab[i].ring_off = ring_off; + svc_fb_claim_tab[i].userd_off = userd_off; + return; + } + } + if (svc_fb_claim_n < ARRAY_SIZE(svc_fb_claim_tab)) { + svc_fb_claim_tab[svc_fb_claim_n].h = h; + svc_fb_claim_tab[svc_fb_claim_n].ring_off = ring_off; + svc_fb_claim_tab[svc_fb_claim_n].userd_off = userd_off; + svc_fb_claim_n++; + } +} + +/* + * true = the candidate FB block (ring at ring_off, USERD at userd_off if + * known) is already claimed by a DIFFERENT channel, or is a live CE + * rank's convention block even before that rank latched, i.e. it must not + * be latched or stolen. Same-channel re-latches pass. + */ +static bool svc_fb_claim_foreign(struct nv_vgpu_device *vgpu, u32 self_h, + u64 ring_off, u64 userd_off) +{ + u32 i; + unsigned long flags; + int k; + + for (i = 0; i < svc_fb_claim_n; i++) { + if (svc_fb_claim_tab[i].h == self_h) + continue; + if (svc_fb_claim_tab[i].ring_off == ring_off) + return true; + if (userd_off && svc_fb_claim_tab[i].userd_off == userd_off) + return true; + } + /* + * SESSION 144b: reserve every live CE rank's convention block even + * before that rank has latched. r809: CE rank 1's block at fb + * 0xa0000 was still unclaimed when GR rank 0's userd-shape scan ran + * (rank 1 latched through the plain autopin path, not the + * first-submission capture) and the GR channel stole it again. + */ + spin_lock_irqsave(&svc.chan_lock, flags); + for (k = 0; k < SVC_MAX_CHANNELS; k++) { + u64 conv; + + if (!svc.chan[k].valid || svc.chan[k].vgpu != vgpu || + !svc.chan[k].engine_type || + svc.chan[k].h_object == self_h || + svc.chan[k].ce_index >= 32) + continue; + conv = SVC_CE_RING_FB_BASE + + SVC_CE_RING_FB_STRIDE * svc.chan[k].ce_index; + if (ring_off == conv || + (userd_off && userd_off == conv + 0x10000)) { + spin_unlock_irqrestore(&svc.chan_lock, flags); + return true; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + return false; +} + +static void svc_fb_claim_reset(void) +{ + svc_fb_claim_n = 0; + memset(svc_fb_claim_tab, 0, sizeof(svc_fb_claim_tab)); +} + /* * ISSUE-0002 (r47): deferred FB sweep for a CE channel whose layout USERD * probe fails at discovery. Session/reset-scoped, scheduled on the first @@ -13133,6 +13234,7 @@ static void svc_session_flush(void) memset(svc_ce_firstsub_n, 0, sizeof(svc_ce_firstsub_n)); memset(svc_ce_hold_since, 0, sizeof(svc_ce_hold_since)); memset(svc_ce_firstsub_off, 0, sizeof(svc_ce_firstsub_off)); + svc_fb_claim_reset(); svc_userd_sweep_log = 0; memset(svc_userd_sweep_at, 0, sizeof(svc_userd_sweep_at)); memset(svc_userd_sweep_n, 0, sizeof(svc_userd_sweep_n)); @@ -14778,7 +14880,8 @@ static bool svc_ce_firstsub_try_off(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u8 *sbase, u64 lim, u64 off, u32 *scan_hits, u64 *latch_off, u64 *latch_upa, - u32 *latch_put, u32 *latch_get) + u32 *latch_put, u32 *latch_get, + bool own_block) { static const u64 udeltas[] = { 0x10000, 0x2000, 0x1000, 0x0 }; u32 h0, h1, hlen; @@ -14804,8 +14907,24 @@ static bool svc_ce_firstsub_try_off(struct nv_vgpu_device *vgpu, SVC_APERTURE_FB, ch->gpfifo_entries, &p3, &g3); - if (prc && prc != -EAGAIN) - continue; + /* + * SESSION 144d: the rank's OWN convention block (own_block, + * the +0x10000 candidate at the rank's convention offset) is + * latched even when its stale content probes incoherent + * (g > p) or idle - the guest rewrites put/get on its next + * submission and the pump's per-channel last_get processes + * entry0 exactly once. r813: client #2's convention block at + * fb 0x90000 read put=1/get=146 (g > p -> -EINVAL) at the + * capture instant, the coherence gate rejected it, and the + * never-used 0x82000 zero block was latched instead, so the + * client's first push never drained and it hung. The gates + * below stay for every heuristic candidate (other deltas and + * the fallback scan), preserving the SESSION 135 protection. + */ + if (!own_block || ui != 0) { + if (prc && prc != -EAGAIN) + continue; + } /* * SESSION 135: at the FIRST-SUBMISSION instant the channel's * submission is by definition unconsumed - the notifier clear @@ -14819,7 +14938,20 @@ static bool svc_ce_firstsub_try_off(struct nv_vgpu_device *vgpu, * old session's, the client stalled on its expectations). * Idle 0/0 stays acceptable (svc_userd_probe's own rule). */ - if (p3 && p3 == g3) + /* + * SESSION 144c: this consumed-state rejection is for FOREIGN + * blocks found by the heuristic deltas. The rank's OWN + * convention block (+0x10000) legitimately carries the + * previous session's consumed state until the guest + * re-initialises the reused allocation - rejecting it made + * the r812 second client fall through to a never-used + * 0x82000 block and stall forever (cefirst row: off=0x80000 + * upa=0x82000 put=0 get=0 while the client waited for the + * first push's completion). The pump's per-channel last_get + * still processes entry0 exactly once, so latching the + * convention block with stale put/get is correct. + */ + if (!own_block && p3 && p3 == g3) continue; /* latch exactly like the CE autopin */ svc_pin_store(ch->gpfifo_va, off, SVC_APERTURE_FB); @@ -14904,7 +15036,8 @@ static void svc_ce_firstsub_capture(struct nv_vgpu_device *vgpu, latched = svc_ce_firstsub_try_off(vgpu, ch, sbase, lim, exp, &scan_hits, &latch_off, &latch_upa, - &latch_put, &latch_get); + &latch_put, &latch_get, + true); for (off = 0; !latched && off + 0x90 <= lim; off += 0x100) { unsigned int ri; @@ -14923,16 +15056,35 @@ static void svc_ce_firstsub_capture(struct nv_vgpu_device *vgpu, } if (foreign) continue; + /* + * SESSION 144: symmetric protection - a CE + * fallback scan must not steal a block another + * channel (CE rank or GR) already claims. + */ + if (svc_fb_claim_foreign(vgpu, ch->h_object, off, + off + 0x10000)) + continue; latched = svc_ce_firstsub_try_off(vgpu, ch, sbase, lim, off, &scan_hits, &latch_off, &latch_upa, &latch_put, - &latch_get); + &latch_get, + false); } - if (latched) + if (latched) { svc_ce_firstsub_off[ch->ce_index] = latch_off; + /* + * SESSION 144: publish the claim so a GR rank + * whose own ring is not written yet cannot + * latch this CE block (r805-r808: every CUDA + * GR rank stole the CE rank-0 block and the + * launch was never captured). + */ + svc_fb_claim_add(ch->h_object, latch_off, + latch_upa); + } } } @@ -16458,6 +16610,9 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, ch->userd_as = cap; ch->userd_ok = true; svc_gr_ring_base = cand; + if (cap == SVC_APERTURE_FB) + svc_fb_claim_add(ch->h_object, cand, + ch->userd_pa); set_bit(ch->gr_index, &svc_gr_ring_done); svc_ring_n++; pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx ap=%d (userd-derived d=0x%llx; entry0 va=0x%llx len=%u put=%u get=%u)\n", @@ -16497,6 +16652,8 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, ch->userd_as = SVC_APERTURE_FB; ch->userd_ok = true; svc_gr_ring_base = base; + svc_fb_claim_add(ch->h_object, pa, + pa + SVC_GR_USERD_OFF); set_bit(ch->gr_index, &svc_gr_ring_done); svc_ring_n++; pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> fb 0x%llx (entry0 va=0x%llx len=%u put=%u get=%u)\n", @@ -16549,6 +16706,10 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, ch->userd_as = svc_ap_of(pin_ap); ch->userd_ok = true; svc_gr_ring_base = base; + if (svc_ap_of(pin_ap) == SVC_APERTURE_FB) + svc_fb_claim_add(ch->h_object, pin_pa, + pin_pa + + SVC_GR_USERD_OFF); set_bit(ch->gr_index, &svc_gr_ring_done); svc_ring_n++; pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx ap=%d (entry0 va=0x%llx len=%u put=%u get=%u)\n", @@ -16584,6 +16745,9 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, ch->userd_as = wap; ch->userd_ok = true; svc_gr_ring_base = wpa; + if (wap == SVC_APERTURE_FB) + svc_fb_claim_add(ch->h_object, wpa, + wpa + SVC_GR_USERD_OFF); set_bit(ch->gr_index, &svc_gr_ring_done); svc_ring_n++; pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx ap=%d (trusted walk; entry0 va=0x%llx len=%u put=%u get=%u)\n", @@ -16673,6 +16837,19 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, continue; if (svc_ring_entry_dead(vgpu, h0, h1)) continue; + /* + * SESSION 144: never latch a foreign + * block. Without this every CUDA GR + * rank stole the CE rank-0 block while + * its own ring was unwritten + * (r805-r808); staying unlatched here + * lets the FB-convention path latch the + * real ring once the client writes it. + */ + if (svc_fb_claim_foreign(vgpu, + ch->h_object, + roff, off)) + continue; if (svc_userd_probe(vgpu, off, SVC_APERTURE_FB, ch->gpfifo_entries, @@ -16686,6 +16863,7 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, ch->userd_as = SVC_APERTURE_FB; ch->userd_ok = true; svc_gr_ring_base = roff; + svc_fb_claim_add(ch->h_object, roff, off); set_bit(ch->gr_index, &svc_gr_ring_done); set_bit(ch->gr_index, @@ -16723,6 +16901,10 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, if (upa + 4 > lim) continue; + if (svc_fb_claim_foreign(vgpu, + ch->h_object, + off, upa)) + continue; if (svc_userd_probe(vgpu, upa, SVC_APERTURE_FB, ch->gpfifo_entries, @@ -16736,6 +16918,8 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, ch->userd_as = SVC_APERTURE_FB; ch->userd_ok = true; svc_gr_ring_base = off; + svc_fb_claim_add(ch->h_object, off, + upa); set_bit(ch->gr_index, &svc_gr_ring_done); set_bit(ch->gr_index, @@ -16960,6 +17144,13 @@ static void svc_ring_autopin(struct nv_vgpu_device *vgpu, svc_pin_store(ch->gpfifo_va, pa, SVC_APERTURE_FB); svc_lat_store(ch->gpfifo_va, pa, SVC_APERTURE_FB); + /* + * SESSION 144b: record the block ownership here too - CE ranks that + * latch through this plain autopin path (not the first-submission + * capture) were invisible to the GR fallback scan, which then stole + * their block (r809: CE rank 1 at fb 0xa0000). + */ + svc_fb_claim_add(ch->h_object, pa, pa + 0x10000); set_bit(ch->ce_index, &svc_ring_done); svc_ring_n++; pr_info("sema-chan: auto-pinned ring h=0x%x rank=%u va=0x%llx -> fb 0x%llx (entry0 va=0x%llx len=%u)\n", -- cgit v1.2.3