diff options
| -rw-r--r-- | kernel/gsp_phys_booter.c | 46 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 441 | ||||
| -rw-r--r-- | tools/guest-diag/guest_pin_stream.py | 680 |
3 files changed, 1115 insertions, 52 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index f083c99..72cda10 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -17302,6 +17302,17 @@ gr_page_unmapped: le64_to_cpu(pv[s + 2]), le64_to_cpu(pv[s + 3]), le64_to_cpu(pv[s + 4]), le64_to_cpu(pv[s + 5]), le64_to_cpu(pv[s + 6]), le64_to_cpu(pv[s + 7])); + /* m143b: the b (0x130..) and c (0x150..) relocated windows + * and the trailing data window, so an aliased/stale PTE + * that redirects an engine access into another window is + * visible in the same log. */ + for (s = 0x130; s < 0x180; s += 0x20) + pr_info("vgsp-gr: main PT[%03x..%03x]=%016llx %016llx %016llx %016llx %016llx %016llx %016llx %016llx\n", + s, s + 7, + le64_to_cpu(pv[s]), le64_to_cpu(pv[s + 1]), + le64_to_cpu(pv[s + 2]), le64_to_cpu(pv[s + 3]), + le64_to_cpu(pv[s + 4]), le64_to_cpu(pv[s + 5]), + le64_to_cpu(pv[s + 6]), le64_to_cpu(pv[s + 7])); for (s = 0x190; s < 0x1a0; s += 8) pr_info("vgsp-gr: main PT[%03x..%03x]=%016llx %016llx %016llx %016llx %016llx %016llx %016llx %016llx\n", s, s + 7, @@ -17909,6 +17920,36 @@ gr_page_unmapped: GSP_PHYS_PAGE_SIZE); wrote++; } + /* m143b: unconditional first words of the first page of each + * 32-page relocation window (a=0x110000, b=0x130000, c=0x150000) + * after the run, so the engine's read/write footprint per window + * is provable without relying on the changed-page counter. */ + { + u32 wi; + + for (wi = 0; wi + 1 < req->n_data; wi += 32) { + u32 bi = (u32)((req->data[wi].reloc_va - + GSP_PHYS_RMCALL_GR_BOUNCE_VA0) >> + 12); + const u8 *wp; + u32 wc = 0, k; + + if (bi >= GSP_PHYS_RMCALL_GR_MAIL_SLOT) + continue; + wp = gsp_phys_chunk_page_va( + &boot_stage.rmcall_gr_bounce, bi); + for (k = 0; k < GSP_PHYS_PAGE_SIZE; k += 4) + wc += get_unaligned_le32(wp + k); + pr_info("vgsp-gr: window[%u] reloc=0x%llx first=%08x %08x %08x %08x changed=%u\n", + bi, + req->data[wi].reloc_va, + get_unaligned_le32(wp), + get_unaligned_le32(wp + 4), + get_unaligned_le32(wp + 8), + get_unaligned_le32(wp + 12), + wc != bounce_csum_in[bi]); + } + } pr_info("vgsp-gr: bounce copy-out: %u pages written back, %u changed by engine (first bi: %u %u %u %u %u %u %u %u)\n", wrote, changed, chg[0], chg[1], chg[2], chg[3], chg[4], chg[5], chg[6], chg[7]); @@ -17932,8 +17973,9 @@ gr_page_unmapped: &boot_stage.rmcall_gr_qmdwin, 0); if (qw) - pr_info("vgsp-gr: post-OK qmdwin w48..51=%08x %08x %08x %08x\n", - qw[48], qw[49], qw[50], qw[51]); + pr_info("vgsp-gr: post-OK qmdwin w48..59=%08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n", + qw[48], qw[49], qw[50], qw[51], qw[52], qw[53], + qw[54], qw[55], qw[56], qw[57], qw[58], qw[59]); } ret = 0; diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 7d690ab..20dfc54 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -339,6 +339,11 @@ struct svc_channel { * slot a bounded number of times, while a * clobbered slot loops forever (storm * detector) */ + unsigned long gr_hold_until; /* m150: GR data-arg retry window expiry */ + u32 gr_hold_idx; /* GPFIFO slot the window was armed for (identity: + * an entry holds at most once, then is consumed + * even if still unresolved - bounded zeros, never + * a hang) */ }; /* @@ -597,6 +602,14 @@ struct sema_svc { u64 grce_log; /* GRCE inline-write budget */ u32 cur_ch; /* channel of the op being run */ u64 entry_log; /* per-push summary budget */ + /* + * m151: bounded log for CE pushes that were READABLE (nonzero first + * bytes) yet decoded no release. These are the entries the pump holds + * for the 8 s grace; naming their decode state (have_sem, rel) and + * head bytes is the Class-2 diagnostic (r128: correct content, right + * sem block at +0xC0 with RELEASE_ONE_WORD, yet rel_seen false). + */ + u64 entry_norel_log; u64 ops_denied; /* work refused: implausible stream */ u64 false_trans; /* resolvable VA mapped to non-push */ u64 sem_write_reresolve; /* writes re-resolved after read-back */ @@ -759,6 +772,7 @@ struct sema_svc { u64 fwd_gr_ok; u64 fwd_gr_fallback; u64 fwd_gr_refused; + u64 fwd_gr_defer; /* m150: held for data-arg retry */ u64 fwd_gr_timeout; u64 fwd_gr_fault; u64 fwd_gr_bridge; @@ -928,6 +942,42 @@ static int svc_write_ap(struct nv_vgpu_device *vgpu, u64 addr, u32 aperture, static bool svc_gr_content_rearm; /* + * m149 second-client stall evidence: the CE push-buffer window pins are the + * translation source the second session's GPFIFO entries need; log the + * first few per session (reset on channel registration) so a boot shows + * whether they arrived. + */ +static u32 svc_cepin_log; + +/* + * m148 (second-client wedge): re-arm the per-rank one-shot ring/push-buffer + * latches for a (re-)registered channel. The guest driver rebuilds the GSP + * session for every client context and RM reuses the same object handles and + * channel ranks across sessions. The ISSUE-0002 session flush only runs on + * the last-channel-closed transition, so a second session that re-registers + * the same handles while the first session's entries are still present (or + * before the deferred flush executes) inherits the previous session's latch + * bits. The CE/GR ring autopin and the push-buffer pairing then never re-run, + * the new session's ring stays untranslated (svc_lookup fails for its GPFIFO + * VA), the pump counts the entries as nomap and the guest spins forever in + * uvm_tracker_wait_for_entry waiting for its channel-creation push release - + * the second-client wedge. Both autopin paths and the pairing are + * content-validated (entry0 must parse; a discovered base must carry this + * channel's slot), so re-arming is safe and idempotent within a session. + */ +static void svc_chan_rearm_rank(struct svc_channel *ch) +{ + if (!ch || !ch->valid) + return; + if (ch->engine_type) { + clear_bit(ch->ce_index, &svc_ring_done); + clear_bit(ch->ce_index, &svc_pb_done); + } else { + clear_bit(ch->gr_index, &svc_gr_ring_done); + } +} + +/* * Register a guest channel. Called from the GSP-RM emulation when the guest * driver allocates a channel (the allocation carries the GPFIFO GPU VA, its * entry count and the guest-physical USERD/instance blocks), and dropped when @@ -942,8 +992,10 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) if (!desc || !desc->h_object || !desc->gpfifo_va || desc->gpfifo_entries < 2) return; - if (!desc->engine_type) + if (!desc->engine_type) { WRITE_ONCE(svc_gr_content_rearm, true); + svc_cepin_log = 0; + } /* * ISSUE-0002: a session-end flush may still be pending on svc_thread. @@ -1050,6 +1102,13 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) spin_unlock_irqrestore(&svc.chan_lock, flags); if (ch) { + /* + * m148: force the ring/push-buffer discovery latches for this + * rank to re-run for the (possibly new) session, so the new + * ring is translated from its own content instead of inheriting + * the previous session's one-shot state (second-client wedge). + */ + svc_chan_rearm_rank(ch); svc.last_chan_reg = jiffies; pr_info("sema-svc: channel 0x%x registered gpfifo=0x%llx entries=%u userd=0x%llx eng=%u cid=%u notif=0x%llx\n", desc->h_object, desc->gpfifo_va, desc->gpfifo_entries, @@ -1063,7 +1122,6 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) /* P6b wave-1.5 hold state is defined with the forwarding section below. */ static void svc_fwd_hold_abort(u32 h_object); - /* * P6b wave-2 inline software fallback for a withheld (skipped) candidate; * defined with the forwarding section below. Service-thread only. @@ -6499,6 +6557,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u32 cur = 0; int pending = 0; bool have_cur = false; + bool pool_stale_drop = false; bool inc, inc_apply = false; if (page_pa) @@ -6646,6 +6705,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, have_cur = true; if (st.sem_payload > cur) pending = 1; + else + pool_stale_drop = true; } /* * ISSUE-0002 r70: the residue rule @@ -6657,8 +6718,22 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * so a submissions-based freshness * test misses the re-decode; this * flag does not. + * + * r101: a read the stale-drop path + * discarded (payload <= cur, nothing + * applied or deferred) must NOT set + * the flag. The reused pool page of + * a fresh session reads the previous + * session's residue (cur > payload) + * BEFORE the first genuine release is + * translated; that read used to set + * the flag and every later residue + * release was then silently dropped + * (r99/r100: pay=1 cur=2 dropped, the + * client spins at DEVICE). */ - if (!rrc2 && svc_pool_sem_va(sem_va) && ch) + if (!rrc2 && svc_pool_sem_va(sem_va) && ch && + !pool_stale_drop) ch->pool_rel_seen = true; } @@ -6869,6 +6944,36 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, tlen ? tail : NULL); } } + + /* + * m151 Class-2 diagnostic: a CE push with real content that decoded no + * release is the one that pays the 8 s grace. Log its decode state + * (bounded, content-gated so all-zero reads do not wash it out). + */ + if (gsp_sema_dbg && ch->engine_type && ch->h_object && !st.have_rel) { + u8 head[64]; + u32 hlen = len < sizeof(head) ? len : (u32)sizeof(head); + bool nz = false; + u32 i; + + for (i = 0; i + 4 <= hlen; i += 4) { + u32 w; + + memcpy(&w, buf + i, 4); + if (w) { + nz = true; + break; + } + } + if (nz && svc.entry_norel_log < 2048) { + svc.entry_norel_log++; + memcpy(head, buf, hlen); + pr_info("sema-norel: h=0x%x eng=%u va=0x%llx len=%u ops=%u have_sem=%u rel=0x%llx pay=%u head=%*phN\n", + ch->h_object, ch->engine_type, push_va, len, + ops_runs, st.have_sem, st.rel_va, + st.rel_payload, hlen, head); + } + } } /* ------------------------------------------------------------------ */ @@ -6937,11 +7042,14 @@ static void svc_pending_add(u64 va, u32 payload, u32 op) * is reproducibly green. Held releases belong to waiters that no longer * exist; drop them instead of resolving them against the next session. */ +static void svc_gr_diag_reset(void); /* m150: defined with the fill diag */ + static void svc_session_flush(void) { unsigned long flags; u32 i; + svc_gr_diag_reset(); /* m150: per-session diagnostic budgets */ svc.map_n = 0; svc_pb_n = 0; svc_pd_found = 0; @@ -7872,6 +7980,8 @@ static bool svc_push_prefix_ok(const u8 *buf, size_t len) /* Fallback-line budget for the whole boot. */ static u32 svc_pushpg_fb_log; +/* Bounded split-read rejection lines (cross-page CE push, r129 class). */ +static u32 svc_pushpg_split_log; /* * Read a push the way the engine does: every 4 KB page of the range is @@ -7891,11 +8001,16 @@ static u32 svc_pushpg_fb_log; * method stream is rejected and the remaining candidates are tried * (observed: a wrong injected pin for the push page made every push * decode no release, starving the client's tracker); - * 2. validated self-pin (svc_pin_lookup_valid): read-back-validated pages + * 2. the displaced injected pair (svc_ipin_alt_lookup): what the page + * resolved to before a later injection overwrote the primary slot - + * the fall-back when a foreign/stale injection shadowed a good page + * (observed r129: a continuation page of a cross-page CE push under + * an all-FF pin); + * 3. validated self-pin (svc_pin_lookup_valid): read-back-validated pages * (content rescue, first-push pairing), window guesses excluded; - * 3. the channel's linear window mapping (svc_pb_win_lookup): last resort + * 4. the channel's linear window mapping (svc_pb_win_lookup): last resort * for a page no validated source knows; - * 4. outside any window: the remaining trusted sources + * 5. outside any window: the remaining trusted sources * (svc_lookup_trusted), exactly the old behaviour for GR/other pushes. * * For a contiguous allocation every path resolves to the same PA the old @@ -7923,8 +8038,9 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, have_win = !svc_pb_win_lookup(ch, page_va, &win_pa); - for (src = 0; src < 4 && !done; src++) { + for (src = 0; src < 5 && !done; src++) { bool via_ipin = false; + bool untrusted = false; pa = 0; ap = SVC_APERTURE_SYS; @@ -7934,22 +8050,39 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, svc_ipin_lookup(va, &pa, &ap)) continue; via_ipin = true; + untrusted = true; if (have_win) svc_pushpg_note(ch, page_va, pa, win_pa); break; case 1: + /* + * Displaced injected pin (svc_ipin_alt): + * the mapping this page had before a later + * injection overwrote it. The push decode + * is the consumer here, so a pin that no + * longer reads like the page's validated + * content (all-FF continuation chunks, + * see below) can fall back to the pair the + * current injection displaced. + */ + if (svc_ipin_alt_lookup(va, &pa, &ap)) + continue; + untrusted = true; + break; + case 2: if (svc_pin_lookup_valid(va, &pa, &ap)) continue; if (have_win) svc_pushpg_note(ch, page_va, pa, win_pa); break; - case 2: + case 3: if (!have_win) continue; pa = win_pa + (va & (SEMA_PAGE_SIZE - 1)); ap = SVC_APERTURE_SYS; + untrusted = true; break; default: /* learned cache / discovered map / page @@ -7975,18 +8108,45 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, !svc_push_prefix_ok(p, chunk)) continue; /* content says: wrong pin */ /* + * A continuation chunk (not the push start) read from + * an unvalidated source can be a stale/foreign pin: + * observed r129, a 216-byte CE push whose last 24 + * bytes (the SET_SEMAPHORE_A + LAUNCH_DMA) fell in a + * page whose injected pin resolved to an all-0xFF + * page while the first page was correct. A full + * chunk of 0xFF is never a method stream - fresh + * emulation pages read 0xFF - so reject it and try + * the remaining sources (the displaced alt pair + * first). The bounded line names the rejected + * (va, pa, src) for the post-mortem. + */ + if (untrusted && !first && chunk >= 64 && + !memchr_inv(p, 0xff, chunk)) { + if (svc_pushpg_split_log < 8) { + svc_pushpg_split_log++; + pr_info("sema-pushpg: split reject va=0x%llx pa=0x%llx src=%s (all-FF) at page 0x%llx\n", + va, pa, + src == 0 ? "ipin" : + (src == 1 ? "ipin-alt" : + "win"), + page_va); + } + last_rc = -EIO; + continue; + } + /* * ISSUE-0002 (r51): a window guess that reads ALL * ZEROS cannot be right - the paired base page is * real, so a zero first chunk means the linear window * does not describe this page (r49: the second * session's CE push pages read zeros and, because the * read "succeeded", shadowed the per-page resolvers). - * Fall through to src=3 instead. Non-zero window + * Fall through to src=4 instead. Non-zero window * content stays trusted exactly as before (r50: * requiring method-stream plausibility rejected valid * non-method push prefixes and corrupted c1). */ - if (src == 2 && first && !memchr_inv(p, 0, chunk)) + if (src == 3 && first && !memchr_inv(p, 0, chunk)) continue; done = true; } @@ -9635,6 +9795,14 @@ static int svc_rescue_try(struct nv_vgpu_device *vgpu, struct svc_channel *ch, entry_len)) { svc_pin_store(cand_va_page, cand_pa, SVC_APERTURE_SYS); svc_lat_store(cand_va_page, cand_pa, SVC_APERTURE_SYS); + /* + * m150: a content-validated page is stronger evidence than an + * unverified host injection. Replace the injected pair so a + * wrong injection (e.g. an off-by-one watcher pin) cannot + * shadow the validated page on the next read and re-run this + * rescue per pass (the second-session rescue crawl). + */ + svc_ipin_store(cand_va_page, cand_pa, SVC_APERTURE_SYS); svc_mark_hot(cand_pa); pr_info("sema-svc: content rescue ch=0x%x rank=%u slot=%u entry_va=0x%llx len=%u want=%u -> pa=0x%llx va_page=0x%llx (cand %u/%u)\n", ch->h_object, ch->ce_index, 4 * ch->ce_index, entry_va, @@ -9757,6 +9925,12 @@ static int svc_rescue_run_try(struct nv_vgpu_device *vgpu, for (i = 0; i < npages; i++) { svc_pin_store(pin_va[i], pin_pa[i], SVC_APERTURE_SYS); svc_lat_store(pin_va[i], pin_pa[i], SVC_APERTURE_SYS); + /* + * m150: see svc_rescue_try - validated pages replace + * a wrong injection so the rescue is paid once, not + * on every rewind pass. + */ + svc_ipin_store(pin_va[i], pin_pa[i], SVC_APERTURE_SYS); svc_mark_hot(pin_pa[i]); } svc.resc_run++; @@ -12128,6 +12302,78 @@ static bool svc_gr_page_inline_covers(u64 page) #define SVC_GR_DATA_BOUNCE_MAX 100u static u32 svc_gr_data_log; /* m142: bounded per-page attribution log */ +static u32 svc_gr_src_log; /* m150: bounded per-candidate source log */ +static bool svc_gr_data_stalled; /* m150: high data args unresolved */ +static u64 svc_gr_hold_log; /* m150: bounded GR-hold proof lines */ + +/* m150: re-arm both fill diagnostics for each client session. */ +static void svc_gr_diag_reset(void) +{ + svc_gr_data_log = 0; + svc_gr_src_log = 0; +} + +/* + * m150 (second-client zeros diagnosis): identify, read-only, which trusted + * source answers a candidate VA at fill time. Mirrors svc_resolve_va's source + * order without storing anything, so the fill's choice can be attributed in + * the dmesg record (the second client's data VAs resolve through no source - + * this names the one that serves the green client and the one missing). + * Codes: 0 none, 1 ipin, 2 pin, 3 lat, 4 map, 5 published vaspace, + * 6 discovered directory, 7 vidmem window convention. + */ +static const char *svc_gr_src_probe(struct nv_vgpu_device *vgpu, u64 va, + u64 *pa_out, u32 *ap_out) +{ + u64 pa = 0; + int ap = SVC_APERTURE_SYS; + u64 wpa; + int wap; + + if (!svc_ipin_lookup(va, &pa, &ap)) { + *pa_out = pa; + *ap_out = svc_ap_of(ap); + return "ipin"; + } + if (!svc_pin_lookup(va, &pa, &ap)) { + *pa_out = pa; + *ap_out = svc_ap_of(ap); + return "pin"; + } + if (!svc_lat_lookup(va, &pa, &ap)) { + *pa_out = pa; + *ap_out = svc_ap_of(ap); + return "lat"; + } + if (!svc_map_lookup(va, &pa)) { + *pa_out = pa; + *ap_out = SVC_APERTURE_SYS; + return "map"; + } + wpa = 0; + wap = SVC_APERTURE_SYS; + if (!svc_walk_vaspaces(vgpu, va & ~0xfffULL, &wpa, &wap)) { + *pa_out = wpa + (va & 0xfff); + *ap_out = svc_ap_of(wap); + return "vas"; + } + wpa = 0; + wap = SVC_APERTURE_SYS; + if (!svc_walk_discovered(vgpu, va & ~0xfffULL, &wpa, &wap)) { + *pa_out = wpa + (va & 0xfff); + *ap_out = svc_ap_of(wap); + return "disc"; + } + if (va >= SVC_VIDMEM_VA_BASE && + va - SVC_VIDMEM_VA_BASE + 4 <= svc_fb_shadow_size(vgpu)) { + *pa_out = va - SVC_VIDMEM_VA_BASE; + *ap_out = SVC_APERTURE_FB; + return "win"; + } + *pa_out = 0; + *ap_out = 0; + return "none"; +} /* * m144 (second-client launch-state divergence): collect device-VA-shaped @@ -12182,12 +12428,23 @@ static void svc_gr_scan_inline(u64 *wins, u32 *n_cand, u64 page_va) static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) { u64 wins[16]; + bool is_arg[16]; /* m150: candidate came from the argument scan */ u32 n_win = 0, n_cand; u32 off = 0, i, w; u8 kind; + /* + * m150: high (user/device-space) argument candidates and how many of + * them produced at least one resolved page. None resolving is the + * second-client zeros signature: the engine would run the kernel + * against unresolved pointers. Such a launch is held for a bounded + * retry instead of forwarded (svc_gr_data_stalled). + */ + u32 n_arg_hi = 0, n_arg_hi_res = 0; svc_fwd_req.gr_n_data = 0; svc_fwd_req.gr_n_data_wins = 0; + svc_gr_data_stalled = false; + memset(is_arg, 0, sizeof(is_arg)); /* * m100: scan every const-bank page the launch carries (QMD words * 32..47 are the bank LO/HI pairs; the demo's LDG.E base comes from @@ -12240,6 +12497,11 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) * pointers. */ svc_gr_scan_inline(wins, &n_cand, pp->va); } + /* m150: everything collected so far came from the c[0x0] argument + * page (or its inline bytes) - these are the pointers the kernel + * actually dereferences. */ + for (w = 0; w < n_cand; w++) + is_arg[w] = true; for (i = 0; i < svc_fwd_req.gr_n_pages; i++) { const struct svc_gr_page *pp = &svc_fwd_req.gr_pages[i]; u64 hpa = 0; @@ -12282,6 +12544,23 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) u64 base = wins[w]; u32 p; u32 before = svc_fwd_req.gr_n_data; + u64 pba = 0; + u32 pba_ap = 0; + const char *pba_src = "?"; + + if (is_arg[w] && base >= 0x40000000000ULL) + n_arg_hi++; + + /* m150: name the source that answers this candidate at fill + * time (read-only probe before the real resolution). */ + if (svc_gr_src_log < 24) { + svc_gr_src_log++; + pba_src = svc_gr_src_probe(vgpu, base, &pba, &pba_ap); + pr_info("vgsp-sema: svc-gr cand[%u]%s base=0x%llx probe=%s pa=0x%llx ap=%u pd_found=%u ipin_n=%u\n", + w, is_arg[w] ? " arg" : "", base, + pba_src, pba, pba_ap, + svc_pd_found, svc.ipin_n); + } for (p = 0; p < SVC_GR_DATA_PAGES_PER_PTR; p++) { u64 va = wins[w] + (u64)p * SEMA_PAGE_SIZE; @@ -12342,12 +12621,56 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) } } (void)base; + if (is_arg[w] && base >= 0x40000000000ULL && + svc_fwd_req.gr_n_data > before) + n_arg_hi_res++; pr_info("vgsp-sema: svc-gr candidate[%u] base=0x%llx -> %u new pages (total %u)\n", w, base, svc_fwd_req.gr_n_data - before, svc_fwd_req.gr_n_data); + /* + * m150: an argument pointer that resolves zero pages is the + * second-client zeros signature; record the walk result per + * published vaspace (pdb/entries/ap + rc) so the post-mortem + * shows whether the client's user VA space directory is + * published at fill time and whether the walk reaches it. + */ + if (is_arg[w] && svc_fwd_req.gr_n_data == before && + svc_gr_src_log < 64) { + int vi; + char vb[240]; + int o = 0; + + svc_gr_src_log++; + for (vi = 0; vi < SVC_MAX_VASPACES; vi++) { + u64 wa = 0; + int wap = SVC_APERTURE_SYS; + int wrc; + + if (!svc.vas[vi].valid) + continue; + wrc = svc_gmmu_walk(vgpu, svc.vas[vi].pdb, + svc.vas[vi].entries, base, + svc.vas[vi].ap, &wa, &wap, + false); + o += snprintf(vb + o, sizeof(vb) - o, + " vas[%d] pdb=0x%llx ent=%u ap=%d rc=%d pa=0x%llx", + vi, svc.vas[vi].pdb, + svc.vas[vi].entries, + svc.vas[vi].ap, wrc, wa); + } + pr_info("vgsp-sema: svc-gr arg-miss base=0x%llx probe=%s n_cand=%u pd_found=%u ipin_n=%u disc=0x%llx/%d%s\n", + base, pba_src, n_cand, svc_pd_found, + svc.ipin_n, svc_pd_base, svc_pd_ap, vb); + } } - pr_info("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved (reloc 0x%llx+)\n", - n_cand, svc_fwd_req.gr_n_data, (u64)SVC_GR_DATA_RELOC_BASE); + svc_gr_data_stalled = (n_arg_hi > 0 && n_arg_hi_res == 0); + if (svc_gr_data_stalled) + pr_info("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved; %u high argument ptr(s), 0 resolved -> launch held for a bounded retry\n", + n_cand, svc_fwd_req.gr_n_data, n_arg_hi); + else + pr_info("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved (reloc 0x%llx+)\n", + n_cand, svc_fwd_req.gr_n_data, + (u64)SVC_GR_DATA_RELOC_BASE); } static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, @@ -12500,6 +12823,8 @@ static void svc_gr_req_bridge_fallback(struct nv_vgpu_device *vgpu) * forwarding can never overlap; the caller engages the release hold on a * true return. Service-thread only. */ +static void svc_handle_map_add(void); + static bool svc_gr_request(struct nv_vgpu_device *vgpu, const struct svc_channel *ch) { @@ -12514,6 +12839,7 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, if (!gsp_phys_gr_forward_enabled() || !svc_gr_cand.valid || svc_gr_cand.h_object != ch->h_object) return false; + svc_gr_data_stalled = false; /* m150: only the fill sets this */ svc.fwd_gr_req++; if (!gsp_phys_backend_gr_ready() || !gsp_phys_guest_mem_enabled()) { svc.fwd_gr_refused++; @@ -12540,6 +12866,17 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, svc_gr_bridge_fallback(vgpu); return false; } + /* + * m143: drain the host injection queue before resolving the launch's + * pages. The watcher's pins for the client's a/b/c buffers can land + * between the service loop's map_add drain and this capture; the fill + * then misses them, nothing is relocated, and the win0 param page the + * SM reads keeps the driver's original guest VAs (the second client's + * all-zero results, rk1: candidate[0..2] -> 0 new pages). Draining + * here closes that window; svc_handle_map_add() runs on this same + * service thread, so there is no race with the loop's own drain. + */ + svc_handle_map_add(); rc = svc_gr_req_fill(vgpu, ch); if (rc) { clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); @@ -12553,6 +12890,26 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, return false; } /* + * m150: the fill found high argument pointers that no source can + * translate (the second-client zeros class). Do NOT forward a launch + * whose kernel would dereference unresolved pointers: leave the + * candidate unconsumed and return false so the pump holds the entry + * and retries it on later passes (svc_gr_data_stalled), bounded by + * the channel's gr_hold window at the call site. Once the window + * expires the entry is consumed exactly as before. + */ + if (svc_gr_data_stalled) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + mutex_unlock(&svc_fwd_lock); + svc.fwd_gr_refused++; + svc.fwd_gr_defer++; + if ((svc.fwd_gr_defer & SVC_FWD_LOG_MASK) == 1) + pr_info("vgsp-work: GR launch held (data args unresolved) h=0x%x push=0x%llx defer=%llu\n", + ch->h_object, svc_gr_cand.push_va, + svc.fwd_gr_defer); + return false; /* candidate kept: the pump retries the entry */ + } + /* * The candidate is consumed: the request copy owns it from here, so a * later capture/replay cannot re-offer it and the bridge is only * reached through the request's own latch. @@ -13791,6 +14148,45 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ fwd_guest_queued = svc_fwd_request(vgpu, ch); + if (fwd_guest_queued && + !ch->engine_type) + ch->gr_hold_idx = ~0u; + /* + * m150: the GR fill held this + * launch because its high + * argument pointers do not + * translate yet. Leave the + * entry unconsumed (do not + * write GP_GET) so the pump + * re-captures and retries it; + * one bounded window per entry, + * then fall through and consume + * exactly as before. The slot's + * identity makes the hold + * one-shot per entry: after + * expiry the same entry is + * never re-held. + */ + if (!fwd_guest_queued && + !ch->engine_type && + svc_gr_data_stalled && + ch->gr_hold_idx != (u32)idx) { + u32 hold_ms = gsp_sema_gr_grace_ms ? + gsp_sema_gr_grace_ms : 8000; + + ch->gr_hold_idx = (u32)idx; + ch->gr_hold_until = jiffies + + msecs_to_jiffies(hold_ms); + svc.fwd_gr_defer++; + if (svc_gr_hold_log < 16) { + svc_gr_hold_log++; + pr_info("sema-svc: GR hold h=0x%x entry=%u (data args unresolved; retry window %u ms)\n", + ch->h_object, + (u32)idx, hold_ms); + } + kfree(pb); + break; + } /* * P6b wave 1.5: when * this pass recorded @@ -14563,6 +14959,14 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.chan[k].unread_idx = ch->unread_idx; svc.chan[k].unread_until = ch->unread_until; svc.chan[k].unread_ipin_pa = ch->unread_ipin_pa; + /* + * m150: the GR data-arg hold window is snapshot + * state too; without the write-back every pass + * would re-arm it and a permanently unresolved + * launch would never be consumed. + */ + svc.chan[k].gr_hold_until = ch->gr_hold_until; + svc.chan[k].gr_hold_idx = ch->gr_hold_idx; svc.chan[k].pushpg_va = ch->pushpg_va; svc.chan[k].pushpg_n = ch->pushpg_n; svc.chan[k].content_max = ch->content_max; @@ -14931,6 +15335,19 @@ static void svc_handle_map_add(void) svc_ipin_store(req.va + off, req.pa + off, ap); svc_lat_store(req.va + off, req.pa + off, ap); } + /* + * m149: bounded per-session evidence that the CE push-buffer + * pins arrived - the source the second session's ring entries + * resolve through. + */ + if (ap == SVC_APERTURE_SYS && req.va >= 0x120000000ULL && + req.va < 0x121000000ULL && svc_cepin_log < 4) { + svc_cepin_log++; + pr_info("sema-svc: CE push-buffer pin va=0x%llx pa=0x%llx len=0x%llx (%u/4)\n", + (unsigned long long)req.va, + (unsigned long long)req.pa, + (unsigned long long)req.len, svc_cepin_log); + } rewound = true; if (gsp_sema_dbg) pr_info_ratelimited("sema-svc: injected mapping va=0x%llx -> pa=0x%llx len=0x%llx (%s)\n", diff --git a/tools/guest-diag/guest_pin_stream.py b/tools/guest-diag/guest_pin_stream.py index 04aba73..6607ead 100644 --- a/tools/guest-diag/guest_pin_stream.py +++ b/tools/guest-diag/guest_pin_stream.py @@ -266,16 +266,28 @@ def _vmalloc_pages_for(kc, va): return 0 -def resolve_mem_pins(kc, mem, out, default_pages=16): - """uvm_rm_mem kptr -> GPU-VA/GPA runs (direct map or page-table walk).""" +def resolve_mem_pins(kc, mem, out, default_pages=16, va_lo=0, va_hi=0, + page_off=0): + """uvm_rm_mem kptr -> GPU-VA/GPA runs (direct map or page-table walk). + + va_lo/va_hi optionally restrict the emitted GPU VA range; the kcore + chain resolver targets the CE push-buffer window only. page_off shifts + the vmap page paired with GPU data page k: the CE push-buffer vmap is + one page longer than its GPU data window on some allocations (a leading + header page), so GPU data page k lives at vmap page k + page_off. + page_off is derived per allocation (mem_page_off): a fixed offset is + wrong (r116/r117 regressed client #1 with a hard +1). + """ if not is_kern_ptr(mem): return 0 vas = [kc.read64(mem + 0x10 + i * 8) for i in range(16)] cpu = next((v for v in vas if is_kern_ptr(v)), None) - gpus = [v for v in vas if is_gpu_va(v)] + gpus = [v for v in vas + if is_gpu_va(v) and (not va_lo or v >= va_lo) + and (not va_hi or v < va_hi)] if not cpu or not gpus: return 0 - size = kc.read64(mem + 0x60) or 0 + size = _mem_size(kc, mem) dm = _direct_map_pa(kc, cpu) emitted = 0 if dm is not None: @@ -284,14 +296,15 @@ def resolve_mem_pins(kc, mem, out, default_pages=16): out.append("0x%x,0x%x,%d\n" % (gv, dm, npages)) emitted += 1 return emitted + area = _vmalloc_pages_for(kc, cpu) npages = (max(1, min(size >> 12, 4098)) if size - else _vmalloc_pages_for(kc, cpu)) - if not npages: - npages = 1 + else area or default_pages) + if area and page_off + npages > area: + npages = max(1, area - page_off) runs = [] cur = None for k in range(npages): - pa = kc.va_to_pa(cpu + k * 0x1000) + pa = kc.va_to_pa(cpu + (k + page_off) * 0x1000) if pa is None: if cur: runs.append(cur) @@ -312,6 +325,152 @@ def resolve_mem_pins(kc, mem, out, default_pages=16): return emitted +def _mem_size(kc, mem): + """uvm_rm_mem size (bytes) from the 570.144 layout, with a fallback. + + The struct is type(4)+pad(4)+mask(8)+vas[33] (264 B)+proxy_vas+ + gpu_owner+size, so size lives at +0x128; +0x60 (inside vas[]) reads 0 + for unused processors and is kept only as a fallback. + """ + for off in (0x128, 0x60): + v = kc.read64(mem + off) or 0 + if 0 < v <= (256 << 20): + return v + return 0 + + +def _push_prefix_ok(buf): + """First-4KB method-stream plausibility, mirroring the host's check.""" + off = 0 + while off + 4 <= len(buf): + hdr = struct.unpack_from('<I', buf, off)[0] + if hdr: + break + off += 4 + if off + 4 > len(buf): + return True # all zeros: not refutable + op = (hdr >> 29) & 0x7 + if op in (4, 7): + return True + if op not in (1, 3, 5): + return False + return ((hdr >> 16) & 0xfff) != 0 + + +def _page_content(kc, va): + pa = kc.va_to_pa(va & ~0xFFF) + if pa is None: + return None + return kc.read(kc.page_offset_base + pa, 64) + + +def mem_page_off(kc, mem): + """Per-allocation vmap offset for a uvm_rm_mem's GPU data pages. + + Derived, never fixed: the vmap area's page count (vmallocinfo) minus + the rm_mem's data page count (size at +0x128). A 4097-page area with + 4096 data pages means one leading header page (offset 1); an area that + exactly matches the data means offset 0. When the size is unreadable, + the two candidate offsets are content-probed with the host's method- + stream plausibility test and the decisively-passing one wins; an + ambiguous result keeps 0 (the status-quo pairing). + """ + vas = [kc.read64(mem + 0x10 + i * 8) for i in range(16)] + cpu = next((v for v in vas if is_kern_ptr(v)), None) + if not cpu or _direct_map_pa(kc, cpu) is not None: + return 0 + area = _vmalloc_pages_for(kc, cpu) + if area <= 1: + return 0 + size = _mem_size(kc, mem) + if size: + data = min(size >> 12, 4098) + if 0 <= area - data <= 2: + return area - data + return 0 + g = [] + for off in (0, 1): + if off >= area: + g.append(0) + continue + buf = _page_content(kc, cpu + off * 0x1000) + g.append(1 if (buf is not None and _push_prefix_ok(buf)) else 0) + if g[0] != g[1]: + return 0 if g[0] else 1 + return 0 + + +def _ce_mem_pages(kc, mem): + """CE mem walk parameters: (gpus, cpu, off, npages) or None. + + Mirrors resolve_mem_pins' derivation so the delta sweeper emits the same + page set the one-shot resolver used: size at +0x128 when readable, else + the vmalloc area length; all CE-window GPU VAs share the CPU mapping. + """ + vas = [kc.read64(mem + 0x10 + i * 8) for i in range(16)] + cpu = next((v for v in vas if is_kern_ptr(v)), None) + gpus = [v for v in vas if v and PB_CE_LO <= v < PB_CE_HI] + if not cpu or not gpus: + return None + if _direct_map_pa(kc, cpu) is not None: + return None # never a vmap: not the push-buffer shape + off = mem_page_off(kc, mem) + area = _vmalloc_pages_for(kc, cpu) + size = _mem_size(kc, mem) + npages = (max(1, min(size >> 12, 4098)) if size + else area or 64) + if area and off + npages > area: + npages = max(1, area - off) + return (gpus, cpu, off, npages) + + +def _emit_ce_mem_delta(kc, st, out, fresh_map=False): + """Emit the CE mem's live {va,pa} pairs as content-validated runs. + + The one-shot emission this replaces pinned the whole 16 MB window + including pages whose content was zero at the time and pages whose vmap + pairing had drifted; the host then read zeros for the other CE ranks' + pushes (the second-client stall class). The sweep now skips zero pages + (a zero page cannot carry a push), re-derives every PA against the live + page tables (optionally bursting the Kcore cache first), and emits only + pairs that are new or changed since the last sweep - so a page the + client writes later, or a mem whose vmap pages moved, is re-emitted + within KC_REEMIT_S and the host's last-store-wins pin sees the truth. + Never drops a previously emitted pair (a stale pin is corrected by the + changed pair, and a failed burst never clears the whole map). + """ + gpus, cpu, off, npages = st["gpus"], st["cpu"], st["page_off"], st["npages"] + last = st["last"] + if fresh_map: + for k in range(npages): + kc._pa_cache.pop(cpu + (k + off) * 0x1000, None) + changed = [] + for k in range(npages): + d = kc.read(cpu + (k + off) * 0x1000, 32) + if not d or not any(d): + continue # zero/unreadable: nothing to pin yet + pa = kc.va_to_pa(cpu + (k + off) * 0x1000) + if pa is None: + continue + for gv in gpus: + if last.get(gv + k * 0x1000) != pa: + changed.append((gv + k * 0x1000, pa)) + changed.sort() + merged = [] + for va, pa in changed: + if merged and va == merged[-1][0] + merged[-1][2] * 0x1000 and \ + pa == merged[-1][1] + merged[-1][2] * 0x1000: + merged[-1][2] += 1 + else: + merged.append([va, pa, 1]) + for va, pa, n in merged: + out.append("0x%x,0x%x,%d\n" % (va, pa, n)) + for va, pa in changed: + last[va] = pa + st["emit_total"] = st.get("emit_total", 0) + len(merged) + return len(merged) + + def _emit_ring_pins(kc, gpu_va, entries_cpu, entries, out): """Pin the GPFIFO ring pages (and the pages after it, where the GPPUT buffer usually sits) from the ring's CPU mapping.""" @@ -438,7 +597,246 @@ def resolve_pb_pins(kc, pb, out): mem = kc.read64(pb + 8) if not mem: return 0 - return resolve_mem_pins(kc, mem, out, default_pages=64) + return resolve_mem_pins(kc, mem, out, default_pages=64, + page_off=mem_page_off(kc, mem)) + + +# The CE push-buffer window is the one UVM region the client never maps into +# its own address space: the pagemap window scan and the pool-pin resolver +# can never see it. The uvbp kprobe was its only supplier, and that event is +# one-shot per push buffer while two trace readers race it - this scanner's +# pid-switch clear_trace() and semfix's trace-tail clear - so when the event +# loses the race the second client's ring entries stay untranslated and the +# service falls back to the ~250 ms/entry content rescue (the second-client +# stall, the same resolution-miss class as the ISSUE-0001 lottery). The +# channel->pool->manager->pushbuffer->rm_mem chain is walkable in kernel +# memory, so resolve the CE push buffer directly with kcore: no trace event, +# nothing to lose. +# +# The kptr-keyed one-shot latches below are NOT object identity: UVM frees +# and re-creates these rm_mems and channels during a session, and the slab +# reuses the same addresses. A resolution taken before such a move pins +# pages that are stale by the time the host reads them (observed r125: the +# pinned CE pages read all-zero while the channel's real pushes lived at +# new vmap pages of the same rm_mem kptr; every ring entry then stalled in +# the host's 8 s grace since no release could be decoded from zeros). So a +# resolved mem is periodically re-validated against the channel's own ring +# entries and re-resolved when it stops matching (KC_PB_STATE/_kc_pb_fresh), +# and when several mem candidates are reachable the one whose pages actually +# hold the ring-entry pushes is chosen (content anchor, _mem_content_ok). +PB_CE_LO = 0x120000000 +PB_CE_HI = 0x121000000 +KC_CHAIN_BACK = 0x2000 +KC_CHAIN_SPAN = 0xA000 +KC_CHAIN_MAX = 400 +RESOLVED_KC_PB = set() +KC_PB_STATE = {} # sem kptr -> resolved-mem state (fingerprint + pins) +KC_REFRESH_S = 2.0 # re-validate a resolved CE mem at most this often +KC_REEMIT_S = 2.0 # delta-sweep the CE mem's live pages this often +KC_REMAP_S = 30.0 # full re-translation sweep (catches remaps) + + +def _chain_mems_at(kc, w): + """rm_mem candidates when kernel pointer w is a pool, manager or pb.""" + mems = [] + mgr = kc.read64(w) + if is_kern_ptr(mgr): + pb = kc.read64(mgr + 8) + if is_kern_ptr(pb): + mem = kc.read64(pb + 8) + if is_kern_ptr(mem): + mems.append(mem) + pb = kc.read64(w + 8) + if is_kern_ptr(pb): + mem = kc.read64(pb + 8) + if is_kern_ptr(mem): + mems.append(mem) + return mems + + +def _mem_gpu_cpu(kc, mem): + """The CE-window GPU VA and CPU alias of an rm_mem (None/None when absent).""" + vas = [kc.read64(mem + 0x10 + i * 8) for i in range(16)] + cpu = next((v for v in vas if is_kern_ptr(v)), None) + gpu = next((v for v in vas + if v and PB_CE_LO <= v < PB_CE_HI), None) + return gpu, cpu + + +def _mem_content_score(kc, mem, gpu): + """Rough 'is this the live CE push buffer' score for a mem candidate. + + The channel's CE rings live in (emulated) video memory, so the guest + cannot read the ring entries to anchor a push VA; but the CE push + buffer's guest-RAM pages are CPU-written method streams during channel + setup and client work. Count nonzero u32 pairs across the first 128 + pages of the candidate's window: a live mem scores high, a freed page + range that a reused kptr still describes reads zeros and scores 0. + """ + cpu = None + for i in range(16): + v = kc.read64(mem + 0x10 + i * 8) + if is_kern_ptr(v): + cpu = v + break + if cpu is None: + return -1 + score = 0 + for k in range(128): + buf = kc.read(cpu + k * 0x1000, 0x1000) + if not buf: + continue + for off in range(0, 0x1000, 16): + if buf[off:off + 8] != b"\x00" * 8: + score += 1 + return score + + +def _kc_pb_fresh(kc, sem): + """The pinned CE mem still matches the channel's live backing. + + Cheap check: the mem's own {cpu,gpu} fields and the guest PA of its + first window page must still match the recorded resolution. A vmap area + never moves while the mem object lives, so either check failing means + the object was freed and (possibly) re-created at the same kptr - the + caller then re-resolves and re-emits the current pages. + """ + st = KC_PB_STATE.get(sem) + if not st: + return False + mem = st.get("mem") + if not mem or not is_kern_ptr(mem): + return False + if (kc.read64(mem + 0x30) or 0) != st.get("cpu") or \ + (kc.read64(mem + 0x38) or 0) != st.get("gpu"): + return False + if not st.get("page_va"): + return True + if not st.get("page_pa"): + # Unknown PA at store time: the manual page-table walk could not + # translate the vmap page (it may have been read before the kernel + # page tables were walkable). Treat the record as fresh instead of + # failing every check - a permanently-failing freshness test made + # the scanner drop the latch and re-resolve in a tight loop, and + # each failed re-resolution left the host with no pins at all + # (observed r131: "kc-chain stale: re-resolving" every second for + # five sems while smiley sat in cudaMalloc). + return True + pa = kc.va_to_pa(st["page_va"]) + if pa is None or pa != st.get("page_pa"): + return False + return True + + +def resolve_pb_chain(kc, sem, out): + """Tracking-sem kptr -> CE push-buffer rm_mem pins (kcore chain walk). + + Returns True when a CE push-buffer mem was found for this sem, so the + caller marks it resolved even when the mem was already pinned; new pin + lines are appended to out. Bounded: one window read plus a few reads per + candidate, and each rm_mem is resolved at most once per session (until a + freshness failure drops it again, see _kc_pb_fresh). + + When several mem candidates are reachable, the content score decides: a + stale candidate (freed memory read through a reused kptr) reads zeros + and is skipped in favour of the live push-buffer pages, so it cannot + shadow them on the host's last-store-wins pin table. An all-zero + candidate set emits nothing: pinning zeros is what made the host read + zero pushes and stall every CE entry; the drift retries (retry_at) and + the periodic refresh pick the pages up as soon as content appears. + """ + if not is_kern_ptr(sem): + return False + data = kc.read(sem - KC_CHAIN_BACK, KC_CHAIN_SPAN) + if not data: + return False + words = struct.unpack_from("<%dQ" % (len(data) // 8), data, 0) + cands = list(dict.fromkeys(w for w in words if is_kern_ptr(w))) + # Channel-anchored expansion (r134 live dump): the tracking sem is a + # member of a uvm_channel_t table; the channel records sit in the sem's + # own window and carry their CE ring GPU VA ([0x121010000, 0x121400000)) + # plus back-to-back 32-bit object handles. The pool/manager/pushbuffer + # pointers hang off those records' targets, which the bare sem-window + # scan never reaches - the second-client stall is exactly this: the only + # live uvmt event is the spinning sem, and no mem candidate is found + # around it. For each ring-VA word, scan the surrounding record for + # kptrs, take those targets, and try the chain shape on both. Additive + # and bounded; content scoring still decides, so a wrong candidate + # cannot shadow the live pages. + extra = [] + for i, w in enumerate(words): + if not w or not (0x121010000 <= w < 0x121400000): + continue + lo = max(0, i - 0x60) + hi = min(len(words), i + 0x60) + for c in words[lo:hi]: + if not is_kern_ptr(c): + continue + extra.append(c) + tgt = kc.read(c, 0x100) + if tgt: + tw = struct.unpack_from("<%dQ" % (len(tgt) // 8), + tgt, 0) + for w2 in tw: + if is_kern_ptr(w2): + extra.append(w2) + if len(extra) >= 512: + break + if extra: + head = [] + for w in extra: + if w not in head and w not in cands: + head.append(w) + cands = head + cands + mems = [] + for w in cands[:KC_CHAIN_MAX]: + for mem in _chain_mems_at(kc, w): + if mem not in mems: + mems.append(mem) + found = False + scored = [] + for mem in mems: + gpu, cpu = _mem_gpu_cpu(kc, mem) + if not gpu or not cpu: + continue + found = True + score = _mem_content_score(kc, mem, gpu) + scored.append((score, mem, gpu, cpu)) + if not scored: + return found + best = max(scored, key=lambda c: c[0]) + if best[0] <= 0: + dbg("kc-chain all candidates zero (sem=0x%x, n=%d): deferring" + % (sem, len(scored))) + return found + if len(scored) > 1: + dbg("kc-chain scores sem=0x%x: %s" % + (sem, ' '.join('%#x=%d' % (m, s) for s, m, _g, _c in scored))) + _score, mem, gpu, cpu = best + if mem in RESOLVED_KC_PB: + return True + RESOLVED_KC_PB.add(mem) + params = _ce_mem_pages(kc, mem) + if not params: + return found + gpus, cpu, off, npages = params + st = {"mem": mem, "gpu": gpu, "cpu": cpu, "gpus": gpus, + "page_va": cpu, "page_pa": kc.va_to_pa(cpu) or 0, + "page_off": off, "npages": npages, "last": {}} + # Fresh translation for the first sweep: the Kcore PA cache may hold + # pairs from before this mem's vmap area was (re)populated. + kc._pa_cache.clear() + n = _emit_ce_mem_delta(kc, st, out) + KC_PB_STATE[sem] = st + try: + with open('/tmp/guest_pin_stream.log', 'a') as f: + f.write('%s kc-chain CE push-buffer mem=0x%x sem=0x%x ' + 'pin-runs=%d off=%d npages=%d size=%d score=%d\n' + % (time.strftime('%T'), mem, sem, n, off, npages, + _mem_size(kc, mem), _score)) + except OSError: + pass + return True # UvmGpuChannelInfo field offsets (nv_uvm_types.h, x86_64): @@ -533,15 +931,72 @@ def resolve_channel_pins(kc, ch, out): -def clear_trace(): - """Drop stale ring events so the fallback tail reader cannot re-learn a - previous client's freed sems after a client switch. The new client's - events are delivered fresh through the consuming trace_pipe reader.""" +def _read_trace_snapshot(max_bytes=1 << 20, budget=0.5): + """Best-effort non-blocking read of the trace ring file. + + Spawning `tail` here left uninterruptible D-state processes when the + seq_file read blocked under load (observed r127: dozens of stuck tails, + load 26); a direct os.read with O_NONBLOCK returns whatever is ready and + never accumulates processes. + """ try: - with open(TRACE, "w") as f: - f.write("") + fd = os.open(TRACE, os.O_RDONLY | os.O_NONBLOCK) except OSError: - pass + return b"" + chunks = [] + got = 0 + t0 = time.time() + try: + while got < (8 << 20) and time.time() - t0 < budget: + try: + d = os.read(fd, 262144) + except BlockingIOError: + break + except OSError: + break + if not d: + break + chunks.append(d) + got += len(d) + finally: + os.close(fd) + return b"".join(chunks)[-max_bytes:] + + +def seed_events_from_trace(seen_sems, seen_pbs, seen_chs): + """One-shot tail scan of the trace ring after a pid switch / start. + + The uvmt/uvbp/uvch kprobe events are one-shot per object, and the + trace_pipe reader only sees events that arrive after it attaches: a + client whose channels are registered in the instant around the scanner's + pid switch would otherwise never appear in the sets (observed r126: + vecadd_loop ran for minutes with sem=0 while its CE entries paid the + 8 s grace). The ring keeps recent events, so scan its tail once and add + every distinct kptr. Stale previous-client pointers that are also + re-learned are harmless: their chains no longer resolve to content + (the mem candidates are scored and deferred). + """ + try: + out = _read_trace_snapshot() + except Exception: + return 0 + n = 0 + for m in re.finditer(rb"sem=0x([0-9a-fA-F]+)", out): + v = int(m.group(1), 16) + if v not in seen_sems: + seen_sems.add(v) + n += 1 + for m in re.finditer(rb"pb=0x([0-9a-fA-F]+)", out): + v = int(m.group(1), 16) + if v not in seen_pbs: + seen_pbs.add(v) + n += 1 + for m in re.finditer(rb"ch=0x([0-9a-fA-F]+)", out): + v = int(m.group(1), 16) + if v not in seen_chs: + seen_chs.add(v) + n += 1 + return n class TraceReader: @@ -611,6 +1066,18 @@ class TraceReader: out, self.lines = self.lines, [] return out + def reset(self): + """Forget parsed-event dedup marks on a client switch. + + The kernel reuses freed objects, so the new client's uvmt/uvbp kptr + can equal one already seen for the previous client; keeping the old + marks suppressed the new client's #pb line entirely (second-client + stall). Queued lines are kept: an event committed just before the + switch may belong to the new client. + """ + with self.lock: + self.seen = set() + def new_sems(seen): """New tracking-sem kptrs (uvmt lines) in the trace tail. @@ -663,9 +1130,7 @@ def cdt_pid(): # scanner must never commit to them (a transient nvidia-smi at boot was # picked once and its exited pagemap read back EOF forever). non_clients = ("nvidia-smi", "nvidia-persistenced", "Xorg", "X") - best = None - uvm = None - gpu = None + cands = [] for d in os.listdir('/proc'): if not d.isdigit(): continue @@ -680,28 +1145,24 @@ def cdt_pid(): rp = st.rfind(')') if rp >= 0 and st[rp + 2:rp + 3] == 'Z': continue - if any(name.startswith(p) for p in prefixes): - best = int(d) - continue if any(name.startswith(p) for p in non_clients): continue # Arbitrary CUDA binaries (any comm) are clients too: match by # the CUDA device they hold open so streaming never depends on - # the binary's name. Prefer a UVM holder (compute client) over a - # plain /dev/nvidia0 holder (nvidia-smi, persistenced); the last - # match wins, which is the newest client when several overlap. + # the binary's name. The newest candidate (highest pid) is the + # client being streamed: preferring a name-prefix match first + # pinned the scanner to a previous client that was still tearing + # down while the new client had already started pushing (its + # first pushes were unreadable, the expectation drifted, and the + # stall followed). kind = nvidia_fds(d) - if kind == 'uvm': - uvm = int(d) - elif kind == 'gpu': - gpu = int(d) + if kind is not None or any(name.startswith(p) for p in prefixes): + cands.append(int(d)) except OSError: pass - if best is not None: - return best - if uvm is not None: - return uvm - return gpu + if not cands: + return None + return max(cands) def scan_window(fd, lo, hi, span, out): @@ -778,10 +1239,19 @@ def scan_vmas(fd, pid, out): try: for line in maps: parts = line.split() - if len(parts) < 6: + if len(parts) < 5: continue - path = parts[5] - if not any(p in path for p in VMA_PATHS): + # Anonymous mappings (no path field) are the client's UVM sysmem + # buffers when the driver maps them without a /dev/zero backing: + # the second client's cudaMalloc'd data buffers show up exactly + # like that (e.g. 75d7b0000000-75d7b0021000 rw-p 00000000 00:00 0) + # and were skipped by the old len<6 check, so their pages never + # reached the host and the forwarded launch could not resolve + # them (all-zero results for the second client). Accept them at + # the high-VA range; the VMA_LO bound and the pagemap present/PFN + # filters keep the set to real GPU mappings. + path = parts[5] if len(parts) >= 6 else '' + if path and not any(p in path for p in VMA_PATHS): continue try: a, b = parts[0].split('-') @@ -857,7 +1327,11 @@ def main(): resolved_sem_chan = set() resolved_pbs = set() resolved_chs = set() + resolved_kc_pb = set() retry_at = {} + kc_refresh_at = {} + kc_reemit_at = {} + kc_remap_at = {} tracer = TraceReader() kc = Kcore() kc_ok = kc.load() @@ -891,13 +1365,26 @@ def main(): resolved_sem_chan = set() resolved_pbs = set() resolved_chs = set() + resolved_kc_pb = set() retry_at = {} + kc_refresh_at = {} + kc_reemit_at = {} + kc_remap_at = {} # Client switch: the old client's channel/put pointers are freed # memory. Clearing the #put map stops stale reads from being # emitted as if they were the new client's counters. CHAN_PUTS.clear() PUT_LAST.clear() - clear_trace() + tracer.reset() + RESOLVED_KC_PB.clear() + KC_PB_STATE.clear() + # One-shot: recover the new client's one-shot channel events + # from the trace ring (they can fall in the switch instant and + # be invisible to the pipe reader; the old clear_trace() made + # that permanent - part of the r126 stall class). + _seeded = seed_events_from_trace(seen_sems, seen_pbs, seen_chs) + if _seeded: + dbg("switch seed: +%d event kptr(s)" % _seeded) if sock is not None: try: sock.sendall(("#pid %d\n" % pid).encode()) @@ -959,6 +1446,103 @@ def main(): added += len(got) else: retry_at[sem] = now_r + 1.0 + # CE push-buffer pin freshness: a resolved mem is re-probed + # against the channel's live ring content. UVM frees and + # re-creates these rm_mems/channels mid-session and the slab + # reuses the same kptrs, so a one-shot resolution can pin + # pages that are stale (observed reading all zeros) while + # the live pushes sit at new pages; dropping the latches + # makes the resolution below re-emit the current truth. + if sem in resolved_kc_pb and \ + now_r >= kc_refresh_at.get(sem, 0): + kc_refresh_at[sem] = now_r + KC_REFRESH_S + try: + _fresh = _kc_pb_fresh(kc, sem) + except Exception: + _fresh = True + if not _fresh: + # Re-resolve, but never lose the latch on a failed + # attempt: a dropped latch means the host keeps no + # pins for this channel at all, which is worse than + # a possibly-stale pin (the host content-checks the + # first page and now rejects all-FF continuation + # chunks, so stale pages are bounded). On failure + # keep the old state and defer the next attempt. + _st = KC_PB_STATE.get(sem) + if _st and _st.get("mem"): + RESOLVED_KC_PB.discard(_st["mem"]) + got = [] + try: + _found2 = resolve_pb_chain(kc, sem, got) + except Exception: + _found2 = False + got = [] + if _found2 and got: + _new = KC_PB_STATE.get(sem) + _newmem = _new.get("mem") if _new else None + dbg("kc-chain stale: re-resolved sem=0x%x " + "new-mem=%s runs=%d" + % (sem, "yes" if _newmem != + (_st.get("mem") if _st else None) + else "no", len(got))) + lines.extend(got) + added += len(got) + resolved_kc_pb.add(sem) + else: + if _st and _st.get("mem"): + RESOLVED_KC_PB.add(_st["mem"]) + if sem not in KC_PB_STATE: + KC_PB_STATE[sem] = _st + kc_refresh_at[sem] = now_r + 5.0 + dbg("kc-chain stale: keep old latch sem=0x%x" + % sem) + else: + # CE mem delta sweep (second-client pins): the one-shot + # emission pinned the mem once; pages the client writes + # later (or whose vmap pairing moved) were never + # re-emitted, so the host kept zeros for the other CE + # ranks' pushes. Sweep every KC_REEMIT_S, emitting only + # new/changed nonzero pairs; burst the Kcore PA cache + # every KC_REMAP_S so a remap is not shadowed. + _st = KC_PB_STATE.get(sem) + if _st is not None and \ + now_r >= kc_reemit_at.get(sem, 0): + _fresh_map = now_r >= kc_remap_at.get(sem, 0) + if _fresh_map: + kc_remap_at[sem] = now_r + KC_REMAP_S + kc_reemit_at[sem] = now_r + KC_REEMIT_S + got = [] + try: + _n2 = _emit_ce_mem_delta(kc, _st, got, + fresh_map=_fresh_map) + except Exception: + _n2 = 0 + got = [] + if got: + lines.extend(got) + added += len(got) + dbg("ce-delta sem=0x%x runs=%d mem=0x%x" + % (sem, _n2, _st.get("mem", 0))) + # CE push-buffer chain (second-client stall fix): resolve + # pool -> manager -> pushbuffer -> rm_mem directly, so the + # push-buffer pages never depend on the racy one-shot uvbp + # event. Independent of the ring/pool gates below (the + # channel gate's continue must not starve this path). + if sem not in resolved_kc_pb and \ + now_r >= retry_at.get(("pk", sem), 0): + got = [] + try: + _found = resolve_pb_chain(kc, sem, got) + except Exception: + _found = False + got = [] + if _found: + resolved_kc_pb.add(sem) + if got: + lines.extend(got) + added += len(got) + else: + retry_at[("pk", sem)] = now_r + 2.0 # ISSUE-0002: the pool pin (mem) and the channel ring are # resolved independently. The sem is marked resolved as soon # as the pool page pins, but the channel_info anchor may only @@ -1006,6 +1590,26 @@ def main(): added += len(got) else: retry_at[ch] = now_r + 1.0 + # A uvch channel kptr is another early seed for the CE + # push-buffer chain (its window contains the pool pointer): + # resolve it here so the pins never depend on the uvmt sem + # events alone. + if ch in resolved_kc_pb or \ + now_r < retry_at.get(("pk", ch), 0): + continue + got = [] + try: + _found = resolve_pb_chain(kc, ch, got) + except Exception: + _found = False + got = [] + if _found: + resolved_kc_pb.add(ch) + if got: + lines.extend(got) + added += len(got) + else: + retry_at[("pk", ch)] = now_r + 2.0 if added: try: with open('/tmp/guest_pin_stream.log', 'a') as f: |