diff options
| -rw-r--r-- | kernel/gsp_sema_service.c | 244 | ||||
| -rw-r--r-- | tools/guest-diag/guest_pin_stream.py | 108 |
2 files changed, 339 insertions, 13 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index d68b3d3..9ab609a 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -339,6 +339,11 @@ struct svc_channel { * slot a bounded number of times, while a * clobbered slot loops forever (storm * detector) */ + unsigned long gr_hold_until; /* m150: GR data-arg retry window expiry */ + u32 gr_hold_idx; /* GPFIFO slot the window was armed for (identity: + * an entry holds at most once, then is consumed + * even if still unresolved - bounded zeros, never + * a hang) */ }; /* @@ -759,6 +764,7 @@ struct sema_svc { u64 fwd_gr_ok; u64 fwd_gr_fallback; u64 fwd_gr_refused; + u64 fwd_gr_defer; /* m150: held for data-arg retry */ u64 fwd_gr_timeout; u64 fwd_gr_fault; u64 fwd_gr_bridge; @@ -6998,11 +7004,14 @@ static void svc_pending_add(u64 va, u32 payload, u32 op) * is reproducibly green. Held releases belong to waiters that no longer * exist; drop them instead of resolving them against the next session. */ +static void svc_gr_diag_reset(void); /* m150: defined with the fill diag */ + static void svc_session_flush(void) { unsigned long flags; u32 i; + svc_gr_diag_reset(); /* m150: per-session diagnostic budgets */ svc.map_n = 0; svc_pb_n = 0; svc_pd_found = 0; @@ -9696,6 +9705,14 @@ static int svc_rescue_try(struct nv_vgpu_device *vgpu, struct svc_channel *ch, entry_len)) { svc_pin_store(cand_va_page, cand_pa, SVC_APERTURE_SYS); svc_lat_store(cand_va_page, cand_pa, SVC_APERTURE_SYS); + /* + * m150: a content-validated page is stronger evidence than an + * unverified host injection. Replace the injected pair so a + * wrong injection (e.g. an off-by-one watcher pin) cannot + * shadow the validated page on the next read and re-run this + * rescue per pass (the second-session rescue crawl). + */ + svc_ipin_store(cand_va_page, cand_pa, SVC_APERTURE_SYS); svc_mark_hot(cand_pa); pr_info("sema-svc: content rescue ch=0x%x rank=%u slot=%u entry_va=0x%llx len=%u want=%u -> pa=0x%llx va_page=0x%llx (cand %u/%u)\n", ch->h_object, ch->ce_index, 4 * ch->ce_index, entry_va, @@ -9818,6 +9835,12 @@ static int svc_rescue_run_try(struct nv_vgpu_device *vgpu, for (i = 0; i < npages; i++) { svc_pin_store(pin_va[i], pin_pa[i], SVC_APERTURE_SYS); svc_lat_store(pin_va[i], pin_pa[i], SVC_APERTURE_SYS); + /* + * m150: see svc_rescue_try - validated pages replace + * a wrong injection so the rescue is paid once, not + * on every rewind pass. + */ + svc_ipin_store(pin_va[i], pin_pa[i], SVC_APERTURE_SYS); svc_mark_hot(pin_pa[i]); } svc.resc_run++; @@ -12189,6 +12212,78 @@ static bool svc_gr_page_inline_covers(u64 page) #define SVC_GR_DATA_BOUNCE_MAX 100u static u32 svc_gr_data_log; /* m142: bounded per-page attribution log */ +static u32 svc_gr_src_log; /* m150: bounded per-candidate source log */ +static bool svc_gr_data_stalled; /* m150: high data args unresolved */ +static u64 svc_gr_hold_log; /* m150: bounded GR-hold proof lines */ + +/* m150: re-arm both fill diagnostics for each client session. */ +static void svc_gr_diag_reset(void) +{ + svc_gr_data_log = 0; + svc_gr_src_log = 0; +} + +/* + * m150 (second-client zeros diagnosis): identify, read-only, which trusted + * source answers a candidate VA at fill time. Mirrors svc_resolve_va's source + * order without storing anything, so the fill's choice can be attributed in + * the dmesg record (the second client's data VAs resolve through no source - + * this names the one that serves the green client and the one missing). + * Codes: 0 none, 1 ipin, 2 pin, 3 lat, 4 map, 5 published vaspace, + * 6 discovered directory, 7 vidmem window convention. + */ +static const char *svc_gr_src_probe(struct nv_vgpu_device *vgpu, u64 va, + u64 *pa_out, u32 *ap_out) +{ + u64 pa = 0; + int ap = SVC_APERTURE_SYS; + u64 wpa; + int wap; + + if (!svc_ipin_lookup(va, &pa, &ap)) { + *pa_out = pa; + *ap_out = svc_ap_of(ap); + return "ipin"; + } + if (!svc_pin_lookup(va, &pa, &ap)) { + *pa_out = pa; + *ap_out = svc_ap_of(ap); + return "pin"; + } + if (!svc_lat_lookup(va, &pa, &ap)) { + *pa_out = pa; + *ap_out = svc_ap_of(ap); + return "lat"; + } + if (!svc_map_lookup(va, &pa)) { + *pa_out = pa; + *ap_out = SVC_APERTURE_SYS; + return "map"; + } + wpa = 0; + wap = SVC_APERTURE_SYS; + if (!svc_walk_vaspaces(vgpu, va & ~0xfffULL, &wpa, &wap)) { + *pa_out = wpa + (va & 0xfff); + *ap_out = svc_ap_of(wap); + return "vas"; + } + wpa = 0; + wap = SVC_APERTURE_SYS; + if (!svc_walk_discovered(vgpu, va & ~0xfffULL, &wpa, &wap)) { + *pa_out = wpa + (va & 0xfff); + *ap_out = svc_ap_of(wap); + return "disc"; + } + if (va >= SVC_VIDMEM_VA_BASE && + va - SVC_VIDMEM_VA_BASE + 4 <= svc_fb_shadow_size(vgpu)) { + *pa_out = va - SVC_VIDMEM_VA_BASE; + *ap_out = SVC_APERTURE_FB; + return "win"; + } + *pa_out = 0; + *ap_out = 0; + return "none"; +} /* * m144 (second-client launch-state divergence): collect device-VA-shaped @@ -12243,12 +12338,23 @@ static void svc_gr_scan_inline(u64 *wins, u32 *n_cand, u64 page_va) static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) { u64 wins[16]; + bool is_arg[16]; /* m150: candidate came from the argument scan */ u32 n_win = 0, n_cand; u32 off = 0, i, w; u8 kind; + /* + * m150: high (user/device-space) argument candidates and how many of + * them produced at least one resolved page. None resolving is the + * second-client zeros signature: the engine would run the kernel + * against unresolved pointers. Such a launch is held for a bounded + * retry instead of forwarded (svc_gr_data_stalled). + */ + u32 n_arg_hi = 0, n_arg_hi_res = 0; svc_fwd_req.gr_n_data = 0; svc_fwd_req.gr_n_data_wins = 0; + svc_gr_data_stalled = false; + memset(is_arg, 0, sizeof(is_arg)); /* * m100: scan every const-bank page the launch carries (QMD words * 32..47 are the bank LO/HI pairs; the demo's LDG.E base comes from @@ -12301,6 +12407,11 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) * pointers. */ svc_gr_scan_inline(wins, &n_cand, pp->va); } + /* m150: everything collected so far came from the c[0x0] argument + * page (or its inline bytes) - these are the pointers the kernel + * actually dereferences. */ + for (w = 0; w < n_cand; w++) + is_arg[w] = true; for (i = 0; i < svc_fwd_req.gr_n_pages; i++) { const struct svc_gr_page *pp = &svc_fwd_req.gr_pages[i]; u64 hpa = 0; @@ -12343,6 +12454,23 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) u64 base = wins[w]; u32 p; u32 before = svc_fwd_req.gr_n_data; + u64 pba = 0; + u32 pba_ap = 0; + const char *pba_src = "?"; + + if (is_arg[w] && base >= 0x40000000000ULL) + n_arg_hi++; + + /* m150: name the source that answers this candidate at fill + * time (read-only probe before the real resolution). */ + if (svc_gr_src_log < 24) { + svc_gr_src_log++; + pba_src = svc_gr_src_probe(vgpu, base, &pba, &pba_ap); + pr_info("vgsp-sema: svc-gr cand[%u]%s base=0x%llx probe=%s pa=0x%llx ap=%u pd_found=%u ipin_n=%u\n", + w, is_arg[w] ? " arg" : "", base, + pba_src, pba, pba_ap, + svc_pd_found, svc.ipin_n); + } for (p = 0; p < SVC_GR_DATA_PAGES_PER_PTR; p++) { u64 va = wins[w] + (u64)p * SEMA_PAGE_SIZE; @@ -12403,12 +12531,56 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) } } (void)base; + if (is_arg[w] && base >= 0x40000000000ULL && + svc_fwd_req.gr_n_data > before) + n_arg_hi_res++; pr_info("vgsp-sema: svc-gr candidate[%u] base=0x%llx -> %u new pages (total %u)\n", w, base, svc_fwd_req.gr_n_data - before, svc_fwd_req.gr_n_data); + /* + * m150: an argument pointer that resolves zero pages is the + * second-client zeros signature; record the walk result per + * published vaspace (pdb/entries/ap + rc) so the post-mortem + * shows whether the client's user VA space directory is + * published at fill time and whether the walk reaches it. + */ + if (is_arg[w] && svc_fwd_req.gr_n_data == before && + svc_gr_src_log < 64) { + int vi; + char vb[240]; + int o = 0; + + svc_gr_src_log++; + for (vi = 0; vi < SVC_MAX_VASPACES; vi++) { + u64 wa = 0; + int wap = SVC_APERTURE_SYS; + int wrc; + + if (!svc.vas[vi].valid) + continue; + wrc = svc_gmmu_walk(vgpu, svc.vas[vi].pdb, + svc.vas[vi].entries, base, + svc.vas[vi].ap, &wa, &wap, + false); + o += snprintf(vb + o, sizeof(vb) - o, + " vas[%d] pdb=0x%llx ent=%u ap=%d rc=%d pa=0x%llx", + vi, svc.vas[vi].pdb, + svc.vas[vi].entries, + svc.vas[vi].ap, wrc, wa); + } + pr_info("vgsp-sema: svc-gr arg-miss base=0x%llx probe=%s n_cand=%u pd_found=%u ipin_n=%u disc=0x%llx/%d%s\n", + base, pba_src, n_cand, svc_pd_found, + svc.ipin_n, svc_pd_base, svc_pd_ap, vb); + } } - pr_info("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved (reloc 0x%llx+)\n", - n_cand, svc_fwd_req.gr_n_data, (u64)SVC_GR_DATA_RELOC_BASE); + svc_gr_data_stalled = (n_arg_hi > 0 && n_arg_hi_res == 0); + if (svc_gr_data_stalled) + pr_info("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved; %u high argument ptr(s), 0 resolved -> launch held for a bounded retry\n", + n_cand, svc_fwd_req.gr_n_data, n_arg_hi); + else + pr_info("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved (reloc 0x%llx+)\n", + n_cand, svc_fwd_req.gr_n_data, + (u64)SVC_GR_DATA_RELOC_BASE); } static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, @@ -12577,6 +12749,7 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, if (!gsp_phys_gr_forward_enabled() || !svc_gr_cand.valid || svc_gr_cand.h_object != ch->h_object) return false; + svc_gr_data_stalled = false; /* m150: only the fill sets this */ svc.fwd_gr_req++; if (!gsp_phys_backend_gr_ready() || !gsp_phys_guest_mem_enabled()) { svc.fwd_gr_refused++; @@ -12627,6 +12800,26 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, return false; } /* + * m150: the fill found high argument pointers that no source can + * translate (the second-client zeros class). Do NOT forward a launch + * whose kernel would dereference unresolved pointers: leave the + * candidate unconsumed and return false so the pump holds the entry + * and retries it on later passes (svc_gr_data_stalled), bounded by + * the channel's gr_hold window at the call site. Once the window + * expires the entry is consumed exactly as before. + */ + if (svc_gr_data_stalled) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + mutex_unlock(&svc_fwd_lock); + svc.fwd_gr_refused++; + svc.fwd_gr_defer++; + if ((svc.fwd_gr_defer & SVC_FWD_LOG_MASK) == 1) + pr_info("vgsp-work: GR launch held (data args unresolved) h=0x%x push=0x%llx defer=%llu\n", + ch->h_object, svc_gr_cand.push_va, + svc.fwd_gr_defer); + return false; /* candidate kept: the pump retries the entry */ + } + /* * The candidate is consumed: the request copy owns it from here, so a * later capture/replay cannot re-offer it and the bridge is only * reached through the request's own latch. @@ -13865,6 +14058,45 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ fwd_guest_queued = svc_fwd_request(vgpu, ch); + if (fwd_guest_queued && + !ch->engine_type) + ch->gr_hold_idx = ~0u; + /* + * m150: the GR fill held this + * launch because its high + * argument pointers do not + * translate yet. Leave the + * entry unconsumed (do not + * write GP_GET) so the pump + * re-captures and retries it; + * one bounded window per entry, + * then fall through and consume + * exactly as before. The slot's + * identity makes the hold + * one-shot per entry: after + * expiry the same entry is + * never re-held. + */ + if (!fwd_guest_queued && + !ch->engine_type && + svc_gr_data_stalled && + ch->gr_hold_idx != (u32)idx) { + u32 hold_ms = gsp_sema_gr_grace_ms ? + gsp_sema_gr_grace_ms : 8000; + + ch->gr_hold_idx = (u32)idx; + ch->gr_hold_until = jiffies + + msecs_to_jiffies(hold_ms); + svc.fwd_gr_defer++; + if (svc_gr_hold_log < 16) { + svc_gr_hold_log++; + pr_info("sema-svc: GR hold h=0x%x entry=%u (data args unresolved; retry window %u ms)\n", + ch->h_object, + (u32)idx, hold_ms); + } + kfree(pb); + break; + } /* * P6b wave 1.5: when * this pass recorded @@ -14637,6 +14869,14 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.chan[k].unread_idx = ch->unread_idx; svc.chan[k].unread_until = ch->unread_until; svc.chan[k].unread_ipin_pa = ch->unread_ipin_pa; + /* + * m150: the GR data-arg hold window is snapshot + * state too; without the write-back every pass + * would re-arm it and a permanently unresolved + * launch would never be consumed. + */ + svc.chan[k].gr_hold_until = ch->gr_hold_until; + svc.chan[k].gr_hold_idx = ch->gr_hold_idx; svc.chan[k].pushpg_va = ch->pushpg_va; svc.chan[k].pushpg_n = ch->pushpg_n; svc.chan[k].content_max = ch->content_max; diff --git a/tools/guest-diag/guest_pin_stream.py b/tools/guest-diag/guest_pin_stream.py index 35cbfb9..1da78fb 100644 --- a/tools/guest-diag/guest_pin_stream.py +++ b/tools/guest-diag/guest_pin_stream.py @@ -266,11 +266,17 @@ def _vmalloc_pages_for(kc, va): return 0 -def resolve_mem_pins(kc, mem, out, default_pages=16, va_lo=0, va_hi=0): +def resolve_mem_pins(kc, mem, out, default_pages=16, va_lo=0, va_hi=0, + page_off=0): """uvm_rm_mem kptr -> GPU-VA/GPA runs (direct map or page-table walk). va_lo/va_hi optionally restrict the emitted GPU VA range; the kcore - chain resolver targets the CE push-buffer window only. + chain resolver targets the CE push-buffer window only. page_off shifts + the vmap page paired with GPU data page k: the CE push-buffer vmap is + one page longer than its GPU data window on some allocations (a leading + header page), so GPU data page k lives at vmap page k + page_off. + page_off is derived per allocation (mem_page_off): a fixed offset is + wrong (r116/r117 regressed client #1 with a hard +1). """ if not is_kern_ptr(mem): return 0 @@ -281,7 +287,7 @@ def resolve_mem_pins(kc, mem, out, default_pages=16, va_lo=0, va_hi=0): and (not va_hi or v < va_hi)] if not cpu or not gpus: return 0 - size = kc.read64(mem + 0x60) or 0 + size = _mem_size(kc, mem) dm = _direct_map_pa(kc, cpu) emitted = 0 if dm is not None: @@ -290,14 +296,15 @@ def resolve_mem_pins(kc, mem, out, default_pages=16, va_lo=0, va_hi=0): out.append("0x%x,0x%x,%d\n" % (gv, dm, npages)) emitted += 1 return emitted + area = _vmalloc_pages_for(kc, cpu) npages = (max(1, min(size >> 12, 4098)) if size - else _vmalloc_pages_for(kc, cpu)) - if not npages: - npages = 1 + else area or default_pages) + if area and page_off + npages > area: + npages = max(1, area - page_off) runs = [] cur = None for k in range(npages): - pa = kc.va_to_pa(cpu + k * 0x1000) + pa = kc.va_to_pa(cpu + (k + page_off) * 0x1000) if pa is None: if cur: runs.append(cur) @@ -318,6 +325,81 @@ def resolve_mem_pins(kc, mem, out, default_pages=16, va_lo=0, va_hi=0): return emitted +def _mem_size(kc, mem): + """uvm_rm_mem size (bytes) from the 570.144 layout, with a fallback. + + The struct is type(4)+pad(4)+mask(8)+vas[33] (264 B)+proxy_vas+ + gpu_owner+size, so size lives at +0x128; +0x60 (inside vas[]) reads 0 + for unused processors and is kept only as a fallback. + """ + for off in (0x128, 0x60): + v = kc.read64(mem + off) or 0 + if 0 < v <= (256 << 20): + return v + return 0 + + +def _push_prefix_ok(buf): + """First-4KB method-stream plausibility, mirroring the host's check.""" + off = 0 + while off + 4 <= len(buf): + hdr = struct.unpack_from('<I', buf, off)[0] + if hdr: + break + off += 4 + if off + 4 > len(buf): + return True # all zeros: not refutable + op = (hdr >> 29) & 0x7 + if op in (4, 7): + return True + if op not in (1, 3, 5): + return False + return ((hdr >> 16) & 0xfff) != 0 + + +def _page_content(kc, va): + pa = kc.va_to_pa(va & ~0xFFF) + if pa is None: + return None + return kc.read(kc.page_offset_base + pa, 64) + + +def mem_page_off(kc, mem): + """Per-allocation vmap offset for a uvm_rm_mem's GPU data pages. + + Derived, never fixed: the vmap area's page count (vmallocinfo) minus + the rm_mem's data page count (size at +0x128). A 4097-page area with + 4096 data pages means one leading header page (offset 1); an area that + exactly matches the data means offset 0. When the size is unreadable, + the two candidate offsets are content-probed with the host's method- + stream plausibility test and the decisively-passing one wins; an + ambiguous result keeps 0 (the status-quo pairing). + """ + vas = [kc.read64(mem + 0x10 + i * 8) for i in range(16)] + cpu = next((v for v in vas if is_kern_ptr(v)), None) + if not cpu or _direct_map_pa(kc, cpu) is not None: + return 0 + area = _vmalloc_pages_for(kc, cpu) + if area <= 1: + return 0 + size = _mem_size(kc, mem) + if size: + data = min(size >> 12, 4098) + if 0 <= area - data <= 2: + return area - data + return 0 + g = [] + for off in (0, 1): + if off >= area: + g.append(0) + continue + buf = _page_content(kc, cpu + off * 0x1000) + g.append(1 if (buf is not None and _push_prefix_ok(buf)) else 0) + if g[0] != g[1]: + return 0 if g[0] else 1 + return 0 + + def _emit_ring_pins(kc, gpu_va, entries_cpu, entries, out): """Pin the GPFIFO ring pages (and the pages after it, where the GPPUT buffer usually sits) from the ring's CPU mapping.""" @@ -444,7 +526,8 @@ def resolve_pb_pins(kc, pb, out): mem = kc.read64(pb + 8) if not mem: return 0 - return resolve_mem_pins(kc, mem, out, default_pages=64) + return resolve_mem_pins(kc, mem, out, default_pages=64, + page_off=mem_page_off(kc, mem)) # The CE push-buffer window is the one UVM region the client never maps into @@ -509,13 +592,16 @@ def resolve_pb_chain(kc, sem, out): if mem in RESOLVED_KC_PB: continue RESOLVED_KC_PB.add(mem) + off = mem_page_off(kc, mem) n = resolve_mem_pins(kc, mem, out, default_pages=64, - va_lo=PB_CE_LO, va_hi=PB_CE_HI) + va_lo=PB_CE_LO, va_hi=PB_CE_HI, + page_off=off) try: with open('/tmp/guest_pin_stream.log', 'a') as f: f.write('%s kc-chain CE push-buffer mem=0x%x sem=0x%x ' - 'pin-runs=%d\n' - % (time.strftime('%T'), mem, sem, n)) + 'pin-runs=%d off=%d size=%d\n' + % (time.strftime('%T'), mem, sem, n, off, + _mem_size(kc, mem))) except OSError: pass return found |