diff options
| -rw-r--r-- | kernel/gsp_sema_service.c | 104 | ||||
| -rw-r--r-- | tools/guest-diag/guest_pin_stream.py | 210 |
2 files changed, 294 insertions, 20 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 9ab609a..20dfc54 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -602,6 +602,14 @@ struct sema_svc { u64 grce_log; /* GRCE inline-write budget */ u32 cur_ch; /* channel of the op being run */ u64 entry_log; /* per-push summary budget */ + /* + * m151: bounded log for CE pushes that were READABLE (nonzero first + * bytes) yet decoded no release. These are the entries the pump holds + * for the 8 s grace; naming their decode state (have_sem, rel) and + * head bytes is the Class-2 diagnostic (r128: correct content, right + * sem block at +0xC0 with RELEASE_ONE_WORD, yet rel_seen false). + */ + u64 entry_norel_log; u64 ops_denied; /* work refused: implausible stream */ u64 false_trans; /* resolvable VA mapped to non-push */ u64 sem_write_reresolve; /* writes re-resolved after read-back */ @@ -6936,6 +6944,36 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, tlen ? tail : NULL); } } + + /* + * m151 Class-2 diagnostic: a CE push with real content that decoded no + * release is the one that pays the 8 s grace. Log its decode state + * (bounded, content-gated so all-zero reads do not wash it out). + */ + if (gsp_sema_dbg && ch->engine_type && ch->h_object && !st.have_rel) { + u8 head[64]; + u32 hlen = len < sizeof(head) ? len : (u32)sizeof(head); + bool nz = false; + u32 i; + + for (i = 0; i + 4 <= hlen; i += 4) { + u32 w; + + memcpy(&w, buf + i, 4); + if (w) { + nz = true; + break; + } + } + if (nz && svc.entry_norel_log < 2048) { + svc.entry_norel_log++; + memcpy(head, buf, hlen); + pr_info("sema-norel: h=0x%x eng=%u va=0x%llx len=%u ops=%u have_sem=%u rel=0x%llx pay=%u head=%*phN\n", + ch->h_object, ch->engine_type, push_va, len, + ops_runs, st.have_sem, st.rel_va, + st.rel_payload, hlen, head); + } + } } /* ------------------------------------------------------------------ */ @@ -7942,6 +7980,8 @@ static bool svc_push_prefix_ok(const u8 *buf, size_t len) /* Fallback-line budget for the whole boot. */ static u32 svc_pushpg_fb_log; +/* Bounded split-read rejection lines (cross-page CE push, r129 class). */ +static u32 svc_pushpg_split_log; /* * Read a push the way the engine does: every 4 KB page of the range is @@ -7961,11 +8001,16 @@ static u32 svc_pushpg_fb_log; * method stream is rejected and the remaining candidates are tried * (observed: a wrong injected pin for the push page made every push * decode no release, starving the client's tracker); - * 2. validated self-pin (svc_pin_lookup_valid): read-back-validated pages + * 2. the displaced injected pair (svc_ipin_alt_lookup): what the page + * resolved to before a later injection overwrote the primary slot - + * the fall-back when a foreign/stale injection shadowed a good page + * (observed r129: a continuation page of a cross-page CE push under + * an all-FF pin); + * 3. validated self-pin (svc_pin_lookup_valid): read-back-validated pages * (content rescue, first-push pairing), window guesses excluded; - * 3. the channel's linear window mapping (svc_pb_win_lookup): last resort + * 4. the channel's linear window mapping (svc_pb_win_lookup): last resort * for a page no validated source knows; - * 4. outside any window: the remaining trusted sources + * 5. outside any window: the remaining trusted sources * (svc_lookup_trusted), exactly the old behaviour for GR/other pushes. * * For a contiguous allocation every path resolves to the same PA the old @@ -7993,8 +8038,9 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, have_win = !svc_pb_win_lookup(ch, page_va, &win_pa); - for (src = 0; src < 4 && !done; src++) { + for (src = 0; src < 5 && !done; src++) { bool via_ipin = false; + bool untrusted = false; pa = 0; ap = SVC_APERTURE_SYS; @@ -8004,22 +8050,39 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, svc_ipin_lookup(va, &pa, &ap)) continue; via_ipin = true; + untrusted = true; if (have_win) svc_pushpg_note(ch, page_va, pa, win_pa); break; case 1: + /* + * Displaced injected pin (svc_ipin_alt): + * the mapping this page had before a later + * injection overwrote it. The push decode + * is the consumer here, so a pin that no + * longer reads like the page's validated + * content (all-FF continuation chunks, + * see below) can fall back to the pair the + * current injection displaced. + */ + if (svc_ipin_alt_lookup(va, &pa, &ap)) + continue; + untrusted = true; + break; + case 2: if (svc_pin_lookup_valid(va, &pa, &ap)) continue; if (have_win) svc_pushpg_note(ch, page_va, pa, win_pa); break; - case 2: + case 3: if (!have_win) continue; pa = win_pa + (va & (SEMA_PAGE_SIZE - 1)); ap = SVC_APERTURE_SYS; + untrusted = true; break; default: /* learned cache / discovered map / page @@ -8045,18 +8108,45 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, !svc_push_prefix_ok(p, chunk)) continue; /* content says: wrong pin */ /* + * A continuation chunk (not the push start) read from + * an unvalidated source can be a stale/foreign pin: + * observed r129, a 216-byte CE push whose last 24 + * bytes (the SET_SEMAPHORE_A + LAUNCH_DMA) fell in a + * page whose injected pin resolved to an all-0xFF + * page while the first page was correct. A full + * chunk of 0xFF is never a method stream - fresh + * emulation pages read 0xFF - so reject it and try + * the remaining sources (the displaced alt pair + * first). The bounded line names the rejected + * (va, pa, src) for the post-mortem. + */ + if (untrusted && !first && chunk >= 64 && + !memchr_inv(p, 0xff, chunk)) { + if (svc_pushpg_split_log < 8) { + svc_pushpg_split_log++; + pr_info("sema-pushpg: split reject va=0x%llx pa=0x%llx src=%s (all-FF) at page 0x%llx\n", + va, pa, + src == 0 ? "ipin" : + (src == 1 ? "ipin-alt" : + "win"), + page_va); + } + last_rc = -EIO; + continue; + } + /* * ISSUE-0002 (r51): a window guess that reads ALL * ZEROS cannot be right - the paired base page is * real, so a zero first chunk means the linear window * does not describe this page (r49: the second * session's CE push pages read zeros and, because the * read "succeeded", shadowed the per-page resolvers). - * Fall through to src=3 instead. Non-zero window + * Fall through to src=4 instead. Non-zero window * content stays trusted exactly as before (r50: * requiring method-stream plausibility rejected valid * non-method push prefixes and corrupted c1). */ - if (src == 2 && first && !memchr_inv(p, 0, chunk)) + if (src == 3 && first && !memchr_inv(p, 0, chunk)) continue; done = true; } diff --git a/tools/guest-diag/guest_pin_stream.py b/tools/guest-diag/guest_pin_stream.py index 97c68f2..6607ead 100644 --- a/tools/guest-diag/guest_pin_stream.py +++ b/tools/guest-diag/guest_pin_stream.py @@ -400,6 +400,77 @@ def mem_page_off(kc, mem): return 0 +def _ce_mem_pages(kc, mem): + """CE mem walk parameters: (gpus, cpu, off, npages) or None. + + Mirrors resolve_mem_pins' derivation so the delta sweeper emits the same + page set the one-shot resolver used: size at +0x128 when readable, else + the vmalloc area length; all CE-window GPU VAs share the CPU mapping. + """ + vas = [kc.read64(mem + 0x10 + i * 8) for i in range(16)] + cpu = next((v for v in vas if is_kern_ptr(v)), None) + gpus = [v for v in vas if v and PB_CE_LO <= v < PB_CE_HI] + if not cpu or not gpus: + return None + if _direct_map_pa(kc, cpu) is not None: + return None # never a vmap: not the push-buffer shape + off = mem_page_off(kc, mem) + area = _vmalloc_pages_for(kc, cpu) + size = _mem_size(kc, mem) + npages = (max(1, min(size >> 12, 4098)) if size + else area or 64) + if area and off + npages > area: + npages = max(1, area - off) + return (gpus, cpu, off, npages) + + +def _emit_ce_mem_delta(kc, st, out, fresh_map=False): + """Emit the CE mem's live {va,pa} pairs as content-validated runs. + + The one-shot emission this replaces pinned the whole 16 MB window + including pages whose content was zero at the time and pages whose vmap + pairing had drifted; the host then read zeros for the other CE ranks' + pushes (the second-client stall class). The sweep now skips zero pages + (a zero page cannot carry a push), re-derives every PA against the live + page tables (optionally bursting the Kcore cache first), and emits only + pairs that are new or changed since the last sweep - so a page the + client writes later, or a mem whose vmap pages moved, is re-emitted + within KC_REEMIT_S and the host's last-store-wins pin sees the truth. + Never drops a previously emitted pair (a stale pin is corrected by the + changed pair, and a failed burst never clears the whole map). + """ + gpus, cpu, off, npages = st["gpus"], st["cpu"], st["page_off"], st["npages"] + last = st["last"] + if fresh_map: + for k in range(npages): + kc._pa_cache.pop(cpu + (k + off) * 0x1000, None) + changed = [] + for k in range(npages): + d = kc.read(cpu + (k + off) * 0x1000, 32) + if not d or not any(d): + continue # zero/unreadable: nothing to pin yet + pa = kc.va_to_pa(cpu + (k + off) * 0x1000) + if pa is None: + continue + for gv in gpus: + if last.get(gv + k * 0x1000) != pa: + changed.append((gv + k * 0x1000, pa)) + changed.sort() + merged = [] + for va, pa in changed: + if merged and va == merged[-1][0] + merged[-1][2] * 0x1000 and \ + pa == merged[-1][1] + merged[-1][2] * 0x1000: + merged[-1][2] += 1 + else: + merged.append([va, pa, 1]) + for va, pa, n in merged: + out.append("0x%x,0x%x,%d\n" % (va, pa, n)) + for va, pa in changed: + last[va] = pa + st["emit_total"] = st.get("emit_total", 0) + len(merged) + return len(merged) + + def _emit_ring_pins(kc, gpu_va, entries_cpu, entries, out): """Pin the GPFIFO ring pages (and the pages after it, where the GPPUT buffer usually sits) from the ring's CPU mapping.""" @@ -559,8 +630,10 @@ KC_CHAIN_BACK = 0x2000 KC_CHAIN_SPAN = 0xA000 KC_CHAIN_MAX = 400 RESOLVED_KC_PB = set() -KC_PB_STATE = {} # sem kptr -> resolved-mem fingerprint (freshness) +KC_PB_STATE = {} # sem kptr -> resolved-mem state (fingerprint + pins) KC_REFRESH_S = 2.0 # re-validate a resolved CE mem at most this often +KC_REEMIT_S = 2.0 # delta-sweep the CE mem's live pages this often +KC_REMAP_S = 30.0 # full re-translation sweep (catches remaps) def _chain_mems_at(kc, w): @@ -639,6 +712,16 @@ def _kc_pb_fresh(kc, sem): return False if not st.get("page_va"): return True + if not st.get("page_pa"): + # Unknown PA at store time: the manual page-table walk could not + # translate the vmap page (it may have been read before the kernel + # page tables were walkable). Treat the record as fresh instead of + # failing every check - a permanently-failing freshness test made + # the scanner drop the latch and re-resolve in a tight loop, and + # each failed re-resolution left the host with no pins at all + # (observed r131: "kc-chain stale: re-resolving" every second for + # five sems while smiley sat in cudaMalloc). + return True pa = kc.va_to_pa(st["page_va"]) if pa is None or pa != st.get("page_pa"): return False @@ -669,6 +752,42 @@ def resolve_pb_chain(kc, sem, out): return False words = struct.unpack_from("<%dQ" % (len(data) // 8), data, 0) cands = list(dict.fromkeys(w for w in words if is_kern_ptr(w))) + # Channel-anchored expansion (r134 live dump): the tracking sem is a + # member of a uvm_channel_t table; the channel records sit in the sem's + # own window and carry their CE ring GPU VA ([0x121010000, 0x121400000)) + # plus back-to-back 32-bit object handles. The pool/manager/pushbuffer + # pointers hang off those records' targets, which the bare sem-window + # scan never reaches - the second-client stall is exactly this: the only + # live uvmt event is the spinning sem, and no mem candidate is found + # around it. For each ring-VA word, scan the surrounding record for + # kptrs, take those targets, and try the chain shape on both. Additive + # and bounded; content scoring still decides, so a wrong candidate + # cannot shadow the live pages. + extra = [] + for i, w in enumerate(words): + if not w or not (0x121010000 <= w < 0x121400000): + continue + lo = max(0, i - 0x60) + hi = min(len(words), i + 0x60) + for c in words[lo:hi]: + if not is_kern_ptr(c): + continue + extra.append(c) + tgt = kc.read(c, 0x100) + if tgt: + tw = struct.unpack_from("<%dQ" % (len(tgt) // 8), + tgt, 0) + for w2 in tw: + if is_kern_ptr(w2): + extra.append(w2) + if len(extra) >= 512: + break + if extra: + head = [] + for w in extra: + if w not in head and w not in cands: + head.append(w) + cands = head + cands mems = [] for w in cands[:KC_CHAIN_MAX]: for mem in _chain_mems_at(kc, w): @@ -697,18 +816,23 @@ def resolve_pb_chain(kc, sem, out): if mem in RESOLVED_KC_PB: return True RESOLVED_KC_PB.add(mem) - off = mem_page_off(kc, mem) - n = resolve_mem_pins(kc, mem, out, default_pages=64, - va_lo=PB_CE_LO, va_hi=PB_CE_HI, - page_off=off) - st = {"mem": mem, "gpu": gpu, "cpu": cpu, - "page_va": cpu, "page_pa": kc.va_to_pa(cpu) or 0, "page_off": 0} + params = _ce_mem_pages(kc, mem) + if not params: + return found + gpus, cpu, off, npages = params + st = {"mem": mem, "gpu": gpu, "cpu": cpu, "gpus": gpus, + "page_va": cpu, "page_pa": kc.va_to_pa(cpu) or 0, + "page_off": off, "npages": npages, "last": {}} + # Fresh translation for the first sweep: the Kcore PA cache may hold + # pairs from before this mem's vmap area was (re)populated. + kc._pa_cache.clear() + n = _emit_ce_mem_delta(kc, st, out) KC_PB_STATE[sem] = st try: with open('/tmp/guest_pin_stream.log', 'a') as f: f.write('%s kc-chain CE push-buffer mem=0x%x sem=0x%x ' - 'pin-runs=%d off=%d size=%d score=%d\n' - % (time.strftime('%T'), mem, sem, n, off, + 'pin-runs=%d off=%d npages=%d size=%d score=%d\n' + % (time.strftime('%T'), mem, sem, n, off, npages, _mem_size(kc, mem), _score)) except OSError: pass @@ -1206,6 +1330,8 @@ def main(): resolved_kc_pb = set() retry_at = {} kc_refresh_at = {} + kc_reemit_at = {} + kc_remap_at = {} tracer = TraceReader() kc = Kcore() kc_ok = kc.load() @@ -1242,6 +1368,8 @@ def main(): resolved_kc_pb = set() retry_at = {} kc_refresh_at = {} + kc_reemit_at = {} + kc_remap_at = {} # Client switch: the old client's channel/put pointers are freed # memory. Clearing the #put map stops stale reads from being # emitted as if they were the new client's counters. @@ -1333,12 +1461,68 @@ def main(): except Exception: _fresh = True if not _fresh: - _st = KC_PB_STATE.pop(sem, None) + # Re-resolve, but never lose the latch on a failed + # attempt: a dropped latch means the host keeps no + # pins for this channel at all, which is worse than + # a possibly-stale pin (the host content-checks the + # first page and now rejects all-FF continuation + # chunks, so stale pages are bounded). On failure + # keep the old state and defer the next attempt. + _st = KC_PB_STATE.get(sem) if _st and _st.get("mem"): RESOLVED_KC_PB.discard(_st["mem"]) - resolved_kc_pb.discard(sem) - retry_at.pop(("pk", sem), None) - dbg("kc-chain stale: re-resolving sem=0x%x" % sem) + got = [] + try: + _found2 = resolve_pb_chain(kc, sem, got) + except Exception: + _found2 = False + got = [] + if _found2 and got: + _new = KC_PB_STATE.get(sem) + _newmem = _new.get("mem") if _new else None + dbg("kc-chain stale: re-resolved sem=0x%x " + "new-mem=%s runs=%d" + % (sem, "yes" if _newmem != + (_st.get("mem") if _st else None) + else "no", len(got))) + lines.extend(got) + added += len(got) + resolved_kc_pb.add(sem) + else: + if _st and _st.get("mem"): + RESOLVED_KC_PB.add(_st["mem"]) + if sem not in KC_PB_STATE: + KC_PB_STATE[sem] = _st + kc_refresh_at[sem] = now_r + 5.0 + dbg("kc-chain stale: keep old latch sem=0x%x" + % sem) + else: + # CE mem delta sweep (second-client pins): the one-shot + # emission pinned the mem once; pages the client writes + # later (or whose vmap pairing moved) were never + # re-emitted, so the host kept zeros for the other CE + # ranks' pushes. Sweep every KC_REEMIT_S, emitting only + # new/changed nonzero pairs; burst the Kcore PA cache + # every KC_REMAP_S so a remap is not shadowed. + _st = KC_PB_STATE.get(sem) + if _st is not None and \ + now_r >= kc_reemit_at.get(sem, 0): + _fresh_map = now_r >= kc_remap_at.get(sem, 0) + if _fresh_map: + kc_remap_at[sem] = now_r + KC_REMAP_S + kc_reemit_at[sem] = now_r + KC_REEMIT_S + got = [] + try: + _n2 = _emit_ce_mem_delta(kc, _st, got, + fresh_map=_fresh_map) + except Exception: + _n2 = 0 + got = [] + if got: + lines.extend(got) + added += len(got) + dbg("ce-delta sem=0x%x runs=%d mem=0x%x" + % (sem, _n2, _st.get("mem", 0))) # CE push-buffer chain (second-client stall fix): resolve # pool -> manager -> pushbuffer -> rm_mem directly, so the # push-buffer pages never depend on the racy one-shot uvbp |