diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 348 |
1 files changed, 339 insertions, 9 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index d68b3d3..20dfc54 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -339,6 +339,11 @@ struct svc_channel { * slot a bounded number of times, while a * clobbered slot loops forever (storm * detector) */ + unsigned long gr_hold_until; /* m150: GR data-arg retry window expiry */ + u32 gr_hold_idx; /* GPFIFO slot the window was armed for (identity: + * an entry holds at most once, then is consumed + * even if still unresolved - bounded zeros, never + * a hang) */ }; /* @@ -597,6 +602,14 @@ struct sema_svc { u64 grce_log; /* GRCE inline-write budget */ u32 cur_ch; /* channel of the op being run */ u64 entry_log; /* per-push summary budget */ + /* + * m151: bounded log for CE pushes that were READABLE (nonzero first + * bytes) yet decoded no release. These are the entries the pump holds + * for the 8 s grace; naming their decode state (have_sem, rel) and + * head bytes is the Class-2 diagnostic (r128: correct content, right + * sem block at +0xC0 with RELEASE_ONE_WORD, yet rel_seen false). + */ + u64 entry_norel_log; u64 ops_denied; /* work refused: implausible stream */ u64 false_trans; /* resolvable VA mapped to non-push */ u64 sem_write_reresolve; /* writes re-resolved after read-back */ @@ -759,6 +772,7 @@ struct sema_svc { u64 fwd_gr_ok; u64 fwd_gr_fallback; u64 fwd_gr_refused; + u64 fwd_gr_defer; /* m150: held for data-arg retry */ u64 fwd_gr_timeout; u64 fwd_gr_fault; u64 fwd_gr_bridge; @@ -6930,6 +6944,36 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, tlen ? tail : NULL); } } + + /* + * m151 Class-2 diagnostic: a CE push with real content that decoded no + * release is the one that pays the 8 s grace. Log its decode state + * (bounded, content-gated so all-zero reads do not wash it out). + */ + if (gsp_sema_dbg && ch->engine_type && ch->h_object && !st.have_rel) { + u8 head[64]; + u32 hlen = len < sizeof(head) ? len : (u32)sizeof(head); + bool nz = false; + u32 i; + + for (i = 0; i + 4 <= hlen; i += 4) { + u32 w; + + memcpy(&w, buf + i, 4); + if (w) { + nz = true; + break; + } + } + if (nz && svc.entry_norel_log < 2048) { + svc.entry_norel_log++; + memcpy(head, buf, hlen); + pr_info("sema-norel: h=0x%x eng=%u va=0x%llx len=%u ops=%u have_sem=%u rel=0x%llx pay=%u head=%*phN\n", + ch->h_object, ch->engine_type, push_va, len, + ops_runs, st.have_sem, st.rel_va, + st.rel_payload, hlen, head); + } + } } /* ------------------------------------------------------------------ */ @@ -6998,11 +7042,14 @@ static void svc_pending_add(u64 va, u32 payload, u32 op) * is reproducibly green. Held releases belong to waiters that no longer * exist; drop them instead of resolving them against the next session. */ +static void svc_gr_diag_reset(void); /* m150: defined with the fill diag */ + static void svc_session_flush(void) { unsigned long flags; u32 i; + svc_gr_diag_reset(); /* m150: per-session diagnostic budgets */ svc.map_n = 0; svc_pb_n = 0; svc_pd_found = 0; @@ -7933,6 +7980,8 @@ static bool svc_push_prefix_ok(const u8 *buf, size_t len) /* Fallback-line budget for the whole boot. */ static u32 svc_pushpg_fb_log; +/* Bounded split-read rejection lines (cross-page CE push, r129 class). */ +static u32 svc_pushpg_split_log; /* * Read a push the way the engine does: every 4 KB page of the range is @@ -7952,11 +8001,16 @@ static u32 svc_pushpg_fb_log; * method stream is rejected and the remaining candidates are tried * (observed: a wrong injected pin for the push page made every push * decode no release, starving the client's tracker); - * 2. validated self-pin (svc_pin_lookup_valid): read-back-validated pages + * 2. the displaced injected pair (svc_ipin_alt_lookup): what the page + * resolved to before a later injection overwrote the primary slot - + * the fall-back when a foreign/stale injection shadowed a good page + * (observed r129: a continuation page of a cross-page CE push under + * an all-FF pin); + * 3. validated self-pin (svc_pin_lookup_valid): read-back-validated pages * (content rescue, first-push pairing), window guesses excluded; - * 3. the channel's linear window mapping (svc_pb_win_lookup): last resort + * 4. the channel's linear window mapping (svc_pb_win_lookup): last resort * for a page no validated source knows; - * 4. outside any window: the remaining trusted sources + * 5. outside any window: the remaining trusted sources * (svc_lookup_trusted), exactly the old behaviour for GR/other pushes. * * For a contiguous allocation every path resolves to the same PA the old @@ -7984,8 +8038,9 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, have_win = !svc_pb_win_lookup(ch, page_va, &win_pa); - for (src = 0; src < 4 && !done; src++) { + for (src = 0; src < 5 && !done; src++) { bool via_ipin = false; + bool untrusted = false; pa = 0; ap = SVC_APERTURE_SYS; @@ -7995,22 +8050,39 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, svc_ipin_lookup(va, &pa, &ap)) continue; via_ipin = true; + untrusted = true; if (have_win) svc_pushpg_note(ch, page_va, pa, win_pa); break; case 1: + /* + * Displaced injected pin (svc_ipin_alt): + * the mapping this page had before a later + * injection overwrote it. The push decode + * is the consumer here, so a pin that no + * longer reads like the page's validated + * content (all-FF continuation chunks, + * see below) can fall back to the pair the + * current injection displaced. + */ + if (svc_ipin_alt_lookup(va, &pa, &ap)) + continue; + untrusted = true; + break; + case 2: if (svc_pin_lookup_valid(va, &pa, &ap)) continue; if (have_win) svc_pushpg_note(ch, page_va, pa, win_pa); break; - case 2: + case 3: if (!have_win) continue; pa = win_pa + (va & (SEMA_PAGE_SIZE - 1)); ap = SVC_APERTURE_SYS; + untrusted = true; break; default: /* learned cache / discovered map / page @@ -8036,18 +8108,45 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, !svc_push_prefix_ok(p, chunk)) continue; /* content says: wrong pin */ /* + * A continuation chunk (not the push start) read from + * an unvalidated source can be a stale/foreign pin: + * observed r129, a 216-byte CE push whose last 24 + * bytes (the SET_SEMAPHORE_A + LAUNCH_DMA) fell in a + * page whose injected pin resolved to an all-0xFF + * page while the first page was correct. A full + * chunk of 0xFF is never a method stream - fresh + * emulation pages read 0xFF - so reject it and try + * the remaining sources (the displaced alt pair + * first). The bounded line names the rejected + * (va, pa, src) for the post-mortem. + */ + if (untrusted && !first && chunk >= 64 && + !memchr_inv(p, 0xff, chunk)) { + if (svc_pushpg_split_log < 8) { + svc_pushpg_split_log++; + pr_info("sema-pushpg: split reject va=0x%llx pa=0x%llx src=%s (all-FF) at page 0x%llx\n", + va, pa, + src == 0 ? "ipin" : + (src == 1 ? "ipin-alt" : + "win"), + page_va); + } + last_rc = -EIO; + continue; + } + /* * ISSUE-0002 (r51): a window guess that reads ALL * ZEROS cannot be right - the paired base page is * real, so a zero first chunk means the linear window * does not describe this page (r49: the second * session's CE push pages read zeros and, because the * read "succeeded", shadowed the per-page resolvers). - * Fall through to src=3 instead. Non-zero window + * Fall through to src=4 instead. Non-zero window * content stays trusted exactly as before (r50: * requiring method-stream plausibility rejected valid * non-method push prefixes and corrupted c1). */ - if (src == 2 && first && !memchr_inv(p, 0, chunk)) + if (src == 3 && first && !memchr_inv(p, 0, chunk)) continue; done = true; } @@ -9696,6 +9795,14 @@ static int svc_rescue_try(struct nv_vgpu_device *vgpu, struct svc_channel *ch, entry_len)) { svc_pin_store(cand_va_page, cand_pa, SVC_APERTURE_SYS); svc_lat_store(cand_va_page, cand_pa, SVC_APERTURE_SYS); + /* + * m150: a content-validated page is stronger evidence than an + * unverified host injection. Replace the injected pair so a + * wrong injection (e.g. an off-by-one watcher pin) cannot + * shadow the validated page on the next read and re-run this + * rescue per pass (the second-session rescue crawl). + */ + svc_ipin_store(cand_va_page, cand_pa, SVC_APERTURE_SYS); svc_mark_hot(cand_pa); pr_info("sema-svc: content rescue ch=0x%x rank=%u slot=%u entry_va=0x%llx len=%u want=%u -> pa=0x%llx va_page=0x%llx (cand %u/%u)\n", ch->h_object, ch->ce_index, 4 * ch->ce_index, entry_va, @@ -9818,6 +9925,12 @@ static int svc_rescue_run_try(struct nv_vgpu_device *vgpu, for (i = 0; i < npages; i++) { svc_pin_store(pin_va[i], pin_pa[i], SVC_APERTURE_SYS); svc_lat_store(pin_va[i], pin_pa[i], SVC_APERTURE_SYS); + /* + * m150: see svc_rescue_try - validated pages replace + * a wrong injection so the rescue is paid once, not + * on every rewind pass. + */ + svc_ipin_store(pin_va[i], pin_pa[i], SVC_APERTURE_SYS); svc_mark_hot(pin_pa[i]); } svc.resc_run++; @@ -12189,6 +12302,78 @@ static bool svc_gr_page_inline_covers(u64 page) #define SVC_GR_DATA_BOUNCE_MAX 100u static u32 svc_gr_data_log; /* m142: bounded per-page attribution log */ +static u32 svc_gr_src_log; /* m150: bounded per-candidate source log */ +static bool svc_gr_data_stalled; /* m150: high data args unresolved */ +static u64 svc_gr_hold_log; /* m150: bounded GR-hold proof lines */ + +/* m150: re-arm both fill diagnostics for each client session. */ +static void svc_gr_diag_reset(void) +{ + svc_gr_data_log = 0; + svc_gr_src_log = 0; +} + +/* + * m150 (second-client zeros diagnosis): identify, read-only, which trusted + * source answers a candidate VA at fill time. Mirrors svc_resolve_va's source + * order without storing anything, so the fill's choice can be attributed in + * the dmesg record (the second client's data VAs resolve through no source - + * this names the one that serves the green client and the one missing). + * Codes: 0 none, 1 ipin, 2 pin, 3 lat, 4 map, 5 published vaspace, + * 6 discovered directory, 7 vidmem window convention. + */ +static const char *svc_gr_src_probe(struct nv_vgpu_device *vgpu, u64 va, + u64 *pa_out, u32 *ap_out) +{ + u64 pa = 0; + int ap = SVC_APERTURE_SYS; + u64 wpa; + int wap; + + if (!svc_ipin_lookup(va, &pa, &ap)) { + *pa_out = pa; + *ap_out = svc_ap_of(ap); + return "ipin"; + } + if (!svc_pin_lookup(va, &pa, &ap)) { + *pa_out = pa; + *ap_out = svc_ap_of(ap); + return "pin"; + } + if (!svc_lat_lookup(va, &pa, &ap)) { + *pa_out = pa; + *ap_out = svc_ap_of(ap); + return "lat"; + } + if (!svc_map_lookup(va, &pa)) { + *pa_out = pa; + *ap_out = SVC_APERTURE_SYS; + return "map"; + } + wpa = 0; + wap = SVC_APERTURE_SYS; + if (!svc_walk_vaspaces(vgpu, va & ~0xfffULL, &wpa, &wap)) { + *pa_out = wpa + (va & 0xfff); + *ap_out = svc_ap_of(wap); + return "vas"; + } + wpa = 0; + wap = SVC_APERTURE_SYS; + if (!svc_walk_discovered(vgpu, va & ~0xfffULL, &wpa, &wap)) { + *pa_out = wpa + (va & 0xfff); + *ap_out = svc_ap_of(wap); + return "disc"; + } + if (va >= SVC_VIDMEM_VA_BASE && + va - SVC_VIDMEM_VA_BASE + 4 <= svc_fb_shadow_size(vgpu)) { + *pa_out = va - SVC_VIDMEM_VA_BASE; + *ap_out = SVC_APERTURE_FB; + return "win"; + } + *pa_out = 0; + *ap_out = 0; + return "none"; +} /* * m144 (second-client launch-state divergence): collect device-VA-shaped @@ -12243,12 +12428,23 @@ static void svc_gr_scan_inline(u64 *wins, u32 *n_cand, u64 page_va) static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) { u64 wins[16]; + bool is_arg[16]; /* m150: candidate came from the argument scan */ u32 n_win = 0, n_cand; u32 off = 0, i, w; u8 kind; + /* + * m150: high (user/device-space) argument candidates and how many of + * them produced at least one resolved page. None resolving is the + * second-client zeros signature: the engine would run the kernel + * against unresolved pointers. Such a launch is held for a bounded + * retry instead of forwarded (svc_gr_data_stalled). + */ + u32 n_arg_hi = 0, n_arg_hi_res = 0; svc_fwd_req.gr_n_data = 0; svc_fwd_req.gr_n_data_wins = 0; + svc_gr_data_stalled = false; + memset(is_arg, 0, sizeof(is_arg)); /* * m100: scan every const-bank page the launch carries (QMD words * 32..47 are the bank LO/HI pairs; the demo's LDG.E base comes from @@ -12301,6 +12497,11 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) * pointers. */ svc_gr_scan_inline(wins, &n_cand, pp->va); } + /* m150: everything collected so far came from the c[0x0] argument + * page (or its inline bytes) - these are the pointers the kernel + * actually dereferences. */ + for (w = 0; w < n_cand; w++) + is_arg[w] = true; for (i = 0; i < svc_fwd_req.gr_n_pages; i++) { const struct svc_gr_page *pp = &svc_fwd_req.gr_pages[i]; u64 hpa = 0; @@ -12343,6 +12544,23 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) u64 base = wins[w]; u32 p; u32 before = svc_fwd_req.gr_n_data; + u64 pba = 0; + u32 pba_ap = 0; + const char *pba_src = "?"; + + if (is_arg[w] && base >= 0x40000000000ULL) + n_arg_hi++; + + /* m150: name the source that answers this candidate at fill + * time (read-only probe before the real resolution). */ + if (svc_gr_src_log < 24) { + svc_gr_src_log++; + pba_src = svc_gr_src_probe(vgpu, base, &pba, &pba_ap); + pr_info("vgsp-sema: svc-gr cand[%u]%s base=0x%llx probe=%s pa=0x%llx ap=%u pd_found=%u ipin_n=%u\n", + w, is_arg[w] ? " arg" : "", base, + pba_src, pba, pba_ap, + svc_pd_found, svc.ipin_n); + } for (p = 0; p < SVC_GR_DATA_PAGES_PER_PTR; p++) { u64 va = wins[w] + (u64)p * SEMA_PAGE_SIZE; @@ -12403,12 +12621,56 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) } } (void)base; + if (is_arg[w] && base >= 0x40000000000ULL && + svc_fwd_req.gr_n_data > before) + n_arg_hi_res++; pr_info("vgsp-sema: svc-gr candidate[%u] base=0x%llx -> %u new pages (total %u)\n", w, base, svc_fwd_req.gr_n_data - before, svc_fwd_req.gr_n_data); + /* + * m150: an argument pointer that resolves zero pages is the + * second-client zeros signature; record the walk result per + * published vaspace (pdb/entries/ap + rc) so the post-mortem + * shows whether the client's user VA space directory is + * published at fill time and whether the walk reaches it. + */ + if (is_arg[w] && svc_fwd_req.gr_n_data == before && + svc_gr_src_log < 64) { + int vi; + char vb[240]; + int o = 0; + + svc_gr_src_log++; + for (vi = 0; vi < SVC_MAX_VASPACES; vi++) { + u64 wa = 0; + int wap = SVC_APERTURE_SYS; + int wrc; + + if (!svc.vas[vi].valid) + continue; + wrc = svc_gmmu_walk(vgpu, svc.vas[vi].pdb, + svc.vas[vi].entries, base, + svc.vas[vi].ap, &wa, &wap, + false); + o += snprintf(vb + o, sizeof(vb) - o, + " vas[%d] pdb=0x%llx ent=%u ap=%d rc=%d pa=0x%llx", + vi, svc.vas[vi].pdb, + svc.vas[vi].entries, + svc.vas[vi].ap, wrc, wa); + } + pr_info("vgsp-sema: svc-gr arg-miss base=0x%llx probe=%s n_cand=%u pd_found=%u ipin_n=%u disc=0x%llx/%d%s\n", + base, pba_src, n_cand, svc_pd_found, + svc.ipin_n, svc_pd_base, svc_pd_ap, vb); + } } - pr_info("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved (reloc 0x%llx+)\n", - n_cand, svc_fwd_req.gr_n_data, (u64)SVC_GR_DATA_RELOC_BASE); + svc_gr_data_stalled = (n_arg_hi > 0 && n_arg_hi_res == 0); + if (svc_gr_data_stalled) + pr_info("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved; %u high argument ptr(s), 0 resolved -> launch held for a bounded retry\n", + n_cand, svc_fwd_req.gr_n_data, n_arg_hi); + else + pr_info("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved (reloc 0x%llx+)\n", + n_cand, svc_fwd_req.gr_n_data, + (u64)SVC_GR_DATA_RELOC_BASE); } static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, @@ -12577,6 +12839,7 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, if (!gsp_phys_gr_forward_enabled() || !svc_gr_cand.valid || svc_gr_cand.h_object != ch->h_object) return false; + svc_gr_data_stalled = false; /* m150: only the fill sets this */ svc.fwd_gr_req++; if (!gsp_phys_backend_gr_ready() || !gsp_phys_guest_mem_enabled()) { svc.fwd_gr_refused++; @@ -12627,6 +12890,26 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, return false; } /* + * m150: the fill found high argument pointers that no source can + * translate (the second-client zeros class). Do NOT forward a launch + * whose kernel would dereference unresolved pointers: leave the + * candidate unconsumed and return false so the pump holds the entry + * and retries it on later passes (svc_gr_data_stalled), bounded by + * the channel's gr_hold window at the call site. Once the window + * expires the entry is consumed exactly as before. + */ + if (svc_gr_data_stalled) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + mutex_unlock(&svc_fwd_lock); + svc.fwd_gr_refused++; + svc.fwd_gr_defer++; + if ((svc.fwd_gr_defer & SVC_FWD_LOG_MASK) == 1) + pr_info("vgsp-work: GR launch held (data args unresolved) h=0x%x push=0x%llx defer=%llu\n", + ch->h_object, svc_gr_cand.push_va, + svc.fwd_gr_defer); + return false; /* candidate kept: the pump retries the entry */ + } + /* * The candidate is consumed: the request copy owns it from here, so a * later capture/replay cannot re-offer it and the bridge is only * reached through the request's own latch. @@ -13865,6 +14148,45 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ fwd_guest_queued = svc_fwd_request(vgpu, ch); + if (fwd_guest_queued && + !ch->engine_type) + ch->gr_hold_idx = ~0u; + /* + * m150: the GR fill held this + * launch because its high + * argument pointers do not + * translate yet. Leave the + * entry unconsumed (do not + * write GP_GET) so the pump + * re-captures and retries it; + * one bounded window per entry, + * then fall through and consume + * exactly as before. The slot's + * identity makes the hold + * one-shot per entry: after + * expiry the same entry is + * never re-held. + */ + if (!fwd_guest_queued && + !ch->engine_type && + svc_gr_data_stalled && + ch->gr_hold_idx != (u32)idx) { + u32 hold_ms = gsp_sema_gr_grace_ms ? + gsp_sema_gr_grace_ms : 8000; + + ch->gr_hold_idx = (u32)idx; + ch->gr_hold_until = jiffies + + msecs_to_jiffies(hold_ms); + svc.fwd_gr_defer++; + if (svc_gr_hold_log < 16) { + svc_gr_hold_log++; + pr_info("sema-svc: GR hold h=0x%x entry=%u (data args unresolved; retry window %u ms)\n", + ch->h_object, + (u32)idx, hold_ms); + } + kfree(pb); + break; + } /* * P6b wave 1.5: when * this pass recorded @@ -14637,6 +14959,14 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.chan[k].unread_idx = ch->unread_idx; svc.chan[k].unread_until = ch->unread_until; svc.chan[k].unread_ipin_pa = ch->unread_ipin_pa; + /* + * m150: the GR data-arg hold window is snapshot + * state too; without the write-back every pass + * would re-arm it and a permanently unresolved + * launch would never be consumed. + */ + svc.chan[k].gr_hold_until = ch->gr_hold_until; + svc.chan[k].gr_hold_idx = ch->gr_hold_idx; svc.chan[k].pushpg_va = ch->pushpg_va; svc.chan[k].pushpg_n = ch->pushpg_n; svc.chan[k].content_max = ch->content_max; |