diff options
| -rw-r--r-- | kernel/gsp_phys_booter.c | 247 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 50 |
2 files changed, 240 insertions, 57 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 5af4acb..b008739 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -5220,6 +5220,7 @@ static bool gsp_phys_mme_fast_sample(void __iomem *bar0, const char *tag, static u32 fast_boot_n; u32 ms = READ_ONCE(gsp_phys_mme_fast_ms); u32 esr_p, ds_p, gf_p, tr_p, n = 0; + u32 fed0_p, fed1_p; ktime_t t_end; bool poisoned = false; @@ -5237,16 +5238,28 @@ static bool gsp_phys_mme_fast_sample(void __iomem *bar0, const char *tag, ds_p = ioread32(bar0 + 0x00405840u); gf_p = ioread32(bar0 + 0x00400504u); tr_p = ioread32(bar0 + 0x00400704u); - pr_info("vgsp-gr: MME-FAST begin %s h=0x%x esr=%08x ds=%08x grfifo=%08x trap=%08x (spin %u ms)\n", - tag, h_object, esr_p, ds_p, gf_p, tr_p, ms); + /* + * SESSION 153-CONTINUED-84: FE_DECODE_STATE_EX(0)/(1) also drive a + * transition log - the MME's macro/generated witness can flip + * there without touching ESR/DS/grfifo, which is exactly the + * "did the MME start a macro after this doorbell" question the + * reloaded windows keep failing. + */ + fed0_p = ioread32(bar0 + 0x0040a630u); + fed1_p = ioread32(bar0 + 0x0040a634u); + pr_info("vgsp-gr: MME-FAST begin %s h=0x%x esr=%08x ds=%08x grfifo=%08x trap=%08x fe0=%08x fe1=%08x (spin %u ms)\n", + tag, h_object, esr_p, ds_p, gf_p, tr_p, fed0_p, fed1_p, ms); t_end = ktime_get() + (s64)ms * 1000000ll; for (;;) { u32 esr = ioread32(bar0 + 0x00404490u); u32 ds = ioread32(bar0 + 0x00405840u); u32 gf = ioread32(bar0 + 0x00400504u); u32 tr = ioread32(bar0 + 0x00400704u); + u32 fed0 = ioread32(bar0 + 0x0040a630u); + u32 fed1 = ioread32(bar0 + 0x0040a634u); - if (esr != esr_p || ds != ds_p || gf != gf_p || tr != tr_p) { + if (esr != esr_p || ds != ds_p || gf != gf_p || tr != tr_p || + fed0 != fed0_p || fed1 != fed1_p) { if (n < 12u) { u32 info = ioread32(bar0 + 0x00404494u); u32 info2 = ioread32(bar0 + 0x00404498u); @@ -5265,7 +5278,7 @@ static bool gsp_phys_mme_fast_sample(void __iomem *bar0, const char *tag, * the launch; slot 1 is the second * lane of the same record set. */ - pr_warn("vgsp-gr: MME-FAST %s h=0x%x esr %08x->%08x info=%08x(pc_valid=%u pc=%u mac=0x%x) info2_ir=%08x ds %08x->%08x grfifo %08x->%08x trap=%08x data=%08x fed=%08x,%08x (pc0=%u mac0=0x%x gen0=%u)\n", + pr_warn("vgsp-gr: MME-FAST %s h=0x%x esr %08x->%08x info=%08x(pc_valid=%u pc=%u mac=0x%x) info2_ir=%08x ds %08x->%08x grfifo %08x->%08x trap=%08x data=%08x fed=%08x,%08x (pc0=%u mac0=0x%x gen0=%u) fe1=%08x (pc1=%u mac1=0x%x gen1=%u)\n", tag, h_object, esr_p, esr, info, (info >> 28) & 1u, info & 0xfffu, @@ -5274,13 +5287,19 @@ static bool gsp_phys_mme_fast_sample(void __iomem *bar0, const char *tag, mmedata, mmeir, mmeir1, mmeir & 0xfffu, (mmeir >> 12) & 0x3ffu, - (mmeir >> 31) & 1u); + (mmeir >> 31) & 1u, + fed1, + fed1 & 0xfffu, + (fed1 >> 12) & 0x3ffu, + (fed1 >> 31) & 1u); } n++; esr_p = esr; ds_p = ds; gf_p = gf; tr_p = tr; + fed0_p = fed0; + fed1_p = fed1; /* * The priv-error poison is the terminal state: the * FECS has already logged and the rest of the @@ -17794,6 +17813,43 @@ MODULE_PARM_DESC(sema_gl_mme_hold_calls, "Hold (elide) window CALL_MME_MACRO/DATA runs until the MME prelude latches for the current generation+channel (default 1; 0 = legacy, real calls run against a partial prelude)"); /* + * SESSION 153-CONTINUED-84: emit NVC597_SET_MME_VERSION=1 (0x234) with the + * prelude step. The channel-init defaults the client pushes once per channel + * (Turing+) are never replayed, so the replayed context's MME stays in its + * reset/disabled state and every CALL_MME_MACRO run parks the front-end + * waiting for a macro that cannot start (r1245: call-bearing launches hang + * with all MME registers idle). Default 1; A/B with 0. + */ +static bool gsp_phys_rmcall_gr_mme_version = true; +module_param_named(sema_gl_mme_version, gsp_phys_rmcall_gr_mme_version, + bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_version, + "Emit SET_MME_VERSION=1 ahead of the replayed MME prelude (default 1; the Turing channel-init method the guest pushes once and the capture never carries)"); + +/* + * SESSION 153-CONTINUED-84: SET_MME_SHADOW_RAM_CONTROL mode before the + * window. The vendor comment (mesh shader bind path) is explicit: + * "MME filtering swallows them all if old and new state values are the + * same" - METHOD_TRACK_WITH_FILTER (1) suppresses every method write whose + * value equals the shadowed one. Our replay repeats the same window every + * launch, so after the first replay every redundant write - including the + * CALL_MME_MACRO/CALL_MME_DATA parameter writes - matches the shadow and + * is swallowed; the macro call never reaches the MME (r1236-r1250: calls + * in the emitted window, all MME registers idle, some launches park the + * front-end). The mode-bearing writes live in the captured PRELUDE + * (0x124 x4 in the client's setup push), which the win_patch window sweep + * does not touch; emit METHOD_TRACK (0 = filtering OFF, semantics + * unchanged: the write simply applies) right before the SA re-arm and the + * window. Value = sema_gl_mme_track (0 = TRACK; 1/2/3 = other modes; -1 = + * do not emit). + */ +static int gsp_phys_rmcall_gr_mme_track; +module_param_named(sema_gl_mme_track, gsp_phys_rmcall_gr_mme_track, + int, 0644); +MODULE_PARM_DESC(sema_gl_mme_track, + "SET_MME_SHADOW_RAM_CONTROL mode emitted before each replayed window (0=TRACK filtering off (default); 1=TRACK_WITH_FILTER; 2=PASSTHROUGH; 3=REPLAY; -1=off)"); + +/* * The same stubbed-slot predicate the launch builder applies (see the * sa_stub emit decision): a slot is stubbed when stubbing is on and either * the all-slots knob is set, the capture holds no SA entry, or the IR @@ -18380,6 +18436,34 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, (0u << 13) | (GSP_PHYS_RMCALL_GR_SET_OBJECT >> 2); *p++ = GSP_PHYS_RMCALL_CLASS_GR_3D; + /* + * SESSION 153-CONTINUED-84: SET_MME_VERSION=1 (0x234). + * + * The Turing channel-init defaults push + * `nvcMethodPusher: NVC597_SET_MME_VERSION <- 1` once per + * channel (class c597, TURING_OR_NEWER), and the replayed + * context otherwise never sees it: the captured MME setup + * families carry IR/SA/shadow-scratch loads but no 0x234 run + * (verified in the injected full-SA stream too). With the + * version register unset the MME never starts a macro - the + * r1245 evidence: call-bearing launches (real {0,2,34,39,40}, + * full 55-slot prelude, SA re-armed before the window) hang + * the front-end (`MME-FAST ... events=0` -> `GR LAUNCH FAILED: + * timeout after 2500 ms` with qmd_rel=0/ch_rel=0) while every + * MME register stays idle (`pc_valid=0`, `gen0=0`, + * `mme_esr=80000000`): the FE waits forever for a macro the + * halted MME cannot run. Emitted with the prelude step (the + * resident program is (re)loaded right after), ahead of the + * window, so the version select has landed before the first + * CALL_MME_MACRO run. Knob: sema_gl_mme_version (default 1; + * A/B with 0 reproduces the parked launches). + */ + if (emit_prelude && READ_ONCE(gsp_phys_rmcall_gr_mme_version)) { + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + (1u << 16) | + (0x234u >> 2); + *p++ = 1u; + } if (emit_prelude) { u32 g0 = pl_gen, g1 = 0, sdrop = 0, pwords = 0; u32 doff = READ_ONCE(gsp_phys_gr_mme_pre_emit_off); @@ -18681,6 +18765,71 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, } } } + /* + * SESSION 153-CONTINUED-84: shadow-RAM mode before the window + * (see the sema_gl_mme_track comment). METHOD_TRACK (0) is + * emitted by default so the repeated CALL/DATA parameter + * writes are not swallowed by the redundant-method filter. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_mme_track) >= 0 && + p + 2u <= pb + GSP_PHYS_RMCALL_GR_METHOD_WORDS) { + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + (1u << 16) | (0x124u >> 2); + *p++ = (u32)READ_ONCE(gsp_phys_rmcall_gr_mme_track); + } + /* + * SESSION 153-CONTINUED-84: SA re-arm BEFORE the window. + * The r1236 evidence: this block was emitted AFTER the + * window copy (contradicting its own "before the window" + * log), so the window's CALL_MME_MACRO(j) runs executed + * against the SA table state left by the prelude - whose + * last captured SA write re-clobbers the slot (r1231: + * SA[34]=0 lands last, every draw call resolved to macro + * 0 and emitted nothing). Because the prelude is + * re-emitted on every launch while any slot is kept, the + * late re-arm was re-clobbered before the next window + * anyway; the re-point only ever held between a window + * and the next prelude re-emission. Emitting the re-arm + * ahead of the window makes every launch's calls resolve + * to the captured SA entry. + */ + { + u32 sa_ent[64]; + u64 sa_mask = gsp_sema_mme_sa_get(sa_ent, 64); + u32 r = 0, rearm = 0; + + while (r < win3d_n && rearm < 32u) { + u32 h = win3d[r]; + u32 op = (h >> 29) & 7u; + u32 cnt = (h >> 16) & 0xfffu; + u32 span = ((op == 1u || op == 3u || + op == 5u) && cnt && + r + 1u + cnt <= win3d_n) ? + 1u + cnt : 1u; + u32 mb = (h & 0xfffu) << 2; + + if (mb >= 0x3800u && mb <= 0x39fcu) { + u32 j = (mb - 0x3800u) / 8u; + + if (j < 64u && + (sa_mask & (1ull << j)) && + sa_ent[j]) { + *p++ = ((u32) + GSP_PHYS_RMCALL_METHOD_IMM + << 29) | + (2u << 16) | + (0x11cu >> 2); + *p++ = j; + *p++ = sa_ent[j]; + rearm++; + } + } + r += span; + } + if (rearm) + pr_info("vgsp-gr: MME-SA-REARM %u slot(s) re-pointed before the window (h=0x%x push=0x%llx)\n", + rearm, req->h_object, req->push_va); + } if (mme_hold_calls) { /* * SESSION 153-CONTINUED-82: the prelude latch for @@ -18735,56 +18884,13 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, p += win3d_n; } /* - * SESSION 153-CONTINUED-83: SA re-arm. The captured prelude can - * end with a later client push's SA-pointer write that re-points - * a slot the window still calls (r1231: the prelude's LAST SA - * write is SA[34]=0 while its earlier write SA[34]=0x265 names - * the draw macro - BEGIN/SET_VERTEX_ARRAY_START/END, per the - * offline replay of the captured SA stream - so every - * CALL_MME_MACRO(34) would resolve to IR entry 0). The capture - * re-emits pushes in first-encounter order, so the late zeroing - * clobbers the entry the window's call needs. Re-emit - * SA[slot]=entry for every slot the emitted window calls, from - * the captured SA table, so the calls resolve to the macros the - * capture proved. Bounded: one 3-word run per called slot. + * SESSION 153-CONTINUED-84: the SA re-arm block used to be + * emitted HERE (after the window). It was moved ahead of the + * window above; see that block's comment for the r1236 + * evidence. An SA[slot]=entry re-point behind the call that + * needs it cannot help that call, and the per-launch + * prelude re-emission re-clobbers it before the next window. */ - { - u32 sa_ent[64]; - u64 sa_mask = gsp_sema_mme_sa_get(sa_ent, 64); - u32 r = 0, rearm = 0; - - while (r < win3d_n && rearm < 32u) { - u32 h = win3d[r]; - u32 op = (h >> 29) & 7u; - u32 cnt = (h >> 16) & 0xfffu; - u32 span = ((op == 1u || op == 3u || - op == 5u) && cnt && - r + 1u + cnt <= win3d_n) ? - 1u + cnt : 1u; - u32 mb = (h & 0xfffu) << 2; - - if (mb >= 0x3800u && mb <= 0x39fcu) { - u32 j = (mb - 0x3800u) / 8u; - - if (j < 64u && - (sa_mask & (1ull << j)) && - sa_ent[j]) { - *p++ = ((u32) - GSP_PHYS_RMCALL_METHOD_IMM - << 29) | - (2u << 16) | - (0x11cu >> 2); - *p++ = j; - *p++ = sa_ent[j]; - rearm++; - } - } - r += span; - } - if (rearm) - pr_info("vgsp-gr: MME-SA-REARM %u slot(s) re-pointed before the window (h=0x%x push=0x%llx)\n", - rearm, req->h_object, req->push_va); - } p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_A, (u32)(sem_va >> 32)); @@ -22040,6 +22146,7 @@ static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out, u32 *n_shadow) { u32 off = 0, wout = 0, reloc = 0, dropped = 0, subdrop = 0, sdm_drop = 0; + u32 wp_match = 0, wp_lowrun = 0; *out_n = n; *n_dropped = 0; @@ -22187,6 +22294,7 @@ static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out, static u32 patch_log; u32 old; + wp_match++; if (span > 1u) { old = out[wout + 1]; out[wout + 1] = pv; @@ -22200,6 +22308,21 @@ static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out, pr_info("vgsp-gr: 3D win-patch addr=0x%x off=%u data 0x%x -> 0x%x\n", addr, off, old, pv); } + } else if (addr >= 0x100u && addr < 0x200u) { + /* + * SESSION 153-CONTINUED-84: the win_patch + * knob (sema_gl_win_patch=124:1) reaches + * this pass only if the window carries a + * run at the patch address; log the low-page + * runs so a decode mismatch is visible + * instead of silent. + */ + wp_lowrun++; + if (wp_lowrun <= 4u && + gsp_phys_rmcall_gr_win_patch && + *gsp_phys_rmcall_gr_win_patch) + pr_info("vgsp-gr: WIN-PATCH lowrun addr=0x%x op=%u cnt=%u hdr=%08x\n", + addr, op, cnt, hdr); } } if ((op == 1 || op == 3 || op == 5) && cnt && @@ -22464,6 +22587,22 @@ static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out, off++; /* header-only / resync, like the service decoder */ wout++; } + /* + * SESSION 153-CONTINUED-84: prove the win_patch knob reached the + * pass (the r1236 "never logged" was a late write, not a decode + * gap - this line makes the distinction explicit on every boot). + */ + if (gsp_phys_rmcall_gr_win_patch && *gsp_phys_rmcall_gr_win_patch) { + static u32 wp_dbg; + + if (wp_dbg < 24u) { + wp_dbg++; + pr_info("vgsp-gr: WIN-PATCH sweep knob=%s runs=%u matched=%u lowruns=%u h=0x%x\n", + gsp_phys_rmcall_gr_win_patch, n, wp_match, + wp_lowrun, + req ? req->h_object : 0u); + } + } *out_n = wout; *n_dropped = dropped; gsp_phys_rmcall_gr_3d_last_subdrop = subdrop; diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 29d28d6..5224dad 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -14433,6 +14433,14 @@ static void svc_mme_prelude_capture(const u8 *buf, u32 len, u64 push_va, { u32 off = 0, added = 0, dups = 0, full = 0; u32 words0 = svc_mme_prelude_n; + /* + * SESSION 153-CONTINUED-84: which 4K pages of the push delivered at + * least one setup run. r1240's carrier read decoded only pages 0 of + * 0x4001004 (15 slots, 933 code words) while r1236's identical + * stream decoded all 16 (55 slots, 2854 code words) - the bitmap + * names the truncation page in one line. + */ + u64 page_bits = 0; if (!READ_ONCE(svc_mme_prelude_flag)) return; @@ -14511,6 +14519,7 @@ static void svc_mme_prelude_capture(const u8 *buf, u32 len, u64 push_va, off += span * 4u; continue; } + page_bits |= 1ull << ((off >> 12) & 63u); sig = svc_mme_run_sig(buf + off, span, hdr); for (i = 0; i < svc_mme_prelude_sig_n; i++) { if (svc_mme_prelude_sigs[i] == sig) { @@ -14572,10 +14581,10 @@ static void svc_mme_prelude_capture(const u8 *buf, u32 len, u64 push_va, u32 k; svc_mme_prelude_dump_n++; - pr_info("sema-mme-prelude: capture h=0x%x va=0x%llx len=%u added=%u words=%u gen=%u (was %u)\n", + pr_info("sema-mme-prelude: capture h=0x%x va=0x%llx len=%u added=%u words=%u gen=%u (was %u) pages=0x%llx\n", h_object, push_va, len, added, svc_mme_prelude_n, svc_mme_prelude_gen, - words0); + words0, (unsigned long long)page_bits); for (k = 0; k + 4u <= svc_mme_prelude_n && k < 512u; k += 4u) pr_info("sema-mme-prelude[%03u] %08x %08x %08x %08x\n", @@ -39405,6 +39414,23 @@ int gsp_sema_service_dump_channels(void) #define SVC_PIN_REWIND 8 #define SVC_MAP_ADD_MAX 4096 +/* + * SESSION 153-CONTINUED-84: optional length guard for injected mappings. + * A stale producer on the host (observed 2026-11-01: a periodic writer + * enqueueing (0x4001004,0x11f168000,0x10000) every ~2 s from an unknown + * leftover process) re-applies a whole-64K contiguous mapping over the + * per-page SA-carrier pins and can win the read of the client's push + * region, leaving every page past the first reading zeros (r1240/r1260: + * the replayed prelude captured only the first 15 SA slots while r1236/ + * r1245/r1250 captured all 55). When set, requests longer than this many + * bytes are refused with a bounded log, so only per-page (4K) pins land. + * 0 = accept any length (the historical behaviour). + */ +static u32 gsp_sema_map_add_max; +module_param_named(sema_map_add_max, gsp_sema_map_add_max, uint, 0644); +MODULE_PARM_DESC(sema_map_add_max, + "Refuse injected mapping requests longer than this many bytes (0 = off; set 4096 to defeat stale whole-window injections)"); + struct svc_map_add_req { u64 va; u64 pa; @@ -39488,6 +39514,24 @@ static void svc_handle_map_add(void) svc_map_add_tail++; spin_unlock_irqrestore(&svc_map_add_lock, flags); + /* + * SESSION 153-CONTINUED-84: refuse oversized requests when + * the guard is set (see the sema_map_add_max comment) - the + * page pins from pin_loop2 (len 0x1000) are unaffected. + */ + if (READ_ONCE(gsp_sema_map_add_max) && + req.len > (u64)READ_ONCE(gsp_sema_map_add_max)) { + static u32 map_rej_log; + + if (map_rej_log < 8) { + map_rej_log++; + pr_info("sema-svc: injected mapping va=0x%llx pa=0x%llx len=0x%llx REFUSED (len > sema_map_add_max=%u)\n", + req.va, req.pa, req.len, + READ_ONCE(gsp_sema_map_add_max)); + } + continue; + } + ap = req.ap ? req.ap : SVC_APERTURE_SYS; for (off = 0; off < (req.len ? req.len : 1); off += SEMA_PAGE_SIZE) { @@ -39511,7 +39555,7 @@ static void svc_handle_map_add(void) */ if (svc_va_hits_chan_ring(va)) { svc_ring_pin_skips++; - if (svc_ring_pin_skips <= 8) + if (svc_ring_pin_skips <= 64) pr_info("sema-svc: injected pin va=0x%llx DROPPED (live channel ring range; ring reads stay on the channel's own translation)\n", va); continue; |