diff options
| -rw-r--r-- | kernel/gsp_phys_booter.c | 220 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 148 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.h | 11 |
3 files changed, 371 insertions, 8 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 4c2b460..ad8f9aa 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -17254,6 +17254,70 @@ static u32 gsp_phys_gr_attr_emit_gen; /* gen whose runs are draining */ static u32 gsp_phys_gr_attr_emit_off; /* word cursor into attr_pre */ /* + * SESSION 153-CONTINUED-62: surface (render-target) prelude. The beef + * client's per-frame draw windows are mid-stream cuts: the colour-target + * bind is sticky context state (the client and the X server write it once + * per context), so the replayed windows carry none - and every replayed + * draw rasterized into an unbound target and stored nothing (the observed + * 0/N retire). The service captures the last C597 colour-target run from + * any trusted push (svc_surf_prelude_capture; the observed carrier is the + * X server's 1280x720 screen bind) and this site replays it - aliased and + * relocated exactly like the window - ahead of any window that carries no + * usable target bind of its own. Emitted per capture generation (the + * patched buffer is reused while the captured run is unchanged, and the + * alias VA is stable per boot); a fresh generation re-patches. A/B via + * the sema_gl_surf_prelude knob (default 1). + */ +#define GSP_PHYS_GR_SURF_PRE_WORDS 32u +static u32 gsp_phys_gr_surf_pre[GSP_PHYS_GR_SURF_PRE_WORDS]; /* patched */ +static u32 gsp_phys_gr_surf_pre_in[GSP_PHYS_GR_SURF_PRE_WORDS]; +static u32 gsp_phys_gr_surf_pre_n; /* patched words, 0 = none */ +static u32 gsp_phys_gr_surf_pre_gen; /* capture gen the patch is from */ +static bool gsp_phys_rmcall_gr_surf_prelude = true; +module_param_named(sema_gl_surf_prelude, gsp_phys_rmcall_gr_surf_prelude, + bool, 0644); +MODULE_PARM_DESC(sema_gl_surf_prelude, + "Replay the captured C597 colour-target bind ahead of 3D windows that carry no render target of their own (default 1)"); + +/* + * SESSION 153-CONTINUED-62: does this window already carry a usable + * colour-target bind? A run with va == 0 (the all-zero degenerate bind + * seen on the client's 0x402094c push) does not count - the surface + * prelude is injected for it too. + */ +static bool gsp_phys_gr_window_has_surface(const u32 *win, u32 n) +{ + u32 off = 0; + + while (off < n) { + u32 hdr = win[off]; + u32 op = (hdr >> 29) & 7u; + u32 cnt = (hdr >> 16) & 0xfffu; + u32 addr = (hdr & 0x1fffu) << 2; + u32 span = 1u; + + if ((op == 1u || op == 3u || op == 5u) && cnt && + off + 1u + cnt <= n) + span = 1u + cnt; + else { + off++; + continue; + } + if (op == 1u && addr >= 0x800u && addr < 0xc00u && + (addr & 0x3fu) == 0u && cnt >= 2u) { + u64 va = ((u64)(win[off + 1] & 0xffu) << 32) | + (u64)(win[off + 2] & 0xfffff000u); + + if (va) + return true; + } + off += span; + } + return false; +} + + +/* * GR launch method block: the inline QMD run at its original shape and the * channel REPORT_SEMAPHORE release. The stock guest push does NOT emit * SET_OBJECT before the inline run (the compute class is bound on the @@ -20403,6 +20467,31 @@ static bool gsp_phys_rmcall_gr_win_patch_find(u32 addr, u32 *out_data) static u32 gsp_phys_rmcall_gr_3d_last_subdrop; static u32 gsp_phys_rmcall_gr_3d_last_sdmdrop; +/* + * SESSION 153-CONTINUED-62: an MME-family payload word that neither the + * launch page set nor the level-2 resolver could translate has to stay + * verbatim in the emitted stream (never guessed) - but a raw client VA in + * an 0x3810 macro payload is exactly the operand the engine faults on + * (`MMU Fault HUBCLIENT_PD`). Until this helper the word was left silent: + * the abort census stayed 0 and the launch went out to fault seconds + * later (r1115: fault @0x0_04350000, the 0x4350900 program-page payload). + * Now the surviving miss counts toward the pre-doorbell MISS-ABORT and + * joins the pin demand set, so the launch is held (fail-closed, bounded) + * until the page resolves, exactly like a window pair miss. The float/ + * data-word class (0x10000000..0x100000000) is excluded - it can never + * resolve and must not pin the census nonzero (the r1109 lesson). + */ +static void gsp_phys_gr_3d_mme_miss(u32 wv) +{ + u64 page = (u64)wv & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + + if (page >= 0x10000000ULL && page < 0x100000000ULL) + return; + gsp_phys_rmcall_gr_3d_miss_n++; + gsp_sema_demand_note(page); +} + + static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out, u32 *out_n, u32 *n_dropped, const struct gsp_phys_gr_submit *req, @@ -20486,8 +20575,30 @@ static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out, u32 subch = (hdr >> 13) & 0x7u; for (k = 0; k <= cnt; k++) { + u32 wv = (k < cnt) ? out[wout + 1u + k] : 0u; + /* + * SESSION 153-CONTINUED-62: the bad predicate + * also covers a torn ACTIVE attribute whose + * COMPONENT_BIT_WIDTHS enum is 0 (no such + * encoding exists): the r1113 setup-push + * capture carried + * `20030458 38400000 38200040 000081a4` + * (A(2)=0x81a4, widths=0) and the replayed + * prelude write trapped the engine - + * `Class Error: Class 0000c597, Offset + * 00001168, Data 000081a4, ErrorCode 0xc` + * -> grfifo=0xbadf1201 poison the moment the + * surf prelude let the draw execute. An + * inactive attribute (SOURCE=1, bit 6) keeps + * its word; only the invalid active encoding + * is dropped, so the attribute keeps its + * prelude/previous value until the window's + * own (valid) write lands. + */ bool bad = (k < cnt) && - (out[wout + 1u + k] == 0u); + (wv == 0u || + (((wv >> 21) & 0x3fu) == 0u && + !(wv & 0x40u))); if (!bad && k < cnt) { if (!L) @@ -20512,7 +20623,7 @@ static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out, if (az_log < 64) { az_log++; - pr_info("vgsp-gr: ATTR-ZERO addr=0x%x word=%u/%u off=%u dropped (torn-read zero; attribute keeps its prelude value)\n", + pr_info("vgsp-gr: ATTR-ZERO addr=0x%x word=%u/%u off=%u dropped (torn zero/invalid-width; attribute keeps its prelude value)\n", addr + 4u * k, k, cnt, off); } @@ -20722,7 +20833,18 @@ static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out, * miss or a shadow VA that * cannot fit the 32-bit slot * stays verbatim (never - * guessed). + * guessed) - and, SESSION + * 153-CONTINUED-62, it now + * COUNTS toward the abort + * census and joins the pin + * demand set: r1115's residual + * fault was exactly this class + * (`MMU Fault HUBCLIENT_PD @ + * 0x0_04350000` 2.5 s after a + * launch whose census read 0 + * because an 0x3810 payload + * word 0x4350900 missed here + * silently). */ struct gsp_sema_gr_reloc r; u64 va2 = (u64)wv; @@ -20731,15 +20853,24 @@ static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out, if (gsp_sema_gr_reloc_resolve( va2, &r) || - !r.hpa) + !r.hpa) { + gsp_phys_gr_3d_mme_miss( + wv); continue; + } if (*n_shadow >= - GSP_PHYS_RMCALL_GR_SHADOW_PAGES) + GSP_PHYS_RMCALL_GR_SHADOW_PAGES) { + gsp_phys_gr_3d_mme_miss( + wv); continue; + } if (gsp_phys_rmcall_gr_map_shadow_locked( *n_shadow, r.hpa, - &sh2, &line2)) + &sh2, &line2)) { + gsp_phys_gr_3d_mme_miss( + wv); continue; + } if (line2 != 0xffffffffu) (*n_shadow)++; sh2 |= va2 & @@ -20747,8 +20878,11 @@ static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out, up = (u32)(sh2 >> 32) & 0xffu; lo = (wv & 0x3u) | ((u32)sh2 & 0xfffffffcu); - if (up != 0) + if (up != 0) { + gsp_phys_gr_3d_mme_miss( + wv); continue; + } d[wi] = lo; reloc++; if (mme_log < 32) { @@ -20759,8 +20893,10 @@ static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out, } continue; } - if (up != 0) + if (up != 0) { + gsp_phys_gr_3d_mme_miss(wv); continue; + } d[wi] = lo; reloc++; if (mme_log < 32) { @@ -22306,6 +22442,74 @@ gr_page_unmapped: gsp_phys_rmcall_gr_3d_last_subdrop, gsp_phys_rmcall_gr_3d_last_sdmdrop); /* + * SESSION 153-CONTINUED-62: surface (render-target) prelude - + * see the gsp_phys_gr_surf_pre comment. Injected only when the + * window carries no usable colour-target bind of its own; the + * alias scan + relocation run exactly as they would for a + * window-carried bind (the captured VA is the surface the + * machine's draws target, X screen observed). The miss census + * feeds the MISS-ABORT below, so an unresolvable surface pair + * holds the launch (fail-closed) instead of faulting. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_surf_prelude) && + !gsp_phys_gr_window_has_surface(req->stream, + req->stream_n)) { + const u32 *sp; + u32 spn = 0, spg = 0; + + sp = gsp_sema_surf_prelude_get(&spn, &spg); + if (sp && spn && spg) { + if (spg != READ_ONCE(gsp_phys_gr_surf_pre_gen)) { + u32 sdrop = 0, sn = spn; + u32 srel; + + if (sn > GSP_PHYS_GR_SURF_PRE_WORDS) + sn = GSP_PHYS_GR_SURF_PRE_WORDS; + memcpy(gsp_phys_gr_surf_pre_in, sp, + (size_t)sn * 4u); + gsp_phys_gr_surface_alias_scan_locked( + gsp_phys_gr_surf_pre_in, sn, + req); + srel = gsp_phys_rmcall_gr_3d_patch_window( + gsp_phys_gr_surf_pre_in, sn, + gsp_phys_gr_surf_pre, + &gsp_phys_gr_surf_pre_n, + &sdrop, req, &n_shadow); + WRITE_ONCE(gsp_phys_gr_surf_pre_gen, + spg); + pr_info("vgsp-gr: SURF-PRELUDE patched gen=%u raw=%u kept=%u dropped=%u reloc=%u h=0x%x push=0x%llx\n", + spg, sn, + gsp_phys_gr_surf_pre_n, + sdrop, srel, + req->h_object, + req->push_va); + } + if (gsp_phys_gr_surf_pre_n && + win3d_n + gsp_phys_gr_surf_pre_n + 2u + + 8u <= + GSP_PHYS_RMCALL_GR_METHOD_WORDS) { + static u32 surf_emit_log; + + memmove(&stream_patch[gsp_phys_gr_surf_pre_n], + stream_patch, + (size_t)win3d_n * 4u); + memcpy(stream_patch, + gsp_phys_gr_surf_pre, + (size_t)gsp_phys_gr_surf_pre_n * + 4u); + win3d_n += gsp_phys_gr_surf_pre_n; + if (surf_emit_log < 16) { + surf_emit_log++; + pr_info("vgsp-gr: SURF-PRELUDE emitted words=%u window=%u h=0x%x push=0x%llx\n", + gsp_phys_gr_surf_pre_n, + win3d_n, + req->h_object, + req->push_va); + } + } + } + } + /* * SESSION 153-CONTINUED-57: patch the attr/pipeline prelude for * this launch (relocation exactly like the window: the * SET_PIPELINE_PROGRAM_ADDRESS pairs and any other recognised diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index c2f3084..4d32caa 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -14372,6 +14372,103 @@ static void svc_attr_prelude_capture(const u8 *buf, u32 len, u64 push_va, } } +/* + * SESSION 153-CONTINUED-62: surface (render-target) setup capture. The + * replayed beef 3D windows are mid-stream cuts of the client's per-frame + * pushes: the C597 colour-target bind (INC 0x800+j*0x40, [upper, lower, + * w, h, fmt, blksz]) is sticky context state the client (and the X + * server) writes once per context/window, so the draw windows themselves + * carry no render target and the replayed draw rasterizes into an + * unbound target - every launch retires with 0/N pages changed while + * every other witness (index buffers, programs, macros) is in place. + * The observed carrier is the X server's 1280x720 X-screen bind (VA + * 0x120400000, fmt 0xe6, 900 pages) on its own channel, so this capture + * is channel-agnostic (any trusted push) and keeps the LAST surface run + * whose shape the booter's alias scan can use. The launch builder + * replays it - aliased/relocated exactly like the window - ahead of any + * window that carries no usable render-target bind of its own; see + * gsp_phys_gr_surf_pre in gsp_phys_booter.c. A/B via sema_surf_prelude_cap. + */ +#define SVC_SURF_PRE_WORDS 16u +static u32 svc_surf_pre_buf[SVC_SURF_PRE_WORDS]; +static u32 svc_surf_pre_n; +static u32 svc_surf_pre_gen; +static u64 svc_surf_pre_sig; +static u32 svc_surf_pre_log; +static u32 svc_surf_pre_flag = 1; +module_param_named(sema_surf_prelude_cap, svc_surf_pre_flag, uint, 0644); +MODULE_PARM_DESC(sema_surf_prelude_cap, + "Capture the last C597 colour-target bind (INC 0x800+j*0x40, fmt 0xe6) from trusted pushes so the GR launch builder can replay it ahead of 3D windows that carry no render target (default 1)"); + +const u32 *gsp_sema_surf_prelude_get(u32 *n, u32 *gen) +{ + if (n) + *n = READ_ONCE(svc_surf_pre_n); + if (gen) + *gen = READ_ONCE(svc_surf_pre_gen); + return svc_surf_pre_buf; +} + +static void svc_surf_prelude_capture(const u8 *buf, u32 len) +{ + u32 off = 0; + + if (!READ_ONCE(svc_surf_pre_flag)) + return; + while (off + 4u <= len) { + u32 hdr, op, cnt, addr, span = 1u, k, n; + u32 v[5] = { 0, 0, 0, 0, 0 }; + u64 va, bytes, npages, sig; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 7u; + cnt = (hdr >> 16) & 0xfffu; + addr = (hdr & 0x1fffu) << 2; + if ((op == 1u || op == 3u || op == 5u) && cnt) { + if ((u64)off + 4u + (u64)cnt * 4u > (u64)len) + break; + span = 1u + cnt; + } + if (op != 1u || cnt < 5u || cnt >= SVC_SURF_PRE_WORDS || + addr < 0x800u || addr >= 0xc00u || + (addr & 0x3fu) != 0u) { + off += span * 4u; + continue; + } + for (k = 0; k < 5u; k++) + memcpy(&v[k], buf + off + 4u + 4u * k, 4); + va = ((u64)(v[0] & 0xffu) << 32) | (u64)(v[1] & 0xfffff000u); + bytes = (u64)v[2] * 4ull * (u64)v[3]; + npages = (bytes + 4095ull) >> 12; + if (!va || (va & 0xfffu) || v[4] != 0xe6u || !v[2] || !v[3] || + v[2] > 8192u || v[3] > 8192u || npages < 2ull || + npages > 4096ull) { + off += span * 4u; + continue; + } + n = span; + if (n > SVC_SURF_PRE_WORDS) + n = SVC_SURF_PRE_WORDS; + sig = svc_mme_run_sig(buf + off, n, hdr); + if (n == svc_surf_pre_n && sig == svc_surf_pre_sig) { + off += span * 4u; + continue; + } + for (k = 0; k < n; k++) + memcpy(&svc_surf_pre_buf[k], buf + off + 4u * k, 4); + WRITE_ONCE(svc_surf_pre_n, n); + svc_surf_pre_sig = sig; + WRITE_ONCE(svc_surf_pre_gen, svc_surf_pre_gen + 1); + if (svc_surf_pre_log < 8) { + svc_surf_pre_log++; + pr_info("sema-surf-pre: captured gen=%u words=%u va=0x%llx %ux%u fmt=0x%x hdr=%08x\n", + svc_surf_pre_gen, n, va, v[2], v[3], v[4], + hdr); + } + off += span * 4u; + } +} + static void svc_push_raw_dump(const u8 *buf, u32 len, u64 push_va, u32 h_object) { @@ -15115,6 +15212,13 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, */ if (ch && trusted) svc_attr_prelude_capture(buf, len, push_va, ch->h_object); + /* + * SESSION 153-CONTINUED-62: surface (render-target) setup capture - + * channel-agnostic, the X-screen bind lives on the X server's own + * channel. See svc_surf_prelude_capture. + */ + if (ch && trusted) + svc_surf_prelude_capture(buf, len); memset(&st, 0, sizeof(st)); st.push_va = push_va; @@ -31530,6 +31634,26 @@ static u64 svc_gr_prelude_gate_defer; static u64 svc_gr_prelude_gate_exp; /* + * SESSION 153-CONTINUED-62: immature-MME skip. The prelude gate above + * HOLDS the forward while the client's macro program is missing, but its + * bounded fail-open window used to forward anyway - and a window whose + * CALL_MME_MACRO runs have no program poisons the engine on contact + * (r1094 and r1114: `MME-PRELUDE emitted gen=1 words=9` -> + * mme_esr=0x80000004 ILLEGAL_OPCODE -> grfifo=0xbadf1201, whole lane + * wedged for the boot). The client needs no engine action to advance + * (the software release lanes author every completion it waits on), so an + * immature-MME beef window is CONSUMED with the software release instead + * of being forwarded into the poison. The moment the setup push lands the + * capture matures and the launches proceed normally. Default 1; A/B with + * 0 (the historical fail-open forward). + */ +static unsigned gsp_sema_mme_immature_skip = 1u; +module_param_named(sema_mme_immature_skip, gsp_sema_mme_immature_skip, + uint, 0644); +MODULE_PARM_DESC(sema_mme_immature_skip, + "Software-complete a beef 3D window instead of forwarding it while the captured MME prelude has no macro program yet (avoids the ILLEGAL_OPCODE engine poison the fail-open forward causes; default 1)"); + +/* * SESSION 153-CONTINUED-59: attr-program settle gate. The first beat * window of a capture generation binds program pairs (method 0x2014/0x2054) * whose operand pages must be IN the launch page set or the builder's @@ -32144,6 +32268,30 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, return false; } /* + * SESSION 153-CONTINUED-62: the gate's fail-open edge (see + * gsp_sema_mme_immature_skip). Past the gate window a beef window + * whose macro program is STILL missing must not be forwarded - the + * engine poisons on the first CALL_MME_MACRO without a program. + * Consume it with the software release instead; the client advances, + * and the lane survives until the setup push matures the capture. + */ + if (READ_ONCE(gsp_sema_mme_immature_skip) && + (ch->h_object & 0xfffff000u) == 0xbeef0000u && + READ_ONCE(svc_mme_prelude_flag) && + !gsp_sema_mme_prelude_mature()) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + mutex_unlock(&svc_fwd_lock); + svc.fwd_gr_refused++; + svc_gr_prelude_gate_defer++; + if (svc_gr_prelude_gate_defer <= 24 || + (svc_gr_prelude_gate_defer & 0xFF) == 1) + pr_info("vgsp-work: MME-IMMATURE-SKIP h=0x%x push=0x%llx (macro program not captured yet; window software-completed, no engine contact) n=%llu\n", + ch->h_object, svc_gr_cand.push_va, + svc_gr_prelude_gate_defer); + svc_gr_bridge_fallback(vgpu); + return false; + } + /* * m143: drain the host injection queue before resolving the launch's * pages. The watcher's pins for the client's a/b/c buffers can land * between the service loop's map_add drain and this capture; the fill diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h index 6b7922a..0f32265 100644 --- a/kernel/gsp_sema_service.h +++ b/kernel/gsp_sema_service.h @@ -496,6 +496,17 @@ const u32 *gsp_sema_attr_prelude_get(u32 *n, u32 *gen); const u64 *gsp_sema_attr_prelude_ptrs(u32 *n); /* + * SESSION 153-CONTINUED-62: surface (render-target) setup accessor. Holds + * the last captured C597 colour-target bind run (INC 800+j*40, [upper, + * lower, w, h, fmt, blksz], fmt 0xe6, 2..4096 pages) from any trusted + * push. The GR launch builder replays it - aliased/relocated exactly like + * the window - ahead of a 3D window that carries no usable render-target + * bind of its own; without it the replayed draw rasterizes into an + * unbound target and stores nothing (the observed 0/N retire). + */ +const u32 *gsp_sema_surf_prelude_get(u32 *n, u32 *gen); + +/* * r378 write guards. Both re-validate a guest-memory write target that was * resolved earlier (ledger/pin/walk/counter registration, or the GR submit's * per-page resolution) against the CURRENT resolution before the bytes land, |