diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 304 |
1 files changed, 288 insertions, 16 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index ed1cb0f..07c63ee 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -5067,6 +5067,97 @@ static void gsp_phys_mme_regs_log(void __iomem *bar0, const char *tag) (info >> 12) & 0xffu, info2, trapped); } +/* + * SESSION 153-CONTINUED-69: pre-reset MME fault capture. Everything sampled + * so far was post-mortem: by the time the 5 ms launch poll ticks, the + * GSP-RM's FECS exception handler has already read NV_PGRAPH_PRI_MME_HWW_ESR + * (0x404490) - the relayed errString "ESR 0x404490=0x80000004" is the only + * surviving trace of the raw ILLEGAL_OPCODE value - and either cleared the + * latch back to 0x80000000 or left the whole PRI block in the 0xbadfxxxx + * priv-error poison. This bounded spin runs in the submit thread immediately + * after the publish/doorbell (execution starts within the first + * milliseconds) and records EVERY transition of the fault registers at + * MMIO read rate, so the faulting ESR (+INFO pc/macro when the valid bit is + * set, and the instruction register INFO2 that names the rejected + * instruction word) is on the record before any clear. Read-only; bounded + * per boot and per launch (the spin costs at most sema_mme_fast_ms of the + * launch thread's time and stops early once the poison is seen). + */ +static u32 gsp_phys_mme_fast_ms = 400u; +module_param_named(sema_mme_fast_ms, gsp_phys_mme_fast_ms, uint, 0644); +MODULE_PARM_DESC(sema_mme_fast_ms, + "Spin-sample the MME fault registers right after each GR launch doorbell for this many ms, logging every transition before the GSP-RM clears them (0 = off; default 400)"); + +static void gsp_phys_mme_fast_sample(void __iomem *bar0, const char *tag, + u32 h_object) +{ + static u32 fast_boot_n; + u32 ms = READ_ONCE(gsp_phys_mme_fast_ms); + u32 esr_p, ds_p, gf_p, tr_p, n = 0; + ktime_t t_end; + bool poisoned = false; + + if (!bar0 || !ms || fast_boot_n >= 48u) + return; + /* + * Beef-family channels get the full budget (the draw-class windows + * are the ones with the MME fault); other channels are sampled only + * a few times so a fast X-channel launch train cannot consume it. + */ + if ((h_object & 0xfffff000u) != 0xbeef0000u && fast_boot_n >= 2u) + return; + fast_boot_n++; + esr_p = ioread32(bar0 + 0x00404490u); + ds_p = ioread32(bar0 + 0x00405840u); + gf_p = ioread32(bar0 + 0x00400504u); + tr_p = ioread32(bar0 + 0x00400704u); + pr_info("vgsp-gr: MME-FAST begin %s h=0x%x esr=%08x ds=%08x grfifo=%08x trap=%08x (spin %u ms)\n", + tag, h_object, esr_p, ds_p, gf_p, tr_p, ms); + t_end = ktime_get() + (s64)ms * 1000000ll; + for (;;) { + u32 esr = ioread32(bar0 + 0x00404490u); + u32 ds = ioread32(bar0 + 0x00405840u); + u32 gf = ioread32(bar0 + 0x00400504u); + u32 tr = ioread32(bar0 + 0x00400704u); + + if (esr != esr_p || ds != ds_p || gf != gf_p || tr != tr_p) { + if (n < 12u) { + u32 info = ioread32(bar0 + 0x00404494u); + u32 info2 = ioread32(bar0 + 0x00404498u); + u32 mmeir = ioread32(bar0 + 0x0040a630u); + u32 mmedata = ioread32(bar0 + 0x00400708u); + + pr_warn("vgsp-gr: MME-FAST %s h=0x%x esr %08x->%08x info=%08x(pc_valid=%u pc=%u mac=0x%x) info2_ir=%08x ds %08x->%08x grfifo %08x->%08x trap=%08x data=%08x fed0=%08x\n", + tag, h_object, esr_p, esr, + info, (info >> 28) & 1u, + info & 0xfffu, + (info >> 12) & 0xffu, info2, + ds_p, ds, gf_p, gf, tr, + mmedata, mmeir); + } + n++; + esr_p = esr; + ds_p = ds; + gf_p = gf; + tr_p = tr; + /* + * The priv-error poison is the terminal state: the + * FECS has already logged and the rest of the + * registers are 0xbadfxxxx. Stop early so the launch + * thread resumes its normal poll. + */ + if ((gf & 0xffff0000u) == 0xbadf0000u) { + poisoned = true; + break; + } + } + if (ktime_after(ktime_get(), t_end)) + break; + } + pr_info("vgsp-gr: MME-FAST end %s h=0x%x events=%u%s\n", + tag, h_object, n, poisoned ? " (poison)" : ""); +} + static void gsp_phys_pbdma_fault_dump(u32 chid) { static u32 dump_n; @@ -17338,18 +17429,25 @@ MODULE_PARM_DESC(sema_gl_mme_prelude, */ #define GSP_PHYS_RMCALL_GR_MME_STUB_PC 0xC0u /* - * SESSION 153-CONTINUED-68: the stub body. The r1134/r1136 builds used a - * bare [0, 0, 1] (end_next with every other bit zero) and the engine still - * raised ILLEGAL_OPCODE with pc_valid=0 - a decode failure at the macro's - * FIRST instruction, i.e. the bare END is not an encoding the MME accepts. - * This body is a byte-exact copy of a real compiler-emitted instruction - * from the client's own captured program (macro(0) instruction 3, - * 20208c00 38c40046 038c4301): both ALUs write the discarded ZERO - * register (no side effects), no outputs are emitted, end_next=1 - a - * known-good macro terminator. - */ -static const u32 gsp_phys_gr_mme_stub_code[3] = { - 0x20208c00u, 0x38c40046u, 0x038c4301u + * SESSION 153-CONTINUED-69: the stub body, take 3. The CONT-68 body + * (20208c00 38c40046 038c4301) was copied from macro(0) instruction 3 - + * but that is a MID-MACRO instruction: the offline MME64 simulator shows + * it carries two output writes with mthd=NONE, so in isolation (method + * register still 0) it EMITS SET_OBJECT <- 0 twice - exactly the relayed + * trap "ChID 0001, Class 0000c597, Offset 00000000, Data 00000000" - and + * its end_next=1 requires the following instruction, so the macro also + * runs past the stub into whatever the IR holds there. + * + * The real minimal terminator is what Mesa's mme_tu104_builder_finish() + * emits: two default NOPs with end_next set on the FIRST (end_next - + * "end after the next instruction" - so the second NOP executes and then + * the macro returns). Both NOPs encode as w2=0x318c0300 (pred=ZERO, + * ALU0/1 dst=ZERO src=ZERO,ZERO, outs NONE) and emit NOTHING. The + * end_next bit is w2 bit 0. + */ +static const u32 gsp_phys_gr_mme_stub_code[6] = { + 0x00000000u, 0x318c0000u, 0x318c0301u, /* NOP, end_next=1 */ + 0x00000000u, 0x318c0000u, 0x318c0300u, /* NOP tail (executed) */ }; static bool gsp_phys_rmcall_gr_sa_stub = true; module_param_named(sema_gl_mme_sa_stub, gsp_phys_rmcall_gr_sa_stub, bool, @@ -17375,6 +17473,113 @@ static inline bool gsp_sema_gr_sa_stub_value(void) } /* + * SESSION 153-CONTINUED-69: optional explicit stub IR address override. + * 0 = derive from the capture (ir_end, floor 0xC0). Nonzero pins the stub + * body/call target to this instruction address - the stub-landing A/B for + * the IR-boundary/ONEINC pointer-unit question. + */ +static u32 gsp_phys_rmcall_gr_mme_stub_pc; +module_param_named(sema_gl_mme_stub_pc, gsp_phys_rmcall_gr_mme_stub_pc, uint, + 0644); +MODULE_PARM_DESC(sema_gl_mme_stub_pc, + "Explicit MME stub IR instruction address (0 = derive from capture ir_end; try 0xf0/0x100 for the IR-boundary A/B)"); + +/* + * SESSION 153-CONTINUED-69: no-stall CALL drop. With the stub terminating + * macros as pure NOPs the macro consumes none of the CALL_MME_DATA + * parameter words the window pushes; the MME parameter port then + * backpressures on the window's larger calls - the r1142 stall parked the + * PBDMA mid-block with HDR_SHADOW = the 16-arg CALL_MME_MACRO(57) + * ONEINC run and the block never completed (while every <=4-arg window + * retired in ~100 us). Drop every CALL_MME_MACRO/CALL_MME_DATA run of a + * STUBBED slot from the emitted window instead: no call, no parameter + * words, no backpressure. Covered slots keep their calls (their real code + * consumes the parameters). Default 1; A/B with 0. + */ +static bool gsp_phys_rmcall_gr_call_drop = true; +module_param_named(sema_gl_mme_call_drop, gsp_phys_rmcall_gr_call_drop, bool, + 0644); +MODULE_PARM_DESC(sema_gl_mme_call_drop, + "Drop CALL_MME_MACRO/DATA runs of stubbed slots from emitted 3D windows (prevents the noop-stub parameter-port stall; default 1)"); + +/* + * The same stubbed-slot predicate the launch builder applies (see the + * sa_stub emit decision): a slot is stubbed when stubbing is on and either + * the all-slots knob is set, the capture holds no SA entry, or the IR + * instruction the entry names never entered the replay. + */ +static bool gsp_phys_gr_slot_stubbed(u32 j, u64 sa_mask, const u32 *sa_ent) +{ + if (j >= 64u || !gsp_sema_gr_sa_stub_value()) + return false; + if (READ_ONCE(gsp_phys_rmcall_gr_sa_stub_all)) + return true; + if (!(sa_mask & (1ull << j))) + return true; + return !gsp_sema_mme_ir_covered(sa_ent[j]); +} + +/* + * In-place filter for the relocated 3D window: remove every run that + * writes the CALL_MME_MACRO(j)/CALL_MME_DATA(j) method pair of a stubbed + * slot. Pure run-level removal (runs are self-contained: op 1/3/5 carry + * 1+cnt data words; 0/2/4/6 are single words), so the surviving method + * sequence is unchanged and both the F2A window walk and the engine see + * the same stream shape with the calls elided. + */ +static u32 gsp_phys_gr_mme_drop_calls(u32 *win, u32 n, u64 sa_mask, + const u32 *sa_ent, u64 *dropped_mask) +{ + u32 r = 0, w = 0, dropped = 0; + u64 dmask = 0; + + while (r < n) { + u32 h = win[r]; + u32 op = (h >> 29) & 7u; + u32 cnt = (h >> 16) & 0xfffu; + u32 span = 1u, k, m; + bool iscall = false; + + if ((op == 1u || op == 3u || op == 5u) && cnt && + r + 1u + cnt <= n) + span = 1u + cnt; + /* + * Decode the touched methods the way the engine walks them: + * multi-word runs land on m + 4k; single-word entries carry + * their method in the low bits (IMMD and the SDM family both + * put it at 11:0 for our purposes). + */ + for (k = 0; k < span; k++) { + m = ((h & 0xfffu) << 2) + 4u * k; + if (k && op != 1u && op != 3u) + break; + if (m >= 0x3800u && m <= 0x39fcu) { + u32 j = (m - 0x3800u) / 8u; + + if (gsp_phys_gr_slot_stubbed(j, sa_mask, + sa_ent)) { + iscall = true; + dmask |= 1ull << j; + break; + } + } + } + if (iscall) { + dropped += span; + r += span; + continue; + } + if (w != r) + memmove(&win[w], &win[r], (size_t)span * 4u); + w += span; + r += span; + } + if (dropped_mask) + *dropped_mask = dmask; + return dropped; +} + +/* * SESSION 153-CONTINUED-61: generation of the MME prelude already emitted * into the engine context (0 = none). The r1110 evidence: the prelude was * emitted once on the first beef launch (gen=1, 512 words), that launch @@ -17653,7 +17858,7 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, so += sspan; } if (sa_nstub) - sa_block = 5u + 3u * sa_nstub; + sa_block = 8u + 3u * sa_nstub; } /* @@ -17666,11 +17871,24 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, */ { static u32 win_dump_n; + static u32 win_dump_beef; - if (win3d && win3d_n && win_dump_n < 4u) { + /* + * SESSION 153-CONTINUED-69: the first-4 budget was + * consumed by the X channel's early windows in every + * boot, so the beef draw-class window (the one that + * faults) never got its run-header census. Two extra + * dumps are reserved for beef-family windows. + */ + if (win3d && win3d_n && + (win_dump_n < 4u || + (((req->h_object & 0xfffff000u) == 0xbeef0000u) && + win_dump_beef < 2u))) { u32 wo = 0, wr = 0; win_dump_n++; + if ((req->h_object & 0xfffff000u) == 0xbeef0000u) + win_dump_beef++; pr_info("vgsp-gr: MME-WINRUN dump h=0x%x push=0x%llx window=%u\n", req->h_object, req->push_va, win3d_n); while (wo < win3d_n && wr < 200u) { @@ -17789,17 +18007,35 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, if (stub_pc < GSP_PHYS_RMCALL_GR_MME_STUB_PC) stub_pc = GSP_PHYS_RMCALL_GR_MME_STUB_PC; /* + * SESSION 153-CONTINUED-69: explicit stub-PC override + * (sema_gl_mme_stub_pc). The ir_end-derived PC sits at + * the first instruction past the replayed program; if + * the MME's IR allocation/ONEINC pointer semantics + * differ (instruction vs word units, allocation + * boundary), the stub body may land where the engine + * does not look. This knob moves the stub (e.g. to + * 0xf0/0x100, or back to the 0xC0 floor) so the + * landing can be A/B-tested without a rebuild. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_mme_stub_pc)) + stub_pc = READ_ONCE(gsp_phys_rmcall_gr_mme_stub_pc); + /* * The IR upload MUST use the client's own SEC_OP (5, * ONE_INC): with op=1 the method auto-increment walked * the data words onto 0x11c/0x120 and clobbered SA[0] - * while writing no usable stub code (r1131). + * while writing no usable stub code (r1131). count = 7 + * = 1 pointer + 6 code words (two 3-word NOPs; see the + * gsp_phys_gr_mme_stub_code comment). */ *p++ = (GSP_PHYS_RMCALL_METHOD_ONEINC << 29) | - (4u << 16) | (0x114u >> 2); + (7u << 16) | (0x114u >> 2); *p++ = stub_pc; *p++ = gsp_phys_gr_mme_stub_code[0]; *p++ = gsp_phys_gr_mme_stub_code[1]; *p++ = gsp_phys_gr_mme_stub_code[2]; + *p++ = gsp_phys_gr_mme_stub_code[3]; + *p++ = gsp_phys_gr_mme_stub_code[4]; + *p++ = gsp_phys_gr_mme_stub_code[5]; /* * Each [slot, stub] pair is its own INC 0x11c run: the * pair methods are re-written per slot (a single longer @@ -22913,6 +23149,32 @@ gr_page_unmapped: win3d = stream_patch; win3d_n = win_n; + /* + * SESSION 153-CONTINUED-69: drop the CALL_MME_MACRO/DATA runs + * of stubbed slots (see gsp_phys_gr_mme_drop_calls). With the + * stub as a pure NOP terminator the parameter port no longer + * drains; the r1142 stall parked the PBDMA on the 16-arg + * CALL_MME_MACRO(57) run and the block never completed. The + * drop turns those calls into nothing instead of a call whose + * parameters have nowhere to go. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_call_drop) && + gsp_sema_gr_sa_stub_value()) { + u32 sa_ent[64]; + u64 sa_mask = gsp_sema_mme_sa_get(sa_ent, 64); + u64 dmask = 0; + u32 ndrop_c = gsp_phys_gr_mme_drop_calls( + stream_patch, win3d_n, sa_mask, sa_ent, + &dmask); + + if (ndrop_c) { + win3d_n -= ndrop_c; + pr_info("vgsp-gr: MME-CALL-DROP %u word(s) of stubbed-slot CALL runs (window=%u) dropped_mask=0x%llx h=0x%x push=0x%llx\n", + ndrop_c, win3d_n, + (unsigned long long)dmask, + req->h_object, req->push_va); + } + } pr_info("vgsp-gr: 3D window relocated %u operand pair(s) to shadow VAs (window=%u shadow=%u dropped=%u subdrop=%u sdmdrop=%u)\n", nrel, win3d_n, n_shadow, ndrop, gsp_phys_rmcall_gr_3d_last_subdrop, @@ -23562,6 +23824,16 @@ gr_page_unmapped: (unsigned long long)boot_stage.rmcall_guest_pt_win_mask, qmd_win_ret); + /* + * SESSION 153-CONTINUED-69: pre-reset MME fault sample. The engine + * faults within the first tens of ms of the launch; sample the MME + * ESR/INFO/INFO2 + DS ESR + GRFIFO/trap regs at MMIO rate from the + * moment the doorbell rang (in publish) so the faulting values are + * caught before the GSP-RM clears them. + */ + if (stream3d && bar0) + gsp_phys_mme_fast_sample(bar0, "launch", req->h_object); + poll_start_us = (u32)ktime_to_us(ktime_get()); deadline = jiffies + msecs_to_jiffies(poll_ms); for (;;) { |