diff options
| -rw-r--r-- | kernel/gsp_phys_booter.c | 658 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 628 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.h | 16 | ||||
| -rw-r--r-- | kernel/main.c | 11 | ||||
| -rw-r--r-- | tools/guest-diag/pin_gr_stream.py | 15 |
5 files changed, 1292 insertions, 36 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index d78e0df..bb75a8c 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -4977,7 +4977,14 @@ static u64 gsp_phys_gr_last_push_va; * fetch position shows how far the walk got. Read-only MMIO + a racy read * of the pb page (best effort, diagnostic only); bounded per boot. */ -#define GSP_PHYS_PBDMA_DUMP_MAX 6u +#define GSP_PHYS_PBDMA_DUMP_MAX 64u + +/* + * SESSION 153-CONTINUED-52: the published method block's DMA address, so the + * PBDMA fault dump can name the exact word the PBDMA stopped on + * (offset = PB_FETCH - last block DMA). Set beside gsp_phys_gr_last_words. + */ +static u64 gsp_phys_gr_last_pb_dma; static const char *gsp_phys_pbdma_type_name(u32 t) { @@ -4987,6 +4994,32 @@ static const char *gsp_phys_pbdma_type_name(u32 t) return names[t & 7u]; } +/* + * SESSION 153-CONTINUED-52: MME fault forensics. The C597 replay's dominant + * failure on draw-bearing windows is a real MME exception + * (NV_PGRAPH_PRI_MME_HWW_ESR 0x404490 latched ILLEGAL_OPCODE / + * MISSING_MACRO_DATA / ... - "unknown instruction fetched from the MME + * Instruction RAM", see gr_error_gk104.c grErrorLogMmeException). INFO + * carries the faulting MME PC and the macro address; INFO2 the instruction + * register. Read them post-failure (the latch survives) so the exact macro + * fetch point is on record. Read-only; bounded by its call sites. + */ +static void gsp_phys_mme_regs_log(void __iomem *bar0, const char *tag) +{ + u32 esr, info, info2, trapped; + + if (!bar0) + return; + esr = ioread32(bar0 + 0x00404490u); + info = ioread32(bar0 + 0x00404494u); + info2 = ioread32(bar0 + 0x00404498u); + trapped = ioread32(bar0 + 0x00400704u); + pr_warn("vgsp-gr: MME-REGS %s esr=%08x(pc_valid=%u illegal=%u miss=%u extra=%u) pc=%u macro_addr=0x%x info2_ir=%08x trapped=%08x\n", + tag, esr, (info >> 28) & 1u, (esr >> 2) & 1u, + (esr >> 0) & 1u, (esr >> 1) & 1u, info & 0xfffu, + (info >> 12) & 0xffu, info2, trapped); +} + static void gsp_phys_pbdma_fault_dump(u32 chid) { static u32 dump_n; @@ -5011,15 +5044,43 @@ static void gsp_phys_pbdma_fault_dump(u32 chid) u32 pbd0 = readl(bar0 + b + 0x08cu); u32 hdrsh = readl(bar0 + b + 0x118u); u32 fetch64 = ((u64)(pbfhi & 0xffu) << 32) | (pbf & ~3u); + u32 trig; + const char *trig_src; + s64 off_words = -1; if (!intr0 && !pbf && !hdrsh && !pbhd) continue; - pr_info("vgsp-gr: PB-FAULT-DUMP P%u intr0=%08x gpf=%u pbfetch=0x%llx len=%u hdr=%08x(%s m=0x%x cnt=%u sub=%u) cnt=%08x data0=%08x shadow=%08x\n", - i, intr0, gpf, fetch64, pbfhi >> 11, + /* + * SESSION 153-CONTINUED-52: the real driver's rule for the + * instruction that triggered the PBDMA interrupt: if the + * DECODED header (PB_HEADER) type is IMMD or PB_COUNT's value + * field is zero, the raw instruction is PB_DATA0; otherwise it + * is HDR_SHADOW (fifo_tu102.c: "If the TYPE field of + * NV_PPBDMA_PB_HEADER is IMMD or the VALUE field of + * NV_PPBDMA_PB_COUNT is zero, then the raw PB instruction + * stored in NV_PPBDMA_PB_DATA0 is the one that triggered the + * interrupt. Otherwise ... HDR_SHADOW"). + */ + if (((pbhd >> 29) & 7u) == 4u || (pbcnt & 0x1fffu) == 0u) { + trig = pbd0; + trig_src = "DATA0"; + } else { + trig = hdrsh; + trig_src = "HDRSHADOW"; + } + if (gsp_phys_gr_last_pb_dma && + fetch64 >= gsp_phys_gr_last_pb_dma && + fetch64 < gsp_phys_gr_last_pb_dma + + (u64)gsp_phys_gr_last_words * 4u) + off_words = (s64)(fetch64 - gsp_phys_gr_last_pb_dma) / 4; + pr_info("vgsp-gr: PB-FAULT-DUMP P%u intr0=%08x gpf=%u pbfetch=0x%llx len=%u off=%lld hdr=%08x(%s m=0x%x cnt=%u sub=%u) cnt=%08x data0=%08x shadow=%08x pbhd=%08x(type=%s) TRIG=%08x(%s)\n", + i, intr0, gpf, fetch64, pbfhi >> 11, off_words, hdrsh, gsp_phys_pbdma_type_name(hdrsh >> 29), (hdrsh & 0x1fffu) << 2, (hdrsh >> 16) & 0xfffu, (hdrsh >> 13) & 7u, - pbcnt, pbd0, hdrsh); + pbcnt, pbd0, hdrsh, pbhd, + gsp_phys_pbdma_type_name(pbhd >> 29), + trig, trig_src); } { struct gsp_phys_boot_chunk *pbch = @@ -16728,8 +16789,12 @@ EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_ce_copy); #define GSP_PHYS_RMCALL_PRI_SKED_DEBUG_1 0x00407040u #define GSP_PHYS_RMCALL_PRI_SKED_DEBUG_1_EN_CWD_SLOT_COUNT 0x800u /* Method block bound: init(152) + inline run(67) + PCAS(4) + REPORT_SEM(8) + - * the optional QMD-virt pair(2). */ -#define GSP_PHYS_RMCALL_GR_METHOD_WORDS 512u + * the optional QMD-virt pair(2). Raised 512 -> 1024 (SESSION 153- + * CONTINUED-53) so a 3D stream replay can carry the captured MME setup + * prelude (<=512 words) ahead of the largest chunked window (<=448) in one + * block; the block still lands inside the one 4 KB chunk page it is copied + * to. */ +#define GSP_PHYS_RMCALL_GR_METHOD_WORDS 1024u static u64 gsp_phys_gr_submitted; static u64 gsp_phys_gr_ok; @@ -17045,6 +17110,26 @@ MODULE_PARM_DESC(gsp_phys_rmcall_gr_3d, "Replay a captured raw C597 method window verbatim instead of the compute prelude (default 1)"); /* + * SESSION 153-CONTINUED-53: MME context-setup prelude. The r1073-r1077 + * in-flight capture proved the draw-window replay raises a real MME + * ILLEGAL_OPCODE on CALL_MME_MACRO(2)=0x3810 (mme_pc=0, mme_ir=1, mme_mac + * =2) because the ladder context never received the client's macro + * program. The client does upload it (0x11c/0x114 pointer pairs, 0x118 + * code, 0x120 entry addresses, 0x124 shadow control; see + * svc_mme_prelude_capture); with this knob the launch builder preplays + * those captured runs verbatim - once per capture generation - right + * after the SET_OBJECT C597 bind and ahead of the window, so the macros + * the window calls are resident. Declared before build_gr_launch (its + * reader); the matching capture-side knob is sema_mme_prelude_cap. + * Default 1; A/B with 0. + */ +static bool gsp_phys_rmcall_gr_mme_prelude = true; +module_param_named(sema_gl_mme_prelude, gsp_phys_rmcall_gr_mme_prelude, bool, + 0644); +MODULE_PARM_DESC(sema_gl_mme_prelude, + "Preplay the captured client MME context-setup runs once per capture generation ahead of the first 3D window replay (default 1)"); + +/* * GR launch method block: the inline QMD run at its original shape and the * channel REPORT_SEMAPHORE release. The stock guest push does NOT emit * SET_OBJECT before the inline run (the compute class is bound on the @@ -17076,10 +17161,59 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, */ if (gsp_phys_rmcall_gr_3d && req && win3d && win3d_n && win3d_n + 2u + 8u <= GSP_PHYS_RMCALL_GR_METHOD_WORDS) { + /* + * SESSION 153-CONTINUED-53: MME context-setup prelude (see the + * sema_gl_mme_prelude knob above). Preplay the captured client + * setup runs once per capture generation, right after the + * SET_OBJECT C597 bind and ahead of the window, so + * CALL_MME_MACRO() runs execute against a loaded macro + * program instead of pc=0 -> ILLEGAL_OPCODE. The read is + * seqchecked (generation before/after): a concurrent append + * only defers emission to the next launch. A window too large + * to also carry the prelude replays without it (logged) - the + * emission then happens on the next launch that fits. + */ + static u32 mme_prelude_emitted_gen; + bool emit_prelude = false; + const u32 *pl = NULL; + u32 pl_n = 0, pl_gen = 0; + + if (READ_ONCE(gsp_phys_rmcall_gr_mme_prelude)) { + pl = gsp_sema_mme_prelude_get(&pl_n, &pl_gen); + if (pl && pl_n && pl_gen && + pl_gen != READ_ONCE(mme_prelude_emitted_gen)) { + if (win3d_n + 2u + 8u + pl_n <= + GSP_PHYS_RMCALL_GR_METHOD_WORDS) { + emit_prelude = true; + } else { + pr_info("vgsp-gr: MME-PRELUDE deferred (window=%u prelude=%u exceeds block; replaying window alone) h=0x%x\n", + win3d_n, pl_n, req->h_object); + } + } + } + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | (1u << 16) | (0u << 13) | (GSP_PHYS_RMCALL_GR_SET_OBJECT >> 2); *p++ = GSP_PHYS_RMCALL_CLASS_GR_3D; + if (emit_prelude) { + u32 g0 = pl_gen, g1 = 0; + + memcpy(p, pl, (size_t)pl_n * 4); + p += pl_n; + /* seqcheck: a mid-copy append must not be recorded as + * emitted (the copy may be a torn prefix). */ + gsp_sema_mme_prelude_get(NULL, &g1); + if (g1 == g0) { + WRITE_ONCE(mme_prelude_emitted_gen, g0); + pr_info("vgsp-gr: MME-PRELUDE emitted gen=%u words=%u window=%u h=0x%x push=0x%llx\n", + g0, pl_n, win3d_n, req->h_object, + req->push_va); + } else { + pr_info("vgsp-gr: MME-PRELUDE copy raced an append (gen %u->%u); emitted words=%u, will re-emit next launch\n", + g0, g1, pl_n); + } + } memcpy(p, win3d, (size_t)win3d_n * 4); p += win3d_n; p = gsp_phys_rmcall_push_gr(p, @@ -19032,11 +19166,27 @@ gsp_phys_gr_poll_break_log(u32 w0, u32 want_qmd, u32 w1, u32 since_us, u32 usr_gpget = ud ? get_unaligned_le32(ud + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPGET_OFF) : 0; + /* + * SESSION 153-CONTINUED-47: re-read the ring slot the launch + * published (the entry AS THE ENGINE SEES IT) after the wait, so an + * X-vs-beef diff on one boot compares the exact e0/e1 the PBDMA + * fetched, plus the doorbell-visible GP_GET/GP_PUT movement. + */ + struct gsp_phys_boot_chunk *r_c = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO]; + u8 *ring = r_c->pages ? gsp_phys_chunk_page_va(r_c, 0) : NULL; + u32 re0 = 0, re1 = 0; + + if (ring && sub->slot < GSP_PHYS_RMCALL_GPFIFO_ENTRIES) { + re0 = get_unaligned_le32(ring + sub->slot * 8); + re1 = get_unaligned_le32(ring + sub->slot * 8 + 4); + } - pr_info("vgsp-gr: poll break w0=0x%x want=0x%x ch=0x%x after %u us usr_gpget=0x%x gpget=0x%x gpput=0x%x\n", + pr_info("vgsp-gr: poll break w0=0x%x want=0x%x ch=0x%x after %u us usr_gpget=0x%x gpget=0x%x gpput=0x%x ring[%u]=%08x %08x (pub %08x %08x)\n", w0, want_qmd, w1, (u32)ktime_to_us(ktime_get()) - since_us, - usr_gpget, sub->gpget, sub->gpput); + usr_gpget, sub->gpget, sub->gpput, + sub->slot, re0, re1, sub->e0, sub->e1); /* r425: the healthy-launch SKED/FECS baseline for the wedge compare. */ gsp_phys_gr_sked_wit(gsp_phys_gpu_bar0(), "post-poll"); } @@ -19613,6 +19763,71 @@ static bool gsp_phys_rmcall_gr_3d_reloc(u32 *up_w, u32 *lo_w, * in the state binds/method-stream shape. Empty = off (historical * behaviour); live-tunable via sysfs so one boot can sweep the variants. */ +/* + * SESSION 153-CONTINUED-47 (A/B knob): gates the newly recognised pair + * classes (SET_INDEX_BUFFER_A/B 0x17c8 and the vertex-stream banks + * 0x1c20..0x1cf0) in the verbatim-replay relocation pass. Default 0 keeps + * the r1035-r1041 baseline relocation set exactly; flipping it live + * (`echo Y > /sys/module/vgsp/parameters/sema_gl_idx_reloc`) rewrites the + * next launches' index/vertex operands to shadow VAs, so one boot can A/B + * the effect instead of comparing across boots. + */ +static bool gsp_phys_rmcall_gr_idx_reloc; +module_param_named(sema_gl_idx_reloc, gsp_phys_rmcall_gr_idx_reloc, bool, 0644); +MODULE_PARM_DESC(sema_gl_idx_reloc, + "Relocate SET_INDEX_BUFFER (0x17c8) and vertex-stream banks 0x1c20..0x1cf0 operands to shadow VAs in 3D replays (default 0 = baseline set only)"); + +/* + * SESSION 153-CONTINUED-48: MME-family payload relocation. The C597 class + * carries CALL_MME_MACRO(j) = 0x3800+j*8 and CALL_MME_DATA(j) = 0x3804+j*8 + * (class/clc597.h); the GLX client's macro(2) run (observed at byte + * address 0x3810, cnt=5) inlines raw 32-bit client VAs as macro arguments + * (`... 0450f800 ... 0450f000` in the first beef window of r1042/r1043). + * Those words are the last raw-operand class left in the faulting window + * after the index/vertex relocation: the engine walks the macro data + * through its own fetch path, and a raw client VA has no PTE chain the + * GPC engines can see (the r993/r1043 FAULT_PDE->PRI-fault class). The + * pass rewrites any payload word of a run whose byte address lies in + * [0x3800, 0x3a00) through the same vetted shadow mapping the pair + * classes use: only a word whose page the launch's own page set (or the + * big-surface alias) resolves is rewritten (MISS stays verbatim), the + * rewrite maps the SAME HPA at a shadow VA so the value is + * content-equivalent, and a shadow VA that would need an upper byte (a + * 64-bit alias in a 32-bit slot) is refused. Default 0 (baseline windows + * byte-identical); live-tunable so one boot can A/B. + */ +static bool gsp_phys_rmcall_gr_mme_reloc; +module_param_named(sema_gl_mme_reloc, gsp_phys_rmcall_gr_mme_reloc, bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_reloc, + "Relocate CALL_MME_MACRO/DATA(j) run payload words whose page resolves in the launch page set to shadow VAs in 3D replays (default 0 = off)"); + +/* (SESSION 153-CONTINUED-53 MME prelude knob is declared before + * gsp_phys_rmcall_build_gr_launch, its reader.) */ + +/* + * SESSION 153-CONTINUED-48: ladder-wedge recovery. CONT-47 measured that + * the FIRST beef launch per boot whose window carries an unresolvable raw + * operand PRI-faults the GR: every SKED/FIFO register then reads the + * priv-error poison (cp. kernel_falcon_ga102.c `privErrVal = 0xbadf0000`) + * and the ladder channel stays wedged for the whole boot (engine GP_GET + * stuck; every later publish unconsumed, even an emptied window) - which + * makes content bisection a one-shot per boot. When enabled, the launch + * poll watches the GR FIFO status (0x400504, the sked recorder's + * `grfifo`) for the 0xbadf-tagged poison and performs ONE bounded + * recovery per launch: the r445 hard re-arm (TSG PREEMPT bWait=1 + the + * m155 lane-enable tail + GPFIFO/TSG schedules + doorbell) followed by + * the warm GR semaphore probe, exactly the mechanism behind + * gsp_phys_gr_wedge_rearm. Bounded: one detection per launch, poison + * gated, all RPCs keep their own timeouts. Default 1 (the poisoned state + * is already a guaranteed-dead boot; the recovery only adds the chance + * that later launches in the same boot still execute). + */ +static bool gsp_phys_gr_wedge_recover = true; +module_param_named(sema_gr_wedge_recover, gsp_phys_gr_wedge_recover, bool, + 0644); +MODULE_PARM_DESC(sema_gr_wedge_recover, + "On the 0xbadf priv-error poison in the GR FIFO status, run one bounded TSG-preempt+resched hard re-arm per launch (default 1)"); + static char *gsp_phys_rmcall_gr_win_drop; module_param_named(sema_gl_win_drop, gsp_phys_rmcall_gr_win_drop, charp, 0644); MODULE_PARM_DESC(sema_gl_win_drop, @@ -19659,6 +19874,48 @@ static bool gsp_phys_rmcall_gr_win_drop_match(u32 addr) } if (!digits) break; + /* + * SESSION 153-CONTINUED-47: `lo-hi` drops every method + * byte-address in the closed range (both ends accept the + * 0x prefix). The content bisect needs whole families + * (`0x3800-0x3a00` MME, `0x3400-0x3500` MME scratch) and + * the emptied-window control (`0-0x7ffc`): with the window + * stripped whole, a launch that still retires proves the + * SET_OBJECT + release tail path is healthy and that the + * wedge lives in the stripped content. + */ + if (*p == '-') { + const char *q = p + 1; + u32 hi = 0; + u32 hd = 0; + + if (q[0] == '0' && (q[1] == 'x' || q[1] == 'X')) + q += 2; + for (;;) { + char c = *q; + u32 h; + + if (c >= '0' && c <= '9') + h = c - '0'; + else if (c >= 'a' && c <= 'f') + h = c - 'a' + 10; + else if (c >= 'A' && c <= 'F') + h = c - 'A' + 10; + else + break; + if (hi <= 0xfffffffu) + hi = (hi << 4) | h; + q++; + hd++; + } + if (hd) { + p = q; + n++; + if (addr >= v && addr <= hi) + return true; + continue; + } + } n++; if (v == addr) return true; @@ -19705,7 +19962,30 @@ static bool gsp_phys_rmcall_gr_win_drop_auto_match(u32 addr) { if (!READ_ONCE(gsp_phys_gr_win_drop_auto)) return false; - return addr == 0x010cu || addr == 0x01b0u || addr == 0x05e0u; + /* + * SESSION 153-CONTINUED-48: NVC597_SET_REPORT_SEMAPHORE_A=0x1b00 - + * the C597 twin of the 0x10c NOTIFY trigger. Every glcore draw + * window ends with 4-word runs [A=upper(7:0), B=lower(31:0), + * C=payload, D=flags] whose B word is a raw client VA (observed + * 0x41a5bb0/0x759e870/0x251800; payloads = frame counters 0xb4f3+). + * The replayed engine executing them writes a client VA the GPC + * engines cannot reach - the VIRT_WRITE fault class (r1061: the + * kept [0,0x2000) window whose only fetch/write triggers besides + * the index binds are these raw 0x1b00 writes is exactly the one + * that still faulted with grfifo=0xbadf1201; r1062: the full window + * stalls the front-end right after the PBDMA consumes it and + * fecs_cur never loads). Their page cannot join the launch page set + * (n_ptr is capped at 7 by the QMD pointer region and the draw's + * program/index/vertex/MME pages already fill it), so relocation is + * impossible; dropping is the same call the 0x10c NOTIFY drop + * already makes: the module appends its own SET_REPORT_SEMAPHORE + * tail and the host-side completion lanes author every completion + * the client actually waits on (the r1061 keep-prefix window + * retired with ch_rel=0xc0ffee02 and the client kept pushing + * frames without any 0x1b00 in the replay). + */ + return addr == 0x010cu || addr == 0x01b0u || addr == 0x05e0u || + addr == 0x1b00u; } /* @@ -19786,12 +20066,14 @@ static bool gsp_phys_rmcall_gr_win_patch_find(u32 addr, u32 *out_data) return false; } +static u32 gsp_phys_rmcall_gr_3d_last_subdrop; + static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out, u32 *out_n, u32 *n_dropped, const struct gsp_phys_gr_submit *req, u32 *n_shadow) { - u32 off = 0, wout = 0, reloc = 0, dropped = 0; + u32 off = 0, wout = 0, reloc = 0, dropped = 0, subdrop = 0; *out_n = n; *n_dropped = 0; @@ -19800,11 +20082,38 @@ static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out, u32 op = (hdr >> 29) & 0x7u; u32 cnt = (hdr >> 16) & 0xfffu; u32 addr = (hdr & 0x1fffu) << 2; + u32 sub = (hdr >> 13) & 0x7u; u32 span = 1u; if ((op == 1 || op == 3 || op == 5) && cnt && off + 1u + cnt <= n) span = 1u + cnt; + /* + * SESSION 153-CONTINUED-52: drop METHOD WRITES aimed at a + * non-zero subchannel. The glcore draw window ends with + * `2001c00d 70000000/68000000` = INC m=0x34 cnt=1 SUBCHANNEL 6 + * (channel-class MEM_OP_D lane) two-word runs. The ladder + * channel has nothing bound on subchannel 6, so the physical + * PBDMA hands those methods to its host-software lane and + * raises NV_PPBDMA_INTR_0_METHOD (bit 21): the r1075 relay + * records `Channel ID 00000004 intr 00200000` at every launch, + * and the real host driver's software-method handler answers + * "Invalid host sofware method received" -> rcRequired -> + * robust-channel error recovery kills the channel mid-window + * (fifo_gk104.c). The methods are channel-level barriers whose + * effects the module's own release/fence lanes already author, + * so dropping them from the replay is the same call the 0x10c + * NOTIFY / 0x1b0 LAUNCH_DMA auto-drops make. Only count- + * carrying writes (op 1/3/5) are candidates: the single-word + * TERT encodings (op 0/2/4/7) reuse bits 15:13 as mask/type + * payload. + */ + if ((op == 1u || op == 3u || op == 5u) && sub != 0u) { + dropped++; + subdrop++; + off += span; + continue; + } if (gsp_phys_rmcall_gr_win_drop_match(addr) || gsp_phys_rmcall_gr_win_drop_auto_match(addr)) { dropped++; @@ -19867,6 +20176,54 @@ static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out, * reads. Observed [00001000 1 2150b790]. */ lo_i = 1; pair = true; + } else if (READ_ONCE(gsp_phys_rmcall_gr_idx_reloc) && + op == 1 && addr >= 0x1c20 && addr <= 0x1cf0 && + (addr & 0xfu) == 0u && cnt >= 3) { + /* SESSION 153-CONTINUED-47 (A/B knob): widen + * the vertex-stream banks from j=0/1 to every + * FORMAT slot (0x1c20..0x1cf0). The GLX + * client's 3D window binds streams j>=2 that + * svc_gl_field already collects into the page + * set but the window never had rewritten. */ + lo_i = 1; + pair = true; + } else if (READ_ONCE(gsp_phys_rmcall_gr_idx_reloc) && + op == 1 && addr == 0x17c8 && cnt >= 2) { + /* SESSION 153-CONTINUED-47 (A/B knob): SET_ + * INDEX_BUFFER_A/B [ADDRESS_UPPER(7:0), + * ADDRESS_LOWER(31:0)] - the indexed draw + * fetches its indices through this VA. The + * r1042 boot showed an X window carrying + * index binds PRI-stalling once relocated + * (relocation of exactly this pair is the + * only kernel delta vs the r1035 baseline + * where index-bearing windows did not exist + * and every X window retired), so the + * rewrite is gated off by default and A/B-ed + * live per boot. Observed [00000000 + * 0450fcc8] -> VA 0x450fcc8. */ + lo_i = 0; + pair = true; + } else if (READ_ONCE(gsp_phys_rmcall_gr_idx_reloc) && + op == 1 && addr == 0x1b00 && cnt >= 2) { + /* SESSION 153-CONTINUED-48: NVC597_SET_REPORT_ + * SEMAPHORE_A/B..D (0x1b00) - the engine's own + * completion write [upper(7:0), lower(31:0), + * payload, flags] (observed [0, 0x4181bb0, + * 0x4f9f6, 0x1000f010] at the end of every + * glcore window). With the VA raw the replayed + * engine writes a client VA the GPC engines + * cannot reach - the VIRT_WRITE fault class + * (r1061: the kept [0,0x2000) window that + * faults is exactly the one still carrying the + * raw 0x1b00 VAs). Rewrite the pair to the + * shadow VA (same HPA, page collected by + * svc_gl_field 0x1b00), so the write lands in + * the client's own page - its wait then sees a + * real hardware completion. MISS stays + * verbatim like every other pair. */ + lo_i = 0; + pair = true; } else if (op == 1 && addr == 0x2380 && cnt >= 3) { /* SET_CONSTANT_BUFFER_SELECTOR_A/B: * [size, upper, lower]; observed @@ -19914,6 +20271,105 @@ static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out, (u64)(d[lo_i + 1] & 0xfffffffcu)); } } + /* + * SESSION 153-CONTINUED-48: MME-family payload + * relocation (see the sema_gl_mme_reloc param). The + * C597 MME macro/data family spans 0x3800..0x3a00; + * its runs carry the macro arguments as raw payload + * words, which include 32-bit client VAs + * (0x450f800/0x450f000 observed in the beef macro(2) + * run at 0x3810). Rewrite only a word whose page the + * launch's own page set resolves (the same vetted + * shadow mapping the pairs use); a MISS, an + * unaligned/too-small word, or a shadow VA that + * cannot fit the 32-bit slot stays verbatim. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_mme_reloc) && + addr >= 0x3800u && addr < 0x3a00u) { + u32 wi; + + for (wi = 0; wi < cnt; wi++) { + u32 wv = d[wi]; + u32 up = 0, lo = wv; + static u32 mme_log; + + if (wv < 0x10000u || (wv & 3u)) + continue; + if (!gsp_phys_rmcall_gr_3d_reloc( + &up, &lo, req, n_shadow)) { + /* + * SESSION 153-CONTINUED-51: + * the payload page missed the + * launch's page set (the QMD + * pointer region caps the set + * far below a full window's + * operand count - r1071: the + * macro payload pages + * 0x44fd200..0x44fde00 were + * ~12 distinct pages while the + * set held 7; the words stayed + * RAW and the engine fetched an + * untranslatable guest VA on + * every full window - the + * 0xbadf1201 priv-error + * poison). Resolve the page + * through the SAME trusted + * ledger/pin/walk chain the + * launch pages use (the + * RELOC-FIXED resolver) and + * shadow-map it exactly like a + * matched page. A resolve/map + * miss or a shadow VA that + * cannot fit the 32-bit slot + * stays verbatim (never + * guessed). + */ + struct gsp_sema_gr_reloc r; + u64 va2 = (u64)wv; + u64 sh2 = 0; + u32 line2 = 0; + + if (gsp_sema_gr_reloc_resolve( + va2, &r) || + !r.hpa) + continue; + if (*n_shadow >= + GSP_PHYS_RMCALL_GR_SHADOW_PAGES) + continue; + if (gsp_phys_rmcall_gr_map_shadow_locked( + *n_shadow, r.hpa, + &sh2, &line2)) + continue; + if (line2 != 0xffffffffu) + (*n_shadow)++; + sh2 |= va2 & + (GSP_PHYS_PAGE_SIZE - 1); + up = (u32)(sh2 >> 32) & 0xffu; + lo = (wv & 0x3u) | + ((u32)sh2 & 0xfffffffcu); + if (up != 0) + continue; + d[wi] = lo; + reloc++; + if (mme_log < 32) { + mme_log++; + pr_info("vgsp-gr: mme-reloc-rsv addr=0x%x wi=%u w=0x%x -> 0x%x\n", + addr, wi, wv, + lo); + } + continue; + } + if (up != 0) + continue; + d[wi] = lo; + reloc++; + if (mme_log < 32) { + mme_log++; + pr_info("vgsp-gr: mme-reloc addr=0x%x wi=%u w=0x%x -> 0x%x\n", + addr, wi, wv, lo); + } + } + } wout += span; off += 1u + cnt; continue; @@ -19923,6 +20379,7 @@ static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out, } *out_n = wout; *n_dropped = dropped; + gsp_phys_rmcall_gr_3d_last_subdrop = subdrop; return reloc; } @@ -20172,8 +20629,8 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, u32 qmd_rel_off = 0, ch_rel_off = 0; u32 poll = 0, w0 = 0, w1 = 0, grpoll_n = 0; bool sem_gated = false; - static u32 sked_prev[14]; - static u32 sked_sample[14]; + static u32 sked_prev[20]; + static u32 sked_sample[20]; static u32 sked_tick; static bool sked_first; /* @@ -20209,6 +20666,14 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, bool rearmed = false; u32 attempts = 0; /* + * SESSION 153-CONTINUED-48: one bounded priv-error recovery attempt + * per launch (sema_gr_wedge_recover), with a per-boot tally for the + * report. Static like the other submit scratch: the single-inflight + * ordered work item is the only caller. + */ + bool wedge_recover_done = false; + static u32 wedge_recoveries; + /* * F2/F4 state: the fast first-timeout re-arm runs once per launch; the * deposit readback is one-shot per launch (the first timeout is the * evidence window - the old r198 post-mortem only ran after attempt 3, @@ -21397,8 +21862,9 @@ gr_page_unmapped: win3d = stream_patch; win3d_n = win_n; - pr_info("vgsp-gr: 3D window relocated %u operand pair(s) to shadow VAs (window=%u shadow=%u dropped=%u)\n", - nrel, win3d_n, n_shadow, ndrop); + pr_info("vgsp-gr: 3D window relocated %u operand pair(s) to shadow VAs (window=%u shadow=%u dropped=%u subdrop=%u)\n", + nrel, win3d_n, n_shadow, ndrop, + gsp_phys_rmcall_gr_3d_last_subdrop); if (ndrop) { static u32 gr3d_drop_log; @@ -21421,7 +21887,7 @@ gr_page_unmapped: static u32 gr3d_win_log; u32 wi; - if (gr3d_win_log < 4) { + if (gr3d_win_log < 48) { gr3d_win_log++; pr_info("vgsp-gr: 3D window dump h=0x%x push=0x%llx words=%u\n", req->h_object, req->push_va, win3d_n); @@ -21450,6 +21916,59 @@ gr_page_unmapped: WRITE_ONCE(gsp_phys_gr_last_words, words); WRITE_ONCE(gsp_phys_gr_last_h, req->h_object); WRITE_ONCE(gsp_phys_gr_last_push_va, req->push_va); + WRITE_ONCE(gsp_phys_gr_last_pb_dma, + (u64)gsp_phys_chunk_page_dma(pb, 0)); + /* + * SESSION 153-CONTINUED-52: stream validator for the block the engine + * will walk. The r1072 fault class ("Channel ID 4 intr0 00040000" = + * PBDMA PBENTRY, "invalid method encoding used in method", + * fifo_tu102.c) is raised by a method header the PBDMA refuses, or by + * a run whose data is not fully present so the walk desyncs into + * following words. Walk the exact published dwords with the FIFO DMA + * grammar (dev_ram.ref NV_FIFO_DMA_*: SEC_OP 31:29; 1/3/5 carry 1+cnt + * data words, 4/7 and TERT mask ops are single-word, 6 is RESERVED) + * and flag the first occurrences of a reserved encoding or a run + * overrunning the block. Read-only; bounded per boot. + */ + { + static u32 blkval_log; + const u32 *bv = methods; + u32 bo = 0, nbad = 0; + + if (blkval_log < 48) { + while (bo < words && nbad < 4) { + u32 h = bv[bo]; + u32 bop = (h >> 29) & 7u; + u32 bcnt = (h >> 16) & 0xfffu; + u32 bspan = 1u; + + if (bop == 1u || bop == 3u || bop == 5u) + bspan = 1u + bcnt; + if (bop == 6u) { + nbad++; + pr_warn("vgsp-gr: BLOCKVAL reserved-encoding word=%u/%u val=%08x (h=%x push=0x%llx prev=%08x %08x)\n", + bo, words, h, req->h_object, + req->push_va, + bo ? bv[bo - 1] : 0, + bo + 1 < words ? + bv[bo + 1] : 0); + } + if (bo + bspan > words) { + nbad++; + pr_warn("vgsp-gr: BLOCKVAL run-overrun word=%u/%u hdr=%08x span=%u (needs %u more) (h=%x push=0x%llx)\n", + bo, words, h, bspan, + bo + bspan - words, + req->h_object, req->push_va); + } + bo += bspan; + } + blkval_log++; + if (nbad) + pr_warn("vgsp-gr: BLOCKVAL done words=%u bad=%u (h=%x push=0x%llx)\n", + words, nbad, req->h_object, + req->push_va); + } + } pr_info("vgsp-gr: qmd w0..3=%08x %08x %08x %08x w4..7=%08x %08x %08x %08x w23..25=%08x %08x %08x w32/33=%08x %08x w48/49=%08x %08x\n", qmd[0], qmd[1], qmd[2], qmd[3], qmd[4], qmd[5], qmd[6], qmd[7], qmd[23], qmd[24], qmd[25], qmd[32], qmd[33], qmd[48], qmd[49]); @@ -21769,9 +22288,10 @@ gr_page_unmapped: break; } } - pr_info("vgsp-gr: GR LAUNCH submitted h=0x%x push=0x%llx len=%u pages=%u shadow=%u qmd_va=0x%llx slot=%u words=%u prelude=%d qmdvirt=%d token=0x%x; polling qmd_rel=%u ch_rel=0x%08x (<=%u ms)%s\n", + pr_info("vgsp-gr: GR LAUNCH submitted h=0x%x push=0x%llx len=%u pages=%u shadow=%u qmd_va=0x%llx slot=%u words=%u e0=0x%08x e1=0x%08x prelude=%d qmdvirt=%d token=0x%x; polling qmd_rel=%u ch_rel=0x%08x (<=%u ms)%s\n", req->h_object, req->push_va, req->push_len, req->n_pages, n_shadow, req->qmd_va, sub.slot, words, + sub.e0, sub.e1, gsp_phys_rmcall_gr_prelude_value(), gsp_phys_rmcall_gr_qmdvirt_value(), boot_stage.rmcall_token, @@ -21842,10 +22362,35 @@ gr_page_unmapped: sked_sample[11] = ioread32(bar0 + 0x00409c00u); sked_sample[12] = ioread32(bar0 + 0x00409c18u); sked_sample[13] = ioread32(bar0 + 0x00409008u); + /* + * SESSION 153-CONTINUED-52: in-flight fault registers. + * The DS shader-program-header error latch (0x405840), + * the MME exception latch (0x404490) with its PC/macro + * address (0x404494) and instruction register + * (0x404498) are cleared by the real RM within ~ms of + * the fault, so the post-timeout reads are all clean. + * Sampling them every poll tick records the faulting + * values at the first change. + */ + sked_sample[14] = ioread32(bar0 + 0x00405840u); + sked_sample[15] = ioread32(bar0 + 0x00404490u); + sked_sample[16] = ioread32(bar0 + 0x00404494u); + sked_sample[17] = ioread32(bar0 + 0x00404498u); + /* + * SESSION 153-CONTINUED-52: the PBDMA host-software + * METHOD latch (NV_PPBDMA_METHOD0 0x4000c0: ADDR + * 13:2, SUBCH 18:16, INCR 0) - the method whose + * delivery raises NV_PPBDMA_INTR_0_METHOD (bit 21). + * The real driver clears the entry when it processes + * the interrupt, so only an in-flight sample can name + * it. + */ + sked_sample[18] = ioread32(bar0 + 0x004000c0u); + sked_sample[19] = ioread32(bar0 + 0x004000c4u); if (sked_first || memcmp(sked_sample, sked_prev, sizeof(sked_sample))) { - pr_info("vgsp-gr: sked rec +%ums cwd0=%08x cwd1=%08x cwd2=%08x info1=%08x info2=%08x act=%08x esr=%08x grfifo=%08x fecs_cur=%08x fecs_new=%08x ctxsw1=%08x fe0=%08x fehost=%08x fecsirq=%08x\n", + pr_info("vgsp-gr: sked rec +%ums cwd0=%08x cwd1=%08x cwd2=%08x info1=%08x info2=%08x act=%08x esr=%08x grfifo=%08x fecs_cur=%08x fecs_new=%08x ctxsw1=%08x fe0=%08x fehost=%08x fecsirq=%08x ds_esr=%08x mme_esr=%08x mme_pc=%u mme_mac=0x%x mme_ir=%08x mth0=%08x mth1=%08x(pbaddr=0x%x sub=%u incr=%u)\n", sked_tick * GSP_PHYS_RMCALL_POLL_MS, sked_sample[0], sked_sample[1], @@ -21854,12 +22399,52 @@ gr_page_unmapped: sked_sample[6], sked_sample[7], sked_sample[8], sked_sample[9], sked_sample[10], sked_sample[11], - sked_sample[12], sked_sample[13]); + sked_sample[12], sked_sample[13], + sked_sample[14], sked_sample[15], + sked_sample[16] & 0xfffu, + (sked_sample[16] >> 12) & 0xffu, + sked_sample[17], + sked_sample[18], sked_sample[19], + (sked_sample[18] >> 2) << 2, + (sked_sample[18] >> 16) & 7u, + sked_sample[18] & 1u); memcpy(sked_prev, sked_sample, sizeof(sked_sample)); sked_first = false; } } + /* + * SESSION 153-CONTINUED-48: priv-error (0xbadf-tagged) wedge + * recovery - see sema_gr_wedge_recover. The poison latch is + * checked before the completion break and the ladder so a + * launch whose window faulted the GR hands the engine to the + * next launch recovered instead of wedged. One attempt per + * launch (local flag); the hard re-arm is the r445 mechanism, + * bounded by its own RPC and sem-probe timeouts. + */ + if (READ_ONCE(gsp_phys_gr_wedge_recover) && bar0 && + !wedge_recover_done) { + u32 g0 = ioread32(bar0 + 0x00400504u); + + if ((g0 & 0xffff0000u) == 0xbadf0000u) { + u32 g1; + + wedge_recover_done = true; + wedge_recoveries++; + pr_warn("vgsp-gr: WEDGE-RECOVER priv-error poison seen (grfifo=0x%08x) h=0x%x push=0x%llx - TSG preempt + resched\n", + g0, req->h_object, req->push_va); + gsp_phys_mme_regs_log(bar0, "wedge-poison"); + mutex_lock(&gsp_phys_rm_lock); + gsp_phys_gr_wedge_hard_rearm_locked(bar0); + mutex_unlock(&gsp_phys_rm_lock); + g1 = ioread32(bar0 + 0x00400504u); + pr_warn("vgsp-gr: WEDGE-RECOVER after re-arm grfifo=0x%08x (%s); recovery #%u this boot\n", + g1, + (g1 & 0xffff0000u) == 0xbadf0000u ? + "still poisoned" : "cleared", + wedge_recoveries); + } + } if (qmd_rel) { if (qmd_rel_hpa) w0 = READ_ONCE(*(u32 *)((u8 *)__va(qmd_rel_hpa) + @@ -21881,7 +22466,7 @@ gr_page_unmapped: static u32 winprobe_log; u32 pi; - if (winprobe_log < 10) { + if (winprobe_log < 64) { winprobe_log++; pr_info("vgsp-gr: WINPROBE +%ums push=0x%llx words=%u block_dma=0x%llx grfifo=%08x trapped=%08x trapped_lo=%08x\n", sked_tick * GSP_PHYS_RMCALL_POLL_MS, @@ -21896,18 +22481,33 @@ gr_page_unmapped: u32 pbf = ioread32(bar0 + b + 0x054u); u32 pbfhi = ioread32(bar0 + b + 0x058u); u32 hdrsh = ioread32(bar0 + b + 0x118u); + u32 wbh = ioread32(bar0 + b + 0x084u); + u32 wbcnt = ioread32(bar0 + b + 0x088u); + u32 wbd0 = ioread32(bar0 + b + 0x08cu); u64 fetch64 = ((u64)(pbfhi & 0xffu) << 32) | (u64)(pbf & ~3u); + s64 woff = -1; if (!intr0 && !gpf && !pbf && !hdrsh) continue; - pr_info("vgsp-gr: WINPROBE P%u intr0=%08x gp_put=%u pbfetch=0x%llx hdr=%08x(%s m=0x%x cnt=%u sub=%u)\n", - pi, intr0, gpf, fetch64, hdrsh, + if (gsp_phys_gr_last_pb_dma && + fetch64 >= gsp_phys_gr_last_pb_dma && + fetch64 < + gsp_phys_gr_last_pb_dma + + (u64)gsp_phys_gr_last_words * + 4u) + woff = (s64)(fetch64 - + gsp_phys_gr_last_pb_dma) / + 4; + pr_info("vgsp-gr: WINPROBE P%u intr0=%08x gp_put=%u pbfetch=0x%llx off=%lld hdr=%08x(%s m=0x%x cnt=%u sub=%u) pbhd=%08x(type=%s) pbcnt=%08x pbd0=%08x\n", + pi, intr0, gpf, fetch64, woff, hdrsh, gsp_phys_pbdma_type_name(hdrsh >> 29), (hdrsh & 0x1fffu) << 2, (hdrsh >> 16) & 0xfffu, - (hdrsh >> 13) & 7u); + (hdrsh >> 13) & 7u, wbh, + gsp_phys_pbdma_type_name(wbh >> 29), + wbcnt, wbd0); } } } @@ -21937,6 +22537,7 @@ gr_page_unmapped: pr_err("vgsp-gr: GR LAUNCH FAILED: wait budget crossed (qmd_rel=0x%08x want=0x%08x ch_rel=0x%08x want=0x%08x) h=0x%x push=0x%llx; ladder stopped\n", w0, want_qmd, w1, want_ch, req->h_object, req->push_va); + gsp_phys_mme_regs_log(bar0, "wait-max"); ret = -ETIMEDOUT; goto out; } @@ -21948,6 +22549,7 @@ gr_page_unmapped: pr_err("vgsp-gr: GR LAUNCH FAILED: timeout after %u ms (qmd_rel=0x%08x want=0x%08x ch_rel=0x%08x want=0x%08x) h=0x%x push=0x%llx\n", poll_ms, w0, want_qmd, w1, want_ch, req->h_object, req->push_va); + gsp_phys_mme_regs_log(bar0, "timeout"); /* * F4-b: one-shot deposit readback at the FIRST timeout * (the old post-mortem only ran after attempt 3, which @@ -25649,6 +26251,18 @@ static int gsp_phys_reg_probe_show(struct seq_file *m, void *unused) 0x0040070cu); fe_hww_esr = gsp_phys_reg_probe_line(m, bar0, "FE_HWW_ESR", 0x00404000u); + /* + * SESSION 153-CONTINUED-52: MME exception latches (see + * gsp_phys_mme_regs_log). + */ + (void)gsp_phys_reg_probe_line(m, bar0, "PGRAPH_PRI_MME_HWW_ESR", + 0x00404490u); + (void)gsp_phys_reg_probe_line(m, bar0, + "PGRAPH_PRI_MME_HWW_ESR_INFO", + 0x00404494u); + (void)gsp_phys_reg_probe_line(m, bar0, + "PGRAPH_PRI_MME_HWW_ESR_INFO2", + 0x00404498u); fe_chip_def = gsp_phys_reg_probe_line(m, bar0, "FE_CHIP_DEF_INFO", 0x00404030u); fe_go_idle = gsp_phys_reg_probe_line(m, bar0, "FE_GO_IDLE_INFO", diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 4d497d9..25598bd 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -476,6 +476,15 @@ struct svc_channel { unsigned long push_hold_until; /* absolute deadline of the bounded * work-before-release hold for an * unreadable CE entry (0 = none) */ + unsigned long gr_setup_hold_until; /* SESSION 153-CONTINUED-54: absolute + * deadline of the GR setup-push hold: + * an unreadable/all-zero MME-setup + * (or >= 12 KB) GR push stays pending + * past the 1-failed-rescue consume + * until the demanded guest pin lands + * (0 = none) */ + u64 gr_setup_hold_va; /* entry VA the setup hold was armed for */ + u32 gr_setup_hold_idx; /* GPFIFO slot the setup hold was armed for */ unsigned long defer_since; /* first failure of the pending GR entry * (0 = none): the entry is left pending * for the GR grace window so a late @@ -8022,8 +8031,24 @@ static void svc_gl_add_ptr(u64 va) if (!va) return; + /* + * SESSION 153-CONTINUED-48: dedup by 4 KB page, not exact VA. + * The QMD pointer region holds only seven pairs (words 34..47, + * SVC_GR_POINTER_MAX - 1); a window that binds several different + * offsets of one buffer (r1058 chunk1: five index-buffer VAs + * across three pages) used to consume one slot per exact VA and + * hit the cap, so no later operand page (the raw CALL_MME_MACRO + * payload VAs 0x43545f0/0xb94f700) could join the set and the + * shadow relocation left them raw - the raw operand the engine + * faulted on (grfifo=0xbadf1201). One pointer per page carries + * the same page set with slots left for the remaining operand + * families; the first VA seen for a page is kept so the value + * stays a real client VA (the fill resolves pages, and the + * relocation matches by page). + */ for (k = 0; k < svc_gl.n_ptr; k++) - if (svc_gl.ptr[k] == va) + if (svc_gl.ptr[k] == va || + (svc_gl.ptr[k] & ~(u64)0xfffu) == (va & ~(u64)0xfffu)) return; if (svc_gl.n_ptr < SVC_GR_POINTER_MAX - 1) svc_gl.ptr[svc_gl.n_ptr++] = va; @@ -8049,6 +8074,26 @@ static void svc_gl_field(u32 addr, u32 count, const u8 *data, u32 dlen) * [ENABLE(0) | TYPE(7:4)]. Collected per slot so a bound program VA * can be typed (VTG vs PS) for the program-bytes delivery. */ + /* + * SESSION 153-CONTINUED-48: NVC597_SET_REPORT_SEMAPHORE_A..D = 0x1b00, + * observed as 4-word runs [A=upper(7:0), B=lower(31:0), C=payload, + * D=flags] at the end of every glcore draw window (payload = the + * client's frame counter, e.g. 0x4f9f6; VA e.g. 0x4181bb0/0xc91e6c0). + * These are the engine's own completion writes; with the VA raw the + * replayed engine writes a client VA the GPC engines cannot reach - + * the same VIRT_WRITE fault class the 0x10c NOTIFY drop exists for + * (r1061: the kept [0,0x2000) window that still faults is exactly + * the one carrying these raw 0x1b00 VAs). Collect the semaphore page + * so the window relocation can rewrite the pair to a shadow VA: the + * write then lands in the SAME HPA and the client's wait sees the + * real hardware completion. + */ + if (addr == 0x1b00) { + if (count >= 2) + svc_gl_add_ptr(((u64)(v[0] & 0xffu) << 32) | + (u64)(v[1] & 0xfffffffcu)); + return; + } if (addr >= 0x2000 && addr < 0x2800 && (addr & 0x3f) == 0) { u32 j = (addr - 0x2000) / 0x40; @@ -8056,6 +8101,37 @@ static void svc_gl_field(u32 addr, u32 count, const u8 *data, u32 dlen) svc_gl_shader_type[j] = (u8)((v[0] >> 4) & 0xf); return; } + /* + * SESSION 153-CONTINUED-48: CALL_MME_MACRO(j)=0x3800+j*8 / + * CALL_MME_DATA(j)=0x3804+j*8 runs carry raw 32-bit client VAs as + * macro arguments (r1058 chunk1: the macro(2) run payloads + * 0x44ff800/0x44ff000 relocated only because the page set already + * held 0x44ff000 via the index bind; the sibling payloads + * 0x43545f0/0x4353f60/0xb94f700/0xb941578 stayed raw because their + * pages missed the capped pointer list - the raw operand class the + * GR faulted on). Collect every VA-shaped payload word so the page + * set - and therefore the 3D-window shadow relocation - covers the + * macro arguments too. This must run before the 0x2014..0x3fff + * program-pair recognizer below, whose (addr & 0x3f) == 0x14 shape + * would otherwise swallow a CALL_MME_DATA(2)=0x3814 run. Shape + * gates: 4-byte aligned; >= 64 KB excludes the method/flag scalars + * (0x0c/0x53/0x338); < 512 MB excludes the float constants the + * macro(57) runs carry (1.0f=0x3f800000) while keeping every + * observed client operand VA (max 0xb94f700). + */ + if (addr >= 0x3800 && addr < 0x3a00) { + for (k = 0; k < count; k++) { + u32 w; + + if (4 * (k + 1) > dlen) + break; + memcpy(&w, data + 4 * k, 4); + if (w >= 0x10000u && w < 0x20000000u && + !(w & 3u)) + svc_gl_add_ptr((u64)w); + } + return; + } /* SET_PIPELINE_PROGRAM_ADDRESS_A/B(j), A=upper(7:0), B=lower(31:0). * Observed: 0x20020815 (j=1) [00000001 21201b00] -> 0x121201b00. * SESSION 153-CONTINUED-11: EVERY pair is recorded (the engine @@ -8905,7 +8981,7 @@ static void svc_gl_stream_chunks(struct svc_channel *ch, const u8 *buf, { static u32 chunkdump_log; - if (chunkdump_log < 4 && svc_gl.start_off + 8u <= at_off) { + if (chunkdump_log < 8 && svc_gl.start_off + 8u <= at_off) { u32 cur2 = svc_gl.start_off, end2, k2, wcnt; u32 cw2 = svc_gl_chunk_words_param ? svc_gl_chunk_words_param : @@ -13676,14 +13752,240 @@ static u64 svc_pass_hash; /* r391f: this decode pass's push content hash */ */ unsigned gsp_sema_push_raw; unsigned gsp_sema_push_raw_small; +static u32 svc_mme_census_flag; module_param_named(sema_push_raw_small, gsp_sema_push_raw_small, uint, 0644); MODULE_PARM_DESC(sema_push_raw_small, "Verbatim decode dump for small GR pushes (16..511 B), bounded count (default 0)"); +module_param_named(sema_mme_census, svc_mme_census_flag, uint, 0644); +MODULE_PARM_DESC(sema_mme_census, + "Census MME-family method ids (0x114..0x124, 0x234, 0x550..0x568, 0x3400..0x34fc, 0x3800..0x39fc) in beef pushes, bounded (default 0)"); static u64 svc_pushraw_last_va; static u32 svc_pushraw_last_len; static u32 svc_pushraw_n; static u32 svc_pushraw_small_n; +/* + * SESSION 153-CONTINUED-52: MME-family method census for the beef (GL 3D + * client) channel. The replayed draw windows raise a real MME ILLEGAL_OPCODE + * (NV_PGRAPH_PRI_MME_HWW_ESR bit 2) - the MME executed a macro whose code it + * could not fetch. The macro calls (CALL_MME_MACRO/DATA 0x3800..0x39fc) carry + * macro code addresses, so the macro CONTEXT state - the MME Instruction/ + * Start-address RAM loads (0x114/0x118/0x11c/0x120), the shadow RAM control + * (0x124), MME version (0x234), the MME DMA window (0x550..0x568) and the + * shadow scratch (0x3400..0x34fc) - must have reached the ladder context too. + * If those setup methods never appear in ANY client push the MME can never + * have its macro library and the replay is doomed; if they do appear, the + * next question is which push forwards them. This census answers that: it + * scans every beef push for those method ids and logs a bounded one-line + * summary (first-hit offsets + first data word). Read-only; bounded. + */ +static u32 svc_mme_census_n; + +static void svc_mme_census(const u8 *buf, u32 len, u64 push_va, u32 h_object) +{ + u32 off = 0; + u32 nhit = 0; + char line[512]; + int lc = 0; + + if (!READ_ONCE(svc_mme_census_flag) || svc_mme_census_n >= 128) + return; + if ((h_object & 0xfffff000u) != 0xbeef0000u) + return; + while (off + 4 <= len && nhit < 16) { + u32 hdr, op, cnt, addr; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 7u; + cnt = (hdr >> 16) & 0xfffu; + addr = (hdr & 0x1fffu) << 2; + if (op == 1u || op == 3u || op == 5u) + /* multi-word run: classify by its method id */ + ; + else if (op == 4u || op == 7u) + cnt = 0; + else + cnt = 0; + if ((addr >= 0x114u && addr <= 0x124u) || + addr == 0x234u || + (addr >= 0x550u && addr <= 0x568u) || + (addr >= 0x3400u && addr <= 0x34fcu) || + (addr >= 0x3800u && addr <= 0x39fcu)) { + u32 v0 = 0; + + if (off + 8 <= len) + memcpy(&v0, buf + off + 4, 4); + lc += scnprintf(line + lc, sizeof(line) - lc, + "%s0x%x@0x%x(%08x)", + nhit ? "," : "", addr, off, v0); + nhit++; + } + off += (op == 1u || op == 3u || op == 5u) && cnt ? + (1u + cnt) * 4u : 4u; + } + if (nhit) { + svc_mme_census_n++; + pr_info("sema-mme-census: h=0x%x va=0x%llx len=%u n=%u hits=[%s]\n", + h_object, push_va, len, nhit, line); + } +} + +/* + * SESSION 153-CONTINUED-53: MME context-setup prelude buffer. + * + * The draw-window replay raises a real MME ILLEGAL_OPCODE + * (NV_PGRAPH_PRI_MME_HWW_ESR bit 2) on CALL_MME_MACRO(2)=0x3810 with + * mme_pc=0/mme_ir=1: the MME executed a macro call with no macro program + * loaded. The client DOES upload the program (the census records the + * 0x11c/0x114 pointer pairs and the 0x124 shadow-control runs), but the + * replay only ever forwards draw-window slices, so the ladder context + * never sees the setup. This buffer captures the setup runs verbatim - + * every method run whose id is in the C597 MME setup families: + * + * 0x114 LOAD_MME_INSTRUCTION_RAM_POINTER + * 0x118 LOAD_MME_INSTRUCTION_RAM (the macro code words) + * 0x11c LOAD_MME_START_ADDRESS_RAM_POINTER + * 0x120 LOAD_MME_START_ADDRESS_RAM (per-macro entry pc) + * 0x124 SET_MME_SHADOW_RAM_CONTROL (METHOD_TRACK[_WITH_FILTER]) + * 0x234 SET_MME_VERSION + * 0x550..0x568 MME mem address / data RAM / DMA / reduction + * 0x3400..0x34fc SET_MME_SHADOW_SCRATCH + * + * - in first-encounter order, so the booter can preplay them ahead of the + * first 3D window replay of a capture generation (see + * gsp_sema_mme_prelude_get and the gsp_phys_rmcall_gr_mme_prelude knob). + * Identical runs (a re-upload of the same program) are skipped by an + * exact-run signature table, so repeated setup pushes do not grow the + * buffer; genuinely new content appends and bumps the generation so the + * next window re-preplays. The writer is the single service thread and + * appends strictly in order; the generation only increments after a + * capture that appended, giving the booter a seqcheck against torn views. + */ +#define SVC_MME_PRELUDE_WORDS 512u +#define SVC_MME_PRELUDE_SIGS 256u +static u32 svc_mme_prelude_buf[SVC_MME_PRELUDE_WORDS]; +static u32 svc_mme_prelude_n; +static u32 svc_mme_prelude_gen; +static u64 svc_mme_prelude_sigs[SVC_MME_PRELUDE_SIGS]; +static u32 svc_mme_prelude_sig_n; +static u32 svc_mme_prelude_dump_n; +static u32 svc_mme_prelude_flag = 1; +module_param_named(sema_mme_prelude_cap, svc_mme_prelude_flag, uint, 0644); +MODULE_PARM_DESC(sema_mme_prelude_cap, + "Capture the client's MME context-setup method runs (0x114..0x124, 0x234, 0x550..0x568, 0x3400..0x34fc) from trusted beef pushes for the GR launch builder's one-time preplay (default 1)"); + +static bool svc_mme_setup_run_addr(u32 addr) +{ + return (addr >= 0x114u && addr <= 0x124u) || addr == 0x234u || + (addr >= 0x550u && addr <= 0x568u) || + (addr >= 0x3400u && addr <= 0x34fcu); +} + +static u64 svc_mme_run_sig(const u8 *p, u32 span, u32 hdr) +{ + u64 h = 1469598103934665603ULL ^ hdr; + u32 i; + + for (i = 1u; i < span; i++) { + u32 v; + + memcpy(&v, p + 4u * i, 4); + h ^= v; + h *= 1099511628211ULL; + } + return h; +} + +const u32 *gsp_sema_mme_prelude_get(u32 *n, u32 *gen) +{ + if (n) + *n = READ_ONCE(svc_mme_prelude_n); + if (gen) + *gen = READ_ONCE(svc_mme_prelude_gen); + return svc_mme_prelude_buf; +} + +static void svc_mme_prelude_capture(const u8 *buf, u32 len, u64 push_va, + u32 h_object) +{ + u32 off = 0, added = 0, dups = 0, full = 0; + u32 words0 = svc_mme_prelude_n; + + if (!READ_ONCE(svc_mme_prelude_flag)) + return; + if ((h_object & 0xfffff000u) != 0xbeef0000u) + return; + while (off + 4u <= len) { + u32 hdr, op, cnt, addr, span = 1u, i; + u64 sig; + bool dup = false; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 7u; + cnt = (hdr >> 16) & 0xfffu; + addr = (hdr & 0x1fffu) << 2; + if ((op == 1u || op == 3u || op == 5u) && cnt) { + if ((u64)off + 4u + (u64)cnt * 4u > (u64)len) + break; + span = 1u + cnt; + } + if (!svc_mme_setup_run_addr(addr)) { + off += span * 4u; + continue; + } + sig = svc_mme_run_sig(buf + off, span, hdr); + for (i = 0; i < svc_mme_prelude_sig_n; i++) { + if (svc_mme_prelude_sigs[i] == sig) { + dup = true; + break; + } + } + if (dup) { + dups++; + off += span * 4u; + continue; + } + if (svc_mme_prelude_n + span > SVC_MME_PRELUDE_WORDS || + svc_mme_prelude_sig_n >= SVC_MME_PRELUDE_SIGS) { + full++; + off += span * 4u; + continue; + } + memcpy(&svc_mme_prelude_buf[svc_mme_prelude_n], buf + off, + span * 4u); + svc_mme_prelude_n += span; + svc_mme_prelude_sigs[svc_mme_prelude_sig_n++] = sig; + added++; + off += span * 4u; + } + if (added) { + WRITE_ONCE(svc_mme_prelude_gen, svc_mme_prelude_gen + 1); + if (svc_mme_prelude_dump_n < 4) { + u32 k; + + svc_mme_prelude_dump_n++; + pr_info("sema-mme-prelude: capture h=0x%x va=0x%llx len=%u added=%u words=%u gen=%u (was %u)\n", + h_object, push_va, len, added, + svc_mme_prelude_n, svc_mme_prelude_gen, + words0); + for (k = 0; k + 4u <= svc_mme_prelude_n && + k < 512u; k += 4u) + pr_info("sema-mme-prelude[%03u] %08x %08x %08x %08x\n", + k, svc_mme_prelude_buf[k], + svc_mme_prelude_buf[k + 1], + svc_mme_prelude_buf[k + 2], + svc_mme_prelude_buf[k + 3]); + } + if (full) + pr_info("sema-mme-prelude: buffer full=%u words=%u dup=%u\n", + full, svc_mme_prelude_n, dups); + } else if (dups && svc_mme_prelude_dump_n < 8) { + svc_mme_prelude_dump_n++; + pr_info("sema-mme-prelude: re-upload all-duplicate h=0x%x va=0x%llx dup=%u words=%u\n", + h_object, push_va, dups, svc_mme_prelude_n); + } +} + static void svc_push_raw_dump(const u8 *buf, u32 len, u64 push_va, u32 h_object) { @@ -14410,6 +14712,18 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, len >= 16 && len < 512) svc_push_raw_dump(buf, len, push_va, ch->h_object); + /* SESSION 153-CONTINUED-52: MME-family census for the beef channel. */ + if (ch) + svc_mme_census(buf, len, push_va, ch->h_object); + + /* + * SESSION 153-CONTINUED-53: MME context-setup prelude capture. Only a + * trusted resolution feeds the buffer (the bytes are replayed into a + * hardware context); identical re-uploads are deduped inside. + */ + if (ch && trusted) + svc_mme_prelude_capture(buf, len, push_va, ch->h_object); + memset(&st, 0, sizeof(st)); st.push_va = push_va; /* @@ -18857,6 +19171,101 @@ static u64 svc_push_pa_ceiling(void) return top; } +/* + * SESSION 153-CONTINUED-51: deterministic push-page resolution. + * + * The direct GL client's 5-7 KB window pushes routinely read back all + * zero through the trusted chain on some boots ("sema-pushzero ... + * rescued=0", r1033-class) while the SAME VA pages answer real content + * on other boots - the learned cache (lat) can shadow a page-table walk + * that has since moved, and the client reuses/reallocates its push VAs + * across frames. The launch-page chain (svc_gr_page_note) resolves + * through the ledger then a FRESH walk; the push read chain consults the + * learned cache FIRST, so a stale lat/pin entry shadows the fresh walk + * forever. When the trusted read answered an all-zero page, consult the + * same ledger/walk chain the launch pages use and, when a DIFFERENT page + * with non-zero content answers, adopt it AND heal the read sources + * (validated pin + lat) so every later read of this page is deterministic + * instead of a per-boot lottery. + * + * Only a page whose content differs from all-zero is adopted: a + * legitimately-zero page (padding) must never steal a correct + * translation. Returns 0 when a replacement page was adopted (pa_out / + * ap_out updated and `p` refilled), -ENOENT when nothing better exists. + */ +static int svc_push_page_heal(struct nv_vgpu_device *vgpu, u32 h_object, + u64 va, u64 net_pa, int net_ap, u8 *p, + size_t chunk, u64 *pa_out, int *ap_out) +{ + u64 page_va = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 off = va & (SEMA_PAGE_SIZE - 1); + u64 cur_page = net_pa & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 lpa = 0, wpa = 0; + u32 lap = 0; + int wap = SVC_APERTURE_SYS; + static u32 heal_log; + + /* 1. The ledger: the launch-page chain's landing authority. A page a + * host write or an earlier launch resolution landed on is the + * page the client's own chain agreed about - the read chain does + * not consult it at all. */ + if (!svc_ledger_lookup(page_va, &lpa, &lap) && + (lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != cur_page) { + u32 map_ap = svc_ap_of((int)lap); + + if (svc_read_ap(vgpu, lpa + off, + map_ap == SVC_APERTURE_FB ? + SVC_APERTURE_FB : SVC_APERTURE_SYS, + p, chunk) == 0 && + memchr_inv(p, 0, chunk)) { + svc_pin_store(page_va, lpa & ~(u64)(SEMA_PAGE_SIZE - 1), + (int)map_ap); + svc_lat_store(va, lpa + off, (int)map_ap); + *pa_out = lpa + off; + *ap_out = (int)map_ap; + if (heal_log < 64) { + heal_log++; + pr_info("sema-pushheal: h=0x%x va=0x%llx ledger pa=0x%llx (was 0x%llx, zero)\n", + h_object, (unsigned long long)va, + (unsigned long long)(lpa + off), + (unsigned long long)net_pa); + } + return 0; + } + } + + /* 2. A FRESH walk, bypassing the learned cache: the guest may have + * remapped the VA after the lat entry was written. */ + if (svc_walk_vaspaces(vgpu, page_va, &wpa, &wap) == 0 && + wpa != 0 && (wpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != cur_page) { + u32 map_ap = svc_ap_of(wap); + + if (svc_read_ap(vgpu, wpa + off, + map_ap == SVC_APERTURE_FB ? + SVC_APERTURE_FB : SVC_APERTURE_SYS, + p, chunk) == 0 && + memchr_inv(p, 0, chunk)) { + svc_pin_store(page_va, wpa & ~(u64)(SEMA_PAGE_SIZE - 1), + (int)map_ap); + svc_lat_store(va, wpa + off, (int)map_ap); + *pa_out = wpa + off; + *ap_out = (int)map_ap; + if (heal_log < 64) { + heal_log++; + pr_info("sema-pushheal: h=0x%x va=0x%llx walk pa=0x%llx (was 0x%llx, zero)\n", + h_object, (unsigned long long)va, + (unsigned long long)(wpa + off), + (unsigned long long)net_pa); + } + return 0; + } + } + /* Nothing better: restore the zero state exactly (the caller's read + * filled p with zeros and must see zeros back). */ + memset(p, 0, chunk); + return -ENOENT; +} + static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 va, void *buf, size_t len, bool skip_ipin) @@ -19037,11 +19446,12 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, svc_read_last_pa = rpa; svc_lat_store(page_va, ppa, pap); - if (pushzero_log < 32) { + if (pushzero_log < 1024) { pushzero_log++; - pr_info("sema-pushzero: h=0x%x va=0x%llx src=%u pa=0x%llx promo pa=0x%llx rescued=1\n", + pr_info("sema-pushzero: h=0x%x va=0x%llx len=%u src=%u pa=0x%llx promo pa=0x%llx rescued=1\n", ch->h_object, (unsigned long long)va, + (unsigned)len, src, (unsigned long long)pa, (unsigned long long)ppa); @@ -19050,25 +19460,58 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, /* promo page unusable: put the * zero state back exactly. */ memset(p, 0, chunk); - if (!rrc && pushzero_log < 32) { + if (!rrc && pushzero_log < 1024) { pushzero_log++; - pr_info("sema-pushzero: h=0x%x va=0x%llx src=%u pa=0x%llx promo pa=0x%llx rescued=0 (promo content zero)\n", + pr_info("sema-pushzero: h=0x%x va=0x%llx len=%u src=%u pa=0x%llx promo pa=0x%llx rescued=0 (promo content zero)\n", ch->h_object, (unsigned long long)va, + (unsigned)len, src, (unsigned long long)pa, (unsigned long long)ppa); } } - } else if (pushzero_log < 32) { + } else if (pushzero_log < 1024) { pushzero_log++; - pr_info("sema-pushzero: h=0x%x va=0x%llx src=%u pa=0x%llx rescued=0 (no distinct promo booking)\n", + pr_info("sema-pushzero: h=0x%x va=0x%llx len=%u src=%u pa=0x%llx rescued=0 (no distinct promo booking)\n", ch->h_object, - (unsigned long long)va, src, + (unsigned long long)va, + (unsigned)len, src, (unsigned long long)pa); } } /* + * SESSION 153-CONTINUED-51: when the zero-content + * rescue above did not produce non-zero bytes, the + * page must still be resolved deterministically - + * run the SAME ledger/fresh-walk chain the launch + * pages use (svc_gr_page_note) for this page. A + * different page with non-zero content is adopted, + * tagged src=10, and pinned so every later read of + * the page (continuation chunks and future frames) + * answers from the healed source instead of the + * learned cache. This is the r1069 class: every + * window push read zeros for a whole boot while the + * launch pages resolved fine. + */ + if (src >= 2 && + (ch->h_object & 0xffff0000u) == 0xbeef0000u && + (first ? len >= 512u : chunk >= 512u) && + !memchr_inv(p, 0, chunk)) { + u64 hpa = pa; + int hap = ap; + + if (!svc_push_page_heal(vgpu, ch->h_object, + va, pa, ap, p, + chunk, &hpa, &hap)) { + pa = hpa; + svc_read_last_src = 10; + svc_read_last_pa = hpa; + } + /* Not healed: the original (zero) source and + * its label stay; nothing was adopted. */ + } + /* * r820: name whether the trusted answer came from a * driver-declared PROMOTE_CTX buffer; the decode uses * it for the zero-NOP acceptance (see the flag). @@ -21591,8 +22034,111 @@ unsigned gsp_sema_ce_grace_ms = 25000; */ unsigned gsp_sema_push_hold_ms = 2500; +/* + * SESSION 153-CONTINUED-54: GR setup-push hold. + * + * The client's 12288 B C597 MME-setup push ("LOAD_MME_* program upload") + * reads unreadable/all-zero on the first-read lottery boots and used to be + * consumed ("skipping stale entry ... after 1 failed rescues") before the + * demand channel's injected pin could land, so the MME prelude capture + * missed the macro program for the whole boot. When such a push's first + * read fails, demand every 4K VA page of the push through the stalled-VA + * demand channel (pin_gr_stream harvests the rows and the guest resolves + + * injects the pages) and hold the entry pending PAST the rescue-consume + * until the pin lands (the cheap-wait guard re-reads when it does) or this + * window expires. Bounded by an absolute per-(va,slot) deadline; 0 + * disables the hold (the demand rows are still emitted). + * + * r1083-r1087: DEFAULT 0 (off). A 20 s hold blocks the whole ring behind + * the held entry, and on the three hold-active boots (r1083/r1084/r1085) + * the GL client derailed into the black-dud/churn class, while the r1087 + * boot (hold=0) read the beef windows fine (the wall moved back to the + * incomplete MME prelude + dispatch stall). The demand channel also cannot + * yet resolve the GL pushbuffer VA families (0x4000000+ / 0x42xxxxx), so + * the pin the hold waits for never lands and the hold only delays. Re-enable + * (e.g. sema_gr_setup_hold_ms=20000) once the guest content resolver covers + * those VA families. + */ +unsigned gsp_sema_gr_setup_hold_ms; + static unsigned long svc_gr_ring_tried; /* ranks that got a diagnostic line */ +/* + * Is this a GR push whose readability the MME-setup capture depends on? + * The client's C597 setup push is 12288 B; any GR push of that size is a + * candidate. A smaller push qualifies only when it is READABLE and its + * method runs hit the MME setup families (the unreadable case has no bytes + * to scan, hence the length rule alone). + */ +static bool svc_gr_setup_push(u32 len, const u8 *buf) +{ + u32 off = 0; + + if (len >= 12288u) + return true; + if (!buf) + return false; + while (off + 4u <= len) { + u32 hdr, op, cnt, addr; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 7u; + cnt = (hdr >> 16) & 0xfffu; + addr = (hdr & 0x1fffu) << 2; + if ((op == 1u || op == 3u || op == 5u) && cnt && + svc_mme_setup_run_addr(addr)) + return true; + off += ((op == 1u || op == 3u || op == 5u) && cnt) ? + (1u + cnt) * 4u : 4u; + } + return false; +} + +/* + * Demand every 4K page of the push and arm the per-(va,slot) hold. The + * demand is the existing stalled-VA channel: svc_log_stalled_va rows are + * deduped and harvested by tools/guest-diag/pin_gr_stream.py, which asks + * the guest scanner to resolve the VA set and inject the pages. Only one + * arm line per (va,slot) is logged. `page_cap` bounds the VA rows (the + * 12288 B setup push needs 3; the cap keeps a bulk push from eating the + * 256-entry stalled-VA budget). + */ +static void svc_gr_setup_hold_arm(struct svc_channel *ch, u64 va, u32 idx, + u32 len) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 poff; + + if (!va || !gsp_sema_gr_setup_hold_ms) + return; + if (ch->gr_setup_hold_va != va || ch->gr_setup_hold_idx != idx) { + ch->gr_setup_hold_va = va; + ch->gr_setup_hold_idx = idx; + ch->gr_setup_hold_until = + jiffies + msecs_to_jiffies(gsp_sema_gr_setup_hold_ms); + pr_info("sema-chan: GR setup hold h=0x%x entry=%u va=0x%llx len=%u armed (demanding 4K pages; pending until the pin lands, window %u ms)\n", + ch->h_object, idx, va, len, gsp_sema_gr_setup_hold_ms); + } + for (poff = 0; poff < len && poff < 0x8000u; poff += SEMA_PAGE_SIZE) + svc_log_stalled_va(page + poff); +} + +static bool svc_gr_setup_hold_active(const struct svc_channel *ch, u64 va, + u32 idx) +{ + return ch->gr_setup_hold_until && + ch->gr_setup_hold_va == va && + ch->gr_setup_hold_idx == idx && + time_before(jiffies, ch->gr_setup_hold_until); +} + +static void svc_gr_setup_hold_clear(struct svc_channel *ch) +{ + ch->gr_setup_hold_until = 0; + ch->gr_setup_hold_va = 0; + ch->gr_setup_hold_idx = ~0u; +} + static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 *pa_out) { @@ -33002,6 +33548,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) c->unread_va = 0; c->unread_until = 0; c->defer_since = 0; + svc_gr_setup_hold_clear(c); } } /* @@ -33843,6 +34390,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->defer_since = 0; ch->unread_va = 0; ch->unread_until = 0; + svc_gr_setup_hold_clear(ch); svc_replay_note(ch, (u32)ch->last_get, put); ch->last_get = (ch->last_get + 1) % ch->gpfifo_entries; @@ -34065,7 +34613,9 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * exists for the page, the put advances (a new * push at the same VA), or the window expires. */ - if (ch->engine_type && ch->unread_va == va && + if ((ch->engine_type || + svc_gr_setup_hold_active(ch, va, (u32)idx)) && + ch->unread_va == va && ch->unread_idx == idx && time_before(jiffies, ch->unread_until)) { u64 tpa = 0, vpa = 0; @@ -34564,6 +35114,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (!unreadable) { ch->push_hold_until = 0; + svc_gr_setup_hold_clear(ch); /* * The entry words repeated, but the * push bytes may not have: a client @@ -35433,13 +35984,33 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) 1 : 0; ch->gr_nocap_since = jiffies; + /* + * SESSION 153-CONTINUED-54: an + * all-zero setup push gets the + * same 4K-page demand + pending + * hold as the unreadable one. + */ + if (svc_gr_setup_push(len, pb)) + svc_gr_setup_hold_arm(ch, + va, + (u32)idx, + len); } - if (gsp_sema_gr_nocap_hold_ms && + { + u32 nocap_ms = + gsp_sema_gr_nocap_hold_ms; + + if (svc_gr_setup_push(len, pb) && + gsp_sema_gr_setup_hold_ms > + nocap_ms) + nocap_ms = + gsp_sema_gr_setup_hold_ms; + if (nocap_ms && !ch->gr_ram_ring && time_before(jiffies, ch->gr_nocap_since + msecs_to_jiffies( - gsp_sema_gr_nocap_hold_ms)) && + nocap_ms)) && !svc_hold_bypass()) { /* Force a fresh capture on * the retry: a replay decode @@ -35458,7 +36029,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc_gr_cand_push_owed(va) ? 1 : 0, svc_gr_qmd_seen, - gsp_sema_gr_nocap_hold_ms); + nocap_ms); } kfree(pb); break; @@ -35479,6 +36050,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->h_object, (u32)idx, va, len, owed_done); + } } } /* @@ -35723,6 +36295,16 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) idx, va, len); svc_log_stalled_va(va); /* + * SESSION 153-CONTINUED-54: + * demand the setup push's 4K + * pages and hold it pending. + */ + if (!ch->engine_type && + svc_gr_setup_push(len, NULL)) + svc_gr_setup_hold_arm(ch, va, + (u32)idx, + len); + /* * GR stale-skip clock: the * rescue-failure skip gate below * requires defer_since, yet every @@ -35771,6 +36353,19 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->unread_ipin_pa = ipa; } /* + * SESSION 153-CONTINUED-54: a held + * GR setup push waits cheaply for + * the whole hold window (the fresh- + * pin check still releases it the + * moment the demanded pin lands). + */ + if (svc_gr_setup_hold_active(ch, va, + (u32)idx) && + time_before(ch->unread_until, + ch->gr_setup_hold_until)) + ch->unread_until = + ch->gr_setup_hold_until; + /* * Bounded work-before-release * hold (CE): the deferred retry * probe shows the page's @@ -36091,7 +36686,9 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->defer_since && time_after(jiffies, ch->defer_since + - msecs_to_jiffies(gsp_sema_gr_grace_ms))) { + msecs_to_jiffies(gsp_sema_gr_grace_ms)) && + !svc_gr_setup_hold_active(ch, va, + (u32)idx)) { /* * The slot no longer holds this * push (overwritten buffer): the @@ -36136,6 +36733,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->defer_since = 0; ch->unread_va = 0; ch->unread_until = 0; + svc_gr_setup_hold_clear(ch); /* * m177: deliver any * withheld set even @@ -36235,6 +36833,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->gr_nocap_va = 0; ch->unread_va = 0; ch->unread_until = 0; + svc_gr_setup_hold_clear(ch); /* GRFIX FIX1c (r214): do NOT reset the per-launch * bound record here - a consume of an unrelated entry * would re-arm the held launch's bound every pass @@ -37786,6 +38385,7 @@ static void svc_stall_watch_check(void) continue; ch->unread_va = 0; ch->unread_until = 0; + svc_gr_setup_hold_clear(ch); ch->gr_hold_idx = ~0u; ch->gr_hold_until = 0; ch->gr_nocap_idx = ~0u; diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h index 69d6f33..e47e2e8 100644 --- a/kernel/gsp_sema_service.h +++ b/kernel/gsp_sema_service.h @@ -303,6 +303,8 @@ extern u32 gsp_phys_gr_peek; * retry probe). Absolute deadline per entry, never reset; 0 disables. */ extern unsigned gsp_sema_push_hold_ms; +/* SESSION 153-CONTINUED-54: GR setup-push (MME-setup / >=12 KB) hold. */ +extern unsigned gsp_sema_gr_setup_hold_ms; /* * Grace window (ms) an untranslatable GR GPFIFO entry is left pending before @@ -448,6 +450,20 @@ struct gsp_sema_gr_reloc { int gsp_sema_gr_reloc_resolve(u64 va, struct gsp_sema_gr_reloc *out); /* + * SESSION 153-CONTINUED-53: MME context-setup prelude accessor for the GR + * launch builder. Returns the captured setup runs (C597 MME family + * 0x114..0x124/0x234/0x550..0x568/0x3400..0x34fc, verbatim, first- + * encounter order), the word count in *n and the capture generation in + * *gen. The builder preplays the words once per generation, ahead of the + * first 3D window replay, so the ladder context's MME has its macro + * program before CALL_MME_MACRO runs. Reading the generation before and + * after the copy detects a concurrent append (the service thread only + * appends; a mismatch just defers emission to the next launch). *n is 0 + * until the client's setup push has been decoded. + */ +const u32 *gsp_sema_mme_prelude_get(u32 *n, u32 *gen); + +/* * r378 write guards. Both re-validate a guest-memory write target that was * resolved earlier (ledger/pin/walk/counter registration, or the GR submit's * per-page resolution) against the CURRENT resolution before the bytes land, diff --git a/kernel/main.c b/kernel/main.c index b5c7105..c96abf8 100644 --- a/kernel/main.c +++ b/kernel/main.c @@ -2798,6 +2798,17 @@ module_param_named(sema_push_hold_ms, gsp_sema_push_hold_ms, uint, 0644); MODULE_PARM_DESC(sema_push_hold_ms, "CE unreadable-entry work-before-release hold in ms (default 2500, 0 disables)"); /* + * SESSION 153-CONTINUED-54: GR setup-push (client MME-setup upload / any + * >= 12 KB GR push) pending hold (ms). An unreadable/all-zero setup push + * keeps its ring entry pending past the failed-rescue consume until the + * demanded 4K-page guest pin lands; 0 disables (demand rows still emitted). + * Default 0: the demand channel cannot yet resolve the GL pushbuffer VA + * families, so the hold only delays the ring (see gsp_sema_service.c). + */ +module_param_named(sema_gr_setup_hold_ms, gsp_sema_gr_setup_hold_ms, uint, 0644); +MODULE_PARM_DESC(sema_gr_setup_hold_ms, "GR setup-push (MME-setup/>=12KB) pending hold in ms (default 0 = off; 20000 = aggressive, needs a resolvable demand family)"); + +/* * Grace window (ms) an untranslatable GR entry stays pending before the pump * consumes it as stale, so a late push-buffer pin can still make it readable. */ diff --git a/tools/guest-diag/pin_gr_stream.py b/tools/guest-diag/pin_gr_stream.py index 776b440..0bd6759 100644 --- a/tools/guest-diag/pin_gr_stream.py +++ b/tools/guest-diag/pin_gr_stream.py @@ -261,6 +261,18 @@ def _demand_loop(): (r.stdout or "")[-200000:]): va = int(m.group(1) or m.group(2) or m.group(3), 16) track.setdefault(va, [now, 0.0]) + # SESSION 153-CONTINUED-51: the pushzero line now carries the + # push length; a zero-head WINDOW push spans 1-2 pages, and the + # demand of a single page leaves the capture zero - expand the + # range and demand every page so the guest pagemap resolves the + # whole push at once (bulk demand instead of one page per cycle). + for m in re.finditer( + r"sema-pushzero: h=0x[0-9a-f]+ va=(0x[0-9a-f]+) len=(\d+)", + (r.stdout or "")[-200000:]): + zva = int(m.group(1), 16) + zlen = int(m.group(2)) + for zoff in range(0, min(zlen, 0x8000), 0x1000): + track.setdefault(zva + zoff, [now, 0.0]) for va in list(track): first, last = track[va] if now - first > 300.0: @@ -268,6 +280,9 @@ def _demand_loop(): todo = [va for va, (f, l) in track.items() if now - l > 20.0] if not todo: continue + # CONT-51: bound one batch's command size (a page-expanded push + # range can add many pages; the remainder goes next cycle). + todo = todo[:48] cmd = _demand_write_cmd(todo) try: vexec(cmd, 30) |