diff options
| -rw-r--r-- | kernel/gsp_phys_booter.c | 172 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 71 |
2 files changed, 239 insertions, 4 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 1ce8820..b2d759e 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -19643,6 +19643,54 @@ module_param_named(sema_gl_idx_reloc, gsp_phys_rmcall_gr_idx_reloc, bool, 0644); MODULE_PARM_DESC(sema_gl_idx_reloc, "Relocate SET_INDEX_BUFFER (0x17c8) and vertex-stream banks 0x1c20..0x1cf0 operands to shadow VAs in 3D replays (default 0 = baseline set only)"); +/* + * SESSION 153-CONTINUED-48: MME-family payload relocation. The C597 class + * carries CALL_MME_MACRO(j) = 0x3800+j*8 and CALL_MME_DATA(j) = 0x3804+j*8 + * (class/clc597.h); the GLX client's macro(2) run (observed at byte + * address 0x3810, cnt=5) inlines raw 32-bit client VAs as macro arguments + * (`... 0450f800 ... 0450f000` in the first beef window of r1042/r1043). + * Those words are the last raw-operand class left in the faulting window + * after the index/vertex relocation: the engine walks the macro data + * through its own fetch path, and a raw client VA has no PTE chain the + * GPC engines can see (the r993/r1043 FAULT_PDE->PRI-fault class). The + * pass rewrites any payload word of a run whose byte address lies in + * [0x3800, 0x3a00) through the same vetted shadow mapping the pair + * classes use: only a word whose page the launch's own page set (or the + * big-surface alias) resolves is rewritten (MISS stays verbatim), the + * rewrite maps the SAME HPA at a shadow VA so the value is + * content-equivalent, and a shadow VA that would need an upper byte (a + * 64-bit alias in a 32-bit slot) is refused. Default 0 (baseline windows + * byte-identical); live-tunable so one boot can A/B. + */ +static bool gsp_phys_rmcall_gr_mme_reloc; +module_param_named(sema_gl_mme_reloc, gsp_phys_rmcall_gr_mme_reloc, bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_reloc, + "Relocate CALL_MME_MACRO/DATA(j) run payload words whose page resolves in the launch page set to shadow VAs in 3D replays (default 0 = off)"); + +/* + * SESSION 153-CONTINUED-48: ladder-wedge recovery. CONT-47 measured that + * the FIRST beef launch per boot whose window carries an unresolvable raw + * operand PRI-faults the GR: every SKED/FIFO register then reads the + * priv-error poison (cp. kernel_falcon_ga102.c `privErrVal = 0xbadf0000`) + * and the ladder channel stays wedged for the whole boot (engine GP_GET + * stuck; every later publish unconsumed, even an emptied window) - which + * makes content bisection a one-shot per boot. When enabled, the launch + * poll watches the GR FIFO status (0x400504, the sked recorder's + * `grfifo`) for the 0xbadf-tagged poison and performs ONE bounded + * recovery per launch: the r445 hard re-arm (TSG PREEMPT bWait=1 + the + * m155 lane-enable tail + GPFIFO/TSG schedules + doorbell) followed by + * the warm GR semaphore probe, exactly the mechanism behind + * gsp_phys_gr_wedge_rearm. Bounded: one detection per launch, poison + * gated, all RPCs keep their own timeouts. Default 1 (the poisoned state + * is already a guaranteed-dead boot; the recovery only adds the chance + * that later launches in the same boot still execute). + */ +static bool gsp_phys_gr_wedge_recover = true; +module_param_named(sema_gr_wedge_recover, gsp_phys_gr_wedge_recover, bool, + 0644); +MODULE_PARM_DESC(sema_gr_wedge_recover, + "On the 0xbadf priv-error poison in the GR FIFO status, run one bounded TSG-preempt+resched hard re-arm per launch (default 1)"); + static char *gsp_phys_rmcall_gr_win_drop; module_param_named(sema_gl_win_drop, gsp_phys_rmcall_gr_win_drop, charp, 0644); MODULE_PARM_DESC(sema_gl_win_drop, @@ -19777,7 +19825,30 @@ static bool gsp_phys_rmcall_gr_win_drop_auto_match(u32 addr) { if (!READ_ONCE(gsp_phys_gr_win_drop_auto)) return false; - return addr == 0x010cu || addr == 0x01b0u || addr == 0x05e0u; + /* + * SESSION 153-CONTINUED-48: NVC597_SET_REPORT_SEMAPHORE_A=0x1b00 - + * the C597 twin of the 0x10c NOTIFY trigger. Every glcore draw + * window ends with 4-word runs [A=upper(7:0), B=lower(31:0), + * C=payload, D=flags] whose B word is a raw client VA (observed + * 0x41a5bb0/0x759e870/0x251800; payloads = frame counters 0xb4f3+). + * The replayed engine executing them writes a client VA the GPC + * engines cannot reach - the VIRT_WRITE fault class (r1061: the + * kept [0,0x2000) window whose only fetch/write triggers besides + * the index binds are these raw 0x1b00 writes is exactly the one + * that still faulted with grfifo=0xbadf1201; r1062: the full window + * stalls the front-end right after the PBDMA consumes it and + * fecs_cur never loads). Their page cannot join the launch page set + * (n_ptr is capped at 7 by the QMD pointer region and the draw's + * program/index/vertex/MME pages already fill it), so relocation is + * impossible; dropping is the same call the 0x10c NOTIFY drop + * already makes: the module appends its own SET_REPORT_SEMAPHORE + * tail and the host-side completion lanes author every completion + * the client actually waits on (the r1061 keep-prefix window + * retired with ch_rel=0xc0ffee02 and the client kept pushing + * frames without any 0x1b00 in the replay). + */ + return addr == 0x010cu || addr == 0x01b0u || addr == 0x05e0u || + addr == 0x1b00u; } /* @@ -19967,6 +20038,26 @@ static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out, * 0450fcc8] -> VA 0x450fcc8. */ lo_i = 0; pair = true; + } else if (READ_ONCE(gsp_phys_rmcall_gr_idx_reloc) && + op == 1 && addr == 0x1b00 && cnt >= 2) { + /* SESSION 153-CONTINUED-48: NVC597_SET_REPORT_ + * SEMAPHORE_A/B..D (0x1b00) - the engine's own + * completion write [upper(7:0), lower(31:0), + * payload, flags] (observed [0, 0x4181bb0, + * 0x4f9f6, 0x1000f010] at the end of every + * glcore window). With the VA raw the replayed + * engine writes a client VA the GPC engines + * cannot reach - the VIRT_WRITE fault class + * (r1061: the kept [0,0x2000) window that + * faults is exactly the one still carrying the + * raw 0x1b00 VAs). Rewrite the pair to the + * shadow VA (same HPA, page collected by + * svc_gl_field 0x1b00), so the write lands in + * the client's own page - its wait then sees a + * real hardware completion. MISS stays + * verbatim like every other pair. */ + lo_i = 0; + pair = true; } else if (op == 1 && addr == 0x2380 && cnt >= 3) { /* SET_CONSTANT_BUFFER_SELECTOR_A/B: * [size, upper, lower]; observed @@ -20014,6 +20105,44 @@ static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out, (u64)(d[lo_i + 1] & 0xfffffffcu)); } } + /* + * SESSION 153-CONTINUED-48: MME-family payload + * relocation (see the sema_gl_mme_reloc param). The + * C597 MME macro/data family spans 0x3800..0x3a00; + * its runs carry the macro arguments as raw payload + * words, which include 32-bit client VAs + * (0x450f800/0x450f000 observed in the beef macro(2) + * run at 0x3810). Rewrite only a word whose page the + * launch's own page set resolves (the same vetted + * shadow mapping the pairs use); a MISS, an + * unaligned/too-small word, or a shadow VA that + * cannot fit the 32-bit slot stays verbatim. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_mme_reloc) && + addr >= 0x3800u && addr < 0x3a00u) { + u32 wi; + + for (wi = 0; wi < cnt; wi++) { + u32 wv = d[wi]; + u32 up = 0, lo = wv; + static u32 mme_log; + + if (wv < 0x10000u || (wv & 3u)) + continue; + if (!gsp_phys_rmcall_gr_3d_reloc( + &up, &lo, req, n_shadow)) + continue; + if (up != 0) + continue; + d[wi] = lo; + reloc++; + if (mme_log < 32) { + mme_log++; + pr_info("vgsp-gr: mme-reloc addr=0x%x wi=%u w=0x%x -> 0x%x\n", + addr, wi, wv, lo); + } + } + } wout += span; off += 1u + cnt; continue; @@ -20309,6 +20438,14 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, bool rearmed = false; u32 attempts = 0; /* + * SESSION 153-CONTINUED-48: one bounded priv-error recovery attempt + * per launch (sema_gr_wedge_recover), with a per-boot tally for the + * report. Static like the other submit scratch: the single-inflight + * ordered work item is the only caller. + */ + bool wedge_recover_done = false; + static u32 wedge_recoveries; + /* * F2/F4 state: the fast first-timeout re-arm runs once per launch; the * deposit readback is one-shot per launch (the first timeout is the * evidence window - the old r198 post-mortem only ran after attempt 3, @@ -21521,7 +21658,7 @@ gr_page_unmapped: static u32 gr3d_win_log; u32 wi; - if (gr3d_win_log < 4) { + if (gr3d_win_log < 10) { gr3d_win_log++; pr_info("vgsp-gr: 3D window dump h=0x%x push=0x%llx words=%u\n", req->h_object, req->push_va, win3d_n); @@ -21961,6 +22098,37 @@ gr_page_unmapped: sked_first = false; } } + /* + * SESSION 153-CONTINUED-48: priv-error (0xbadf-tagged) wedge + * recovery - see sema_gr_wedge_recover. The poison latch is + * checked before the completion break and the ladder so a + * launch whose window faulted the GR hands the engine to the + * next launch recovered instead of wedged. One attempt per + * launch (local flag); the hard re-arm is the r445 mechanism, + * bounded by its own RPC and sem-probe timeouts. + */ + if (READ_ONCE(gsp_phys_gr_wedge_recover) && bar0 && + !wedge_recover_done) { + u32 g0 = ioread32(bar0 + 0x00400504u); + + if ((g0 & 0xffff0000u) == 0xbadf0000u) { + u32 g1; + + wedge_recover_done = true; + wedge_recoveries++; + pr_warn("vgsp-gr: WEDGE-RECOVER priv-error poison seen (grfifo=0x%08x) h=0x%x push=0x%llx - TSG preempt + resched\n", + g0, req->h_object, req->push_va); + mutex_lock(&gsp_phys_rm_lock); + gsp_phys_gr_wedge_hard_rearm_locked(bar0); + mutex_unlock(&gsp_phys_rm_lock); + g1 = ioread32(bar0 + 0x00400504u); + pr_warn("vgsp-gr: WEDGE-RECOVER after re-arm grfifo=0x%08x (%s); recovery #%u this boot\n", + g1, + (g1 & 0xffff0000u) == 0xbadf0000u ? + "still poisoned" : "cleared", + wedge_recoveries); + } + } if (qmd_rel) { if (qmd_rel_hpa) w0 = READ_ONCE(*(u32 *)((u8 *)__va(qmd_rel_hpa) + diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 4d497d9..789cf27 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -8022,8 +8022,24 @@ static void svc_gl_add_ptr(u64 va) if (!va) return; + /* + * SESSION 153-CONTINUED-48: dedup by 4 KB page, not exact VA. + * The QMD pointer region holds only seven pairs (words 34..47, + * SVC_GR_POINTER_MAX - 1); a window that binds several different + * offsets of one buffer (r1058 chunk1: five index-buffer VAs + * across three pages) used to consume one slot per exact VA and + * hit the cap, so no later operand page (the raw CALL_MME_MACRO + * payload VAs 0x43545f0/0xb94f700) could join the set and the + * shadow relocation left them raw - the raw operand the engine + * faulted on (grfifo=0xbadf1201). One pointer per page carries + * the same page set with slots left for the remaining operand + * families; the first VA seen for a page is kept so the value + * stays a real client VA (the fill resolves pages, and the + * relocation matches by page). + */ for (k = 0; k < svc_gl.n_ptr; k++) - if (svc_gl.ptr[k] == va) + if (svc_gl.ptr[k] == va || + (svc_gl.ptr[k] & ~(u64)0xfffu) == (va & ~(u64)0xfffu)) return; if (svc_gl.n_ptr < SVC_GR_POINTER_MAX - 1) svc_gl.ptr[svc_gl.n_ptr++] = va; @@ -8049,6 +8065,26 @@ static void svc_gl_field(u32 addr, u32 count, const u8 *data, u32 dlen) * [ENABLE(0) | TYPE(7:4)]. Collected per slot so a bound program VA * can be typed (VTG vs PS) for the program-bytes delivery. */ + /* + * SESSION 153-CONTINUED-48: NVC597_SET_REPORT_SEMAPHORE_A..D = 0x1b00, + * observed as 4-word runs [A=upper(7:0), B=lower(31:0), C=payload, + * D=flags] at the end of every glcore draw window (payload = the + * client's frame counter, e.g. 0x4f9f6; VA e.g. 0x4181bb0/0xc91e6c0). + * These are the engine's own completion writes; with the VA raw the + * replayed engine writes a client VA the GPC engines cannot reach - + * the same VIRT_WRITE fault class the 0x10c NOTIFY drop exists for + * (r1061: the kept [0,0x2000) window that still faults is exactly + * the one carrying these raw 0x1b00 VAs). Collect the semaphore page + * so the window relocation can rewrite the pair to a shadow VA: the + * write then lands in the SAME HPA and the client's wait sees the + * real hardware completion. + */ + if (addr == 0x1b00) { + if (count >= 2) + svc_gl_add_ptr(((u64)(v[0] & 0xffu) << 32) | + (u64)(v[1] & 0xfffffffcu)); + return; + } if (addr >= 0x2000 && addr < 0x2800 && (addr & 0x3f) == 0) { u32 j = (addr - 0x2000) / 0x40; @@ -8056,6 +8092,37 @@ static void svc_gl_field(u32 addr, u32 count, const u8 *data, u32 dlen) svc_gl_shader_type[j] = (u8)((v[0] >> 4) & 0xf); return; } + /* + * SESSION 153-CONTINUED-48: CALL_MME_MACRO(j)=0x3800+j*8 / + * CALL_MME_DATA(j)=0x3804+j*8 runs carry raw 32-bit client VAs as + * macro arguments (r1058 chunk1: the macro(2) run payloads + * 0x44ff800/0x44ff000 relocated only because the page set already + * held 0x44ff000 via the index bind; the sibling payloads + * 0x43545f0/0x4353f60/0xb94f700/0xb941578 stayed raw because their + * pages missed the capped pointer list - the raw operand class the + * GR faulted on). Collect every VA-shaped payload word so the page + * set - and therefore the 3D-window shadow relocation - covers the + * macro arguments too. This must run before the 0x2014..0x3fff + * program-pair recognizer below, whose (addr & 0x3f) == 0x14 shape + * would otherwise swallow a CALL_MME_DATA(2)=0x3814 run. Shape + * gates: 4-byte aligned; >= 64 KB excludes the method/flag scalars + * (0x0c/0x53/0x338); < 512 MB excludes the float constants the + * macro(57) runs carry (1.0f=0x3f800000) while keeping every + * observed client operand VA (max 0xb94f700). + */ + if (addr >= 0x3800 && addr < 0x3a00) { + for (k = 0; k < count; k++) { + u32 w; + + if (4 * (k + 1) > dlen) + break; + memcpy(&w, data + 4 * k, 4); + if (w >= 0x10000u && w < 0x20000000u && + !(w & 3u)) + svc_gl_add_ptr((u64)w); + } + return; + } /* SET_PIPELINE_PROGRAM_ADDRESS_A/B(j), A=upper(7:0), B=lower(31:0). * Observed: 0x20020815 (j=1) [00000001 21201b00] -> 0x121201b00. * SESSION 153-CONTINUED-11: EVERY pair is recorded (the engine @@ -8905,7 +8972,7 @@ static void svc_gl_stream_chunks(struct svc_channel *ch, const u8 *buf, { static u32 chunkdump_log; - if (chunkdump_log < 4 && svc_gl.start_off + 8u <= at_off) { + if (chunkdump_log < 8 && svc_gl.start_off + 8u <= at_off) { u32 cur2 = svc_gl.start_off, end2, k2, wcnt; u32 cw2 = svc_gl_chunk_words_param ? svc_gl_chunk_words_param : |