diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 1334 | ||||
| -rw-r--r-- | kernel/gsp_phys_booter.h | 10 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 1513 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.h | 48 | ||||
| -rw-r--r-- | kernel/main.c | 11 |
5 files changed, 2874 insertions, 42 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index d78e0df..4c2b460 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -4977,7 +4977,14 @@ static u64 gsp_phys_gr_last_push_va; * fetch position shows how far the walk got. Read-only MMIO + a racy read * of the pb page (best effort, diagnostic only); bounded per boot. */ -#define GSP_PHYS_PBDMA_DUMP_MAX 6u +#define GSP_PHYS_PBDMA_DUMP_MAX 64u + +/* + * SESSION 153-CONTINUED-52: the published method block's DMA address, so the + * PBDMA fault dump can name the exact word the PBDMA stopped on + * (offset = PB_FETCH - last block DMA). Set beside gsp_phys_gr_last_words. + */ +static u64 gsp_phys_gr_last_pb_dma; static const char *gsp_phys_pbdma_type_name(u32 t) { @@ -4987,6 +4994,32 @@ static const char *gsp_phys_pbdma_type_name(u32 t) return names[t & 7u]; } +/* + * SESSION 153-CONTINUED-52: MME fault forensics. The C597 replay's dominant + * failure on draw-bearing windows is a real MME exception + * (NV_PGRAPH_PRI_MME_HWW_ESR 0x404490 latched ILLEGAL_OPCODE / + * MISSING_MACRO_DATA / ... - "unknown instruction fetched from the MME + * Instruction RAM", see gr_error_gk104.c grErrorLogMmeException). INFO + * carries the faulting MME PC and the macro address; INFO2 the instruction + * register. Read them post-failure (the latch survives) so the exact macro + * fetch point is on record. Read-only; bounded by its call sites. + */ +static void gsp_phys_mme_regs_log(void __iomem *bar0, const char *tag) +{ + u32 esr, info, info2, trapped; + + if (!bar0) + return; + esr = ioread32(bar0 + 0x00404490u); + info = ioread32(bar0 + 0x00404494u); + info2 = ioread32(bar0 + 0x00404498u); + trapped = ioread32(bar0 + 0x00400704u); + pr_warn("vgsp-gr: MME-REGS %s esr=%08x(pc_valid=%u illegal=%u miss=%u extra=%u) pc=%u macro_addr=0x%x info2_ir=%08x trapped=%08x\n", + tag, esr, (info >> 28) & 1u, (esr >> 2) & 1u, + (esr >> 0) & 1u, (esr >> 1) & 1u, info & 0xfffu, + (info >> 12) & 0xffu, info2, trapped); +} + static void gsp_phys_pbdma_fault_dump(u32 chid) { static u32 dump_n; @@ -5011,15 +5044,56 @@ static void gsp_phys_pbdma_fault_dump(u32 chid) u32 pbd0 = readl(bar0 + b + 0x08cu); u32 hdrsh = readl(bar0 + b + 0x118u); u32 fetch64 = ((u64)(pbfhi & 0xffu) << 32) | (pbf & ~3u); + u32 trig; + const char *trig_src; + s64 off_words = -1; + s64 voff = -1; if (!intr0 && !pbf && !hdrsh && !pbhd) continue; - pr_info("vgsp-gr: PB-FAULT-DUMP P%u intr0=%08x gpf=%u pbfetch=0x%llx len=%u hdr=%08x(%s m=0x%x cnt=%u sub=%u) cnt=%08x data0=%08x shadow=%08x\n", - i, intr0, gpf, fetch64, pbfhi >> 11, + /* + * SESSION 153-CONTINUED-52: the real driver's rule for the + * instruction that triggered the PBDMA interrupt: if the + * DECODED header (PB_HEADER) type is IMMD or PB_COUNT's value + * field is zero, the raw instruction is PB_DATA0; otherwise it + * is HDR_SHADOW (fifo_tu102.c: "If the TYPE field of + * NV_PPBDMA_PB_HEADER is IMMD or the VALUE field of + * NV_PPBDMA_PB_COUNT is zero, then the raw PB instruction + * stored in NV_PPBDMA_PB_DATA0 is the one that triggered the + * interrupt. Otherwise ... HDR_SHADOW"). + */ + if (((pbhd >> 29) & 7u) == 4u || (pbcnt & 0x1fffu) == 0u) { + trig = pbd0; + trig_src = "DATA0"; + } else { + trig = hdrsh; + trig_src = "HDRSHADOW"; + } + if (gsp_phys_gr_last_pb_dma && + fetch64 >= gsp_phys_gr_last_pb_dma && + fetch64 < gsp_phys_gr_last_pb_dma + + (u64)gsp_phys_gr_last_words * 4u) + off_words = (s64)(fetch64 - gsp_phys_gr_last_pb_dma) / 4; + /* + * SESSION 153-CONTINUED-59: the SAME offset computed against + * the GPU VA the engine fetches (the r1102 WINPROBE/PB-FAULT + * agreement: pbfetch 0x101ebc with pb_va 0x101000 and 943 + * words = exactly the block end). voff >= words is the + * past-the-block-end prefetch position. + */ + voff = (s64)-1; + if (pb_va && fetch64 >= pb_va && + fetch64 < pb_va + (u64)GSP_PHYS_PAGE_SIZE) + voff = (s64)(fetch64 - pb_va) / 4; + pr_info("vgsp-gr: PB-FAULT-DUMP P%u intr0=%08x gpf=%u pbfetch=0x%llx len=%u off=%lld voff=%lld/%u hdr=%08x(%s m=0x%x cnt=%u sub=%u) cnt=%08x data0=%08x shadow=%08x pbhd=%08x(type=%s) TRIG=%08x(%s)\n", + i, intr0, gpf, fetch64, pbfhi >> 11, off_words, voff, + gsp_phys_gr_last_words, hdrsh, gsp_phys_pbdma_type_name(hdrsh >> 29), (hdrsh & 0x1fffu) << 2, (hdrsh >> 16) & 0xfffu, (hdrsh >> 13) & 7u, - pbcnt, pbd0, hdrsh); + pbcnt, pbd0, hdrsh, pbhd, + gsp_phys_pbdma_type_name(pbhd >> 29), + trig, trig_src); } { struct gsp_phys_boot_chunk *pbch = @@ -5044,6 +5118,34 @@ static void gsp_phys_pbdma_fault_dump(u32 chid) w + 12 < n ? pw[w + 12] : 0, w + 13 < n ? pw[w + 13] : 0, w + 14 < n ? pw[w + 14] : 0, w + 15 < n ? pw[w + 15] : 0); } + /* + * SESSION 153-CONTINUED-59: the block TAIL and the + * words PAST the block end (the r1102 fault fetch + * sits exactly at the end - voff == words). The past + * words are what a prefetching PBDMA decodes next; + * the tail shows whether the block itself ends at an + * entry boundary. + */ + { + u32 words = gsp_phys_gr_last_words; + u32 lo = words > 48u ? words - 48u : 0u; + u32 hi = words + 32u; + + if (hi > 1024u) + hi = 1024u; + for (w = lo; w < hi; w += 16) { + pr_info("vgsp-gr: PB-FAULT-DUMP %s[%3u]= %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n", + w < words ? "tail" : "past", w, + pw[w], w + 1 < 1024u ? pw[w + 1] : 0, + w + 2 < 1024u ? pw[w + 2] : 0, w + 3 < 1024u ? pw[w + 3] : 0, + w + 4 < 1024u ? pw[w + 4] : 0, w + 5 < 1024u ? pw[w + 5] : 0, + w + 6 < 1024u ? pw[w + 6] : 0, w + 7 < 1024u ? pw[w + 7] : 0, + w + 8 < 1024u ? pw[w + 8] : 0, w + 9 < 1024u ? pw[w + 9] : 0, + w + 10 < 1024u ? pw[w + 10] : 0, w + 11 < 1024u ? pw[w + 11] : 0, + w + 12 < 1024u ? pw[w + 12] : 0, w + 13 < 1024u ? pw[w + 13] : 0, + w + 14 < 1024u ? pw[w + 14] : 0, w + 15 < 1024u ? pw[w + 15] : 0); + } + } } } } @@ -16728,13 +16830,25 @@ EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_ce_copy); #define GSP_PHYS_RMCALL_PRI_SKED_DEBUG_1 0x00407040u #define GSP_PHYS_RMCALL_PRI_SKED_DEBUG_1_EN_CWD_SLOT_COUNT 0x800u /* Method block bound: init(152) + inline run(67) + PCAS(4) + REPORT_SEM(8) + - * the optional QMD-virt pair(2). */ -#define GSP_PHYS_RMCALL_GR_METHOD_WORDS 512u + * the optional QMD-virt pair(2). Raised 512 -> 1024 (SESSION 153- + * CONTINUED-53) so a 3D stream replay can carry the captured MME setup + * prelude (<=512 words) ahead of the largest chunked window (<=448) in one + * block; the block still lands inside the one 4 KB chunk page it is copied + * to. */ +#define GSP_PHYS_RMCALL_GR_METHOD_WORDS 1024u static u64 gsp_phys_gr_submitted; static u64 gsp_phys_gr_ok; static u64 gsp_phys_gr_refused; static u64 gsp_phys_gr_timed_out; +/* + * SESSION 153-CONTINUED-60: 3D-stream launches whose window was all-zero + * (the client's push page caught mid-rewrite). They are never handed to the + * engine: the zero method stream trips the PBDMA (trap 0x1168 -> priv-error + * poison, see the ZEROWIN-SKIP site). Counted so a boot's zero-window rate + * is visible without dmesg archaeology. + */ +static u64 gsp_phys_gr_zerowin_skips; static u64 gsp_phys_gr_retries; /* fresh-entry launch retries published */ static u64 gsp_phys_gr_pool_exhausted; /* m156 (E5): zero-write retries where the warm sem push proved the engine @@ -17045,6 +17159,101 @@ MODULE_PARM_DESC(gsp_phys_rmcall_gr_3d, "Replay a captured raw C597 method window verbatim instead of the compute prelude (default 1)"); /* + * SESSION 153-CONTINUED-59: drop single-word SDM-type entries (SEC_OP 0 + * SSDM / 2 STORE_SDM / 6 USE_SDM, tu102 dev_esched_pbdma.h type map) from + * every replayed stream: the 3D window patch, the MME prelude and the attr + * prelude emission. The r1102 PBENTRY first-sighting forensics localized + * the fault decode point to the MME prelude head (WINPROBE/PB-FAULT-DUMP + * P0 HDR_SHADOW=04666d14 = prelude[2], PB_DATA0=04646d00 = prelude[3], + * pbfetch=0x101ebc = block end after the whole 943-word block was + * prefetched, intr0 bit 18 in the guest relay): the client's setup push + * carries a 21-word run of `04666d00 04666d10 04666d14 04646d00 ...` + * (SDM-family single-word encodings, method fields 0x3400/0x3440/0x3450) + * that svc_mme_prelude_capture stored verbatim; the physical PBDMA refuses + * them ("invalid method encoding used in method", NV_PPBDMA_INTR_0 + * PBENTRY), which also explains why the whole block was fetched but the + * REPORT_SEMAPHORE never retired. The guest-vGPU parts (subdevice masks) + * are meaningless on the single-subdevice ladder channel - the same call + * the sub!=0 multi-word drop makes. Zero words are kept (benign padding). + * Default 1; A/B with 0. + */ +static bool gsp_phys_rmcall_gr_sdm_drop = true; +module_param_named(sema_gl_sdm_drop, gsp_phys_rmcall_gr_sdm_drop, bool, + 0644); +MODULE_PARM_DESC(sema_gl_sdm_drop, + "Drop single-word SDM-type entries (SEC_OP 0/2/6) from 3D window replays, the MME prelude and the attr prelude (default 1)"); + +/* + * SESSION 153-CONTINUED-53: MME context-setup prelude. The r1073-r1077 + * in-flight capture proved the draw-window replay raises a real MME + * ILLEGAL_OPCODE on CALL_MME_MACRO(2)=0x3810 (mme_pc=0, mme_ir=1, mme_mac + * =2) because the ladder context never received the client's macro + * program. The client does upload it (0x11c/0x114 pointer pairs, 0x118 + * code, 0x120 entry addresses, 0x124 shadow control; see + * svc_mme_prelude_capture); with this knob the launch builder preplays + * those captured runs verbatim - once per capture generation - right + * after the SET_OBJECT C597 bind and ahead of the window, so the macros + * the window calls are resident. Declared before build_gr_launch (its + * reader); the matching capture-side knob is sema_mme_prelude_cap. + * Default 1; A/B with 0. + */ +static bool gsp_phys_rmcall_gr_mme_prelude = true; +module_param_named(sema_gl_mme_prelude, gsp_phys_rmcall_gr_mme_prelude, bool, + 0644); +MODULE_PARM_DESC(sema_gl_mme_prelude, + "Preplay the captured client MME context-setup runs once per capture generation ahead of the first 3D window replay (default 1)"); + +/* + * SESSION 153-CONTINUED-61: generation of the MME prelude already emitted + * into the engine context (0 = none). The r1110 evidence: the prelude was + * emitted once on the first beef launch (gen=1, 512 words), that launch + * priv-errored (grfifo=0xbadf1201) and WEDGE-RECOVER ran TSG preempt + + * resched - which resets the engine context and with it the resident MME + * program - yet the latch kept suppressing re-emission, so every later + * window replay ran its macros against an unloaded MME (mme_pc=513 class + * stall, no retire). The recovery clears the latch so the next launch + * re-preloads the context. + */ +static u32 gsp_phys_gr_mme_prelude_gen; + +/* + * SESSION 153-CONTINUED-57: attr/pipeline state prelude. The r1091 + * full-MME-prelude boot cleared MME (mme_esr=80000000) but the first beef + * window then latched a draw-shader program-header error + * (NV_PGRAPH_PRI_DS_HWW_ESR 0x405840 = 0xa2040a00: SPH9|SPH11|SPH18, + * M_PIPE_STAGE = VSB|PS) and a class error at SET_VERTEX_ATTRIBUTE_A(2) + * (0x1168, INVALID_ENUM): the replayed window is a mid-stream state cut - + * the attribute/pipeline record the client set in EARLIER windows (the + * missing stages' SET_PIPELINE_PROGRAM_ADDRESS binds and the attribute + * format state) was never replayed, so the DS validated program headers + * that were never bound. With this knob the launch builder preplays the + * captured runs from the service's attr prelude buffer (see + * svc_attr_prelude_capture), relocated exactly like the window itself + * (the patched copy is computed by the caller - it needs the launch page + * set and the live shadow counter), run-aligned, drained across the first + * launches of a capture generation. Declared before build_gr_launch (its + * reader); the matching capture-side knob is sema_attr_prelude_cap. + * Default 1; A/B with 0. + */ +static bool gsp_phys_rmcall_gr_attr_prelude = true; +module_param_named(sema_gl_attr_prelude, gsp_phys_rmcall_gr_attr_prelude, + bool, 0644); +MODULE_PARM_DESC(sema_gl_attr_prelude, + "Preplay the captured client attr/pipeline runs (0x1100..0x11fc, 0x2000..0x205c, relocation applied) ahead of the first 3D window replays of a capture generation (default 1)"); + +/* + * The patched (relocation-applied) attr prelude for the current capture + * generation, built by the submit path just before build_gr_launch. Single + * caller (the ordered single-inflight sema work item), so plain statics. + */ +#define GSP_PHYS_GR_ATTR_PRE_WORDS 512u +static u32 gsp_phys_gr_attr_pre[GSP_PHYS_GR_ATTR_PRE_WORDS]; +static u32 gsp_phys_gr_attr_pre_n; +static u32 gsp_phys_gr_attr_pre_gen; /* capture gen the patch is from */ +static u32 gsp_phys_gr_attr_emit_gen; /* gen whose runs are draining */ +static u32 gsp_phys_gr_attr_emit_off; /* word cursor into attr_pre */ + +/* * GR launch method block: the inline QMD run at its original shape and the * channel REPORT_SEMAPHORE release. The stock guest push does NOT emit * SET_OBJECT before the inline run (the compute class is bound on the @@ -17055,7 +17264,9 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, const struct gsp_phys_gr_submit *req, const u32 *qmd, u64 qmd_va, u64 sem_va, u32 sem_payload, - const u32 *win3d, u32 win3d_n) + const u32 *win3d, u32 win3d_n, + const u32 *attr_pre, + u32 attr_pre_n) { u32 *p = pb; int prelude = gsp_phys_rmcall_gr_prelude_value(); @@ -17076,10 +17287,162 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, */ if (gsp_phys_rmcall_gr_3d && req && win3d && win3d_n && win3d_n + 2u + 8u <= GSP_PHYS_RMCALL_GR_METHOD_WORDS) { + /* + * SESSION 153-CONTINUED-53: MME context-setup prelude (see the + * sema_gl_mme_prelude knob above). Preplay the captured client + * setup runs once per capture generation, right after the + * SET_OBJECT C597 bind and ahead of the window, so + * CALL_MME_MACRO() runs execute against a loaded macro + * program instead of pc=0 -> ILLEGAL_OPCODE. The read is + * seqchecked (generation before/after): a concurrent append + * only defers emission to the next launch. A window too large + * to also carry the prelude replays without it (logged) - the + * emission then happens on the next launch that fits. + */ + bool emit_prelude = false; + const u32 *pl = NULL; + u32 pl_n = 0, pl_gen = 0; + + if (READ_ONCE(gsp_phys_rmcall_gr_mme_prelude)) { + pl = gsp_sema_mme_prelude_get(&pl_n, &pl_gen); + if (pl && pl_n && pl_gen && + pl_gen != READ_ONCE(gsp_phys_gr_mme_prelude_gen)) { + if (win3d_n + 2u + 8u + pl_n <= + GSP_PHYS_RMCALL_GR_METHOD_WORDS) { + emit_prelude = true; + } else { + pr_info("vgsp-gr: MME-PRELUDE deferred (window=%u prelude=%u exceeds block; replaying window alone) h=0x%x\n", + win3d_n, pl_n, req->h_object); + } + } + } + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | (1u << 16) | (0u << 13) | (GSP_PHYS_RMCALL_GR_SET_OBJECT >> 2); *p++ = GSP_PHYS_RMCALL_CLASS_GR_3D; + if (emit_prelude) { + u32 g0 = pl_gen, g1 = 0, sdrop = 0, pwords = pl_n; + + if (READ_ONCE(gsp_phys_rmcall_gr_sdm_drop)) { + u32 r = 0, w = 0; + + /* + * SESSION 153-CONTINUED-59: SDM sanitize (see the + * gsp_phys_rmcall_gr_sdm_drop comment). Walk the + * prelude verbatim but drop single-word SDM-family + * encodings; they are the r1102 PBENTRY offenders. + */ + while (r < pl_n) { + u32 h = pl[r]; + u32 op = (h >> 29) & 7u; + u32 cnt = (h >> 16) & 0xfffu; + u32 span = ((op == 1u || op == 3u || + op == 5u) && cnt && + r + 1u + cnt <= pl_n) ? + 1u + cnt : 1u; + + if (h && (op == 0u || op == 2u || + op == 6u)) { + sdrop++; + r += span; + continue; + } + memcpy(&p[w], &pl[r], (size_t)span * 4u); + w += span; + r += span; + } + p += w; + pwords = w; + } else { + memcpy(p, pl, (size_t)pl_n * 4); + p += pl_n; + } + /* seqcheck: a mid-copy append must not be recorded as + * emitted (the copy may be a torn prefix). */ + gsp_sema_mme_prelude_get(NULL, &g1); + if (g1 == g0) { + WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, g0); + pr_info("vgsp-gr: MME-PRELUDE emitted gen=%u words=%u window=%u h=0x%x push=0x%llx sdm_drop=%u\n", + g0, pwords, win3d_n, req->h_object, + req->push_va, sdrop); + } else { + pr_info("vgsp-gr: MME-PRELUDE copy raced an append (gen %u->%u); emitted words=%u, will re-emit next launch\n", + g0, g1, pl_n); + } + } + /* + * SESSION 153-CONTINUED-57: attr/pipeline state prelude (see the + * sema_gl_attr_prelude knob above). Emit the patched captured + * runs after the MME prelude and ahead of the window, + * run-aligned, draining across launches when the remaining + * block budget cannot take the whole remainder. The cursor + * resets when a new capture generation is patched. A window + * with no room at all defers the whole step to the next launch + * (logged by the patch site). + */ + if (attr_pre && attr_pre_n && + READ_ONCE(gsp_phys_rmcall_gr_attr_prelude)) { + u32 avail = GSP_PHYS_RMCALL_GR_METHOD_WORDS - + (u32)(p - pb) - (win3d_n + 2u + 8u); + u32 off, copied = 0; + + if (gsp_phys_gr_attr_emit_gen != + READ_ONCE(gsp_phys_gr_attr_pre_gen)) { + WRITE_ONCE(gsp_phys_gr_attr_emit_gen, + READ_ONCE(gsp_phys_gr_attr_pre_gen)); + WRITE_ONCE(gsp_phys_gr_attr_emit_off, 0); + } + off = READ_ONCE(gsp_phys_gr_attr_emit_off); + while (off < attr_pre_n) { + u32 hdr = attr_pre[off]; + u32 op = (hdr >> 29) & 7u; + u32 cnt = (hdr >> 16) & 0xfffu; + u32 span = ((op == 1u || op == 3u || + op == 5u) && cnt) ? + 1u + cnt : 1u; + + if (off + span > attr_pre_n) + span = attr_pre_n - off; + /* + * SESSION 153-CONTINUED-59: the same SDM + * sanitize as the MME prelude (see + * gsp_phys_rmcall_gr_sdm_drop); the cursor + * still advances so the drop is permanent. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_sdm_drop) && + hdr && (op == 0u || op == 2u || + op == 6u)) { + off += span; + continue; + } + if (copied + span > avail) + break; + memcpy(&p[copied], &attr_pre[off], + (size_t)span * 4u); + copied += span; + off += span; + } + if (copied) { + p += copied; + WRITE_ONCE(gsp_phys_gr_attr_emit_off, off); + pr_info("vgsp-gr: ATTR-PRELUDE emitted gen=%u words=%u/%u window=%u h=0x%x push=0x%llx%s\n", + READ_ONCE(gsp_phys_gr_attr_pre_gen), + off, attr_pre_n, win3d_n, + req->h_object, req->push_va, + off < attr_pre_n ? + " (continuation next launch)" : ""); + } else if (off < attr_pre_n) { + static u32 attr_defer_log; + + if (attr_defer_log < 8) { + attr_defer_log++; + pr_info("vgsp-gr: ATTR-PRELUDE deferred (no room: window=%u avail=%u) h=0x%x\n", + win3d_n, avail, + req->h_object); + } + } + } memcpy(p, win3d, (size_t)win3d_n * 4); p += win3d_n; p = gsp_phys_rmcall_push_gr(p, @@ -19032,11 +19395,27 @@ gsp_phys_gr_poll_break_log(u32 w0, u32 want_qmd, u32 w1, u32 since_us, u32 usr_gpget = ud ? get_unaligned_le32(ud + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPGET_OFF) : 0; + /* + * SESSION 153-CONTINUED-47: re-read the ring slot the launch + * published (the entry AS THE ENGINE SEES IT) after the wait, so an + * X-vs-beef diff on one boot compares the exact e0/e1 the PBDMA + * fetched, plus the doorbell-visible GP_GET/GP_PUT movement. + */ + struct gsp_phys_boot_chunk *r_c = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO]; + u8 *ring = r_c->pages ? gsp_phys_chunk_page_va(r_c, 0) : NULL; + u32 re0 = 0, re1 = 0; - pr_info("vgsp-gr: poll break w0=0x%x want=0x%x ch=0x%x after %u us usr_gpget=0x%x gpget=0x%x gpput=0x%x\n", + if (ring && sub->slot < GSP_PHYS_RMCALL_GPFIFO_ENTRIES) { + re0 = get_unaligned_le32(ring + sub->slot * 8); + re1 = get_unaligned_le32(ring + sub->slot * 8 + 4); + } + + pr_info("vgsp-gr: poll break w0=0x%x want=0x%x ch=0x%x after %u us usr_gpget=0x%x gpget=0x%x gpput=0x%x ring[%u]=%08x %08x (pub %08x %08x)\n", w0, want_qmd, w1, (u32)ktime_to_us(ktime_get()) - since_us, - usr_gpget, sub->gpget, sub->gpput); + usr_gpget, sub->gpget, sub->gpput, + sub->slot, re0, re1, sub->e0, sub->e1); /* r425: the healthy-launch SKED/FECS baseline for the wedge compare. */ gsp_phys_gr_sked_wit(gsp_phys_gpu_bar0(), "post-poll"); } @@ -19544,6 +19923,24 @@ static u64 gsp_phys_gr_surf_dummy_dma_locked(void) * verbatim (it faults later and is named in the drain). Called with * gsp_phys_rm_lock held. */ +/* + * SESSION 153-CONTINUED-59: window/attr operand-miss state for the + * pre-doorbell abort (see gsp_phys_rmcall_gr_3d_reloc's level-2 fallback + * and the MISS-ABORT site in gsp_phys_backend_guest_gr_submit). Reset at + * the start of each 3D-window patch pass; a nonzero count at the submit + * point means the emitted block still carries a raw client VA the physical + * engine cannot resolve, which is the r1104/r1105 lane-killing class. + * sema_gl_miss_abort=0 restores the historical submit-anyway behaviour. + */ +static u32 gsp_phys_rmcall_gr_3d_miss_n; +static u64 gsp_phys_rmcall_gr_3d_junk_n; +static bool gsp_phys_rmcall_gr_miss_abort = true; +static u64 gsp_phys_gr_miss_abort_n; +module_param_named(sema_gl_miss_abort, gsp_phys_rmcall_gr_miss_abort, bool, + 0644); +MODULE_PARM_DESC(sema_gl_miss_abort, + "Abort a 3D-window launch before the doorbell while a window/attr operand relocation MISSes (page demanded via the pin path; retried on the next forward; fail-open after 3000 aborts). Default 1"); + static bool gsp_phys_rmcall_gr_3d_reloc(u32 *up_w, u32 *lo_w, const struct gsp_phys_gr_submit *req, u32 *n_shadow) @@ -19592,6 +19989,69 @@ static bool gsp_phys_rmcall_gr_3d_reloc(u32 *up_w, u32 *lo_w, *up_w = (*up_w & ~0xffu) | ((u32)(sh >> 32) & 0xffu); return true; } + /* + * SESSION 153-CONTINUED-59: level-2 late resolution, the same trusted + * chain the MME payload branch uses (ipin/pin/lat/map/walk then shadow + * map). The r1104 first beat window MISSed the program pair 0x2054 + * (va=0x4350900) and the r1105 windows MISSed the index-buffer pairs + * (0x17c8, va=0x44fedf0...) whose pages were not in the launch page + * set at build time; the raw guest VA then let the engine fault the + * MMU (HUBCLIENT_PD) and the guest driver reset the channel. Resolve + * the page here if any stage knows it, so a MISS only survives when + * every source fails; a surviving MISS is recorded for the pin demand + * set and counts toward the pre-doorbell abort. + */ + { + struct gsp_sema_gr_reloc r; + u32 line2 = 0; + + if (!gsp_sema_gr_reloc_resolve(va, &r) && r.hpa && + *n_shadow < GSP_PHYS_RMCALL_GR_SHADOW_PAGES && + !gsp_phys_rmcall_gr_map_shadow_locked(*n_shadow, r.hpa, + &sh, &line2)) { + if (line2 != 0xffffffffu) + (*n_shadow)++; + sh |= va & (GSP_PHYS_PAGE_SIZE - 1); + *lo_w = (*lo_w & 0x3u) | ((u32)sh & 0xfffffffcu); + *up_w = (*up_w & ~0xffu) | ((u32)(sh >> 32) & 0xffu); + return true; + } + } + /* + * SESSION 153-CONTINUED-61: junk-operand filter. The window + * recognizer is byte-shape based, so float/immediate DATA words in + * the method stream also parse as address pairs. The observed junk + * class sits in the 256 MB..4 GB numeric range (0x3d000000..0xc2...... + * float bit patterns: 0x3f800000 = 1.0f, 0xbf800000 = -1.0f, ...) + * and can NEVER resolve - r1109 measured miss=20 constant across + * every attempt, all of it this class, which pinned the abort census + * nonzero forever and blocked every submit. Real client VAs of this + * driver live below 0x5000000 (the 64-80 MB GL/heap range, e.g. + * 0x4250000/0x4350000/0x462fc40) or at/above 0x1_00000000 (the 4G+ + * push-buffer/UVM ranges). A miss outside both is a data-word + * misidentification, not a raw client VA the engine would read as an + * address: it is neither demanded (the demand set was 60 % junk by + * r1109, diverting the pin cycle) nor counted toward the pre-doorbell + * abort. Witnessed per class, bounded. + */ + if (page >= 0x10000000ULL && page < 0x100000000ULL) { + static u32 gr_missjunk_log; + + gsp_phys_rmcall_gr_3d_junk_n++; + if (gr_missjunk_log < 24 || + !(gsp_phys_rmcall_gr_3d_junk_n % 512u)) { + gr_missjunk_log++; + pr_info("vgsp-gr: MISS-JUNK va=0x%llx (data-word class; not demanded, not counted toward the abort) n=%llu\n", + page, (u64)gsp_phys_rmcall_gr_3d_junk_n); + } + return false; + } + /* + * Unresolvable: demand the page for the pin path and count the miss + * (the submit path aborts pre-doorbell while the count is nonzero). + */ + gsp_phys_rmcall_gr_3d_miss_n++; + gsp_sema_demand_note(page); return false; } @@ -19613,6 +20073,95 @@ static bool gsp_phys_rmcall_gr_3d_reloc(u32 *up_w, u32 *lo_w, * in the state binds/method-stream shape. Empty = off (historical * behaviour); live-tunable via sysfs so one boot can sweep the variants. */ +/* + * SESSION 153-CONTINUED-47 (A/B knob): gates the newly recognised pair + * classes (SET_INDEX_BUFFER_A/B 0x17c8 and the vertex-stream banks + * 0x1c20..0x1cf0) in the verbatim-replay relocation pass. Default 0 keeps + * the r1035-r1041 baseline relocation set exactly; flipping it live + * (`echo Y > /sys/module/vgsp/parameters/sema_gl_idx_reloc`) rewrites the + * next launches' index/vertex operands to shadow VAs, so one boot can A/B + * the effect instead of comparing across boots. + */ +static bool gsp_phys_rmcall_gr_idx_reloc; +module_param_named(sema_gl_idx_reloc, gsp_phys_rmcall_gr_idx_reloc, bool, 0644); +MODULE_PARM_DESC(sema_gl_idx_reloc, + "Relocate SET_INDEX_BUFFER (0x17c8) and vertex-stream banks 0x1c20..0x1cf0 operands to shadow VAs in 3D replays (default 0 = baseline set only)"); + +/* + * SESSION 153-CONTINUED-48: MME-family payload relocation. The C597 class + * carries CALL_MME_MACRO(j) = 0x3800+j*8 and CALL_MME_DATA(j) = 0x3804+j*8 + * (class/clc597.h); the GLX client's macro(2) run (observed at byte + * address 0x3810, cnt=5) inlines raw 32-bit client VAs as macro arguments + * (`... 0450f800 ... 0450f000` in the first beef window of r1042/r1043). + * Those words are the last raw-operand class left in the faulting window + * after the index/vertex relocation: the engine walks the macro data + * through its own fetch path, and a raw client VA has no PTE chain the + * GPC engines can see (the r993/r1043 FAULT_PDE->PRI-fault class). The + * pass rewrites any payload word of a run whose byte address lies in + * [0x3800, 0x3a00) through the same vetted shadow mapping the pair + * classes use: only a word whose page the launch's own page set (or the + * big-surface alias) resolves is rewritten (MISS stays verbatim), the + * rewrite maps the SAME HPA at a shadow VA so the value is + * content-equivalent, and a shadow VA that would need an upper byte (a + * 64-bit alias in a 32-bit slot) is refused. Default 0 (baseline windows + * byte-identical); live-tunable so one boot can A/B. + */ +static bool gsp_phys_rmcall_gr_mme_reloc; +module_param_named(sema_gl_mme_reloc, gsp_phys_rmcall_gr_mme_reloc, bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_reloc, + "Relocate CALL_MME_MACRO/DATA(j) run payload words whose page resolves in the launch page set to shadow VAs in 3D replays (default 0 = off)"); + +/* + * SESSION 153-CONTINUED-58: torn-read zero guard for the vertex-attribute + * families. The r1098/r1099 first-launch class error + * (`Class Error: ChId 0001, Class 0000c597, Offset 00001168, + * Data 00000000, ErrorCode 00000005` = SET_VERTEX_ATTRIBUTE_A(2) written + * with NUMERICAL_TYPE=0, the header's UNUSED_ENUM_DO_NOT_USE value -> + * INVALID_ENUM) is written by a replayed window run + * `20030458 38400000 38200040 00000000`: a cnt=3 INC run at method 0x1160 + * whose THIRD data word is exactly zero and is immediately followed by a + * 52-word zero region - the read caught the client's push buffer after + * part of it had been reused/cleared. The same client writes valid A(2) + * values in the other windows of the same push (0x38200040/0x38200600/ + * 0x38400600), so the zero is a torn read, not client content; and + * NUMERICAL_TYPE=0 is never a legitimate client write. Guard: for INC + * runs in the SET_VERTEX_ATTRIBUTE_A/B families (0x1160..0x11dc) drop a + * data word that is exactly zero, splitting the run so the remaining + * words keep their methods (the attribute then keeps the value the + * ATTR-PRELUDE or the previous frame wrote). Default 1; live-tunable. + */ +static u32 gsp_phys_rmcall_gr_attr_zero_fix = 1; +module_param_named(sema_gl_attr_zero_fix, gsp_phys_rmcall_gr_attr_zero_fix, uint, 0644); +MODULE_PARM_DESC(sema_gl_attr_zero_fix, + "Drop torn-read zero data words in the SET_VERTEX_ATTRIBUTE_A/B runs (0x1160..0x11dc) of 3D replays, splitting the run (default 1)"); + +/* (SESSION 153-CONTINUED-53 MME prelude knob is declared before + * gsp_phys_rmcall_build_gr_launch, its reader.) */ + +/* + * SESSION 153-CONTINUED-48: ladder-wedge recovery. CONT-47 measured that + * the FIRST beef launch per boot whose window carries an unresolvable raw + * operand PRI-faults the GR: every SKED/FIFO register then reads the + * priv-error poison (cp. kernel_falcon_ga102.c `privErrVal = 0xbadf0000`) + * and the ladder channel stays wedged for the whole boot (engine GP_GET + * stuck; every later publish unconsumed, even an emptied window) - which + * makes content bisection a one-shot per boot. When enabled, the launch + * poll watches the GR FIFO status (0x400504, the sked recorder's + * `grfifo`) for the 0xbadf-tagged poison and performs ONE bounded + * recovery per launch: the r445 hard re-arm (TSG PREEMPT bWait=1 + the + * m155 lane-enable tail + GPFIFO/TSG schedules + doorbell) followed by + * the warm GR semaphore probe, exactly the mechanism behind + * gsp_phys_gr_wedge_rearm. Bounded: one detection per launch, poison + * gated, all RPCs keep their own timeouts. Default 1 (the poisoned state + * is already a guaranteed-dead boot; the recovery only adds the chance + * that later launches in the same boot still execute). + */ +static bool gsp_phys_gr_wedge_recover = true; +module_param_named(sema_gr_wedge_recover, gsp_phys_gr_wedge_recover, bool, + 0644); +MODULE_PARM_DESC(sema_gr_wedge_recover, + "On the 0xbadf priv-error poison in the GR FIFO status, run one bounded TSG-preempt+resched hard re-arm per launch (default 1)"); + static char *gsp_phys_rmcall_gr_win_drop; module_param_named(sema_gl_win_drop, gsp_phys_rmcall_gr_win_drop, charp, 0644); MODULE_PARM_DESC(sema_gl_win_drop, @@ -19659,6 +20208,48 @@ static bool gsp_phys_rmcall_gr_win_drop_match(u32 addr) } if (!digits) break; + /* + * SESSION 153-CONTINUED-47: `lo-hi` drops every method + * byte-address in the closed range (both ends accept the + * 0x prefix). The content bisect needs whole families + * (`0x3800-0x3a00` MME, `0x3400-0x3500` MME scratch) and + * the emptied-window control (`0-0x7ffc`): with the window + * stripped whole, a launch that still retires proves the + * SET_OBJECT + release tail path is healthy and that the + * wedge lives in the stripped content. + */ + if (*p == '-') { + const char *q = p + 1; + u32 hi = 0; + u32 hd = 0; + + if (q[0] == '0' && (q[1] == 'x' || q[1] == 'X')) + q += 2; + for (;;) { + char c = *q; + u32 h; + + if (c >= '0' && c <= '9') + h = c - '0'; + else if (c >= 'a' && c <= 'f') + h = c - 'a' + 10; + else if (c >= 'A' && c <= 'F') + h = c - 'A' + 10; + else + break; + if (hi <= 0xfffffffu) + hi = (hi << 4) | h; + q++; + hd++; + } + if (hd) { + p = q; + n++; + if (addr >= v && addr <= hi) + return true; + continue; + } + } n++; if (v == addr) return true; @@ -19705,7 +20296,30 @@ static bool gsp_phys_rmcall_gr_win_drop_auto_match(u32 addr) { if (!READ_ONCE(gsp_phys_gr_win_drop_auto)) return false; - return addr == 0x010cu || addr == 0x01b0u || addr == 0x05e0u; + /* + * SESSION 153-CONTINUED-48: NVC597_SET_REPORT_SEMAPHORE_A=0x1b00 - + * the C597 twin of the 0x10c NOTIFY trigger. Every glcore draw + * window ends with 4-word runs [A=upper(7:0), B=lower(31:0), + * C=payload, D=flags] whose B word is a raw client VA (observed + * 0x41a5bb0/0x759e870/0x251800; payloads = frame counters 0xb4f3+). + * The replayed engine executing them writes a client VA the GPC + * engines cannot reach - the VIRT_WRITE fault class (r1061: the + * kept [0,0x2000) window whose only fetch/write triggers besides + * the index binds are these raw 0x1b00 writes is exactly the one + * that still faulted with grfifo=0xbadf1201; r1062: the full window + * stalls the front-end right after the PBDMA consumes it and + * fecs_cur never loads). Their page cannot join the launch page set + * (n_ptr is capped at 7 by the QMD pointer region and the draw's + * program/index/vertex/MME pages already fill it), so relocation is + * impossible; dropping is the same call the 0x10c NOTIFY drop + * already makes: the module appends its own SET_REPORT_SEMAPHORE + * tail and the host-side completion lanes author every completion + * the client actually waits on (the r1061 keep-prefix window + * retired with ch_rel=0xc0ffee02 and the client kept pushing + * frames without any 0x1b00 in the replay). + */ + return addr == 0x010cu || addr == 0x01b0u || addr == 0x05e0u || + addr == 0x1b00u; } /* @@ -19786,12 +20400,15 @@ static bool gsp_phys_rmcall_gr_win_patch_find(u32 addr, u32 *out_data) return false; } +static u32 gsp_phys_rmcall_gr_3d_last_subdrop; +static u32 gsp_phys_rmcall_gr_3d_last_sdmdrop; + static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out, u32 *out_n, u32 *n_dropped, const struct gsp_phys_gr_submit *req, u32 *n_shadow) { - u32 off = 0, wout = 0, reloc = 0, dropped = 0; + u32 off = 0, wout = 0, reloc = 0, dropped = 0, subdrop = 0, sdm_drop = 0; *out_n = n; *n_dropped = 0; @@ -19800,11 +20417,53 @@ static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out, u32 op = (hdr >> 29) & 0x7u; u32 cnt = (hdr >> 16) & 0xfffu; u32 addr = (hdr & 0x1fffu) << 2; + u32 sub = (hdr >> 13) & 0x7u; u32 span = 1u; if ((op == 1 || op == 3 || op == 5) && cnt && off + 1u + cnt <= n) span = 1u + cnt; + /* + * SESSION 153-CONTINUED-52: drop METHOD WRITES aimed at a + * non-zero subchannel. The glcore draw window ends with + * `2001c00d 70000000/68000000` = INC m=0x34 cnt=1 SUBCHANNEL 6 + * (channel-class MEM_OP_D lane) two-word runs. The ladder + * channel has nothing bound on subchannel 6, so the physical + * PBDMA hands those methods to its host-software lane and + * raises NV_PPBDMA_INTR_0_METHOD (bit 21): the r1075 relay + * records `Channel ID 00000004 intr 00200000` at every launch, + * and the real host driver's software-method handler answers + * "Invalid host sofware method received" -> rcRequired -> + * robust-channel error recovery kills the channel mid-window + * (fifo_gk104.c). The methods are channel-level barriers whose + * effects the module's own release/fence lanes already author, + * so dropping them from the replay is the same call the 0x10c + * NOTIFY / 0x1b0 LAUNCH_DMA auto-drops make. Only count- + * carrying writes (op 1/3/5) are candidates: the single-word + * TERT encodings (op 0/2/4/7) reuse bits 15:13 as mask/type + * payload. + */ + if ((op == 1u || op == 3u || op == 5u) && sub != 0u) { + dropped++; + subdrop++; + off += span; + continue; + } + /* + * SESSION 153-CONTINUED-59: single-word SDM-type entries + * (SEC_OP 0/2/6, see gsp_phys_rmcall_gr_sdm_drop). The + * retiring X-family windows never yield one; this guards a + * window that carries the client's SDM markers (the + * 0x1fff0-class marker words or the prelude 0x046x6dXX + * family), which the physical PBDMA refuses with PBENTRY. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_sdm_drop) && + hdr && (op == 0u || op == 2u || op == 6u)) { + dropped++; + sdm_drop++; + off += span; + continue; + } if (gsp_phys_rmcall_gr_win_drop_match(addr) || gsp_phys_rmcall_gr_win_drop_auto_match(addr)) { dropped++; @@ -19812,6 +20471,56 @@ static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out, continue; } memcpy(&out[wout], &in[off], (size_t)span * 4); + /* + * SESSION 153-CONTINUED-58: torn-read zero guard - see the + * sema_gl_attr_zero_fix comment. Only INC runs (op 1) whose + * data words advance one method each; the attribute runs are + * exactly this shape. Rebuild the run in place, skipping + * exactly-zero words and re-targeting the surviving words' + * methods (a zero word's method is simply never written, so + * the attribute keeps its prelude/previous-frame value). + */ + if (READ_ONCE(gsp_phys_rmcall_gr_attr_zero_fix) && + op == 1u && cnt && addr >= 0x1160u && addr <= 0x11dcu) { + u32 k, k0 = 0, L = 0, wr = wout; + u32 subch = (hdr >> 13) & 0x7u; + + for (k = 0; k <= cnt; k++) { + bool bad = (k < cnt) && + (out[wout + 1u + k] == 0u); + + if (!bad && k < cnt) { + if (!L) + k0 = k; + out[wr + 1u] = out[wout + 1u + k]; + wr++; + L++; + continue; + } + if (L) { + u32 naddr = addr + 4u * k0; + u32 nhdr = (1u << 29) | + (L << 16) | + ((naddr >> 2) & 0x1fffu) | + (subch << 13); + + out[wr - L] = nhdr; + L = 0; + } + if (bad) { + static u32 az_log; + + if (az_log < 64) { + az_log++; + pr_info("vgsp-gr: ATTR-ZERO addr=0x%x word=%u/%u off=%u dropped (torn-read zero; attribute keeps its prelude value)\n", + addr + 4u * k, k, + cnt, off); + } + } + } + /* +1: the rebuilt run still needs its (new) header word */ + span = (wr - wout) + 1u; + } { /* * SESSION 153-CONTINUED-14: live data patch (BEGIN @@ -19867,6 +20576,54 @@ static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out, * reads. Observed [00001000 1 2150b790]. */ lo_i = 1; pair = true; + } else if (READ_ONCE(gsp_phys_rmcall_gr_idx_reloc) && + op == 1 && addr >= 0x1c20 && addr <= 0x1cf0 && + (addr & 0xfu) == 0u && cnt >= 3) { + /* SESSION 153-CONTINUED-47 (A/B knob): widen + * the vertex-stream banks from j=0/1 to every + * FORMAT slot (0x1c20..0x1cf0). The GLX + * client's 3D window binds streams j>=2 that + * svc_gl_field already collects into the page + * set but the window never had rewritten. */ + lo_i = 1; + pair = true; + } else if (READ_ONCE(gsp_phys_rmcall_gr_idx_reloc) && + op == 1 && addr == 0x17c8 && cnt >= 2) { + /* SESSION 153-CONTINUED-47 (A/B knob): SET_ + * INDEX_BUFFER_A/B [ADDRESS_UPPER(7:0), + * ADDRESS_LOWER(31:0)] - the indexed draw + * fetches its indices through this VA. The + * r1042 boot showed an X window carrying + * index binds PRI-stalling once relocated + * (relocation of exactly this pair is the + * only kernel delta vs the r1035 baseline + * where index-bearing windows did not exist + * and every X window retired), so the + * rewrite is gated off by default and A/B-ed + * live per boot. Observed [00000000 + * 0450fcc8] -> VA 0x450fcc8. */ + lo_i = 0; + pair = true; + } else if (READ_ONCE(gsp_phys_rmcall_gr_idx_reloc) && + op == 1 && addr == 0x1b00 && cnt >= 2) { + /* SESSION 153-CONTINUED-48: NVC597_SET_REPORT_ + * SEMAPHORE_A/B..D (0x1b00) - the engine's own + * completion write [upper(7:0), lower(31:0), + * payload, flags] (observed [0, 0x4181bb0, + * 0x4f9f6, 0x1000f010] at the end of every + * glcore window). With the VA raw the replayed + * engine writes a client VA the GPC engines + * cannot reach - the VIRT_WRITE fault class + * (r1061: the kept [0,0x2000) window that + * faults is exactly the one still carrying the + * raw 0x1b00 VAs). Rewrite the pair to the + * shadow VA (same HPA, page collected by + * svc_gl_field 0x1b00), so the write lands in + * the client's own page - its wait then sees a + * real hardware completion. MISS stays + * verbatim like every other pair. */ + lo_i = 0; + pair = true; } else if (op == 1 && addr == 0x2380 && cnt >= 3) { /* SET_CONSTANT_BUFFER_SELECTOR_A/B: * [size, upper, lower]; observed @@ -19914,6 +20671,105 @@ static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out, (u64)(d[lo_i + 1] & 0xfffffffcu)); } } + /* + * SESSION 153-CONTINUED-48: MME-family payload + * relocation (see the sema_gl_mme_reloc param). The + * C597 MME macro/data family spans 0x3800..0x3a00; + * its runs carry the macro arguments as raw payload + * words, which include 32-bit client VAs + * (0x450f800/0x450f000 observed in the beef macro(2) + * run at 0x3810). Rewrite only a word whose page the + * launch's own page set resolves (the same vetted + * shadow mapping the pairs use); a MISS, an + * unaligned/too-small word, or a shadow VA that + * cannot fit the 32-bit slot stays verbatim. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_mme_reloc) && + addr >= 0x3800u && addr < 0x3a00u) { + u32 wi; + + for (wi = 0; wi < cnt; wi++) { + u32 wv = d[wi]; + u32 up = 0, lo = wv; + static u32 mme_log; + + if (wv < 0x10000u || (wv & 3u)) + continue; + if (!gsp_phys_rmcall_gr_3d_reloc( + &up, &lo, req, n_shadow)) { + /* + * SESSION 153-CONTINUED-51: + * the payload page missed the + * launch's page set (the QMD + * pointer region caps the set + * far below a full window's + * operand count - r1071: the + * macro payload pages + * 0x44fd200..0x44fde00 were + * ~12 distinct pages while the + * set held 7; the words stayed + * RAW and the engine fetched an + * untranslatable guest VA on + * every full window - the + * 0xbadf1201 priv-error + * poison). Resolve the page + * through the SAME trusted + * ledger/pin/walk chain the + * launch pages use (the + * RELOC-FIXED resolver) and + * shadow-map it exactly like a + * matched page. A resolve/map + * miss or a shadow VA that + * cannot fit the 32-bit slot + * stays verbatim (never + * guessed). + */ + struct gsp_sema_gr_reloc r; + u64 va2 = (u64)wv; + u64 sh2 = 0; + u32 line2 = 0; + + if (gsp_sema_gr_reloc_resolve( + va2, &r) || + !r.hpa) + continue; + if (*n_shadow >= + GSP_PHYS_RMCALL_GR_SHADOW_PAGES) + continue; + if (gsp_phys_rmcall_gr_map_shadow_locked( + *n_shadow, r.hpa, + &sh2, &line2)) + continue; + if (line2 != 0xffffffffu) + (*n_shadow)++; + sh2 |= va2 & + (GSP_PHYS_PAGE_SIZE - 1); + up = (u32)(sh2 >> 32) & 0xffu; + lo = (wv & 0x3u) | + ((u32)sh2 & 0xfffffffcu); + if (up != 0) + continue; + d[wi] = lo; + reloc++; + if (mme_log < 32) { + mme_log++; + pr_info("vgsp-gr: mme-reloc-rsv addr=0x%x wi=%u w=0x%x -> 0x%x\n", + addr, wi, wv, + lo); + } + continue; + } + if (up != 0) + continue; + d[wi] = lo; + reloc++; + if (mme_log < 32) { + mme_log++; + pr_info("vgsp-gr: mme-reloc addr=0x%x wi=%u w=0x%x -> 0x%x\n", + addr, wi, wv, lo); + } + } + } wout += span; off += 1u + cnt; continue; @@ -19923,6 +20779,8 @@ static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out, } *out_n = wout; *n_dropped = dropped; + gsp_phys_rmcall_gr_3d_last_subdrop = subdrop; + gsp_phys_rmcall_gr_3d_last_sdmdrop = sdm_drop; return reloc; } @@ -20172,8 +21030,8 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, u32 qmd_rel_off = 0, ch_rel_off = 0; u32 poll = 0, w0 = 0, w1 = 0, grpoll_n = 0; bool sem_gated = false; - static u32 sked_prev[14]; - static u32 sked_sample[14]; + static u32 sked_prev[28]; + static u32 sked_sample[28]; static u32 sked_tick; static bool sked_first; /* @@ -20209,6 +21067,14 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, bool rearmed = false; u32 attempts = 0; /* + * SESSION 153-CONTINUED-48: one bounded priv-error recovery attempt + * per launch (sema_gr_wedge_recover), with a per-boot tally for the + * report. Static like the other submit scratch: the single-inflight + * ordered work item is the only caller. + */ + bool wedge_recover_done = false; + static u32 wedge_recoveries; + /* * F2/F4 state: the fast first-timeout re-arm runs once per launch; the * deposit readback is one-shot per launch (the first timeout is the * evidence window - the old r198 post-mortem only ran after attempt 3, @@ -20323,6 +21189,41 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, ret = -EIO; goto out; } + /* + * SESSION 153-CONTINUED-60: all-zero 3D-window skip. The client's push + * page is cyclically zeroed/rewritten (CONT-45); a capture that lands + * on the zero phase still forwards an all-zero method window. Live + * r1107: such a window launched (3D raw dump words=252, every word 0), + * the engine trapped SET_VERTEX_ATTRIBUTE_A(2) (trap=00001168 + * data_lo=0x50) and escalated to a priv-error poison (grfifo= + * 0xbadf1201, mme_pc=513) that wedged the GR lane for the rest of the + * boot; r1031 had already seen the same class fault the PBDMA on the + * zero lead (PB-FAULT-DUMP P0). A zero window carries no method to + * execute and nothing to store: skip the hardware handoff entirely - + * no shadow relocation, no engine contact, no wedge - and return the + * distinct rc -EBADMSG so the service authors the software release and + * suppresses the synthetic fault (a frame with no content deserves no + * client-visible engine error). + */ + if (stream3d) { + u32 zwi; + bool zerowin = req->stream_n != 0; + + for (zwi = 0; zwi < req->stream_n; zwi++) { + if (req->stream[zwi]) { + zerowin = false; + break; + } + } + if (zerowin) { + gsp_phys_gr_zerowin_skips++; + pr_warn_ratelimited("vgsp-gr: ZEROWIN-SKIP h=0x%x push=0x%llx window=%u all-zero (no engine submit; software release) n=%llu\n", + req->h_object, req->push_va, req->stream_n, + gsp_phys_gr_zerowin_skips); + ret = -EBADMSG; + goto out; + } + } gsp_phys_gr_submitted++; gsp_phys_gr_override_apply_param(); @@ -21391,14 +22292,54 @@ gr_page_unmapped: */ gsp_phys_gr_surface_alias_scan_locked(req->stream, req->stream_n, req); + /* SESSION 153-CONTINUED-59: fresh miss census for this pass + * (window + attr patch below both feed it). */ + gsp_phys_rmcall_gr_3d_miss_n = 0; nrel = gsp_phys_rmcall_gr_3d_patch_window( req->stream, req->stream_n, stream_patch, &win_n, &ndrop, req, &n_shadow); win3d = stream_patch; win3d_n = win_n; - pr_info("vgsp-gr: 3D window relocated %u operand pair(s) to shadow VAs (window=%u shadow=%u dropped=%u)\n", - nrel, win3d_n, n_shadow, ndrop); + pr_info("vgsp-gr: 3D window relocated %u operand pair(s) to shadow VAs (window=%u shadow=%u dropped=%u subdrop=%u sdmdrop=%u)\n", + nrel, win3d_n, n_shadow, ndrop, + gsp_phys_rmcall_gr_3d_last_subdrop, + gsp_phys_rmcall_gr_3d_last_sdmdrop); + /* + * SESSION 153-CONTINUED-57: patch the attr/pipeline prelude for + * this launch (relocation exactly like the window: the + * SET_PIPELINE_PROGRAM_ADDRESS pairs and any other recognised + * operand land in engine-visible shadow VAs; sub!=0 and + * win_drop runs are removed by the same pass). Only done when + * the capture generation moved - the shadow VAs are stable + * per HPA for the boot, so an older patch stays valid while it + * drains across launches. See gsp_phys_gr_attr_pre*. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_attr_prelude)) { + const u32 *ap; + u32 apn = 0, apg = 0; + + ap = gsp_sema_attr_prelude_get(&apn, &apg); + if (ap && apn && apg && + apg != READ_ONCE(gsp_phys_gr_attr_pre_gen)) { + u32 adrop = 0, an = apn; + + if (an > GSP_PHYS_GR_ATTR_PRE_WORDS) + an = GSP_PHYS_GR_ATTR_PRE_WORDS; + gsp_phys_rmcall_gr_3d_patch_window(ap, an, + gsp_phys_gr_attr_pre, + &gsp_phys_gr_attr_pre_n, + &adrop, req, &n_shadow); + if (gsp_phys_gr_attr_pre_n) { + WRITE_ONCE(gsp_phys_gr_attr_pre_gen, + apg); + pr_info("vgsp-gr: ATTR-PRELUDE patched gen=%u raw=%u kept=%u dropped=%u h=0x%x push=0x%llx\n", + apg, an, gsp_phys_gr_attr_pre_n, + adrop, req->h_object, + req->push_va); + } + } + } if (ndrop) { static u32 gr3d_drop_log; @@ -21412,6 +22353,87 @@ gr_page_unmapped: } } /* + * SESSION 153-CONTINUED-59: pre-doorbell MISS abort (see the + * gsp_phys_rmcall_gr_miss_abort comment). A window/attr + * operand whose relocation stayed a raw client VA after the + * level-2 resolver would make the physical engine read an + * unmapped page (r1104 MMU Fault HUBCLIENT_PD @0x4350000 -> + * guest channel reset; r1105 index-buffer misses). Ride the + * same failure path as RELOC-GAP-STRICT: the launch is not + * submitted, the caller falls back and authors the software + * release, the page is already in the pin demand set, and + * the next forward retries. Bounded: after 3000 aborts the + * check fails open so a page the resolver can never see + * cannot livelock the lane forever. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_miss_abort) && + gsp_phys_rmcall_gr_3d_miss_n && + gsp_phys_gr_miss_abort_n < 3000u) { + static u32 miss_abort_log; + + gsp_phys_gr_miss_abort_n++; + if (miss_abort_log < 16u || + !(gsp_phys_gr_miss_abort_n % 256u)) { + miss_abort_log++; + pr_info("vgsp-gr: MISS-ABORT h=0x%x push=0x%llx miss=%u n=%llu (operand page unresolvable; demanded via pin path, launch not submitted - retry next forward)\n", + req->h_object, req->push_va, + gsp_phys_rmcall_gr_3d_miss_n, + gsp_phys_gr_miss_abort_n); + } + mutex_unlock(&gsp_phys_rm_lock); + gsp_phys_gr_refused++; + /* + * SESSION 153-CONTINUED-61: the dedicated verdict lets + * the sema service keep the captured candidate armed + * and re-forward it (re-arm) instead of consuming it + * with a software completion: the demanded pages + * resolve through the pin path and the retry then runs + * the real launch. + */ + ret = GSP_PHYS_GR_RC_MISSABORT; + goto out; + } + /* + * SESSION 153-CONTINUED-56: bounded RAW (pre-drop, pre-reloc) + * window dump so the emitted window can be diffed against the + * client's authentic bytes. r1091 faulted at method 0x1168 + * (SET_VERTEX_ATTRIBUTE_A(2)) with Data 00000000 in the emitted + * stream; this copy tells whether that zero is the client's own + * value or an emission artifact (drop/reloc/align). + */ + { + static u32 gr3d_raw_log; + u32 wi; + + if (gr3d_raw_log < 8) { + gr3d_raw_log++; + pr_info("vgsp-gr: 3D raw dump h=0x%x push=0x%llx words=%u\n", + req->h_object, req->push_va, + req->stream_n); + for (wi = 0; + wi + 16 <= req->stream_n && wi < 448; + wi += 16) + pr_info("vgsp-gr: 3D raw[%3u]: %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n", + wi, + req->stream[wi + 0], + req->stream[wi + 1], + req->stream[wi + 2], + req->stream[wi + 3], + req->stream[wi + 4], + req->stream[wi + 5], + req->stream[wi + 6], + req->stream[wi + 7], + req->stream[wi + 8], + req->stream[wi + 9], + req->stream[wi + 10], + req->stream[wi + 11], + req->stream[wi + 12], + req->stream[wi + 13], + req->stream[wi + 14], + req->stream[wi + 15]); + } + } + /* * SESSION 153-CONTINUED-11: bounded raw window dump - the * exact C597 method stream the engine walks (program binds * 0x2014/0x2018+j*0x40, SET_PIPELINE_SHADER 0x2000+j*0x40, @@ -21421,7 +22443,7 @@ gr_page_unmapped: static u32 gr3d_win_log; u32 wi; - if (gr3d_win_log < 4) { + if (gr3d_win_log < 48) { gr3d_win_log++; pr_info("vgsp-gr: 3D window dump h=0x%x push=0x%llx words=%u\n", req->h_object, req->push_va, win3d_n); @@ -21444,12 +22466,67 @@ gr_page_unmapped: words = gsp_phys_rmcall_build_gr_launch(methods, req, qmd, qmd_dst, ch_sem_va, want_ch, - win3d, win3d_n); + win3d, win3d_n, + gsp_phys_gr_attr_pre, + gsp_phys_gr_attr_pre_n); memcpy(gsp_phys_chunk_page_va(pb, 0), methods, words * 4); /* SESSION 153-CONTINUED-12: identify this block to a later PBDMA dump. */ WRITE_ONCE(gsp_phys_gr_last_words, words); WRITE_ONCE(gsp_phys_gr_last_h, req->h_object); WRITE_ONCE(gsp_phys_gr_last_push_va, req->push_va); + WRITE_ONCE(gsp_phys_gr_last_pb_dma, + (u64)gsp_phys_chunk_page_dma(pb, 0)); + /* + * SESSION 153-CONTINUED-52: stream validator for the block the engine + * will walk. The r1072 fault class ("Channel ID 4 intr0 00040000" = + * PBDMA PBENTRY, "invalid method encoding used in method", + * fifo_tu102.c) is raised by a method header the PBDMA refuses, or by + * a run whose data is not fully present so the walk desyncs into + * following words. Walk the exact published dwords with the FIFO DMA + * grammar (dev_ram.ref NV_FIFO_DMA_*: SEC_OP 31:29; 1/3/5 carry 1+cnt + * data words, 4/7 and TERT mask ops are single-word, 6 is RESERVED) + * and flag the first occurrences of a reserved encoding or a run + * overrunning the block. Read-only; bounded per boot. + */ + { + static u32 blkval_log; + const u32 *bv = methods; + u32 bo = 0, nbad = 0; + + if (blkval_log < 48) { + while (bo < words && nbad < 4) { + u32 h = bv[bo]; + u32 bop = (h >> 29) & 7u; + u32 bcnt = (h >> 16) & 0xfffu; + u32 bspan = 1u; + + if (bop == 1u || bop == 3u || bop == 5u) + bspan = 1u + bcnt; + if (bop == 6u) { + nbad++; + pr_warn("vgsp-gr: BLOCKVAL reserved-encoding word=%u/%u val=%08x (h=%x push=0x%llx prev=%08x %08x)\n", + bo, words, h, req->h_object, + req->push_va, + bo ? bv[bo - 1] : 0, + bo + 1 < words ? + bv[bo + 1] : 0); + } + if (bo + bspan > words) { + nbad++; + pr_warn("vgsp-gr: BLOCKVAL run-overrun word=%u/%u hdr=%08x span=%u (needs %u more) (h=%x push=0x%llx)\n", + bo, words, h, bspan, + bo + bspan - words, + req->h_object, req->push_va); + } + bo += bspan; + } + blkval_log++; + if (nbad) + pr_warn("vgsp-gr: BLOCKVAL done words=%u bad=%u (h=%x push=0x%llx)\n", + words, nbad, req->h_object, + req->push_va); + } + } pr_info("vgsp-gr: qmd w0..3=%08x %08x %08x %08x w4..7=%08x %08x %08x %08x w23..25=%08x %08x %08x w32/33=%08x %08x w48/49=%08x %08x\n", qmd[0], qmd[1], qmd[2], qmd[3], qmd[4], qmd[5], qmd[6], qmd[7], qmd[23], qmd[24], qmd[25], qmd[32], qmd[33], qmd[48], qmd[49]); @@ -21769,9 +22846,10 @@ gr_page_unmapped: break; } } - pr_info("vgsp-gr: GR LAUNCH submitted h=0x%x push=0x%llx len=%u pages=%u shadow=%u qmd_va=0x%llx slot=%u words=%u prelude=%d qmdvirt=%d token=0x%x; polling qmd_rel=%u ch_rel=0x%08x (<=%u ms)%s\n", + pr_info("vgsp-gr: GR LAUNCH submitted h=0x%x push=0x%llx len=%u pages=%u shadow=%u qmd_va=0x%llx slot=%u words=%u e0=0x%08x e1=0x%08x prelude=%d qmdvirt=%d token=0x%x; polling qmd_rel=%u ch_rel=0x%08x (<=%u ms)%s\n", req->h_object, req->push_va, req->push_len, req->n_pages, n_shadow, req->qmd_va, sub.slot, words, + sub.e0, sub.e1, gsp_phys_rmcall_gr_prelude_value(), gsp_phys_rmcall_gr_qmdvirt_value(), boot_stage.rmcall_token, @@ -21842,10 +22920,68 @@ gr_page_unmapped: sked_sample[11] = ioread32(bar0 + 0x00409c00u); sked_sample[12] = ioread32(bar0 + 0x00409c18u); sked_sample[13] = ioread32(bar0 + 0x00409008u); + /* + * SESSION 153-CONTINUED-52: in-flight fault registers. + * The DS shader-program-header error latch (0x405840), + * the MME exception latch (0x404490) with its PC/macro + * address (0x404494) and instruction register + * (0x404498) are cleared by the real RM within ~ms of + * the fault, so the post-timeout reads are all clean. + * Sampling them every poll tick records the faulting + * values at the first change. + */ + sked_sample[14] = ioread32(bar0 + 0x00405840u); + sked_sample[15] = ioread32(bar0 + 0x00404490u); + sked_sample[16] = ioread32(bar0 + 0x00404494u); + sked_sample[17] = ioread32(bar0 + 0x00404498u); + /* + * SESSION 153-CONTINUED-52: the PBDMA host-software + * METHOD latch (NV_PPBDMA_METHOD0 0x4000c0: ADDR + * 13:2, SUBCH 18:16, INCR 0) - the method whose + * delivery raises NV_PPBDMA_INTR_0_METHOD (bit 21). + * The real driver clears the entry when it processes + * the interrupt, so only an in-flight sample can name + * it. + */ + sked_sample[18] = ioread32(bar0 + 0x004000c0u); + sked_sample[19] = ioread32(bar0 + 0x004000c4u); + /* + * SESSION 153-CONTINUED-57: the class-error trapped + * register set (gr_error_gk104.c grGetExceptionChannel): + * NV_PGRAPH_TRAPPED_ADDR 0x40704 (MTHD offset + SUBCH + + * INVALID status bit), TRAPPED_DATA_LOW 0x40708 (the + * class error's "Data" field), TRAPPED_DATA_HIGH + * 0x4070c, and TRAPPED_DATA_MME 0x40710 (PC 11:0 + + * MACRO_ADDRESS 19:12) - the last one names whether the + * offending write came from MME macro execution. The + * real RM clears these within ~ms of the fault; a + * per-tick sample catches the first change. The + * persistent 0x1168/Data=0 class error (r1091/r1098) + * needs exactly this decode. + */ + sked_sample[20] = ioread32(bar0 + 0x00400704u); + sked_sample[21] = ioread32(bar0 + 0x00400708u); + sked_sample[22] = ioread32(bar0 + 0x0040070cu); + sked_sample[23] = ioread32(bar0 + 0x00400710u); + /* + * SESSION 153-CONTINUED-58: the front-end DECODE + * STATE EX registers (NV_PGRAPH_PRI_FE_DECODE_STATE_EX + * 0x40a630..0x40a63c, 4 slots): MME_PC 11:0, + * MME_MACRO_ADDR 21:12, MME_GENERATED bit 31 - the + * front end's own record of the last MME-generated + * method write, which names the macro (address) and + * its pc for the class error even when the trapped + * registers are already back to the 0xbadf priv + * poison. Sampled per tick like the trap set. + */ + sked_sample[24] = ioread32(bar0 + 0x0040a630u); + sked_sample[25] = ioread32(bar0 + 0x0040a634u); + sked_sample[26] = ioread32(bar0 + 0x0040a638u); + sked_sample[27] = ioread32(bar0 + 0x0040a63cu); if (sked_first || memcmp(sked_sample, sked_prev, sizeof(sked_sample))) { - pr_info("vgsp-gr: sked rec +%ums cwd0=%08x cwd1=%08x cwd2=%08x info1=%08x info2=%08x act=%08x esr=%08x grfifo=%08x fecs_cur=%08x fecs_new=%08x ctxsw1=%08x fe0=%08x fehost=%08x fecsirq=%08x\n", + pr_info("vgsp-gr: sked rec +%ums cwd0=%08x cwd1=%08x cwd2=%08x info1=%08x info2=%08x act=%08x esr=%08x grfifo=%08x fecs_cur=%08x fecs_new=%08x ctxsw1=%08x fe0=%08x fehost=%08x fecsirq=%08x ds_esr=%08x mme_esr=%08x mme_pc=%u mme_mac=0x%x mme_ir=%08x mth0=%08x mth1=%08x(pbaddr=0x%x sub=%u incr=%u) trap=%08x(data_lo=%08x hi=%08x sub=%u mmegen=%u dhv=%u) mmepc=%u mmemac=0x%x\n", sked_tick * GSP_PHYS_RMCALL_POLL_MS, sked_sample[0], sked_sample[1], @@ -21854,12 +22990,80 @@ gr_page_unmapped: sked_sample[6], sked_sample[7], sked_sample[8], sked_sample[9], sked_sample[10], sked_sample[11], - sked_sample[12], sked_sample[13]); + sked_sample[12], sked_sample[13], + sked_sample[14], sked_sample[15], + sked_sample[16] & 0xfffu, + (sked_sample[16] >> 12) & 0xffu, + sked_sample[17], + sked_sample[18], sked_sample[19], + (sked_sample[18] >> 2) << 2, + (sked_sample[18] >> 16) & 7u, + sked_sample[18] & 1u, + sked_sample[20], sked_sample[21], + sked_sample[22], + (sked_sample[20] >> 16) & 7u, + (sked_sample[20] >> 20) & 1u, + (sked_sample[20] >> 24) & 1u, + sked_sample[23] & 0xfffu, + (sked_sample[23] >> 12) & 0xffu); + if (sked_sample[24] || sked_sample[25] || + sked_sample[26] || sked_sample[27]) + pr_info("vgsp-gr: sked fed0=%08x fed1=%08x fed2=%08x fed3=%08x (pc0=%u mac0=0x%x gen0=%u)\n", + sked_sample[24], + sked_sample[25], + sked_sample[26], + sked_sample[27], + sked_sample[24] & 0xfffu, + (sked_sample[24] >> 12) & 0x3ffu, + (sked_sample[24] >> 31) & 1u); memcpy(sked_prev, sked_sample, sizeof(sked_sample)); sked_first = false; } } + /* + * SESSION 153-CONTINUED-48: priv-error (0xbadf-tagged) wedge + * recovery - see sema_gr_wedge_recover. The poison latch is + * checked before the completion break and the ladder so a + * launch whose window faulted the GR hands the engine to the + * next launch recovered instead of wedged. One attempt per + * launch (local flag); the hard re-arm is the r445 mechanism, + * bounded by its own RPC and sem-probe timeouts. + */ + if (READ_ONCE(gsp_phys_gr_wedge_recover) && bar0 && + !wedge_recover_done) { + u32 g0 = ioread32(bar0 + 0x00400504u); + + if ((g0 & 0xffff0000u) == 0xbadf0000u) { + u32 g1; + + wedge_recover_done = true; + wedge_recoveries++; + pr_warn("vgsp-gr: WEDGE-RECOVER priv-error poison seen (grfifo=0x%08x) h=0x%x push=0x%llx - TSG preempt + resched\n", + g0, req->h_object, req->push_va); + gsp_phys_mme_regs_log(bar0, "wedge-poison"); + mutex_lock(&gsp_phys_rm_lock); + gsp_phys_gr_wedge_hard_rearm_locked(bar0); + mutex_unlock(&gsp_phys_rm_lock); + g1 = ioread32(bar0 + 0x00400504u); + /* + * SESSION 153-CONTINUED-61: the TSG preempt + + * resched above resets the engine context, so + * the MME program the emitted prelude loaded is + * gone. Clear the emission latch: the next + * launch that can carry the prelude re-preloads + * it (without this every later replay ran its + * macros against an unloaded MME and stalled + * pre-retire - the r1110 mme_pc=513 class). + */ + WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, 0); + pr_warn("vgsp-gr: WEDGE-RECOVER after re-arm grfifo=0x%08x (%s); recovery #%u this boot; MME prelude latch cleared (context re-preload owed)\n", + g1, + (g1 & 0xffff0000u) == 0xbadf0000u ? + "still poisoned" : "cleared", + wedge_recoveries); + } + } if (qmd_rel) { if (qmd_rel_hpa) w0 = READ_ONCE(*(u32 *)((u8 *)__va(qmd_rel_hpa) + @@ -21868,6 +23072,55 @@ gr_page_unmapped: w0 = READ_ONCE(*(u32 *)sem_va); } /* + * SESSION 153-CONTINUED-59: PBENTRY first-sighting watch. + * The r1102 fault was only captured via the GSP's 0x1006 + * relay (seconds later, intr0 already cleared by the RM): + * here every 5 ms poll of a 3D-stream launch scans the 12 + * PBDMA instances for NV_PPBDMA_INTR_0 bit 18 (PBENTRY) and + * on the FIRST sighting dumps the live PB_HEADER / PB_COUNT + * / PB_DATA0 / HDR_SHADOW set plus the raw 0x100..0x120 + * block for the faulting instance, before any re-arm or RM + * clear can overwrite them. Bounded (8 lines per boot); + * read-only, no behavior change to the launch. + */ + if (stream3d && bar0) { + static u32 pbentry_log; + u32 pi2; + + for (pi2 = 0; pi2 < 12 && pbentry_log < 8; pi2++) { + u32 b2 = 0x40000u + pi2 * 0x2000u; + u32 i0 = ioread32(bar0 + b2 + 0x108u); + + if (!(i0 & 0x00040000u)) + continue; + pbentry_log++; + pr_warn("vgsp-gr: PBENTRY-FIRST-SIGHTING P%u intr0=%08x gpf=%u pbfetch=0x%llx hdr=%08x cnt=%08x data0=%08x pbhd=%08x h=%x push=0x%llx words=%u tick=%u\n", + pi2, i0, + ioread32(bar0 + b2 + 0x050u), + ((u64)(ioread32(bar0 + b2 + 0x058u) & + 0xffu) << 32) | + (ioread32(bar0 + b2 + 0x054u) & + ~3u), + ioread32(bar0 + b2 + 0x118u), + ioread32(bar0 + b2 + 0x088u), + ioread32(bar0 + b2 + 0x08cu), + ioread32(bar0 + b2 + 0x084u), + req->h_object, req->push_va, words, + sked_tick); + pr_info("vgsp-gr: PBENTRY-FIRST-SIGHTING P%u raw[0x100]=%08x [104]=%08x [10c]=%08x [110]=%08x [114]=%08x [11c]=%08x [120]=%08x\n", + pi2, + ioread32(bar0 + b2 + 0x100u), + ioread32(bar0 + b2 + 0x104u), + ioread32(bar0 + b2 + 0x10cu), + ioread32(bar0 + b2 + 0x110u), + ioread32(bar0 + b2 + 0x114u), + ioread32(bar0 + b2 + 0x11cu), + ioread32(bar0 + b2 + 0x120u)); + gsp_phys_pbdma_fault_dump(0xffffffffu); + break; + } + } + /* * SESSION 153-CONTINUED-13: early PBDMA fetch probe for a 3D * stream launch - where is the front end while a DRAW window * does not retire? One shot per launch at ~150 ms and ~1.2 s @@ -21881,7 +23134,7 @@ gr_page_unmapped: static u32 winprobe_log; u32 pi; - if (winprobe_log < 10) { + if (winprobe_log < 64) { winprobe_log++; pr_info("vgsp-gr: WINPROBE +%ums push=0x%llx words=%u block_dma=0x%llx grfifo=%08x trapped=%08x trapped_lo=%08x\n", sked_tick * GSP_PHYS_RMCALL_POLL_MS, @@ -21896,18 +23149,33 @@ gr_page_unmapped: u32 pbf = ioread32(bar0 + b + 0x054u); u32 pbfhi = ioread32(bar0 + b + 0x058u); u32 hdrsh = ioread32(bar0 + b + 0x118u); + u32 wbh = ioread32(bar0 + b + 0x084u); + u32 wbcnt = ioread32(bar0 + b + 0x088u); + u32 wbd0 = ioread32(bar0 + b + 0x08cu); u64 fetch64 = ((u64)(pbfhi & 0xffu) << 32) | (u64)(pbf & ~3u); + s64 woff = -1; if (!intr0 && !gpf && !pbf && !hdrsh) continue; - pr_info("vgsp-gr: WINPROBE P%u intr0=%08x gp_put=%u pbfetch=0x%llx hdr=%08x(%s m=0x%x cnt=%u sub=%u)\n", - pi, intr0, gpf, fetch64, hdrsh, + if (gsp_phys_gr_last_pb_dma && + fetch64 >= gsp_phys_gr_last_pb_dma && + fetch64 < + gsp_phys_gr_last_pb_dma + + (u64)gsp_phys_gr_last_words * + 4u) + woff = (s64)(fetch64 - + gsp_phys_gr_last_pb_dma) / + 4; + pr_info("vgsp-gr: WINPROBE P%u intr0=%08x gp_put=%u pbfetch=0x%llx off=%lld hdr=%08x(%s m=0x%x cnt=%u sub=%u) pbhd=%08x(type=%s) pbcnt=%08x pbd0=%08x\n", + pi, intr0, gpf, fetch64, woff, hdrsh, gsp_phys_pbdma_type_name(hdrsh >> 29), (hdrsh & 0x1fffu) << 2, (hdrsh >> 16) & 0xfffu, - (hdrsh >> 13) & 7u); + (hdrsh >> 13) & 7u, wbh, + gsp_phys_pbdma_type_name(wbh >> 29), + wbcnt, wbd0); } } } @@ -21937,6 +23205,7 @@ gr_page_unmapped: pr_err("vgsp-gr: GR LAUNCH FAILED: wait budget crossed (qmd_rel=0x%08x want=0x%08x ch_rel=0x%08x want=0x%08x) h=0x%x push=0x%llx; ladder stopped\n", w0, want_qmd, w1, want_ch, req->h_object, req->push_va); + gsp_phys_mme_regs_log(bar0, "wait-max"); ret = -ETIMEDOUT; goto out; } @@ -21948,6 +23217,7 @@ gr_page_unmapped: pr_err("vgsp-gr: GR LAUNCH FAILED: timeout after %u ms (qmd_rel=0x%08x want=0x%08x ch_rel=0x%08x want=0x%08x) h=0x%x push=0x%llx\n", poll_ms, w0, want_qmd, w1, want_ch, req->h_object, req->push_va); + gsp_phys_mme_regs_log(bar0, "timeout"); /* * F4-b: one-shot deposit readback at the FIRST timeout * (the old post-mortem only ran after attempt 3, which @@ -22730,7 +24000,9 @@ gr_page_unmapped: methods, req, qmd, qmd_dst, ch_sem_va, want_ch, win3d, - win3d_n); + win3d_n, + gsp_phys_gr_attr_pre, + gsp_phys_gr_attr_pre_n); memcpy(gsp_phys_chunk_page_va(pb, 0), methods, words * 4); } @@ -25649,6 +26921,18 @@ static int gsp_phys_reg_probe_show(struct seq_file *m, void *unused) 0x0040070cu); fe_hww_esr = gsp_phys_reg_probe_line(m, bar0, "FE_HWW_ESR", 0x00404000u); + /* + * SESSION 153-CONTINUED-52: MME exception latches (see + * gsp_phys_mme_regs_log). + */ + (void)gsp_phys_reg_probe_line(m, bar0, "PGRAPH_PRI_MME_HWW_ESR", + 0x00404490u); + (void)gsp_phys_reg_probe_line(m, bar0, + "PGRAPH_PRI_MME_HWW_ESR_INFO", + 0x00404494u); + (void)gsp_phys_reg_probe_line(m, bar0, + "PGRAPH_PRI_MME_HWW_ESR_INFO2", + 0x00404498u); fe_chip_def = gsp_phys_reg_probe_line(m, bar0, "FE_CHIP_DEF_INFO", 0x00404030u); fe_go_idle = gsp_phys_reg_probe_line(m, bar0, "FE_GO_IDLE_INFO", diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h index 14cf482..39824e3 100644 --- a/kernel/gsp_phys_booter.h +++ b/kernel/gsp_phys_booter.h @@ -369,6 +369,16 @@ void gsp_phys_backend_relay_counters(u64 *requested, u64 *ok, u64 *verify_fail, #define GSP_PHYS_GR_QMD_SENTINEL_WORD 63u #define GSP_PHYS_GR_QMD_WORD_NONE 0xffffffffu /* + * SESSION 153-CONTINUED-61: dedicated backend verdict for the 3D-window + * MISS-ABORT (gsp_phys_rmcall_gr_miss_abort): the launch was NOT submitted + * because a window/attr operand's page is still unresolved. The sema + * service uses this code to keep the captured candidate armed (re-arm) + * instead of consuming it with a software completion, so the forward path + * retries the same chunk once the pin demand resolves the page. -ENOMSG + * is returned by no other path on this interface. + */ +#define GSP_PHYS_GR_RC_MISSABORT (-ENOMSG) +/* * r929: grown 256 -> 512 dwords so a whole GL 3D section push (the observed * X server one-shot is 1524 B = 381 dwords) survives in the captured raw * tail; the raw window is the source of the 3D method-stream replay. diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 4d497d9..c2f3084 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -44,6 +44,8 @@ #include <linux/workqueue.h> #include <linux/mm.h> #include <linux/vmalloc.h> +#include <linux/debugfs.h> +#include <linux/seq_file.h> #include <asm/unaligned.h> #include "nvidia-vgpu-vfio.h" @@ -476,6 +478,15 @@ struct svc_channel { unsigned long push_hold_until; /* absolute deadline of the bounded * work-before-release hold for an * unreadable CE entry (0 = none) */ + unsigned long gr_setup_hold_until; /* SESSION 153-CONTINUED-54: absolute + * deadline of the GR setup-push hold: + * an unreadable/all-zero MME-setup + * (or >= 12 KB) GR push stays pending + * past the 1-failed-rescue consume + * until the demanded guest pin lands + * (0 = none) */ + u64 gr_setup_hold_va; /* entry VA the setup hold was armed for */ + u32 gr_setup_hold_idx; /* GPFIFO slot the setup hold was armed for */ unsigned long defer_since; /* first failure of the pending GR entry * (0 = none): the entry is left pending * for the GR grace window so a late @@ -1511,6 +1522,33 @@ struct sema_svc { #define SVC_STALLED_MAX 256 u64 stalled_vas[SVC_STALLED_MAX]; u32 stalled_va_count; + + /* + * SESSION 153-CONTINUED-56: the armed setup-hold's demand VA set + * (page-aligned, deduped per boot), exposed read-only to the demand + * tool via /sys/kernel/debug/vgsp/sema_setup_demand. State, not a + * log row: under vgsp-sema spam the `stalled VA #N` dmesg rows + * rotate out of the ring before the 3 s harvest reads them, so the + * setup-push pages were lost (r1090 needed a manual demand); the pin + * tool now reads this file directly each pass instead. + * + * SESSION 153-CONTINUED-60: the set is a RING. At 64 entries the + * r1106 MISS wall (push=0xcf3d724 miss=11..17/pass, abort loop + * n=2..9) never converged because every later gsp_sema_demand_note + * (the MISS pages themselves, the attr-prelude program pages) was + * silently dropped once the 64 slots were exhausted -> the pin tool + * never demanded them -> never resolved -> perpetual pre-doorbell + * abort. Capacity is raised to 512 and a full set overwrites the + * oldest slot instead of refusing the note, so the resident set is + * the demand-paging working set and a later note is never lost; + * total/evicted counters are published for observability. + */ +#define SVC_SETUP_DEMAND_MAX 512 + u64 setup_demand_vas[SVC_SETUP_DEMAND_MAX]; + u32 setup_demand_count; + u32 setup_demand_head; /* ring write cursor (valid once full) */ + u64 setup_demand_total; /* notes accepted (dedup hits excluded) */ + u64 setup_demand_evicted; /* oldest-slot overwrites */ }; static struct sema_svc svc; @@ -1544,6 +1582,145 @@ static void svc_log_stalled_va(u64 va) } /* + * SESSION 153-CONTINUED-56: record one page-aligned VA of the armed + * setup-hold demand set. Same caller as svc_log_stalled_va (the service + * thread); the debugfs reader publishes entries with a write barrier on the + * count, so a reader can never observe a torn entry (the count never + * decreases and an entry is stable once its slot is covered by the count). + * + * SESSION 153-CONTINUED-60: full-set behaviour is ring overwrite (oldest + * slot) instead of drop, so the r1106 MISS pages can never be refused a + * slot by a stale 64-entry set; see the member comment. Entry stores are + * aligned u64 on a 64-bit kernel, so the in-place overwrite cannot tear for + * the seq reader (worst case it sees the previous resident VA once more). + */ +static void svc_setup_demand_record(u64 va) +{ + u32 i, n = svc.setup_demand_count; + + if (!va) + return; + for (i = 0; i < n; i++) + if (svc.setup_demand_vas[i] == va) + return; + if (n < SVC_SETUP_DEMAND_MAX) { + svc.setup_demand_vas[n] = va; + smp_wmb(); + svc.setup_demand_count = n + 1; + } else { + static u32 ring_log; + u32 w = svc.setup_demand_head; + + svc.setup_demand_vas[w] = va; + svc.setup_demand_head = (w + 1u) % SVC_SETUP_DEMAND_MAX; + svc.setup_demand_evicted++; + if (ring_log < 4u || !(svc.setup_demand_evicted % 4096u)) { + ring_log++; + pr_info("sema-svc: demand ring full; evicting oldest slot %u for va=0x%llx (evicted=%llu total=%llu)\n", + w, va, svc.setup_demand_evicted, + svc.setup_demand_total + 1u); + } + } + svc.setup_demand_total++; +} + +/* + * SESSION 153-CONTINUED-59: cross-file entry for the GR launch builder (see + * gsp_sema_service.h). Records the page of a VA whose window/attr operand + * relocation MISSed, so the pin demand path fetches it before the next + * forward; the builder then either resolves it (RELOC level-2 fallback) or + * aborts the launch pre-doorbell instead of letting the engine read a raw + * unmapped guest VA (the r1104 MMU Fault HUBCLIENT_PD class). + */ +void gsp_sema_demand_note(u64 va) +{ + if (va) + svc_setup_demand_record(va & ~(u64)0xfffu); +} + +/* + * Read-only view of the setup-hold demand set for the pin tool: + * # sema setup-hold demand VAs (page-aligned): N total=T evicted=E + * 0x4021000 + * ... + * `total` counts every accepted note (dedup hits excluded); `evicted` counts + * ring overwrites of the oldest slot (SESSION 153-CONTINUED-60). + */ +static int svc_setup_demand_show(struct seq_file *m, void *v) +{ + u32 i, n = READ_ONCE(svc.setup_demand_count); + + seq_printf(m, "# sema setup-hold demand VAs (page-aligned): %u total=%llu evicted=%llu\n", + n, (unsigned long long)svc.setup_demand_total, + (unsigned long long)svc.setup_demand_evicted); + for (i = 0; i < n && i < SVC_SETUP_DEMAND_MAX; i++) + seq_printf(m, "0x%llx\n", + (unsigned long long)svc.setup_demand_vas[i]); + return 0; +} + +static int svc_setup_demand_open(struct inode *inode, struct file *file) +{ + return single_open(file, svc_setup_demand_show, NULL); +} + +static const struct file_operations svc_setup_demand_fops = { + .owner = THIS_MODULE, + .open = svc_setup_demand_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +/* True when this module created the vgsp debugfs root (mirrors cuda bridge). */ +static bool svc_setup_demand_root_owned; + +static int svc_setup_demand_debugfs_init(void) +{ + struct dentry *root, *f; + + svc_setup_demand_root_owned = false; + root = debugfs_lookup("vgsp", NULL); + if (IS_ERR(root)) + root = NULL; + if (!root) { + root = debugfs_create_dir("vgsp", NULL); + if (IS_ERR_OR_NULL(root)) { + pr_warn("sema-svc: debugfs dir vgsp unavailable; sema_setup_demand not exposed\n"); + return -ENOENT; + } + svc_setup_demand_root_owned = true; + } + f = debugfs_create_file("sema_setup_demand", 0444, root, NULL, + &svc_setup_demand_fops); + if (IS_ERR_OR_NULL(f)) { + pr_warn("sema-svc: debugfs sema_setup_demand creation failed\n"); + if (svc_setup_demand_root_owned) + debugfs_lookup_and_remove("vgsp", NULL); + svc_setup_demand_root_owned = false; + return -ENOMEM; + } + dput(root); + pr_info("sema-svc: debugfs /sys/kernel/debug/vgsp/sema_setup_demand ready\n"); + return 0; +} + +static void svc_setup_demand_debugfs_fini(void) +{ + struct dentry *root; + + root = debugfs_lookup("vgsp", NULL); + if (IS_ERR_OR_NULL(root)) + return; + debugfs_lookup_and_remove("sema_setup_demand", root); + dput(root); + if (svc_setup_demand_root_owned) { + debugfs_lookup_and_remove("vgsp", NULL); + svc_setup_demand_root_owned = false; + } +} + +/* * Deferred readability probe for unreadable CE pushes that were consumed via * the fake-release fallback. The pump fakes immediately (work-before-release * cannot hold the channel); this queue re-attempts the read later in the @@ -8022,8 +8199,24 @@ static void svc_gl_add_ptr(u64 va) if (!va) return; + /* + * SESSION 153-CONTINUED-48: dedup by 4 KB page, not exact VA. + * The QMD pointer region holds only seven pairs (words 34..47, + * SVC_GR_POINTER_MAX - 1); a window that binds several different + * offsets of one buffer (r1058 chunk1: five index-buffer VAs + * across three pages) used to consume one slot per exact VA and + * hit the cap, so no later operand page (the raw CALL_MME_MACRO + * payload VAs 0x43545f0/0xb94f700) could join the set and the + * shadow relocation left them raw - the raw operand the engine + * faulted on (grfifo=0xbadf1201). One pointer per page carries + * the same page set with slots left for the remaining operand + * families; the first VA seen for a page is kept so the value + * stays a real client VA (the fill resolves pages, and the + * relocation matches by page). + */ for (k = 0; k < svc_gl.n_ptr; k++) - if (svc_gl.ptr[k] == va) + if (svc_gl.ptr[k] == va || + (svc_gl.ptr[k] & ~(u64)0xfffu) == (va & ~(u64)0xfffu)) return; if (svc_gl.n_ptr < SVC_GR_POINTER_MAX - 1) svc_gl.ptr[svc_gl.n_ptr++] = va; @@ -8049,6 +8242,26 @@ static void svc_gl_field(u32 addr, u32 count, const u8 *data, u32 dlen) * [ENABLE(0) | TYPE(7:4)]. Collected per slot so a bound program VA * can be typed (VTG vs PS) for the program-bytes delivery. */ + /* + * SESSION 153-CONTINUED-48: NVC597_SET_REPORT_SEMAPHORE_A..D = 0x1b00, + * observed as 4-word runs [A=upper(7:0), B=lower(31:0), C=payload, + * D=flags] at the end of every glcore draw window (payload = the + * client's frame counter, e.g. 0x4f9f6; VA e.g. 0x4181bb0/0xc91e6c0). + * These are the engine's own completion writes; with the VA raw the + * replayed engine writes a client VA the GPC engines cannot reach - + * the same VIRT_WRITE fault class the 0x10c NOTIFY drop exists for + * (r1061: the kept [0,0x2000) window that still faults is exactly + * the one carrying these raw 0x1b00 VAs). Collect the semaphore page + * so the window relocation can rewrite the pair to a shadow VA: the + * write then lands in the SAME HPA and the client's wait sees the + * real hardware completion. + */ + if (addr == 0x1b00) { + if (count >= 2) + svc_gl_add_ptr(((u64)(v[0] & 0xffu) << 32) | + (u64)(v[1] & 0xfffffffcu)); + return; + } if (addr >= 0x2000 && addr < 0x2800 && (addr & 0x3f) == 0) { u32 j = (addr - 0x2000) / 0x40; @@ -8056,6 +8269,37 @@ static void svc_gl_field(u32 addr, u32 count, const u8 *data, u32 dlen) svc_gl_shader_type[j] = (u8)((v[0] >> 4) & 0xf); return; } + /* + * SESSION 153-CONTINUED-48: CALL_MME_MACRO(j)=0x3800+j*8 / + * CALL_MME_DATA(j)=0x3804+j*8 runs carry raw 32-bit client VAs as + * macro arguments (r1058 chunk1: the macro(2) run payloads + * 0x44ff800/0x44ff000 relocated only because the page set already + * held 0x44ff000 via the index bind; the sibling payloads + * 0x43545f0/0x4353f60/0xb94f700/0xb941578 stayed raw because their + * pages missed the capped pointer list - the raw operand class the + * GR faulted on). Collect every VA-shaped payload word so the page + * set - and therefore the 3D-window shadow relocation - covers the + * macro arguments too. This must run before the 0x2014..0x3fff + * program-pair recognizer below, whose (addr & 0x3f) == 0x14 shape + * would otherwise swallow a CALL_MME_DATA(2)=0x3814 run. Shape + * gates: 4-byte aligned; >= 64 KB excludes the method/flag scalars + * (0x0c/0x53/0x338); < 512 MB excludes the float constants the + * macro(57) runs carry (1.0f=0x3f800000) while keeping every + * observed client operand VA (max 0xb94f700). + */ + if (addr >= 0x3800 && addr < 0x3a00) { + for (k = 0; k < count; k++) { + u32 w; + + if (4 * (k + 1) > dlen) + break; + memcpy(&w, data + 4 * k, 4); + if (w >= 0x10000u && w < 0x20000000u && + !(w & 3u)) + svc_gl_add_ptr((u64)w); + } + return; + } /* SET_PIPELINE_PROGRAM_ADDRESS_A/B(j), A=upper(7:0), B=lower(31:0). * Observed: 0x20020815 (j=1) [00000001 21201b00] -> 0x121201b00. * SESSION 153-CONTINUED-11: EVERY pair is recorded (the engine @@ -8905,7 +9149,7 @@ static void svc_gl_stream_chunks(struct svc_channel *ch, const u8 *buf, { static u32 chunkdump_log; - if (chunkdump_log < 4 && svc_gl.start_off + 8u <= at_off) { + if (chunkdump_log < 8 && svc_gl.start_off + 8u <= at_off) { u32 cur2 = svc_gl.start_off, end2, k2, wcnt; u32 cw2 = svc_gl_chunk_words_param ? svc_gl_chunk_words_param : @@ -13676,14 +13920,458 @@ static u64 svc_pass_hash; /* r391f: this decode pass's push content hash */ */ unsigned gsp_sema_push_raw; unsigned gsp_sema_push_raw_small; +static u32 svc_mme_census_flag; module_param_named(sema_push_raw_small, gsp_sema_push_raw_small, uint, 0644); MODULE_PARM_DESC(sema_push_raw_small, "Verbatim decode dump for small GR pushes (16..511 B), bounded count (default 0)"); +module_param_named(sema_mme_census, svc_mme_census_flag, uint, 0644); +MODULE_PARM_DESC(sema_mme_census, + "Census MME-family method ids (0x114..0x124, 0x234, 0x550..0x568, 0x3400..0x34fc, 0x3800..0x39fc) in beef pushes, bounded (default 0)"); static u64 svc_pushraw_last_va; static u32 svc_pushraw_last_len; static u32 svc_pushraw_n; static u32 svc_pushraw_small_n; +/* + * SESSION 153-CONTINUED-52: MME-family method census for the beef (GL 3D + * client) channel. The replayed draw windows raise a real MME ILLEGAL_OPCODE + * (NV_PGRAPH_PRI_MME_HWW_ESR bit 2) - the MME executed a macro whose code it + * could not fetch. The macro calls (CALL_MME_MACRO/DATA 0x3800..0x39fc) carry + * macro code addresses, so the macro CONTEXT state - the MME Instruction/ + * Start-address RAM loads (0x114/0x118/0x11c/0x120), the shadow RAM control + * (0x124), MME version (0x234), the MME DMA window (0x550..0x568) and the + * shadow scratch (0x3400..0x34fc) - must have reached the ladder context too. + * If those setup methods never appear in ANY client push the MME can never + * have its macro library and the replay is doomed; if they do appear, the + * next question is which push forwards them. This census answers that: it + * scans every beef push for those method ids and logs a bounded one-line + * summary (first-hit offsets + first data word). Read-only; bounded. + */ +static u32 svc_mme_census_n; + +static void svc_mme_census(const u8 *buf, u32 len, u64 push_va, u32 h_object) +{ + u32 off = 0; + u32 nhit = 0; + char line[512]; + int lc = 0; + + if (!READ_ONCE(svc_mme_census_flag) || svc_mme_census_n >= 128) + return; + if ((h_object & 0xfffff000u) != 0xbeef0000u) + return; + while (off + 4 <= len && nhit < 16) { + u32 hdr, op, cnt, addr; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 7u; + cnt = (hdr >> 16) & 0xfffu; + addr = (hdr & 0x1fffu) << 2; + if (op == 1u || op == 3u || op == 5u) + /* multi-word run: classify by its method id */ + ; + else if (op == 4u || op == 7u) + cnt = 0; + else + cnt = 0; + if ((addr >= 0x114u && addr <= 0x124u) || + addr == 0x234u || + (addr >= 0x550u && addr <= 0x568u) || + (addr >= 0x3400u && addr <= 0x34fcu) || + (addr >= 0x3800u && addr <= 0x39fcu)) { + u32 v0 = 0; + + if (off + 8 <= len) + memcpy(&v0, buf + off + 4, 4); + lc += scnprintf(line + lc, sizeof(line) - lc, + "%s0x%x@0x%x(%08x)", + nhit ? "," : "", addr, off, v0); + nhit++; + } + off += (op == 1u || op == 3u || op == 5u) && cnt ? + (1u + cnt) * 4u : 4u; + } + if (nhit) { + svc_mme_census_n++; + pr_info("sema-mme-census: h=0x%x va=0x%llx len=%u n=%u hits=[%s]\n", + h_object, push_va, len, nhit, line); + } +} + +/* + * SESSION 153-CONTINUED-53: MME context-setup prelude buffer. + * + * The draw-window replay raises a real MME ILLEGAL_OPCODE + * (NV_PGRAPH_PRI_MME_HWW_ESR bit 2) on CALL_MME_MACRO(2)=0x3810 with + * mme_pc=0/mme_ir=1: the MME executed a macro call with no macro program + * loaded. The client DOES upload the program (the census records the + * 0x11c/0x114 pointer pairs and the 0x124 shadow-control runs), but the + * replay only ever forwards draw-window slices, so the ladder context + * never sees the setup. This buffer captures the setup runs verbatim - + * every method run whose id is in the C597 MME setup families: + * + * 0x114 LOAD_MME_INSTRUCTION_RAM_POINTER + * 0x118 LOAD_MME_INSTRUCTION_RAM (the macro code words) + * 0x11c LOAD_MME_START_ADDRESS_RAM_POINTER + * 0x120 LOAD_MME_START_ADDRESS_RAM (per-macro entry pc) + * 0x124 SET_MME_SHADOW_RAM_CONTROL (METHOD_TRACK[_WITH_FILTER]) + * 0x234 SET_MME_VERSION + * 0x550..0x568 MME mem address / data RAM / DMA / reduction + * 0x3400..0x34fc SET_MME_SHADOW_SCRATCH + * + * - in first-encounter order, so the booter can preplay them ahead of the + * first 3D window replay of a capture generation (see + * gsp_sema_mme_prelude_get and the gsp_phys_rmcall_gr_mme_prelude knob). + * Identical runs (a re-upload of the same program) are skipped by an + * exact-run signature table, so repeated setup pushes do not grow the + * buffer; genuinely new content appends and bumps the generation so the + * next window re-preplays. The writer is the single service thread and + * appends strictly in order; the generation only increments after a + * capture that appended, giving the booter a seqcheck against torn views. + */ +#define SVC_MME_PRELUDE_WORDS 512u +#define SVC_MME_PRELUDE_SIGS 256u +static u32 svc_mme_prelude_buf[SVC_MME_PRELUDE_WORDS]; +static u32 svc_mme_prelude_n; +static u32 svc_mme_prelude_gen; +static u64 svc_mme_prelude_sigs[SVC_MME_PRELUDE_SIGS]; +static u32 svc_mme_prelude_sig_n; +static u32 svc_mme_prelude_dump_n; +static u32 svc_mme_prelude_flag = 1; +/* + * SESSION 153-CONTINUED-57: macro-program payload words seen by the + * capture (sum of the 0x114 LOAD_MME_INSTRUCTION_RAM run counts). The + * r1094 failure mode was a launch emitted with the capture still at 9 + * words (scratch only, zero code words): the MME executed macro(2) with + * pc=0 -> ILLEGAL_OPCODE -> poison 0.6 s before the client's real setup + * push became readable. gsp_sema_mme_prelude_mature() lets the forward + * path gate the first beef launch on the program actually being present. + */ +static u32 svc_mme_prelude_code_words; +module_param_named(sema_mme_prelude_cap, svc_mme_prelude_flag, uint, 0644); +MODULE_PARM_DESC(sema_mme_prelude_cap, + "Capture the client's MME context-setup method runs (0x114..0x124, 0x234, 0x550..0x568, 0x3400..0x34fc) from trusted beef pushes for the GR launch builder's one-time preplay (default 1)"); + +static bool svc_mme_setup_run_addr(u32 addr) +{ + return (addr >= 0x114u && addr <= 0x124u) || addr == 0x234u || + (addr >= 0x550u && addr <= 0x568u) || + (addr >= 0x3400u && addr <= 0x34fcu); +} + +static u64 svc_mme_run_sig(const u8 *p, u32 span, u32 hdr) +{ + u64 h = 1469598103934665603ULL ^ hdr; + u32 i; + + for (i = 1u; i < span; i++) { + u32 v; + + memcpy(&v, p + 4u * i, 4); + h ^= v; + h *= 1099511628211ULL; + } + return h; +} + +const u32 *gsp_sema_mme_prelude_get(u32 *n, u32 *gen) +{ + if (n) + *n = READ_ONCE(svc_mme_prelude_n); + if (gen) + *gen = READ_ONCE(svc_mme_prelude_gen); + return svc_mme_prelude_buf; +} + +static void svc_mme_prelude_capture(const u8 *buf, u32 len, u64 push_va, + u32 h_object) +{ + u32 off = 0, added = 0, dups = 0, full = 0; + u32 words0 = svc_mme_prelude_n; + + if (!READ_ONCE(svc_mme_prelude_flag)) + return; + if ((h_object & 0xfffff000u) != 0xbeef0000u) + return; + while (off + 4u <= len) { + u32 hdr, op, cnt, addr, span = 1u, i; + u64 sig; + bool dup = false; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 7u; + cnt = (hdr >> 16) & 0xfffu; + addr = (hdr & 0x1fffu) << 2; + if ((op == 1u || op == 3u || op == 5u) && cnt) { + if ((u64)off + 4u + (u64)cnt * 4u > (u64)len) + break; + span = 1u + cnt; + } + if (!svc_mme_setup_run_addr(addr)) { + off += span * 4u; + continue; + } + sig = svc_mme_run_sig(buf + off, span, hdr); + for (i = 0; i < svc_mme_prelude_sig_n; i++) { + if (svc_mme_prelude_sigs[i] == sig) { + dup = true; + break; + } + } + if (dup) { + dups++; + off += span * 4u; + continue; + } + if (svc_mme_prelude_n + span > SVC_MME_PRELUDE_WORDS || + svc_mme_prelude_sig_n >= SVC_MME_PRELUDE_SIGS) { + full++; + off += span * 4u; + continue; + } + memcpy(&svc_mme_prelude_buf[svc_mme_prelude_n], buf + off, + span * 4u); + /* + * SESSION 153-CONTINUED-57: count the macro-program payload - + * the 0x114 instruction-RAM uploads are the only runs that + * make the prelude functionally complete; the maturity gate + * keys on them. + */ + if (addr == 0x114u && span > 1u) + svc_mme_prelude_code_words += span - 1u; + svc_mme_prelude_n += span; + svc_mme_prelude_sigs[svc_mme_prelude_sig_n++] = sig; + added++; + off += span * 4u; + } + if (added) { + WRITE_ONCE(svc_mme_prelude_gen, svc_mme_prelude_gen + 1); + if (svc_mme_prelude_dump_n < 4) { + u32 k; + + svc_mme_prelude_dump_n++; + pr_info("sema-mme-prelude: capture h=0x%x va=0x%llx len=%u added=%u words=%u gen=%u (was %u)\n", + h_object, push_va, len, added, + svc_mme_prelude_n, svc_mme_prelude_gen, + words0); + for (k = 0; k + 4u <= svc_mme_prelude_n && + k < 512u; k += 4u) + pr_info("sema-mme-prelude[%03u] %08x %08x %08x %08x\n", + k, svc_mme_prelude_buf[k], + svc_mme_prelude_buf[k + 1], + svc_mme_prelude_buf[k + 2], + svc_mme_prelude_buf[k + 3]); + } + if (full) + pr_info("sema-mme-prelude: buffer full=%u words=%u dup=%u\n", + full, svc_mme_prelude_n, dups); + } else if (dups && svc_mme_prelude_dump_n < 8) { + svc_mme_prelude_dump_n++; + pr_info("sema-mme-prelude: re-upload all-duplicate h=0x%x va=0x%llx dup=%u words=%u\n", + h_object, push_va, dups, svc_mme_prelude_n); + } +} + +/* + * SESSION 153-CONTINUED-57: is the captured MME setup complete enough to + * launch against? The r1091 (works) / r1094 (poisons) A/B: a mature capture + * holds the client's macro program (>= 64 code words = at least four + * 0x114 uploads); the premature r1094 capture held 9 scratch-only words + * and the launch raised MME ILLEGAL_OPCODE within milliseconds. The gate + * in svc_gr_request defers beef forwards until this reads true (bounded by + * sema_gr_prelude_gate_ms so a client that never uploads cannot livelock + * the forward path). + */ +bool gsp_sema_mme_prelude_mature(void) +{ + return READ_ONCE(svc_mme_prelude_code_words) >= 64u; +} + +/* + * SESSION 153-CONTINUED-57: attr/pipeline state prelude. + * + * The full-MME-prelude boot (r1091) cleared the MME ILLEGAL_OPCODE but the + * first beef window then faulted the draw shader: NV_PGRAPH_PRI_DS_HWW_ESR + * (0x405840) latched 0xa2040a00 = SPH9|SPH11|SPH18 pending with + * M_PIPE_STAGE = VSB|PS, i.e. draws launched against shader-program-header + * slots whose SET_PIPELINE_PROGRAM_ADDRESS binds live in windows that were + * NEVER replayed (mid-stream state cut), and the same window then raised a + * class error at SET_VERTEX_ATTRIBUTE_A(2) (0x1168, INVALID_ENUM) - the + * window rebinds SOME attribute/pipeline state (byte-faithful, verified) + * but the surrounding record the client set earlier is missing. This + * buffer captures every trusted beef push's method runs in the C597 + * attribute/pipeline families, first-encounter order: + * + * 0x1100..0x11fc attribute formats, skip masks, vertex attribute A/B, + * anti-alias sample positions + * 0x2000..0x205c SET_PIPELINE_SHADER/REGISTER_COUNT/PROGRAM_ADDRESS/ + * BINDING - the per-stage pipeline record + * + * The launch builder preplays the captured runs (run-aligned, relocation + * applied exactly like the window itself) ahead of the first 3D window + * replay of a capture generation. The SET_PIPELINE_PROGRAM_ADDRESS_A/B + * operand VAs the runs bind are also recorded (svc_attr_prelude_ptr) so + * svc_gr_req_fill can put their pages into the launch page set - without + * them the shadow relocation MISSes the pair, the emitted address stays a + * raw client VA, and the DS cannot fetch the SPH through it. Identical + * re-uploads are deduped by an exact-run signature table (same scheme as + * the MME prelude). + */ +#define SVC_ATTR_PRELUDE_WORDS 512u +#define SVC_ATTR_PRELUDE_SIGS 256u +#define SVC_ATTR_PRELUDE_PTRS 8u +static u32 svc_attr_prelude_buf[SVC_ATTR_PRELUDE_WORDS]; +static u32 svc_attr_prelude_n; +static u32 svc_attr_prelude_gen; +static u64 svc_attr_prelude_sigs[SVC_ATTR_PRELUDE_SIGS]; +static u32 svc_attr_prelude_sig_n; +static u32 svc_attr_prelude_dump_n; +static u32 svc_attr_prelude_flag = 1; +static u64 svc_attr_prelude_ptr[SVC_ATTR_PRELUDE_PTRS]; +static u32 svc_attr_prelude_ptr_n; +module_param_named(sema_attr_prelude_cap, svc_attr_prelude_flag, uint, 0644); +MODULE_PARM_DESC(sema_attr_prelude_cap, + "Capture the client's attr/pipeline method runs (0x1100..0x11fc, 0x2000..0x205c) from trusted beef pushes for the GR launch builder's preplay ahead of the first 3D window (default 1)"); + +static bool svc_attr_setup_run_addr(u32 addr) +{ + return (addr >= 0x1100u && addr < 0x1200u) || + (addr >= 0x2000u && addr < 0x2060u); +} + +const u32 *gsp_sema_attr_prelude_get(u32 *n, u32 *gen) +{ + if (n) + *n = READ_ONCE(svc_attr_prelude_n); + if (gen) + *gen = READ_ONCE(svc_attr_prelude_gen); + return svc_attr_prelude_buf; +} + +const u64 *gsp_sema_attr_prelude_ptrs(u32 *n) +{ + if (n) + *n = READ_ONCE(svc_attr_prelude_ptr_n); + return svc_attr_prelude_ptr; +} + +static void svc_attr_prelude_note_ptr(u64 va) +{ + u32 i; + + if (!va || svc_attr_prelude_ptr_n >= SVC_ATTR_PRELUDE_PTRS) + return; + for (i = 0; i < svc_attr_prelude_ptr_n; i++) + if (svc_attr_prelude_ptr[i] == va || + (svc_attr_prelude_ptr[i] & ~(u64)0xfffu) == + (va & ~(u64)0xfffu)) + return; + svc_attr_prelude_ptr[svc_attr_prelude_ptr_n++] = va; + /* + * SESSION 153-CONTINUED-59: demand the program page NOW. The r1104 + * first beat window MISSed its 0x2054 pair operand (va=0x4350900) + * because the page did not resolve at the launch fill; the engine + * read it raw -> MMU Fault HUBCLIENT_PD @0x4350000 -> the guest + * driver reset the channel and the lane died for the boot. Recording + * the page in the pin-demand set here (at capture time, often + * minutes before the first launch) lets pin_gr_stream demand it in + * the guest so the launch fill's note resolves. See also the + * sema_attr_settle_ms launch gate. + */ + svc_setup_demand_record(va & ~(u64)0xfffu); +} + +static void svc_attr_prelude_capture(const u8 *buf, u32 len, u64 push_va, + u32 h_object) +{ + u32 off = 0, added = 0, dups = 0, full = 0; + u32 words0 = svc_attr_prelude_n; + + if (!READ_ONCE(svc_attr_prelude_flag)) + return; + if ((h_object & 0xfffff000u) != 0xbeef0000u) + return; + while (off + 4u <= len) { + u32 hdr, op, cnt, addr, span = 1u, i; + u64 sig; + bool dup = false; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 7u; + cnt = (hdr >> 16) & 0xfffu; + addr = (hdr & 0x1fffu) << 2; + if ((op == 1u || op == 3u || op == 5u) && cnt) { + if ((u64)off + 4u + (u64)cnt * 4u > (u64)len) + break; + span = 1u + cnt; + } + if (!svc_attr_setup_run_addr(addr)) { + off += span * 4u; + continue; + } + /* + * SET_PIPELINE_PROGRAM_ADDRESS_A/B(j) pair [upper(7:0), + * lower(31:0)]: remember the program VA so its page can join + * the launch page set (see the header comment). + */ + if (addr >= 0x2014u && addr < 0x2060u && + (addr & 0x3fu) == 0x14u && span >= 3u) { + u32 v0 = 0, v1 = 0; + + memcpy(&v0, buf + off + 4, 4); + memcpy(&v1, buf + off + 8, 4); + svc_attr_prelude_note_ptr(((u64)(v0 & 0xffu) << 32) | + (u64)(v1 & 0xfffffffcu)); + } + sig = svc_mme_run_sig(buf + off, span, hdr); + for (i = 0; i < svc_attr_prelude_sig_n; i++) { + if (svc_attr_prelude_sigs[i] == sig) { + dup = true; + break; + } + } + if (dup) { + dups++; + off += span * 4u; + continue; + } + if (svc_attr_prelude_n + span > SVC_ATTR_PRELUDE_WORDS || + svc_attr_prelude_sig_n >= SVC_ATTR_PRELUDE_SIGS) { + full++; + off += span * 4u; + continue; + } + memcpy(&svc_attr_prelude_buf[svc_attr_prelude_n], buf + off, + span * 4u); + svc_attr_prelude_n += span; + svc_attr_prelude_sigs[svc_attr_prelude_sig_n++] = sig; + added++; + off += span * 4u; + } + if (added) { + WRITE_ONCE(svc_attr_prelude_gen, svc_attr_prelude_gen + 1); + if (svc_attr_prelude_dump_n < 4) { + u32 k; + + svc_attr_prelude_dump_n++; + pr_info("sema-attr-prelude: capture h=0x%x va=0x%llx len=%u added=%u words=%u gen=%u ptr_n=%u (was %u)\n", + h_object, push_va, len, added, + svc_attr_prelude_n, svc_attr_prelude_gen, + svc_attr_prelude_ptr_n, words0); + for (k = 0; k + 4u <= svc_attr_prelude_n && + k < 512u; k += 4u) + pr_info("sema-attr-prelude[%03u] %08x %08x %08x %08x\n", + k, svc_attr_prelude_buf[k], + svc_attr_prelude_buf[k + 1], + svc_attr_prelude_buf[k + 2], + svc_attr_prelude_buf[k + 3]); + } + if (full) + pr_info("sema-attr-prelude: buffer full=%u words=%u dup=%u\n", + full, svc_attr_prelude_n, dups); + } +} + static void svc_push_raw_dump(const u8 *buf, u32 len, u64 push_va, u32 h_object) { @@ -14410,6 +15098,24 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, len >= 16 && len < 512) svc_push_raw_dump(buf, len, push_va, ch->h_object); + /* SESSION 153-CONTINUED-52: MME-family census for the beef channel. */ + if (ch) + svc_mme_census(buf, len, push_va, ch->h_object); + + /* + * SESSION 153-CONTINUED-53: MME context-setup prelude capture. Only a + * trusted resolution feeds the buffer (the bytes are replayed into a + * hardware context); identical re-uploads are deduped inside. + */ + if (ch && trusted) + svc_mme_prelude_capture(buf, len, push_va, ch->h_object); + /* + * SESSION 153-CONTINUED-57: attr/pipeline state prelude capture (same + * trusted-only rule; the runs are replayed into a hardware context). + */ + if (ch && trusted) + svc_attr_prelude_capture(buf, len, push_va, ch->h_object); + memset(&st, 0, sizeof(st)); st.push_va = push_va; /* @@ -18857,6 +19563,101 @@ static u64 svc_push_pa_ceiling(void) return top; } +/* + * SESSION 153-CONTINUED-51: deterministic push-page resolution. + * + * The direct GL client's 5-7 KB window pushes routinely read back all + * zero through the trusted chain on some boots ("sema-pushzero ... + * rescued=0", r1033-class) while the SAME VA pages answer real content + * on other boots - the learned cache (lat) can shadow a page-table walk + * that has since moved, and the client reuses/reallocates its push VAs + * across frames. The launch-page chain (svc_gr_page_note) resolves + * through the ledger then a FRESH walk; the push read chain consults the + * learned cache FIRST, so a stale lat/pin entry shadows the fresh walk + * forever. When the trusted read answered an all-zero page, consult the + * same ledger/walk chain the launch pages use and, when a DIFFERENT page + * with non-zero content answers, adopt it AND heal the read sources + * (validated pin + lat) so every later read of this page is deterministic + * instead of a per-boot lottery. + * + * Only a page whose content differs from all-zero is adopted: a + * legitimately-zero page (padding) must never steal a correct + * translation. Returns 0 when a replacement page was adopted (pa_out / + * ap_out updated and `p` refilled), -ENOENT when nothing better exists. + */ +static int svc_push_page_heal(struct nv_vgpu_device *vgpu, u32 h_object, + u64 va, u64 net_pa, int net_ap, u8 *p, + size_t chunk, u64 *pa_out, int *ap_out) +{ + u64 page_va = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 off = va & (SEMA_PAGE_SIZE - 1); + u64 cur_page = net_pa & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 lpa = 0, wpa = 0; + u32 lap = 0; + int wap = SVC_APERTURE_SYS; + static u32 heal_log; + + /* 1. The ledger: the launch-page chain's landing authority. A page a + * host write or an earlier launch resolution landed on is the + * page the client's own chain agreed about - the read chain does + * not consult it at all. */ + if (!svc_ledger_lookup(page_va, &lpa, &lap) && + (lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != cur_page) { + u32 map_ap = svc_ap_of((int)lap); + + if (svc_read_ap(vgpu, lpa + off, + map_ap == SVC_APERTURE_FB ? + SVC_APERTURE_FB : SVC_APERTURE_SYS, + p, chunk) == 0 && + memchr_inv(p, 0, chunk)) { + svc_pin_store(page_va, lpa & ~(u64)(SEMA_PAGE_SIZE - 1), + (int)map_ap); + svc_lat_store(va, lpa + off, (int)map_ap); + *pa_out = lpa + off; + *ap_out = (int)map_ap; + if (heal_log < 64) { + heal_log++; + pr_info("sema-pushheal: h=0x%x va=0x%llx ledger pa=0x%llx (was 0x%llx, zero)\n", + h_object, (unsigned long long)va, + (unsigned long long)(lpa + off), + (unsigned long long)net_pa); + } + return 0; + } + } + + /* 2. A FRESH walk, bypassing the learned cache: the guest may have + * remapped the VA after the lat entry was written. */ + if (svc_walk_vaspaces(vgpu, page_va, &wpa, &wap) == 0 && + wpa != 0 && (wpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != cur_page) { + u32 map_ap = svc_ap_of(wap); + + if (svc_read_ap(vgpu, wpa + off, + map_ap == SVC_APERTURE_FB ? + SVC_APERTURE_FB : SVC_APERTURE_SYS, + p, chunk) == 0 && + memchr_inv(p, 0, chunk)) { + svc_pin_store(page_va, wpa & ~(u64)(SEMA_PAGE_SIZE - 1), + (int)map_ap); + svc_lat_store(va, wpa + off, (int)map_ap); + *pa_out = wpa + off; + *ap_out = (int)map_ap; + if (heal_log < 64) { + heal_log++; + pr_info("sema-pushheal: h=0x%x va=0x%llx walk pa=0x%llx (was 0x%llx, zero)\n", + h_object, (unsigned long long)va, + (unsigned long long)(wpa + off), + (unsigned long long)net_pa); + } + return 0; + } + } + /* Nothing better: restore the zero state exactly (the caller's read + * filled p with zeros and must see zeros back). */ + memset(p, 0, chunk); + return -ENOENT; +} + static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 va, void *buf, size_t len, bool skip_ipin) @@ -19037,11 +19838,12 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, svc_read_last_pa = rpa; svc_lat_store(page_va, ppa, pap); - if (pushzero_log < 32) { + if (pushzero_log < 1024) { pushzero_log++; - pr_info("sema-pushzero: h=0x%x va=0x%llx src=%u pa=0x%llx promo pa=0x%llx rescued=1\n", + pr_info("sema-pushzero: h=0x%x va=0x%llx len=%u src=%u pa=0x%llx promo pa=0x%llx rescued=1\n", ch->h_object, (unsigned long long)va, + (unsigned)len, src, (unsigned long long)pa, (unsigned long long)ppa); @@ -19050,25 +19852,58 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, /* promo page unusable: put the * zero state back exactly. */ memset(p, 0, chunk); - if (!rrc && pushzero_log < 32) { + if (!rrc && pushzero_log < 1024) { pushzero_log++; - pr_info("sema-pushzero: h=0x%x va=0x%llx src=%u pa=0x%llx promo pa=0x%llx rescued=0 (promo content zero)\n", + pr_info("sema-pushzero: h=0x%x va=0x%llx len=%u src=%u pa=0x%llx promo pa=0x%llx rescued=0 (promo content zero)\n", ch->h_object, (unsigned long long)va, + (unsigned)len, src, (unsigned long long)pa, (unsigned long long)ppa); } } - } else if (pushzero_log < 32) { + } else if (pushzero_log < 1024) { pushzero_log++; - pr_info("sema-pushzero: h=0x%x va=0x%llx src=%u pa=0x%llx rescued=0 (no distinct promo booking)\n", + pr_info("sema-pushzero: h=0x%x va=0x%llx len=%u src=%u pa=0x%llx rescued=0 (no distinct promo booking)\n", ch->h_object, - (unsigned long long)va, src, + (unsigned long long)va, + (unsigned)len, src, (unsigned long long)pa); } } /* + * SESSION 153-CONTINUED-51: when the zero-content + * rescue above did not produce non-zero bytes, the + * page must still be resolved deterministically - + * run the SAME ledger/fresh-walk chain the launch + * pages use (svc_gr_page_note) for this page. A + * different page with non-zero content is adopted, + * tagged src=10, and pinned so every later read of + * the page (continuation chunks and future frames) + * answers from the healed source instead of the + * learned cache. This is the r1069 class: every + * window push read zeros for a whole boot while the + * launch pages resolved fine. + */ + if (src >= 2 && + (ch->h_object & 0xffff0000u) == 0xbeef0000u && + (first ? len >= 512u : chunk >= 512u) && + !memchr_inv(p, 0, chunk)) { + u64 hpa = pa; + int hap = ap; + + if (!svc_push_page_heal(vgpu, ch->h_object, + va, pa, ap, p, + chunk, &hpa, &hap)) { + pa = hpa; + svc_read_last_src = 10; + svc_read_last_pa = hpa; + } + /* Not healed: the original (zero) source and + * its label stay; nothing was adopted. */ + } + /* * r820: name whether the trusted answer came from a * driver-declared PROMOTE_CTX buffer; the decode uses * it for the zero-NOP acceptance (see the flag). @@ -21591,8 +22426,126 @@ unsigned gsp_sema_ce_grace_ms = 25000; */ unsigned gsp_sema_push_hold_ms = 2500; +/* + * SESSION 153-CONTINUED-54: GR setup-push hold. + * + * The client's 12288 B C597 MME-setup push ("LOAD_MME_* program upload") + * reads unreadable/all-zero on the first-read lottery boots and used to be + * consumed ("skipping stale entry ... after 1 failed rescues") before the + * demand channel's injected pin could land, so the MME prelude capture + * missed the macro program for the whole boot. When such a push's first + * read fails, demand every 4K VA page of the push through the stalled-VA + * demand channel (pin_gr_stream harvests the rows and the guest resolves + + * injects the pages) and hold the entry pending PAST the rescue-consume + * until the pin lands (the cheap-wait guard re-reads when it does) or this + * window expires. Bounded by an absolute per-(va,slot) deadline; 0 + * disables the hold (the demand rows are still emitted). + * + * r1083-r1087: DEFAULT 0 (off). A 20 s hold blocks the whole ring behind + * the held entry, and on the three hold-active boots (r1083/r1084/r1085) + * the GL client derailed into the black-dud/churn class, while the r1087 + * boot (hold=0) read the beef windows fine (the wall moved back to the + * incomplete MME prelude + dispatch stall). The demand channel also cannot + * yet resolve the GL pushbuffer VA families (0x4000000+ / 0x42xxxxx), so + * the pin the hold waits for never lands and the hold only delays. Re-enable + * (e.g. sema_gr_setup_hold_ms=20000) once the guest content resolver covers + * those VA families. + */ +unsigned gsp_sema_gr_setup_hold_ms; + static unsigned long svc_gr_ring_tried; /* ranks that got a diagnostic line */ +/* + * Is this a GR push whose readability the MME-setup capture depends on? + * The client's C597 setup push is 12288 B; any GR push of that size is a + * candidate. A smaller push qualifies only when it is READABLE and its + * method runs hit the MME setup families (the unreadable case has no bytes + * to scan, hence the length rule alone). + */ +static bool svc_gr_setup_push(u32 len, const u8 *buf) +{ + u32 off = 0; + + if (len >= 12288u) + return true; + if (!buf) + return false; + while (off + 4u <= len) { + u32 hdr, op, cnt, addr; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 7u; + cnt = (hdr >> 16) & 0xfffu; + addr = (hdr & 0x1fffu) << 2; + if ((op == 1u || op == 3u || op == 5u) && cnt && + svc_mme_setup_run_addr(addr)) + return true; + off += ((op == 1u || op == 3u || op == 5u) && cnt) ? + (1u + cnt) * 4u : 4u; + } + return false; +} + +/* + * Demand every 4K page of the push and arm the per-(va,slot) hold. The + * demand is the existing stalled-VA channel: svc_log_stalled_va rows are + * deduped and harvested by tools/guest-diag/pin_gr_stream.py, which asks + * the guest scanner to resolve the VA set and inject the pages. Only one + * arm line per (va,slot) is logged. The loop is bounded by `page_cap` (the + * 12288 B setup push needs 3; the cap keeps a bulk push from eating the + * 256-entry stalled-VA budget). + * + * SESSION 153-CONTINUED-55: the cap was 0x8000 (8 pages). The GL client's + * observed setup push is 60216 B (16 pages); pages 1..8 were demanded but + * pages 9..15 never were, so the host's per-chunk read of the entry had no + * source for the tail chunks and the whole read failed (unreadable=1, the + * hold re-armed forever). 0x10000 covers the full 60216 B push within the + * stalled-VA budget. + */ +static void svc_gr_setup_hold_arm(struct svc_channel *ch, u64 va, u32 idx, + u32 len) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 poff; + + if (!va || !gsp_sema_gr_setup_hold_ms) + return; + if (ch->gr_setup_hold_va != va || ch->gr_setup_hold_idx != idx) { + ch->gr_setup_hold_va = va; + ch->gr_setup_hold_idx = idx; + ch->gr_setup_hold_until = + jiffies + msecs_to_jiffies(gsp_sema_gr_setup_hold_ms); + pr_info("sema-chan: GR setup hold h=0x%x entry=%u va=0x%llx len=%u armed (demanding 4K pages; pending until the pin lands, window %u ms)\n", + ch->h_object, idx, va, len, gsp_sema_gr_setup_hold_ms); + } + for (poff = 0; poff < len && poff < 0x10000u; poff += SEMA_PAGE_SIZE) { + svc_log_stalled_va(page + poff); + /* + * SESSION 153-CONTINUED-56: also publish the page in the + * debugfs demand set the pin tool reads directly, so the + * demand no longer depends on the dmesg row surviving the + * ring rotation under vgsp-sema spam. + */ + svc_setup_demand_record(page + poff); + } +} + +static bool svc_gr_setup_hold_active(const struct svc_channel *ch, u64 va, + u32 idx) +{ + return ch->gr_setup_hold_until && + ch->gr_setup_hold_va == va && + ch->gr_setup_hold_idx == idx && + time_before(jiffies, ch->gr_setup_hold_until); +} + +static void svc_gr_setup_hold_clear(struct svc_channel *ch) +{ + ch->gr_setup_hold_until = 0; + ch->gr_setup_hold_va = 0; + ch->gr_setup_hold_idx = ~0u; +} + static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 *pa_out) { @@ -25118,6 +26071,216 @@ struct svc_fwd_req { static struct svc_fwd_req svc_fwd_req; +/* + * SESSION 153-CONTINUED-61: MISS-ABORT re-arm. The pre-doorbell abort + * (GSP_PHYS_GR_RC_MISSABORT) fires when a 3D-window operand's page is still + * unresolved; the page is demanded through the pin path and resolves within + * the pin cycle (tens of seconds). The historical handling consumed the + * captured candidate and software-completed the client (releases + QMD + * word), so no later forward ever re-ran that chunk and a store-bearing + * window never reached the engine (CONT-60). With the re-arm the candidate + * is copied back into the candidate ring unoffered, the client's releases + * are held back, and a bounded retry tick (svc_gr_missre_tick, called from + * svc_pump_channels) re-offers the same chunk; once the pins land the real + * launch runs and the engine (hold success edge) authors the releases. + * Bounded: after SVC_GR_MISSRE_DEAD_MS / SVC_GR_MISSRE_MAX_TRIES the + * historical consume-with-software-completion path stands so a page that + * never resolves cannot hang the client. Service-thread only. + */ +#define SVC_GR_MISSRE_INT_MS 1500u +#define SVC_GR_MISSRE_DEAD_MS 55000u +#define SVC_GR_MISSRE_MAX_TRIES 64u +static struct { + bool active; + u64 push_va; + u32 h_object; + u32 tries; + unsigned long first_at; + unsigned long last_try; +} svc_gr_missre; +static u32 svc_gr_missre_log; +static u64 svc_gr_missre_arm_n, svc_gr_missre_ok_n, svc_gr_missre_exp_n; + +static bool svc_gr_request(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch); + +/* + * Episode complete: the re-armed launch ran for real. Clears the retry + * state so the tick stops re-offering. Called from the success edges + * (svc_fwd_outcome_check / svc_fwd_hold_poll). + */ +static void svc_gr_missre_done(void) +{ + if (!svc_gr_missre.active || + svc_gr_missre.push_va != svc_fwd_req.gr_push_va) + return; + svc_gr_missre.active = false; + svc_gr_missre_ok_n++; + pr_info("vgsp-gr: MISS-RETRY converged push=0x%llx h=0x%x tries=%u ok=%llu (real launch ran after the pins landed)\n", + svc_fwd_req.gr_push_va, svc_fwd_req.gr_h_object, + svc_gr_missre.tries, svc_gr_missre_ok_n); +} + +/* + * Copy the just-aborted request's captured launch back into the candidate + * ring as an unoffered candidate (the re-arm). Returns false when the + * episode bound has expired or the ring has no free slot; the caller then + * falls through to the historical software completion. `rel`/`nrel` are + * the attempt's push-level release records (restored from the park table + * by svc_gr_request and drained by its caller): they are re-parked so the + * retried request restores the same set - the GPFIFO entry is consumed + * with this edge and will not re-decode. + */ +static bool svc_gr_missre_arm(const struct svc_fwd_rel *rel, u8 nrel) +{ + struct svc_gr_cand_rec *c = NULL; + unsigned long now = jiffies; + u32 slot; + + if (!svc_gr_missre.active || + svc_gr_missre.push_va != svc_fwd_req.gr_push_va || + svc_gr_missre.h_object != svc_fwd_req.gr_h_object) { + svc_gr_missre.active = true; + svc_gr_missre.push_va = svc_fwd_req.gr_push_va; + svc_gr_missre.h_object = svc_fwd_req.gr_h_object; + svc_gr_missre.tries = 0; + svc_gr_missre.first_at = now; + } + if (svc_gr_missre.tries >= SVC_GR_MISSRE_MAX_TRIES || + time_after(now, svc_gr_missre.first_at + + msecs_to_jiffies(SVC_GR_MISSRE_DEAD_MS))) { + u32 ri; + + svc_gr_missre_exp_n++; + pr_warn("vgsp-gr: MISS-RETRY episode expired push=0x%llx h=0x%x tries=%u waited_ms=%lu (falling back to software completion)\n", + svc_gr_missre.push_va, svc_gr_missre.h_object, + svc_gr_missre.tries, + jiffies_to_msecs(now - svc_gr_missre.first_at)); + /* + * Drop the lingering armed slot(s) for this launch: an + * unoffered candidate surviving the episode could be offered + * much later with a stale window snapshot (the client has + * since reused its buffers - the double-run class). + */ + for (ri = 0; ri < SVC_GR_CAND_RING; ri++) { + struct svc_gr_cand_rec *cc = &svc_gr_cand_ring[ri]; + + if (cc->valid && !cc->offered && + cc->push_va == svc_gr_missre.push_va && + cc->h_object == svc_gr_missre.h_object) { + cc->offered = true; + cc->valid = false; + } + } + svc_gr_missre.active = false; + return false; + } + for (slot = 0; slot < SVC_GR_CAND_RING; slot++) { + if (!svc_gr_cand_ring[slot].valid || + svc_gr_cand_ring[slot].offered) { + c = &svc_gr_cand_ring[slot]; + break; + } + } + if (!c) + return false; /* ring full: fall back historically */ + memset(c, 0, sizeof(*c)); + c->valid = true; + c->offered = false; + c->rel_done = false; + c->captured_jiffies = now; + c->h_object = svc_fwd_req.gr_h_object; + c->push_va = svc_fwd_req.gr_push_va; + c->push_len = svc_fwd_req.gr_push_len; + c->push_n = svc_fwd_req.gr_push_n; + c->push_qmd_hdr = svc_fwd_req.gr_push_qmd_hdr; + c->push_qmd_end = svc_fwd_req.gr_push_qmd_end; + c->stream_3d = svc_fwd_req.gr_stream_3d; + c->stream_off = svc_fwd_req.gr_stream_off; + c->stream_n = svc_fwd_req.gr_stream_n; + memcpy(c->raw, svc_fwd_req.gr_raw, sizeof(c->raw)); + c->qmd_dev_va = svc_fwd_req.gr_qmd_dev_va; + memcpy(c->qmd, svc_fwd_req.gr_qmd, sizeof(c->qmd)); + c->n_inline = svc_fwd_req.gr_n_inline; + if (c->n_inline > SVC_GR_INLINE_MAX) + c->n_inline = SVC_GR_INLINE_MAX; + memcpy(c->inline_recs, svc_fwd_req.gr_inline, + sizeof(c->inline_recs)); + c->prog_va = svc_fwd_req.gr_prog_va; + c->param_va = svc_fwd_req.gr_param_va; + c->param_size = svc_fwd_req.gr_param_size; + c->rel_va = svc_fwd_req.gr_rel_va; + c->rel_payload = svc_fwd_req.gr_rel_payload; + c->rel_flags = svc_fwd_req.gr_rel_flags; + if (rel && nrel) + svc_grpark_put(svc_fwd_req.vgpu, svc_fwd_req.gr_h_object, + svc_fwd_req.gr_push_va, rel, nrel); + svc_gr_missre_arm_n++; + if (svc_gr_missre_log < 96) { + svc_gr_missre_log++; + pr_info("vgsp-gr: MISS-RETRY re-arm push=0x%llx h=0x%x slot=%u window=%u nrel=%u arm=%llu (candidate kept unoffered; retry tick re-offers after the pins)\n", + c->push_va, c->h_object, slot, c->stream_n, + (unsigned)nrel, svc_gr_missre_arm_n); + } + return true; +} + +/* + * The retry tick (called per pass from svc_pump_channels for the tracked + * channel): re-offer the re-armed candidate on a bounded cadence. Parks + * svc_gr_cand_pos on the tracked slot so svc_gr_request's oldest-unoffered + * scan selects exactly this candidate; svc_gr_request itself defers on + * busy/inflight and no-ops for channels without a matching candidate. + */ +static void svc_gr_missre_tick(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch) +{ + u32 ti; + + if (ch->engine_type || !svc_gr_missre.active || + svc_gr_missre.h_object != ch->h_object || + !gsp_phys_gr_forward_enabled() || + !time_after_eq(jiffies, svc_gr_missre.last_try + + msecs_to_jiffies(SVC_GR_MISSRE_INT_MS))) + return; + if (svc_gr_missre.tries >= SVC_GR_MISSRE_MAX_TRIES || + time_after(jiffies, svc_gr_missre.first_at + + msecs_to_jiffies(SVC_GR_MISSRE_DEAD_MS))) + return; /* the next abort edge expires the episode */ + svc_gr_missre.last_try = jiffies; + svc_gr_missre.tries++; + for (ti = 0; ti < SVC_GR_CAND_RING; ti++) { + struct svc_gr_cand_rec *cc = &svc_gr_cand_ring[ti]; + + if (cc->valid && !cc->offered && + cc->push_va == svc_gr_missre.push_va && + cc->h_object == svc_gr_missre.h_object) { + svc_gr_cand_pos = ti; + break; + } + } + if (svc_gr_missre_log < 192) { + svc_gr_missre_log++; + pr_info("vgsp-gr: MISS-RETRY tick try=%u push=0x%llx h=0x%x (re-offering the armed candidate)\n", + svc_gr_missre.tries, svc_gr_missre.push_va, + svc_gr_missre.h_object); + } + if (svc_gr_request(vgpu, ch)) { + /* + * A tick-queued request never gets an entry hold (the GPFIFO + * entry was consumed with the first abort - there is nothing + * left for the entry walk to hold). The per-push release + * records svc_gr_request just restored are re-parked so they + * survive the work item's lifetime; svc_fwd_outcome_check + * applies them when the retried launch's outcome lands. + */ + if (svc_fwd_defer_armed && svc_fwd_rel_n > 0) + svc_grpark_put(vgpu, ch->h_object, + svc_gr_missre.push_va, + svc_fwd_rel, svc_fwd_rel_n); + } +} + static struct workqueue_struct *svc_fwd_wq; static unsigned long svc_fwd_state; static u64 svc_fwd_requests; @@ -26637,8 +27800,57 @@ static void svc_fwd_outcome_check(struct nv_vgpu_device *vgpu) * semaphore and the client's QMD word is still behind (the same edge * the hold's success path covers for held requests). */ - if (svc_fwd_req.hw_rc == 0 && !svc_fwd_req.gr_rel_unobserved) - return; + if (svc_fwd_req.hw_rc == GSP_PHYS_GR_RC_MISSABORT) { + /* + * SESSION 153-CONTINUED-61: MISS-ABORT without a hold - keep + * the candidate armed for a real re-forward and hold the + * client's release back (it completes on the retried + * launch's own edges). Only the expired episode falls + * through to the historical software completion below. + */ + if (svc_gr_missre_arm(NULL, 0)) { + svc_fwd_req.gr_swrel_done = true; + return; + } + } + /* + * SESSION 153-CONTINUED-61: a tick-queued retried launch carries no + * entry hold, so its per-push release records survive in the park + * table (re-parked by svc_gr_missre_tick). Deliver them here on ANY + * final outcome - success, fault or the expired retry episode - so + * the client's tracking words always complete. Exactly-once: the + * park entry is dropped on delivery and the latch below stops any + * later pass from re-running this outcome. + */ + { + static struct svc_fwd_rel parked[SVC_FWD_REL_MAX]; + static struct svc_channel shim; /* single-threaded service + * path; BSS keeps the kernel + * stack safe */ + u8 pn = 0; + + if (svc_grpark_get(vgpu, svc_fwd_req.gr_h_object, + svc_fwd_req.gr_push_va, parked, &pn)) { + memset(&shim, 0, sizeof(shim)); + shim.valid = true; + shim.h_object = svc_fwd_req.gr_h_object; + svc_gr_copy_pend_run(vgpu); + svc_fwd_rel_apply_all(vgpu, &shim, parked, pn); + svc_grpark_drop(vgpu, svc_fwd_req.gr_h_object, + svc_fwd_req.gr_push_va); + svc.fwd_rel_applied++; + pr_info("vgsp-gr: MISS-RETRY records delivered h=0x%x push=0x%llx nrel=%u (no-entry retry completion)\n", + svc_fwd_req.gr_h_object, + svc_fwd_req.gr_push_va, (unsigned)pn); + } + } + if (svc_fwd_req.hw_rc == 0) { + svc_gr_missre_done(); + if (!svc_fwd_req.gr_rel_unobserved) { + svc_fwd_req.gr_swrel_done = true; + return; + } + } /* * The outcome is final; latch it first so no later pass, hold edge or * consume tail accounts it again (the release write itself is @@ -26662,7 +27874,19 @@ static void svc_fwd_outcome_check(struct nv_vgpu_device *vgpu) svc_fwd_req.gr_rel_va, svc_fwd_req.gr_rel_payload); svc.fwd_gr_stall_swrel++; - if (svc_fwd_req.hw_rc != 0) { + if (svc_fwd_req.hw_rc == -EBADMSG) { + /* + * SESSION 153-CONTINUED-60: the backend skipped an all-zero 3D + * window (ZEROWIN-SKIP) - nothing was submitted, so there is no + * engine error to report. Author the software release and do + * NOT raise the synthetic fault: the client sees a clean + * completion for a frame whose window had no content, and the + * engine is never handed the zero method stream that used to + * poison it (trap 0x1168 -> grfifo=0xbadf1201, r1107). + */ + pr_warn("vgsp-gr: GR-STALL-backend va=0x%llx waited_ms=%lu action=proceed (zero window skipped; QMD release authored in software, no synth fault)\n", + svc_fwd_req.gr_push_va, waited); + } else if (svc_fwd_req.hw_rc != 0) { pr_warn("vgsp-gr: GR-STALL-backend va=0x%llx waited_ms=%lu action=proceed (failed forwarded launch, rc=%d; QMD release authored in software)\n", svc_fwd_req.gr_push_va, waited, svc_fwd_req.hw_rc); /* @@ -26937,6 +28161,20 @@ static void svc_fwd_hold_expire_check(struct nv_vgpu_device *vgpu, done = smp_load_acquire(&svc_fwd_req.done); hv = svc_fwd_hold.vgpu ? svc_fwd_hold.vgpu : vgpu; /* + * SESSION 153-CONTINUED-61: a MISS-ABORT hold expiring with the + * outcome known takes the re-arm edge instead of the software + * completion (the same verdict svc_fwd_hold_poll applies); only an + * episode past its bound falls through to the historical authorship + * below. + */ + if (done && svc_fwd_req.gr && + svc_fwd_req.hw_rc == GSP_PHYS_GR_RC_MISSABORT && + svc_gr_missre_arm(svc_fwd_hold.rel, svc_fwd_hold.nrel)) { + svc_fwd_req.gr_swrel_done = true; + svc_fwd_hold_clear(); + return; + } + /* * P6c C3: a relayed hold whose work item already published a timeout * gets the decoded engine fault delivered before the software releases * land (the late-release recheck inside the helper protects against a @@ -27008,6 +28246,7 @@ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu, if (!done && !expired && !aborted) return false; if (done && svc_fwd_req.hw_rc == 0) { + svc_gr_missre_done(); /* * P6c: on the relay success edge, re-read the guest-visible * semaphore through the software view. Only a word that @@ -27055,6 +28294,23 @@ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu, svc_fwd_hold.nrel); svc.fwd_rel_applied++; } + } else if (done && svc_fwd_req.gr && + svc_fwd_req.hw_rc == GSP_PHYS_GR_RC_MISSABORT && + svc_gr_missre_arm(svc_fwd_hold.rel, svc_fwd_hold.nrel)) { + /* + * SESSION 153-CONTINUED-61: the launch aborted pre-doorbell + * because a window operand's page is still unresolved. Do NOT + * software-complete it: the candidate is re-armed for a real + * re-forward once the pin demand resolves the page (the retry + * tick re-offers it) and the client's releases stay unauthored + * (the retried launch's own edges author them on success). + * The attempt's push-level records were re-parked by the arm. + * Bounded by the episode deadline; an expired episode falls + * into the historical failure path below. gr_swrel_done + * latches the request outcome so the pass-top check cannot + * author the release later. + */ + svc_fwd_req.gr_swrel_done = true; } else { /* * P6b wave 2: failure/timeout/abort fallback - author the @@ -30039,6 +31295,39 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, for (i = 0; i < svc_gr_cand.n_inline; i++) svc_gr_page_note(vgpu, svc_gr_cand.inline_recs[i].dst_va, SVC_GR_QMD_WORD_NONE); + /* + * SESSION 153-CONTINUED-57: attr-prelude operand pages. The captured + * attr/pipeline prelude (see svc_attr_prelude_capture) binds + * SET_PIPELINE_PROGRAM_ADDRESS pairs whose VAs were set in earlier + * windows; the launch builder relocates the emitted pairs to + * shadow-pool VAs - which requires each page to be IN this page set, + * or gsp_phys_rmcall_gr_3d_reloc MISSes and the emitted address stays + * a raw client VA the DS cannot fetch an SPH through. Append them + * (deduped by page inside svc_gr_page_note), soft: an unresolved or + * overflowing page is rolled back so the fill's ENOENT/overflow + * checks cannot refuse the launch over a prelude nicety. + */ + if ((ch->h_object & 0xfffff000u) == 0xbeef0000u) { + u32 apn = 0, ai; + const u64 *aps = gsp_sema_attr_prelude_ptrs(&apn); + static u32 svc_attr_ptr_log; + + for (ai = 0; ai < apn; ai++) { + u32 before = svc_fwd_req.gr_n_pages; + + if (svc_fwd_req.gr_n_pages + 2u >= SVC_GR_PAGES_MAX) + break; + svc_gr_page_note(vgpu, aps[ai], SVC_GR_QMD_WORD_NONE); + if (svc_fwd_req.gr_n_pages > before && + !svc_fwd_req.gr_pages[svc_fwd_req.gr_n_pages - 1].pa) + svc_fwd_req.gr_n_pages = before; + else if (svc_attr_ptr_log < 16) { + svc_attr_ptr_log++; + pr_info("sema-gr: attr-prelude page va=0x%llx noted (n_pages=%u)\n", + aps[ai], svc_fwd_req.gr_n_pages); + } + } + } if (gsp_sema_dbg && svc_gr_pages_log < 24) { svc_gr_pages_log++; pr_info("sema-gr: cand h=0x%x param=0x%llx prog=0x%llx n_ptr=%u inline=%u n_pages=%u push=0x%llx\n", @@ -30212,6 +31501,106 @@ static bool svc_gr_defer_expired(u64 push_va) } /* + * SESSION 153-CONTINUED-57: first-launch prelude-maturity gate. + * + * r1094's second failure mode: the FIRST beef launch of the boot ran with + * `MME-PRELUDE emitted gen=2 words=9` (the client's setup push had not + * become readable yet; the capture matured to 443 words only ~90 s later, + * after the premature launch had already poisoned the lane with MME + * ILLEGAL_OPCODE). The 5 ms sked sampler recorded `mme_esr=80000004 + * mme_mac=0x30` at +5 ms. The mirror of that boot (r1091) read the setup + * push BEFORE the first draw and launched with words=512: MME clean. + * + * This gate withholds a beef forward while the captured MME prelude is + * immature (no macro program yet), so the first window that reaches the + * engine carries the client's real setup. The pump's busy-defer machinery + * holds the entry and retries; once the setup push lands (the setup-hold + * demand/pin path is already armed for it) the capture matures and the + * launch proceeds. Bounded: after sema_gr_prelude_gate_ms the gate stops + * firing for the boot (fail-open, one loud log) so a client that never + * uploads a macro program cannot livelock the forward path. + */ +static unsigned gsp_sema_gr_prelude_gate_ms = 60000u; +module_param_named(sema_gr_prelude_gate_ms, gsp_sema_gr_prelude_gate_ms, + uint, 0644); +MODULE_PARM_DESC(sema_gr_prelude_gate_ms, + "Withhold the first beef GR forwards while the captured MME prelude is immature (no 0x114 macro program yet); fail-open after this many ms (default 60000, 0 = off)"); +static unsigned long svc_gr_prelude_gate_j; +static u64 svc_gr_prelude_gate_defer; +static u64 svc_gr_prelude_gate_exp; + +/* + * SESSION 153-CONTINUED-59: attr-program settle gate. The first beat + * window of a capture generation binds program pairs (method 0x2014/0x2054) + * whose operand pages must be IN the launch page set or the builder's + * shadow relocation MISSes and the emitted address stays a raw client VA + * the engine cannot fetch (r1104: `pair addr=0x2054 va=0x4350900 -> MISS` + * -> MMU Fault HUBCLIENT_PD @0x4350000 -> guest channel reset). The pages + * resolve through the pin demand path with a multi-second latency, so a + * launch that runs within a second of the attr prelude's program-VA set + * last growing races that resolution. This gate withholds beef forwards + * for sema_attr_settle_ms after the set last grew (bounded by the same + * gate window as the prelude gate), giving the pins time; the busy-defer + * keeps the candidate so the launch goes out once settled. + */ +static unsigned gsp_sema_attr_settle_ms = 8000u; +module_param_named(sema_attr_settle_ms, gsp_sema_attr_settle_ms, uint, 0644); +MODULE_PARM_DESC(sema_attr_settle_ms, + "Withhold beef GR forwards for this long after the attr prelude program-VA set last grew, so window pair pages resolve before the first launch (default 8000, 0 = off)"); +static unsigned long svc_attr_settle_j; +static u32 svc_attr_settle_n; + +static bool svc_gr_attr_settle_wait(void) +{ + u32 apn = 0; + + if (!gsp_sema_attr_settle_ms) + return false; + (void)gsp_sema_attr_prelude_ptrs(&apn); + if (!apn) + return false; + if (apn != svc_attr_settle_n) { + svc_attr_settle_n = apn; + svc_attr_settle_j = jiffies; + } + if (!svc_gr_prelude_gate_j) + svc_gr_prelude_gate_j = jiffies; + if (time_after(jiffies, svc_gr_prelude_gate_j + + msecs_to_jiffies(gsp_sema_gr_prelude_gate_ms))) + return false; /* overall fail-open with the prelude gate */ + return time_before(jiffies, svc_attr_settle_j + + msecs_to_jiffies(gsp_sema_attr_settle_ms)); +} + +static bool svc_gr_prelude_gate_active(const struct svc_channel *ch) +{ + if ((ch->h_object & 0xfffff000u) != 0xbeef0000u) + return false; + /* + * SESSION 153-CONTINUED-59: attr-program settle first (fires even + * with the MME prelude already mature - that is the r1104 state). + */ + if (svc_gr_attr_settle_wait()) + return true; + if (!READ_ONCE(svc_mme_prelude_flag) || !gsp_sema_gr_prelude_gate_ms) + return false; + if (gsp_sema_mme_prelude_mature()) + return false; + if (!svc_gr_prelude_gate_j) + svc_gr_prelude_gate_j = jiffies; + if (time_after(jiffies, svc_gr_prelude_gate_j + + msecs_to_jiffies(gsp_sema_gr_prelude_gate_ms))) { + if (!svc_gr_prelude_gate_exp) { + svc_gr_prelude_gate_exp = 1; + pr_warn("vgsp-work: prelude gate EXPIRED h=0x%x (MME prelude still immature after %u ms; forwarding anyway)\n", + ch->h_object, gsp_sema_gr_prelude_gate_ms); + } + return false; + } + return true; +} + +/* * r604: the r543 arg-defer latch and the r556 absolute defer cap are keyed * by push VA ALONE, and the UVM pool reuses the same launch push VAs for * every client of a boot - the exact lifetime bug the session flush already @@ -30733,6 +32122,28 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, return false; } /* + * SESSION 153-CONTINUED-57: first-launch prelude-maturity gate (see + * svc_gr_prelude_gate_active). Defer the forward while the captured + * MME setup is incomplete: the candidate survives (busy-defer), the + * pump re-decodes the entry and retries, and the launch goes out + * with the client's real macro program instead of poisoning the + * lane. Skipped for every non-beef channel and after the bounded + * fail-open window. + */ + if (svc_gr_prelude_gate_active(ch)) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + mutex_unlock(&svc_fwd_lock); + svc_gr_busy_defer = true; + svc.fwd_gr_refused++; + svc_gr_prelude_gate_defer++; + if (svc_gr_prelude_gate_defer <= 16 || + (svc_gr_prelude_gate_defer & 0x3F) == 1) + pr_info("vgsp-work: prelude gate h=0x%x push=0x%llx deferred (MME prelude immature; capture will gate this launch until the setup push lands) n=%llu\n", + ch->h_object, svc_gr_cand.push_va, + svc_gr_prelude_gate_defer); + return false; + } + /* * m143: drain the host injection queue before resolving the launch's * pages. The watcher's pins for the client's a/b/c buffers can land * between the service loop's map_add drain and this capture; the fill @@ -33002,6 +34413,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) c->unread_va = 0; c->unread_until = 0; c->defer_since = 0; + svc_gr_setup_hold_clear(c); } } /* @@ -33246,6 +34658,17 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (svc_fwd_hold.active && svc_fwd_hold.vgpu == vgpu) svc_fwd_hold_expire_check(vgpu, ch); + /* + * SESSION 153-CONTINUED-61: MISS-ABORT retry tick. A re-armed + * candidate (svc_gr_missre) is re-offered on a bounded cadence + * until the pin demand resolves its pages; the full decode path + * is deliberately NOT involved (the candidate ring slot is the + * queue). svc_gr_request no-ops for other channels and defers on + * busy/inflight. Placed before the early `continue`s so the retry + * still runs when the USERD read happens to fail this pass. + */ + svc_gr_missre_tick(vgpu, ch); + if (!ch->gpfifo_va || !ch->gpfifo_entries) continue; @@ -33843,6 +35266,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->defer_since = 0; ch->unread_va = 0; ch->unread_until = 0; + svc_gr_setup_hold_clear(ch); svc_replay_note(ch, (u32)ch->last_get, put); ch->last_get = (ch->last_get + 1) % ch->gpfifo_entries; @@ -34065,7 +35489,9 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * exists for the page, the put advances (a new * push at the same VA), or the window expires. */ - if (ch->engine_type && ch->unread_va == va && + if ((ch->engine_type || + svc_gr_setup_hold_active(ch, va, (u32)idx)) && + ch->unread_va == va && ch->unread_idx == idx && time_before(jiffies, ch->unread_until)) { u64 tpa = 0, vpa = 0; @@ -34564,6 +35990,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (!unreadable) { ch->push_hold_until = 0; + svc_gr_setup_hold_clear(ch); /* * The entry words repeated, but the * push bytes may not have: a client @@ -35433,13 +36860,33 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) 1 : 0; ch->gr_nocap_since = jiffies; + /* + * SESSION 153-CONTINUED-54: an + * all-zero setup push gets the + * same 4K-page demand + pending + * hold as the unreadable one. + */ + if (svc_gr_setup_push(len, pb)) + svc_gr_setup_hold_arm(ch, + va, + (u32)idx, + len); } - if (gsp_sema_gr_nocap_hold_ms && + { + u32 nocap_ms = + gsp_sema_gr_nocap_hold_ms; + + if (svc_gr_setup_push(len, pb) && + gsp_sema_gr_setup_hold_ms > + nocap_ms) + nocap_ms = + gsp_sema_gr_setup_hold_ms; + if (nocap_ms && !ch->gr_ram_ring && time_before(jiffies, ch->gr_nocap_since + msecs_to_jiffies( - gsp_sema_gr_nocap_hold_ms)) && + nocap_ms)) && !svc_hold_bypass()) { /* Force a fresh capture on * the retry: a replay decode @@ -35458,7 +36905,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc_gr_cand_push_owed(va) ? 1 : 0, svc_gr_qmd_seen, - gsp_sema_gr_nocap_hold_ms); + nocap_ms); } kfree(pb); break; @@ -35479,6 +36926,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->h_object, (u32)idx, va, len, owed_done); + } } } /* @@ -35723,6 +37171,16 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) idx, va, len); svc_log_stalled_va(va); /* + * SESSION 153-CONTINUED-54: + * demand the setup push's 4K + * pages and hold it pending. + */ + if (!ch->engine_type && + svc_gr_setup_push(len, NULL)) + svc_gr_setup_hold_arm(ch, va, + (u32)idx, + len); + /* * GR stale-skip clock: the * rescue-failure skip gate below * requires defer_since, yet every @@ -35771,6 +37229,19 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->unread_ipin_pa = ipa; } /* + * SESSION 153-CONTINUED-54: a held + * GR setup push waits cheaply for + * the whole hold window (the fresh- + * pin check still releases it the + * moment the demanded pin lands). + */ + if (svc_gr_setup_hold_active(ch, va, + (u32)idx) && + time_before(ch->unread_until, + ch->gr_setup_hold_until)) + ch->unread_until = + ch->gr_setup_hold_until; + /* * Bounded work-before-release * hold (CE): the deferred retry * probe shows the page's @@ -36091,7 +37562,9 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->defer_since && time_after(jiffies, ch->defer_since + - msecs_to_jiffies(gsp_sema_gr_grace_ms))) { + msecs_to_jiffies(gsp_sema_gr_grace_ms)) && + !svc_gr_setup_hold_active(ch, va, + (u32)idx)) { /* * The slot no longer holds this * push (overwritten buffer): the @@ -36136,6 +37609,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->defer_since = 0; ch->unread_va = 0; ch->unread_until = 0; + svc_gr_setup_hold_clear(ch); /* * m177: deliver any * withheld set even @@ -36235,6 +37709,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->gr_nocap_va = 0; ch->unread_va = 0; ch->unread_until = 0; + svc_gr_setup_hold_clear(ch); /* GRFIX FIX1c (r214): do NOT reset the per-launch * bound record here - a consume of an unrelated entry * would re-arm the held launch's bound every pass @@ -37786,6 +39261,7 @@ static void svc_stall_watch_check(void) continue; ch->unread_va = 0; ch->unread_until = 0; + svc_gr_setup_hold_clear(ch); ch->gr_hold_idx = ~0u; ch->gr_hold_until = 0; ch->gr_nocap_idx = ~0u; @@ -38144,6 +39620,8 @@ int gsp_sema_service_start(void) } if (gsp_cuda_bridge_init()) pr_warn("sema-svc: cuda bridge debugfs unavailable\n"); + if (svc_setup_demand_debugfs_init()) + pr_warn("sema-svc: setup-demand debugfs unavailable\n"); return 0; } @@ -38179,6 +39657,7 @@ void gsp_sema_service_stop(void) } /* Thread is gone: release the bridge's pending jobs and debugfs files. */ gsp_cuda_bridge_fini(); + svc_setup_demand_debugfs_fini(); /* A fresh session must not inherit shadowed bytes from the last one. */ svc_vas_free_all(); } diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h index 69d6f33..6b7922a 100644 --- a/kernel/gsp_sema_service.h +++ b/kernel/gsp_sema_service.h @@ -303,6 +303,8 @@ extern u32 gsp_phys_gr_peek; * retry probe). Absolute deadline per entry, never reset; 0 disables. */ extern unsigned gsp_sema_push_hold_ms; +/* SESSION 153-CONTINUED-54: GR setup-push (MME-setup / >=12 KB) hold. */ +extern unsigned gsp_sema_gr_setup_hold_ms; /* * Grace window (ms) an untranslatable GR GPFIFO entry is left pending before @@ -448,6 +450,52 @@ struct gsp_sema_gr_reloc { int gsp_sema_gr_reloc_resolve(u64 va, struct gsp_sema_gr_reloc *out); /* + * SESSION 153-CONTINUED-59: record one VA (page-aligned internally) in the + * pin-demand set exposed at /sys/kernel/debug/vgsp/sema_setup_demand, so + * pin_gr_stream demands its page in the guest. The GR launch builder calls + * this when a window/attr operand page cannot be resolved at build time + * (the r1104/r1105 MISS -> MMU-fault class); the demand gives the next + * forward a resolvable page instead of a lane-killing raw guest VA. + */ +void gsp_sema_demand_note(u64 va); + +/* + * SESSION 153-CONTINUED-53: MME context-setup prelude accessor for the GR + * launch builder. Returns the captured setup runs (C597 MME family + * 0x114..0x124/0x234/0x550..0x568/0x3400..0x34fc, verbatim, first- + * encounter order), the word count in *n and the capture generation in + * *gen. The builder preplays the words once per generation, ahead of the + * first 3D window replay, so the ladder context's MME has its macro + * program before CALL_MME_MACRO runs. Reading the generation before and + * after the copy detects a concurrent append (the service thread only + * appends; a mismatch just defers emission to the next launch). *n is 0 + * until the client's setup push has been decoded. + */ +const u32 *gsp_sema_mme_prelude_get(u32 *n, u32 *gen); + +/* + * SESSION 153-CONTINUED-57: is the captured MME prelude mature enough to + * launch against - i.e. does it hold the client's macro program (>= 64 + * captured 0x114 code words)? The r1094 premature-emission poison + * (MME ILLEGAL_OPCODE at +5 ms, capture still at 9 scratch words) is + * gated on this: the sema forward path withholds beef launches while it + * reads false, bounded by sema_gr_prelude_gate_ms. + */ +bool gsp_sema_mme_prelude_mature(void); + +/* + * SESSION 153-CONTINUED-57: attr/pipeline state prelude accessor (C597 + * families 0x1100..0x11fc and 0x2000..0x205c, verbatim, first-encounter + * order, deduped). Same contract as gsp_sema_mme_prelude_get: preplay once + * per generation ahead of the first 3D window replay. The runs carry the + * pipeline program binds whose SPH the draw shader validates, so the + * builder also registers the program VAs (gsp_sema_attr_prelude_ptrs) in + * the launch page set to make the address relocation land. + */ +const u32 *gsp_sema_attr_prelude_get(u32 *n, u32 *gen); +const u64 *gsp_sema_attr_prelude_ptrs(u32 *n); + +/* * r378 write guards. Both re-validate a guest-memory write target that was * resolved earlier (ledger/pin/walk/counter registration, or the GR submit's * per-page resolution) against the CURRENT resolution before the bytes land, diff --git a/kernel/main.c b/kernel/main.c index b5c7105..c96abf8 100644 --- a/kernel/main.c +++ b/kernel/main.c @@ -2798,6 +2798,17 @@ module_param_named(sema_push_hold_ms, gsp_sema_push_hold_ms, uint, 0644); MODULE_PARM_DESC(sema_push_hold_ms, "CE unreadable-entry work-before-release hold in ms (default 2500, 0 disables)"); /* + * SESSION 153-CONTINUED-54: GR setup-push (client MME-setup upload / any + * >= 12 KB GR push) pending hold (ms). An unreadable/all-zero setup push + * keeps its ring entry pending past the failed-rescue consume until the + * demanded 4K-page guest pin lands; 0 disables (demand rows still emitted). + * Default 0: the demand channel cannot yet resolve the GL pushbuffer VA + * families, so the hold only delays the ring (see gsp_sema_service.c). + */ +module_param_named(sema_gr_setup_hold_ms, gsp_sema_gr_setup_hold_ms, uint, 0644); +MODULE_PARM_DESC(sema_gr_setup_hold_ms, "GR setup-push (MME-setup/>=12KB) pending hold in ms (default 0 = off; 20000 = aggressive, needs a resolvable demand family)"); + +/* * Grace window (ms) an untranslatable GR entry stays pending before the pump * consumes it as stale, so a late push-buffer pin can still make it readable. */ |