diff options
Diffstat (limited to 'kernel/gsp_phys_booter.c')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 9654 |
1 files changed, 9508 insertions, 146 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index cf5061d..5a5412e 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -46,6 +46,7 @@ #include <linux/vgaarb.h> #include <linux/acpi.h> #include <linux/mm.h> +#include <linux/vmalloc.h> #include <asm/barrier.h> #include <asm/unaligned.h> @@ -57,6 +58,15 @@ #include "gsp_phys_booter_load_img.h" #include "gsp_phys_gsp_rm_boot_img.h" #include "gsp_phys_registry_img.h" +/* + * SESSION 153-CONTINUED-114: golden MME slot library (untracked build + * input, .gitignore'd - see the file header for provenance). Used only + * when sema_gl_mme_golden=1: called-but-uncaptured slots are served + * from the driver's own generated macro code instead of being stubbed, + * so the DRAW_IAB call (slot 34) runs deterministically on every boot + * (the full-capture profile becomes a per-boot lottery otherwise). + */ +#include "golden-mme-tu104.h" #include "gsp_phys_registry_img_srvctx.h" static bool gsp_phys_boot_param; @@ -129,6 +139,16 @@ MODULE_PARM_DESC(gsp_phys_gr_reloc_resolve, * per-launch RELOC-CENSUS line. */ #define GSP_PHYS_GR_COPYWIT_MAX 12u +/* + * CONT-112: the changed-page witness gets its OWN budget. A store-rich + * launch walks the tracked pages in bi order and the unchanged pages come + * first, so the shared 12-line budget was always consumed by the OUT-UNCH + * flood before the loop reached the handful of engine-changed pages - the + * r1385 evidence has "2/900 pages changed" + OUT-ALIAS-WIT for bi=8960/8965 + * but no OUT-PAGE-WIT va->hpa pair for either. With the split counter both + * classes are witnessed on every launch (<=12 unchanged + <=16 changed). + */ +#define GSP_PHYS_GR_COPYWIT_CHG_MAX 16u static bool gsp_phys_gr_copyout_dump_param = true; module_param_named(gsp_phys_gr_copyout_dump, gsp_phys_gr_copyout_dump_param, bool, 0644); @@ -140,6 +160,7 @@ static inline bool gsp_phys_gr_copywit_enabled(void) return READ_ONCE(gsp_phys_gr_copyout_dump_param); } + /* * r405 late-write probes (the r405 "partial write" class: the scale launch * of streams2 wrote 1 of its 32 expected in-place pages and every other @@ -151,9 +172,19 @@ static inline bool gsp_phys_gr_copywit_enabled(void) * "ENGINE STILL WRITING" proves the copy-out shipped before the stores * landed (the fix is a longer settle window); "stable" proves the engine * genuinely stopped, which needs a different treatment. - * - gsp_phys_gr_settle_ms (default 300): the content-stable window the + * - gsp_phys_gr_settle_ms (default 100): the content-stable window the * partial-changed rule requires before it declares the stores settled. * Raise it to give a still-writing engine its time (A/B in one cycle). + * + * r552 (2026-10-23): measured A/B on real boots. Every launch with a + * partial changed-page count (read-mostly kernels: 1..64 of 1000+ pages + * written, so the changed>=n_all shortcut can never fire) paid the full + * window plus scan overhead: 683-901 ms per launch at 300 ms. At 100 ms + * the same boots measured 231-247 ms of retire wait and the launch + * latency p50 dropped 1119 ms -> 651 ms (n=22 launches; baseline 684 ms) + * with the battery unchanged (12/15, 0 hangs) and a green 2-round soak + * on the next boot. The content-stability hash still gates the verdict; + * only the minimum stable duration shrank. */ static bool gsp_phys_gr_late_probe_param; module_param_named(gsp_phys_gr_late_probe, gsp_phys_gr_late_probe_param, @@ -161,11 +192,11 @@ module_param_named(gsp_phys_gr_late_probe, gsp_phys_gr_late_probe_param, MODULE_PARM_DESC(gsp_phys_gr_late_probe, "Diagnostic: after the GR retire wait, re-scan the tracked pages +500 ms later and log whether the engine is still writing (default 0)"); -static int gsp_phys_gr_settle_ms_param = 300; +static int gsp_phys_gr_settle_ms_param = 100; module_param_named(gsp_phys_gr_settle_ms, gsp_phys_gr_settle_ms_param, int, 0644); MODULE_PARM_DESC(gsp_phys_gr_settle_ms, - "GR retire wait: content-stable window (ms) required before a partial changed-page count counts as settled (default 300)"); + "GR retire wait: content-stable window (ms) required before a partial changed-page count counts as settled (default 100; 300 was the pre-r552 value)"); static inline u32 gsp_phys_gr_settle_iters(void) { @@ -1120,18 +1151,61 @@ bool gsp_phys_boot_requested(void) * from gpFifoOffset, the pushbuffer from the GPFIFO ring entry GET, the CE * operands from the method block): SET_SEMAPHORE_A/B are always GPU VAs with * no mode bit (clc5b5.h:39-44; uvm_pascal_ce.c:42-63) and the user channel's - * copy is virtual-mode on both sides (uvm_maxwell_ce.c:179-205). All five - * pages live in one 2 MB PD0 slot below the VAS vaSize limit: with + * copy is virtual-mode on both sides (uvm_maxwell_ce.c:179-205). All the + * ladder pages live in one 2 MB PD0 slot below the VAS vaSize limit: with * VA_BASE = 0x00100000 every parent index (48:47, 46:38, 37:29, 28:21) is 0 - * and the five 4 KB pages take small-PT slots 0x100..0x104 - * (kern_gmmu_fmt_gp10x.c:16-83; uvm_pascal_mmu.c:25-32,172-204). + * and the 4 KB pages take small-PT slots in the single 128 B PTE line + * 0x100..0x10f (kern_gmmu_fmt_gp10x.c:16-83; uvm_pascal_mmu.c:25-32,172-204). + * + * SESSION 153-CONTINUED-66: the module pushbuffer is TWO pages. The MME + * context-setup prelude matured to 901 words while the beef window runs 390 + * words and the SA-RAM stubs add 29..47, so the single-launch block needs + * ~1400 words = 5.6 KB: the 1-page pushbuffer made every prelude-bearing + * launch print "MME-PRELUDE deferred (window=390 prelude=901 sa=47 exceeds + * block; replaying window alone)" and go out prelude=0. The block now spans + * VA_PB..VA_PB+0x2000 with the second page at small-PT slot 0x102. The + * slot 0x102 page used to be CE SRC; SRC moved to 0x109 (it is only read by + * the CE copy path and the GR inline-QMD destination default). Slot 0x10a + * is the dedicated warm-semaphore probe page: the probe used to share the + * pushbuffer's upper half, which a >0x800-byte forwarded block now occupies. */ #define GSP_PHYS_RMCALL_VA_BASE 0x00100000ULL #define GSP_PHYS_RMCALL_VA_GPFIFO GSP_PHYS_RMCALL_VA_BASE #define GSP_PHYS_RMCALL_VA_PB (GSP_PHYS_RMCALL_VA_BASE + GSP_PHYS_PAGE_SIZE) -#define GSP_PHYS_RMCALL_VA_SRC (GSP_PHYS_RMCALL_VA_BASE + 2 * GSP_PHYS_PAGE_SIZE) -#define GSP_PHYS_RMCALL_VA_DST (GSP_PHYS_RMCALL_VA_BASE + 3 * GSP_PHYS_PAGE_SIZE) -#define GSP_PHYS_RMCALL_VA_SEM (GSP_PHYS_RMCALL_VA_BASE + 4 * GSP_PHYS_PAGE_SIZE) +/* SESSION 153-CONTINUED-66: page 2 of the module pushbuffer (same PT line). */ +#define GSP_PHYS_RMCALL_VA_PB2 (GSP_PHYS_RMCALL_VA_BASE + 2 * GSP_PHYS_PAGE_SIZE) +/* + * SESSION 153-CONTINUED-77: page 3 of the module pushbuffer (slot 0x103) + * so a matured MME prelude (cap 2048 words) plus the largest chunked + * window fit one fetch block. The block walk is linear in VA, so the + * three pushbuffer VAs must stay contiguous: VA_DST (previously at + * base+0x3000, slot 0x103) moves to base+0xB000 (slot 0x10b, free). + */ +#define GSP_PHYS_RMCALL_VA_PB3 (GSP_PHYS_RMCALL_VA_BASE + 3 * GSP_PHYS_PAGE_SIZE) +/* + * SESSION 153-CONTINUED-77: page 4 (slot 0x104) after the r1163 cycle + * showed the capture cap-limited at 2048 words (ir_end=0x265, 34 slots + * covered). VA_SEM moves 0x104 -> 0x10c (free) so the four pushbuffer + * pages stay contiguous for the linear fetch. + */ +#define GSP_PHYS_RMCALL_VA_PB4 (GSP_PHYS_RMCALL_VA_BASE + 4 * GSP_PHYS_PAGE_SIZE) +#define GSP_PHYS_RMCALL_VA_DST (GSP_PHYS_RMCALL_VA_BASE + 0xB000ULL) +#define GSP_PHYS_RMCALL_VA_SEM (GSP_PHYS_RMCALL_VA_BASE + 0xC000ULL) +/* SESSION 153-CONTINUED-66: CE SRC moved off the pushbuffer's second page. */ +#define GSP_PHYS_RMCALL_VA_SRC (GSP_PHYS_RMCALL_VA_BASE + 0x9000ULL) +/* SESSION 153-CONTINUED-66: warm-semaphore probe page (own PTE, no sharing). */ +#define GSP_PHYS_RMCALL_VA_GR_PROBE (GSP_PHYS_RMCALL_VA_BASE + 0xA000ULL) +/* + * SESSION 153-CONTINUED-96: secondary CE (COPY0) lane. A COPY0 channel on + * the same VAS translates through the same 2 MiB window PDB, but CE class + * methods are never legal on the GR0 ladder channel, so the ladder's own + * FBMEM readback (gsp_phys_ctx_readback_try) needs its own engine. The + * lane's GPFIFO ring and one-page pushbuffer get their own small-PT slots + * 0x10d/0x10e in the same 128 B PTE line the ladder's pages occupy, planted + * in pdb_build (stage 3, before any engine fetch; there is no invalidate). + */ +#define GSP_PHYS_RMCALL_VA_CE_GPFIFO (GSP_PHYS_RMCALL_VA_BASE + 0xD000ULL) +#define GSP_PHYS_RMCALL_VA_CE_PB (GSP_PHYS_RMCALL_VA_BASE + 0xE000ULL) /* * m69: window-0 shadow pool - 16 module pages holding the Tier-2 copies of * the QMD's program/param pages, so the GR engine never has to read through @@ -1417,6 +1491,24 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RMCALL_HANDLE_USER_LANE_PM_FB 0xcaf000b8u #define GSP_PHYS_RMCALL_HANDLE_USER_LANE_FAULT_MEM0 0xcaf000c0u #define GSP_PHYS_RMCALL_HANDLE_USER_LANE_FAULT_MEM1 0xcaf000c4u +/* + * SESSION 153-CONTINUED-96: the secondary COPY0 lane's TSG/channel handles + * (free in the ladder's client range: 0xcaf000c8/0xcc sit above the mode-35 + * fault-mem handles and below the registered-buffer base 0xcaf01000). + */ +#define GSP_PHYS_RMCALL_HANDLE_CE_TSG 0xcaf000c8u +#define GSP_PHYS_RMCALL_HANDLE_CE_CHANNEL 0xcaf000ccu +/* + * Mode 35, graphics-execution plan §F first experiment: the TURING_A + * (0xc597, GR_OBJECT_TYPE_3D) class object constructed on the V7 + * user-lane channel AFTER its promote - the missing half the mode-35 + * capture never provided ("mode 35: ... the capture carries no 0xc597 on + * the user lane"). Best-effort like the TURING_COMPUTE_A: a rejection is + * logged and the lane keeps running; a construction on V7 must not hit + * the wired-golden deadlock class (m3a-m3g, that channel). 0xcaf00094 is + * free between 0xcaf00090 and 0xcaf000a0. + */ +#define GSP_PHYS_RMCALL_HANDLE_USER_LANE_GR3D 0xcaf00094u /* * Mode-35 V7 channel flags/privilege and fault/PM geometry, decoded from the @@ -1642,6 +1734,19 @@ enum gsp_phys_rmcall_buf { GSP_PHYS_RMCALL_BUF_GOLDEN_PT0, GSP_PHYS_RMCALL_BUF_GOLDEN_PT1, GSP_PHYS_RMCALL_BUF_GOLDEN_PT2, + /* + * SESSION 153-CONTINUED-96: secondary COPY0 lane buffers (one page + * each). The instance/USERD/RAMFC/mthdbuf are the lane's own memdescs + * in the channel alloc (a second channel must never share the GR + * lane's hardware USERD); the GPFIFO ring and pushbuffer page are + * engine-reached through their planted PTE slots. + */ + GSP_PHYS_RMCALL_BUF_CE_GPFIFO, + GSP_PHYS_RMCALL_BUF_CE_USERD, + GSP_PHYS_RMCALL_BUF_CE_INSTANCE, + GSP_PHYS_RMCALL_BUF_CE_RAMFC, + GSP_PHYS_RMCALL_BUF_CE_MTHDBUF, + GSP_PHYS_RMCALL_BUF_CE_PB, GSP_PHYS_RMCALL_BUF_COUNT }; @@ -1657,6 +1762,20 @@ enum gsp_phys_rmcall_buf { #define GSP_PHYS_RMCALL_TIMEOUT_MS 2000u #define GSP_PHYS_RMCALL_POLL_MS 5u /* + * SESSION 148 perf (termbin review, verified in-tree): every RPC reply wait + * polled with msleep(5). With several quanta per round trip and ~8 + * serialized PDE_UPDATE forwards per forwarded GR launch, the 5 ms quantum + * dominated per-launch latency (~95% of the ~189 ms host service time per + * launch). Wait with sub-millisecond quanta instead: the reply is observed + * within one quantum, cutting each round trip to ~1 ms. POLL_MS remains for + * the poll accounting prints. + */ +#define GSP_PHYS_RMCALL_POLL_US 150u +static inline void gsp_phys_rmcall_wait_quantum(void) +{ + usleep_range(GSP_PHYS_RMCALL_POLL_US, GSP_PHYS_RMCALL_POLL_US + 200u); +} +/* * GR launch release poll window. A CE copy completes in single-digit ms, but * a GR launch can force the engine's first context switch for the channel; * the FECS ctxsw path is measured in fractions of a second, so the window is @@ -1708,6 +1827,19 @@ enum gsp_phys_rmcall_buf { #define GSP_PHYS_RMCALL_CE_SUBCHANNEL 4u /* + * SESSION 153-CONTINUED-68: SEC_OP_ONE_INC = 5 (class/cl906f.h:181 + * NV906F_DMA_SEC_OP_ONE_INC, NVC06F_DMA_ONEINCR_* field layout). The + * client emits its MME IR uploads (method 0x114) as op=5 runs whose first + * data word is the IR instruction address and whose remaining words feed + * the IR data port (the r1099/r1116 captures: hdr 0xa0100045 = op 5, + * cnt 16, method 0x114, data [pc, code...]). op=1 (INC_METHOD) walks the + * METHOD address per data word instead (0x114 -> 0x118 -> 0x11c -> + * 0x120), so an IR upload emitted with op=1 writes one stray IR word and + * then clobbers the SA pointer/data registers. + */ +#define GSP_PHYS_RMCALL_METHOD_ONEINC 5u + +/* * CE (TURING_DMA_COPY_A 0xc5b5) method offsets and field values, from * class/clc5b5.h: SET_OBJECT 0x0, SET_SEMAPHORE_A/B/PAYLOAD 0x240/0x244/0x248, * LAUNCH_DMA 0x300, OFFSET_IN_UPPER/LOWER 0x400/0x404, OFFSET_OUT_UPPER/ @@ -2355,6 +2487,16 @@ static struct gsp_phys_boot_stage { u32 rmcall_gr_promote_size; u32 rmcall_gr_hvirt; /* mode-6 NV01_MEMORY_VIRTUAL handle */ u32 rmcall_token; /* GET_WORK_SUBMIT_TOKEN result */ + /* + * SESSION 153-CONTINUED-96: secondary COPY0 lane (same VAS). Alloc'd + * and scheduled only on a GR0 ladder (CE class is illegal on the GR + * channel); rmcall_ce_lane_valid means the token was obtained and the + * lane can accept CE copy blocks. Zero on a COPY0 ladder. + */ + u32 rmcall_ce_tsg; /* KEPLER_CHANNEL_GROUP_A handle */ + u32 rmcall_ce_channel; /* TURING_CHANNEL_GPFIFO_A handle */ + u32 rmcall_ce_token; /* its GET_WORK_SUBMIT_TOKEN result */ + bool rmcall_ce_lane_valid; /* alloc + BIND/schedules + token done */ u32 rmcall_notifier_mem; /* FBMEM error-notifier object handle */ u64 rmcall_notifier_base; /* its returned frame-buffer address */ bool rmcall_notifier_valid; /* the FBMEM notifier alloc succeeded */ @@ -2447,6 +2589,18 @@ static struct gsp_phys_boot_stage { */ struct gsp_phys_boot_chunk rmcall_gr_qmdwin; u64 rmcall_gr_qmdwin_va; /* planted VA; 0 = guest destination */ + /* + * SESSION 153-CONTINUED-66: page 2 of the module pushbuffer plus the + * dedicated warm-semaphore probe page. Allocated lazily by + * gsp_phys_rmcall_pdb_build() and planted in the ladder's 128 B PTE + * line at slots 0x102 (VA_PB2) and 0x10a (VA_GR_PROBE), before the + * engine's first fetch. Page 0 continues the registered BUF_PB chunk + * so stage 2's registration and every existing pushbuffer user are + * unchanged; only the >4 KB GR launch block reaches into these pages. + */ + struct gsp_phys_boot_chunk rmcall_gr_pb2; + struct gsp_phys_boot_chunk rmcall_gr_probe; + bool rmcall_gr_pb2_planted; /* m104: module page holding the injected mail_kernel message. */ struct gsp_phys_boot_chunk rmcall_gr_mail; /* @@ -2623,6 +2777,18 @@ EXPORT_SYMBOL_GPL(gsp_phys_boot_rm_flags); static void *gsp_phys_chunk_page_va(const struct gsp_phys_boot_chunk *c, u32 page) { + /* + * SESSION 141: an unallocated chunk (no per-page array AND no base + * va) must answer NULL. It previously returned + * (u8 *)NULL + page*PAGE_SIZE - a small non-NULL garbage pointer - + * so the caller's NULL guard passed and the empty-set post-mortem + * print dereferenced it: Oops #2 RIP + * gsp_phys_backend_guest_gr_submit+0x4394, CR2=0x1e93c with slot 30 + * (0x1e000 = 30*0x1000 + 0x900 off). min_data=0's no-external-data + * forward hit this on its first pass. + */ + if (!c->page_va && !c->va) + return NULL; if (c->page_va) return c->page_va[page]; return (u8 *)c->va + (size_t)page * GSP_PHYS_PAGE_SIZE; @@ -3169,6 +3335,10 @@ static void gsp_phys_rmcall_reset(void) boot_stage.rmcall_gr_promote_size = 0; boot_stage.rmcall_gr_hvirt = 0; boot_stage.rmcall_token = 0; + boot_stage.rmcall_ce_tsg = 0; + boot_stage.rmcall_ce_channel = 0; + boot_stage.rmcall_ce_token = 0; + boot_stage.rmcall_ce_lane_valid = false; boot_stage.rmcall_notifier_mem = 0; boot_stage.rmcall_notifier_base = 0; boot_stage.rmcall_notifier_valid = false; @@ -3204,6 +3374,11 @@ static void gsp_phys_rmcall_reset(void) memset(&boot_stage.rmcall_gr_qmdwin, 0, sizeof(boot_stage.rmcall_gr_qmdwin)); boot_stage.rmcall_gr_qmdwin_va = 0; + memset(&boot_stage.rmcall_gr_pb2, 0, + sizeof(boot_stage.rmcall_gr_pb2)); + memset(&boot_stage.rmcall_gr_probe, 0, + sizeof(boot_stage.rmcall_gr_probe)); + boot_stage.rmcall_gr_pb2_planted = false; memset(&boot_stage.rmcall_gr_shadow_win0, 0, sizeof(boot_stage.rmcall_gr_shadow_win0)); boot_stage.rmcall_gr_shadow_win0_used = 0; @@ -4911,6 +5086,716 @@ static u32 gsp_phys_engine_type_from_errstr(const char *err) } /* + * SESSION 153-CONTINUED-12: last published GR launch identity, set by the + * single-inflight submit path so a PBDMA fault dump can name the entry the + * PBDMA is stuck on (block length in dwords, guest push VA, object handle). + */ +static u32 gsp_phys_gr_last_words; +static u32 gsp_phys_gr_last_h; +static u64 gsp_phys_gr_last_push_va; + +/* + * SESSION 153-CONTINUED-12: on a robust-channel PBDMA error (exceptType + * 0x20) dump every PBDMA's latched instruction state and, for the PBDMA + * whose PB_FETCH sits inside the module pushbuffer page, the raw block words + * around the fetch point plus the whole block with indices. The latched + * raw instruction (HDR_SHADOW) can be matched against the block dump to + * name the exact word the PBDMA rejected ("invalid method encoding"); the + * fetch position shows how far the walk got. Read-only MMIO + a racy read + * of the pb page (best effort, diagnostic only); bounded per boot. + */ +#define GSP_PHYS_PBDMA_DUMP_MAX 64u + +/* + * SESSION 153-CONTINUED-52: the published method block's DMA address, so the + * PBDMA fault dump can name the exact word the PBDMA stopped on + * (offset = PB_FETCH - last block DMA). Set beside gsp_phys_gr_last_words. + */ +static u64 gsp_phys_gr_last_pb_dma; + +static const char *gsp_phys_pbdma_type_name(u32 t) +{ + static const char *names[8] = { "SSDM", "INC", "STORE_SDM", "NON_INC", + "IMMD", "INC_ONCE", "USE_SDM", "END_SEG" }; + + return names[t & 7u]; +} + +/* + * SESSION 153-CONTINUED-52: MME fault forensics. The C597 replay's dominant + * failure on draw-bearing windows is a real MME exception + * (NV_PGRAPH_PRI_MME_HWW_ESR 0x404490 latched ILLEGAL_OPCODE / + * MISSING_MACRO_DATA / ... - "unknown instruction fetched from the MME + * Instruction RAM", see gr_error_gk104.c grErrorLogMmeException). INFO + * carries the faulting MME PC and the macro address; INFO2 the instruction + * register. Read them post-failure (the latch survives) so the exact macro + * fetch point is on record. Read-only; bounded by its call sites. + */ +static void gsp_phys_mme_regs_log(void __iomem *bar0, const char *tag) +{ + u32 esr, info, info2, info3, info4, trapped; + + if (!bar0) + return; + esr = ioread32(bar0 + 0x00404490u); + info = ioread32(bar0 + 0x00404494u); + /* SESSION 153-CONTINUED-106: INFO2 is 0x40449c (0x404498 is CONFIG). */ + info2 = ioread32(bar0 + 0x0040449cu); + info3 = ioread32(bar0 + 0x004044a8u); + info4 = ioread32(bar0 + 0x004044acu); + trapped = ioread32(bar0 + 0x00400704u); + pr_warn("vgsp-gr: MME-REGS %s esr=%08x(pc_valid=%u illegal=%u miss=%u extra=%u) pc=%u macro_addr=0x%x ir=%08x,%08x,%08x trapped=%08x\n", + tag, esr, (info >> 28) & 1u, (esr >> 2) & 1u, + (esr >> 0) & 1u, (esr >> 1) & 1u, info & 0xfffu, + (info >> 12) & 0xffu, info2, info3, info4, trapped); +} + +/* + * SESSION 153-CONTINUED-107: MME IRAM debug port readback. + * + * CONT-106 found the debug port at 0x4044d0 host-side; the definitive field + * spec (tu104 dev_graphics_nobundle.h:2166 NV_PGRAPH_PRI_MME_DEBUG) is + * bit 0 STATUS (R; 0=RUNNING, 1=STOPPED) + * bit 1 ENABLE (RW; init DISABLED) + * bit 2 IRAM_RD (write 1 = TRIGGER, reads 0 = DONE) + * bits 26:15 IRAM_PC (RW, 12 bits = the IRAM word address) + * and the fetched group lands in NV_PGRAPH_PRI_MME_HWW_ESR_INFO4/3/2 + * (0x4044ac/a8/9c, field IR 31:0 each). CONT-106's r1364 read at PC 0 + * returned the reconstructed image's first three words + * (0x00003463,0x190d82f0,0x39804300), so the data path is the INFO + * triplet. The read is a two-step hardware handshake: set ENABLE, write + * the PC with IRAM_RD=1 (trigger), WAIT for IRAM_RD to read back 0 + * (DONE), then read the group. `a` is an IRAM WORD address (callers pass + * instruction * 3, matching gsp_phys_gr_mme_ir_pack[] indexing). + * Read-only diagnostic; bounded by its call sites. + */ +static void gsp_phys_mme_iram_read3(void __iomem *bar0, u32 a, u32 out[3]) +{ + u32 i; + + iowrite32(0x2u, bar0 + 0x004044d0u); /* ENABLE */ + iowrite32(((a & 0xfffu) << 15) | 6u, bar0 + 0x004044d0u); + for (i = 0; i < 1000u; i++) { + if (!(ioread32(bar0 + 0x004044d0u) & 0x4u)) + break; + cpu_relax(); + } + out[0] = ioread32(bar0 + 0x004044acu); + out[1] = ioread32(bar0 + 0x004044a8u); + out[2] = ioread32(bar0 + 0x0040449cu); +} + +/* + * Defined next to the packed-image array (gsp_phys_gr_mme_ir_pack); forward + * declaration so the post-doorbell forensics spin can call it. + */ +static void gsp_phys_mme_iram_probe(void __iomem *bar0, const char *tag, + u32 h_object); + +/* + * SESSION 153-CONTINUED-107: probe enable (see the sema_gl_mme_iram_probe + * knob beside sema_gl_mme_iram_gate). Declared here for the forensics spin + * above; the matching module_param sits with the gate knob. + */ +static bool gsp_phys_rmcall_gr_mme_iram_probe; + +/* + * SESSION 153-CONTINUED-69: pre-reset MME fault capture. Everything sampled + * so far was post-mortem: by the time the 5 ms launch poll ticks, the + * GSP-RM's FECS exception handler has already read NV_PGRAPH_PRI_MME_HWW_ESR + * (0x404490) - the relayed errString "ESR 0x404490=0x80000004" is the only + * surviving trace of the raw ILLEGAL_OPCODE value - and either cleared the + * latch back to 0x80000000 or left the whole PRI block in the 0xbadfxxxx + * priv-error poison. This bounded spin runs in the submit thread immediately + * after the publish/doorbell (execution starts within the first + * milliseconds) and records EVERY transition of the fault registers at + * MMIO read rate, so the faulting ESR (+INFO pc/macro when the valid bit is + * set, and the instruction register INFO2 that names the rejected + * instruction word) is on the record before any clear. Read-only; bounded + * per boot and per launch (the spin costs at most sema_mme_fast_ms of the + * launch thread's time and stops early once the poison is seen). + */ +static u32 gsp_phys_mme_fast_ms = 400u; +module_param_named(sema_mme_fast_ms, gsp_phys_mme_fast_ms, uint, 0644); +MODULE_PARM_DESC(sema_mme_fast_ms, + "Spin-sample the MME fault registers right after each GR launch doorbell for this many ms, logging every transition before the GSP-RM clears them (0 = off; default 400)"); + +/* + * SESSION 153-CONTINUED-106: post-timeout re-spin. The r1363 chronology + * shows the MME ILLEGAL_OPCODE relay arriving up to ~2.6 s AFTER the sem + * wait failed (launch failed 403088.34, ILLEGAL_OPCODE relayed 403090.96); + * every launch-window spin had already ended, so the latch was never seen + * with its INFO/IR. This bounded second spin runs on the first timeout + * exits of a boot, exactly where the r1363 trap sat. + */ +static u32 gsp_phys_mme_post_ms = 2000u; +module_param_named(sema_mme_post_ms, gsp_phys_mme_post_ms, uint, 0644); +MODULE_PARM_DESC(sema_mme_post_ms, + "Post-timeout MME fault-register re-spin length in ms on the first GR launch-timeout exits (0 = off; default 2000)"); + +/* + * SESSION 153-CONTINUED-83: store-side emission forensics. + * + * The store wall: draws retire but `output retirement wait: 0/N pages + * changed` and gen0=0 (no MME-generated method witnessed). The client's + * Turing draw is EMITTED BY AN MME MACRO (slot 34 = the draw macro: + * BEGIN(0x1618) -> SET_VERTEX_ARRAY_START(0xd74) -> END(0x1614), per the + * offline replay of sa_full_stream2.bin and the nvk reference builder), so + * "no stores" == "the draw macro never emitted". This probe answers three + * questions with one line, at the launch boundary: + * + * 1. DS_HWW_ESR (0x405840) + the per-stage SPH_INFO snapshot + * (0x405c00+si*4 START / 0x405c24+si*4 UPPER / 0x405c1c,c20 REGION): + * which pipeline stages failed SPH validation / what VA they fetched. + * 2. MME_HWW_ESR_INFO (0x404494) PC + MACRO_ADDRESS + PC_VALID, + * INFO_EX (0x4044a4) + IR words INFO2/3/4 (0x40449c/a8/ac): did the + * MME actually execute a macro, and where did it stop. + * 3. FE_DECODE_STATE_EX(i) (0x40a630+i*4): MME_PC 11:0, MACRO 21:12, + * GENERATED 31 - the generated-method witness, all four lanes, at + * the launch end (the retire-time read races the release writes). + * + * Bounded: 24 calls per boot, beef family only, one line per call. + */ +/* + * SESSION 153-CONTINUED-83: MME shadow-RAM read port (bar0 0x404488 index, + * 0x40448c data). The MME tracks class methods in its shadow RAM when the + * class's SET_MME_SHADOW_RAM_CONTROL mode selects tracking; reading the + * port tells us (a) whether the tracked window's methods ever reached the + * MME and (b) what values the MME holds for the draw-relevant methods. + * A read is: index = nvclass(15:0) | method_addr(27:16) | READ(30), then + * read data. Bounded by the caller; host-side diagnostic only. + */ +static u32 gsp_phys_mme_shadow_read(void __iomem *bar0, u32 nvclass, u32 mthd) +{ + iowrite32(nvclass | ((mthd & 0xfffu) << 16) | (1u << 30), + bar0 + 0x00404488u); + return ioread32(bar0 + 0x0040448cu); +} + +static void gsp_phys_store_side_log(void __iomem *bar0, const char *tag, + u32 h_object) +{ + static u32 ss_n; + u32 si, sph[6], sphu[6], rr_a, rr_b, ds, ds2, mminfo, mmex; + u32 mm2, mm3, mm4, fex[4], fed[4], fds[4]; + + if (!bar0 || ss_n >= 40u) + return; + ss_n++; + for (si = 0; si < 6u; si++) { + sph[si] = ioread32(bar0 + 0x00405c00u + si * 4u); + sphu[si] = ioread32(bar0 + 0x00405c24u + si * 4u); + } + for (si = 0; si < 4u; si++) { + fex[si] = ioread32(bar0 + 0x0040a630u + si * 4u); + fed[si] = ioread32(bar0 + 0x0040a620u + si * 4u); + fds[si] = ioread32(bar0 + 0x0040a640u + si * 4u); + } + rr_a = ioread32(bar0 + 0x00405c1cu); + rr_b = ioread32(bar0 + 0x00405c20u); + ds = ioread32(bar0 + 0x00405840u); + ds2 = ioread32(bar0 + 0x00405848u); + mminfo = ioread32(bar0 + 0x00404494u); + mmex = ioread32(bar0 + 0x004044a4u); + mm2 = ioread32(bar0 + 0x0040449cu); + mm3 = ioread32(bar0 + 0x004044a8u); + mm4 = ioread32(bar0 + 0x004044acu); + pr_info("vgsp-store: %s h=0x%x ds=%08x/%08x sph=%08x,%08x,%08x,%08x,%08x,%08x sphU=%08x,%08x,%08x,%08x,%08x,%08x rr=%08x,%08x\n", + tag, h_object, ds, ds2, + sph[0], sph[1], sph[2], sph[3], sph[4], sph[5], + sphu[0], sphu[1], sphu[2], sphu[3], sphu[4], sphu[5], + rr_a, rr_b); + pr_info("vgsp-store: %s h=0x%x mme_esr_info=%08x(pcv=%u pc=%u mac=0x%x) ex=%08x ir=%08x,%08x,%08x fed=%08x,%08x,%08x,%08x fex=%08x,%08x,%08x,%08x\n", + tag, h_object, mminfo, (mminfo >> 28) & 1u, mminfo & 0xfffu, + (mminfo >> 12) & 0xffu, mmex, mm2, mm3, mm4, + fed[0], fed[1], fed[2], fed[3], + fex[0], fex[1], fex[2], fex[3]); + /* + * MME shadow-RAM readback for the draw-relevant C597 methods (tracked + * methods show the last value the MME saw; zeros = never tracked). + * Also probe the MME setup pair and the draw trigger/vertex-start. + */ + pr_info("vgsp-store: %s h=0x%x shadow[c597] 0x114=%08x 0x11c=%08x 0x124=%08x 0x260=%08x 0x840=%08x 0xd74=%08x 0xd78=%08x 0x1618=%08x\n", + tag, h_object, + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x114u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x11cu), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x124u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x260u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x840u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0xd74u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0xd78u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x1618u)); + /* + * SESSION 153-CONTINUED-83 probe 2: the CALL trigger methods the + * window uses (are the calls even reaching the MME?), a repeat read + * (does a read clear?), and the dword-address interpretation of the + * draw/RT methods (0xd74>>2, 0x840>>2) to pin the index units. + */ + pr_info("vgsp-store: %s h=0x%x shadow2[c597] 0x3800=%08x 0x3810=%08x 0x3910=%08x 0x3938=%08x 0x3940=%08x 0x840r=%08x 0xd74dw=%08x 0x840dw=%08x 0x1618r=%08x\n", + tag, h_object, + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x3800u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x3810u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x3910u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x3938u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x3940u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x840u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0xd74u >> 2), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x840u >> 2), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x1618u >> 2)); +} + +static bool gsp_phys_mme_fast_sample(void __iomem *bar0, const char *tag, + u32 h_object, u32 ms_override) +{ + static u32 fast_boot_n; + u32 ms = ms_override ? ms_override : READ_ONCE(gsp_phys_mme_fast_ms); + u32 esr_p, ds_p, gf_p, tr_p, n = 0; + u32 fed0_p, fed1_p; + u32 ds_log_n = 0; + ktime_t t_end; + bool poisoned = false; + + /* + * SESSION 153-CONTINUED-106: the budget was 48, which a beef-lane boot + * with the wedged-engine relaunch train exhausts long before the + * first MME trap is relayed (r1363: 27 launch spins used by +300 s, + * trap traffic still arriving). Raised so every beef-family launch + * of a diagnostic boot keeps its spin. + */ + if (!bar0 || !ms || fast_boot_n >= 256u) + return false; + /* + * Beef-family channels get the full budget (the draw-class windows + * are the ones with the MME fault); other channels are sampled only + * a few times so a fast X-channel launch train cannot consume it. + */ + if ((h_object & 0xfffff000u) != 0xbeef0000u && fast_boot_n >= 2u) + return false; + fast_boot_n++; + esr_p = ioread32(bar0 + 0x00404490u); + ds_p = ioread32(bar0 + 0x00405840u); + gf_p = ioread32(bar0 + 0x00400504u); + tr_p = ioread32(bar0 + 0x00400704u); + /* + * SESSION 153-CONTINUED-109: the poison episode can already be + * latched when the sample's FIRST read happens. r1378's first + * window+live launch: `MME-FAST begin launch ... esr=badf1201 + * ds=badf1201 grfifo=badf1201 trap=badf1201` with the very next + * read clean (`esr badf1201->80000000 ... grfifo badf1201-> + * 00000001`). The historical detector only marks `poisoned` when a + * POST-entry read shows poison, so the episode was never handed to + * the WEDGE-RECOVER decision - no recovery ran, the prelude latch + * and live_gen stayed set, and every later calls-bearing window + * executed macro 57 against the wiped MME context: sked + * `mme_esr=80000004 (ILLEGAL_OPCODE) mme_mac=0x39`, then the FE + * parked at pc=513/mac=0xf1 and the stores stopped (0/905 after the + * 305-page burst). Treat an entry read of poison as the same + * witness; the FECS has already reset the context, so the recovery + * (latch clear + cursor rewind + live_gen invalidate) is exactly + * what is owed. + */ + if (((gf_p & 0xffff0000u) == 0xbadf0000u) || + ((esr_p & 0xffff0000u) == 0xbadf0000u) || + ((ds_p & 0xffff0000u) == 0xbadf0000u)) + poisoned = true; + /* + * SESSION 153-CONTINUED-84: FE_DECODE_STATE_EX(0)/(1) also drive a + * transition log - the MME's macro/generated witness can flip + * there without touching ESR/DS/grfifo, which is exactly the + * "did the MME start a macro after this doorbell" question the + * reloaded windows keep failing. + */ + fed0_p = ioread32(bar0 + 0x0040a630u); + fed1_p = ioread32(bar0 + 0x0040a634u); + pr_info("vgsp-gr: MME-FAST begin %s h=0x%x esr=%08x ds=%08x grfifo=%08x trap=%08x fe0=%08x fe1=%08x (spin %u ms)\n", + tag, h_object, esr_p, ds_p, gf_p, tr_p, fed0_p, fed1_p, ms); + /* + * SESSION 153-CONTINUED-107: IRAM chronology probe (see the + * gsp_phys_mme_iram_probe comment). Bounded to the first 8 fast + * samples of the boot so the launch train cannot flood; knob-gated + * because the debug-port writes disturb a live launch (see the + * sema_gl_mme_iram_probe comment). + */ + if (fast_boot_n <= 8u && + READ_ONCE(gsp_phys_rmcall_gr_mme_iram_probe)) + gsp_phys_mme_iram_probe(bar0, "begin", h_object); + t_end = ktime_get() + (s64)ms * 1000000ll; + for (;;) { + u32 esr = ioread32(bar0 + 0x00404490u); + u32 ds = ioread32(bar0 + 0x00405840u); + u32 gf = ioread32(bar0 + 0x00400504u); + u32 tr = ioread32(bar0 + 0x00400704u); + u32 fed0 = ioread32(bar0 + 0x0040a630u); + u32 fed1 = ioread32(bar0 + 0x0040a634u); + + if (esr != esr_p || ds != ds_p || gf != gf_p || tr != tr_p || + fed0 != fed0_p || fed1 != fed1_p) { + if (n < 12u) { + u32 info = ioread32(bar0 + 0x00404494u); + /* + * SESSION 153-CONTINUED-106: the IR word + * registers are 0x40449c (INFO2), 0x4044a8 + * (INFO3), 0x4044ac (INFO4). The old read at + * 0x404498 was NV_PGRAPH_PRI_MME_CONFIG + * (RW), which is why every "info2_ir" line so + * far showed the shadow-filter enable, not the + * rejected instruction. + */ + u32 info2 = ioread32(bar0 + 0x0040449cu); + u32 info3 = ioread32(bar0 + 0x004044a8u); + u32 info4 = ioread32(bar0 + 0x004044acu); + u32 mmeir = ioread32(bar0 + 0x0040a630u); + u32 mmeir1 = ioread32(bar0 + 0x0040a634u); + u32 mmedata = ioread32(bar0 + 0x00400708u); + /* + * SESSION 153-CONTINUED-106: the MME's own + * macro PC/state at the transition instant + * (0x4044b4: PC 11:0, MACRO 19:12, RUNNING + * 20) - the "is the MME mid-macro at the + * trap" witness. + */ + u32 mmemac = ioread32(bar0 + 0x004044b4u); + + /* + * r1151 (SESSION 153-CONTINUED-72): + * GR-STATE: the FE decode-state EX slot 0 + * records the last MME-GENERATED method + * write (bit31 generated, 21:12 macro + * address, 11:0 pc). gen0=1 with the + * kept slot's macro address is the + * "kept macro executes" witness during + * the launch; slot 1 is the second + * lane of the same record set. + */ + pr_warn("vgsp-gr: MME-FAST %s h=0x%x esr %08x->%08x info=%08x(pc_valid=%u pc=%u mac=0x%x) ir=%08x,%08x,%08x ds %08x->%08x grfifo %08x->%08x trap=%08x data=%08x fed=%08x,%08x (pc0=%u mac0=0x%x gen0=%u) fe1=%08x (pc1=%u mac1=0x%x gen1=%u)\n", + tag, h_object, esr_p, esr, + info, (info >> 28) & 1u, + info & 0xfffu, + (info >> 12) & 0xffu, + info2, info3, info4, + ds_p, ds, gf_p, gf, tr, + mmedata, mmeir, mmeir1, + mmeir & 0xfffu, + (mmeir >> 12) & 0x3ffu, + (mmeir >> 31) & 1u, + fed1, + fed1 & 0xfffu, + (fed1 >> 12) & 0x3ffu, + (fed1 >> 31) & 1u); + pr_warn("vgsp-gr: MME-FAST %s h=0x%x macro-status=%08x(mpc=%u mac=0x%x run=%u)\n", + tag, h_object, mmemac, + mmemac & 0xfffu, + (mmemac >> 12) & 0xffu, + (mmemac >> 20) & 1u); + /* + * SESSION 153-CONTINUED-107: sample the live + * IRAM at every fault-register transition, so + * the chronology (words loaded early vs wiped + * by the poison/DROP machinery) is on record + * for the same instant the ESR changes. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_mme_iram_probe)) + gsp_phys_mme_iram_probe(bar0, "trans", + h_object); + } + /* + * SESSION 153-CONTINUED-106: when the DS latch CHANGES + * into a fault (the SPH validation miss at draw + * dispatch), read the per-stage SPH_INFO START/UPPER + * and PROGRAM_REGION pair IMMEDIATELY - the FECS + * clears the whole block within ~1 ms (r1364: + * 404004.338 poison seen, 404004.339 clean), so this + * is the only in-flight window that names the VA each + * failing stage fetched. + */ + if (ds != ds_p && (ds & 0x7fffffffu) && ds_log_n < 8u) { + ds_log_n++; + u32 s0 = ioread32(bar0 + 0x00405c00u); + u32 s1 = ioread32(bar0 + 0x00405c04u); + u32 s2 = ioread32(bar0 + 0x00405c08u); + u32 s3 = ioread32(bar0 + 0x00405c0cu); + u32 s4 = ioread32(bar0 + 0x00405c10u); + u32 s5 = ioread32(bar0 + 0x00405c14u); + u32 u0 = ioread32(bar0 + 0x00405c24u); + u32 u1 = ioread32(bar0 + 0x00405c28u); + u32 u2 = ioread32(bar0 + 0x00405c2cu); + u32 u3 = ioread32(bar0 + 0x00405c30u); + u32 u4 = ioread32(bar0 + 0x00405c34u); + u32 u5 = ioread32(bar0 + 0x00405c38u); + u32 rr_a = ioread32(bar0 + 0x00405c1cu); + u32 rr_b = ioread32(bar0 + 0x00405c20u); + + pr_warn("vgsp-gr: DS-FAST %s h=0x%x ds %08x->%08x stage=%08x,%08x,%08x,%08x,%08x,%08x upper=%02x,%02x,%02x,%02x,%02x,%02x region=%08x,%08x esr=%08x\n", + tag, h_object, ds_p, ds, + s0, s1, s2, s3, s4, s5, + u0 & 0xffu, u1 & 0xffu, u2 & 0xffu, + u3 & 0xffu, u4 & 0xffu, u5 & 0xffu, + rr_a, rr_b, + ioread32(bar0 + 0x00404490u)); + /* + * SESSION 153-CONTINUED-106: the pending DS + * report poisons the whole PRI block + * (every register reads 0xbadf1201 - the + * first-capture above shows exactly that). + * Clear the report masks and re-read: the + * CONT-91 rule says a DS miss with the masks + * cleared latches without the poison, so the + * per-stage SPH snapshot should become + * readable. + */ + { + u32 m0 = ioread32(bar0 + 0x00405844u); + u32 m2 = ioread32(bar0 + 0x0040584cu); + + iowrite32(0u, bar0 + 0x00405844u); + iowrite32(0u, bar0 + 0x0040584cu); + udelay(80); + pr_warn("vgsp-gr: DS-FAST2 %s h=0x%x masks=%08x/%08x ds_after=%08x stage=%08x,%08x,%08x,%08x,%08x,%08x upper=%02x,%02x,%02x,%02x,%02x,%02x region=%08x,%08x\n", + tag, h_object, m0, m2, + ioread32(bar0 + 0x00405840u), + ioread32(bar0 + 0x00405c00u), + ioread32(bar0 + 0x00405c04u), + ioread32(bar0 + 0x00405c08u), + ioread32(bar0 + 0x00405c0cu), + ioread32(bar0 + 0x00405c10u), + ioread32(bar0 + 0x00405c14u), + ioread32(bar0 + 0x00405c24u) & 0xffu, + ioread32(bar0 + 0x00405c28u) & 0xffu, + ioread32(bar0 + 0x00405c2cu) & 0xffu, + ioread32(bar0 + 0x00405c30u) & 0xffu, + ioread32(bar0 + 0x00405c34u) & 0xffu, + ioread32(bar0 + 0x00405c38u) & 0xffu, + ioread32(bar0 + 0x00405c1cu), + ioread32(bar0 + 0x00405c20u)); + } + } + n++; + esr_p = esr; + ds_p = ds; + gf_p = gf; + tr_p = tr; + fed0_p = fed0; + fed1_p = fed1; + /* + * The priv-error poison is the terminal state: the + * FECS has already logged and the rest of the + * registers are 0xbadfxxxx. Stop early so the launch + * thread resumes its normal poll. + */ + if ((gf & 0xffff0000u) == 0xbadf0000u) { + poisoned = true; + break; + } + } + if (ktime_after(ktime_get(), t_end)) + break; + } + pr_info("vgsp-gr: MME-FAST end %s h=0x%x events=%u%s\n", + tag, h_object, n, poisoned ? " (poison)" : ""); + /* + * SESSION 153-CONTINUED-107: post-spin IRAM sample - the state left + * when the poison (if any) has just cleared, before the poll/retire + * work touches anything. + */ + if (fast_boot_n <= 8u && + READ_ONCE(gsp_phys_rmcall_gr_mme_iram_probe)) + gsp_phys_mme_iram_probe(bar0, "end", h_object); + /* + * SESSION 153-CONTINUED-83 store-side emission forensics: right after + * the launch's retire spin, snapshot the DS shader-program-header + * latches (which pipeline stages failed SPH validation), the MME + * execution state (did the MME run a macro, and where is its PC), and + * the FE decode-state EX slots (did any method decode carry the MME + * GENERATED bit). Bounded per boot (beef-family only). + */ + if ((h_object & 0xfffff000u) == 0xbeef0000u) + gsp_phys_store_side_log(bar0, tag, h_object); + /* + * SESSION 153-CONTINUED-72: return the poison observation to the + * launch site. The r1149 race: the poison flooded during this spin + * and the FECS cleared it before the poll loop's WEDGE-RECOVER check + * read grfifo, so recovery missed, the prelude latch stayed set and + * the engine wedged for the boot. The caller now latches this flag + * into the recovery gate instead of relying on a live register read. + */ + return poisoned; +} + +static void gsp_phys_pbdma_fault_dump(u32 chid) +{ + static u32 dump_n; + void __iomem *bar0 = gsp_phys_gpu_bar0(); + u64 pb_va = boot_stage.rmcall_va_pb; + u32 i; + + if (!bar0 || !gsp_phys_gr_last_words || dump_n >= GSP_PHYS_PBDMA_DUMP_MAX) + return; + dump_n++; + pr_info("vgsp-gr: PB-FAULT-DUMP chid=%u last_block words=%u h=0x%x push=0x%llx pb_va=0x%llx\n", + chid, gsp_phys_gr_last_words, gsp_phys_gr_last_h, + gsp_phys_gr_last_push_va, pb_va); + for (i = 0; i < 12; i++) { + u32 b = 0x40000u + i * 0x2000u; + u32 intr0 = readl(bar0 + b + 0x108u); + u32 gpf = readl(bar0 + b + 0x050u); + u32 pbf = readl(bar0 + b + 0x054u); + u32 pbfhi = readl(bar0 + b + 0x058u); + u32 pbhd = readl(bar0 + b + 0x084u); + u32 pbcnt = readl(bar0 + b + 0x088u); + u32 pbd0 = readl(bar0 + b + 0x08cu); + u32 hdrsh = readl(bar0 + b + 0x118u); + u32 fetch64 = ((u64)(pbfhi & 0xffu) << 32) | (pbf & ~3u); + u32 trig; + const char *trig_src; + s64 off_words = -1; + s64 voff = -1; + + if (!intr0 && !pbf && !hdrsh && !pbhd) + continue; + /* + * SESSION 153-CONTINUED-52: the real driver's rule for the + * instruction that triggered the PBDMA interrupt: if the + * DECODED header (PB_HEADER) type is IMMD or PB_COUNT's value + * field is zero, the raw instruction is PB_DATA0; otherwise it + * is HDR_SHADOW (fifo_tu102.c: "If the TYPE field of + * NV_PPBDMA_PB_HEADER is IMMD or the VALUE field of + * NV_PPBDMA_PB_COUNT is zero, then the raw PB instruction + * stored in NV_PPBDMA_PB_DATA0 is the one that triggered the + * interrupt. Otherwise ... HDR_SHADOW"). + */ + if (((pbhd >> 29) & 7u) == 4u || (pbcnt & 0x1fffu) == 0u) { + trig = pbd0; + trig_src = "DATA0"; + } else { + trig = hdrsh; + trig_src = "HDRSHADOW"; + } + if (gsp_phys_gr_last_pb_dma && + fetch64 >= gsp_phys_gr_last_pb_dma && + fetch64 < gsp_phys_gr_last_pb_dma + + (u64)gsp_phys_gr_last_words * 4u) + off_words = (s64)(fetch64 - gsp_phys_gr_last_pb_dma) / 4; + /* + * SESSION 153-CONTINUED-59: the SAME offset computed against + * the GPU VA the engine fetches (the r1102 WINPROBE/PB-FAULT + * agreement: pbfetch 0x101ebc with pb_va 0x101000 and 943 + * words = exactly the block end). voff >= words is the + * past-the-block-end prefetch position. + */ + voff = (s64)-1; + if (pb_va && fetch64 >= pb_va && + fetch64 < pb_va + (u64)GSP_PHYS_PAGE_SIZE) + voff = (s64)(fetch64 - pb_va) / 4; + pr_info("vgsp-gr: PB-FAULT-DUMP P%u intr0=%08x gpf=%u pbfetch=0x%llx len=%u off=%lld voff=%lld/%u hdr=%08x(%s m=0x%x cnt=%u sub=%u) cnt=%08x data0=%08x shadow=%08x pbhd=%08x(type=%s) TRIG=%08x(%s)\n", + i, intr0, gpf, fetch64, pbfhi >> 11, off_words, voff, + gsp_phys_gr_last_words, + hdrsh, gsp_phys_pbdma_type_name(hdrsh >> 29), + (hdrsh & 0x1fffu) << 2, + (hdrsh >> 16) & 0xfffu, (hdrsh >> 13) & 7u, + pbcnt, pbd0, hdrsh, pbhd, + gsp_phys_pbdma_type_name(pbhd >> 29), + trig, trig_src); + } + { + struct gsp_phys_boot_chunk *pbch = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB]; + u32 *pw = pbch->pages ? gsp_phys_chunk_page_va(pbch, 0) : NULL; + + if (pw && pb_va) { + u32 n = gsp_phys_gr_last_words < 384u ? + gsp_phys_gr_last_words : 384u; + u32 w; + + for (w = 0; w < n; w += 16) { + u32 m; + + pr_info("vgsp-gr: PB-FAULT-DUMP pb[%3u]= %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n", + w, pw[w], w + 1 < n ? pw[w + 1] : 0, + w + 2 < n ? pw[w + 2] : 0, w + 3 < n ? pw[w + 3] : 0, + w + 4 < n ? pw[w + 4] : 0, w + 5 < n ? pw[w + 5] : 0, + w + 6 < n ? pw[w + 6] : 0, w + 7 < n ? pw[w + 7] : 0, + w + 8 < n ? pw[w + 8] : 0, w + 9 < n ? pw[w + 9] : 0, + w + 10 < n ? pw[w + 10] : 0, w + 11 < n ? pw[w + 11] : 0, + w + 12 < n ? pw[w + 12] : 0, w + 13 < n ? pw[w + 13] : 0, + w + 14 < n ? pw[w + 14] : 0, w + 15 < n ? pw[w + 15] : 0); + } + /* + * SESSION 153-CONTINUED-59: the block TAIL and the + * words PAST the block end (the r1102 fault fetch + * sits exactly at the end - voff == words). The past + * words are what a prefetching PBDMA decodes next; + * the tail shows whether the block itself ends at an + * entry boundary. + */ + { + u32 words = gsp_phys_gr_last_words; + u32 lo = words > 48u ? words - 48u : 0u; + u32 hi = words + 32u; + + if (hi > 1024u) + hi = 1024u; + for (w = lo; w < hi; w += 16) { + pr_info("vgsp-gr: PB-FAULT-DUMP %s[%3u]= %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n", + w < words ? "tail" : "past", w, + pw[w], w + 1 < 1024u ? pw[w + 1] : 0, + w + 2 < 1024u ? pw[w + 2] : 0, w + 3 < 1024u ? pw[w + 3] : 0, + w + 4 < 1024u ? pw[w + 4] : 0, w + 5 < 1024u ? pw[w + 5] : 0, + w + 6 < 1024u ? pw[w + 6] : 0, w + 7 < 1024u ? pw[w + 7] : 0, + w + 8 < 1024u ? pw[w + 8] : 0, w + 9 < 1024u ? pw[w + 9] : 0, + w + 10 < 1024u ? pw[w + 10] : 0, w + 11 < 1024u ? pw[w + 11] : 0, + w + 12 < 1024u ? pw[w + 12] : 0, w + 13 < 1024u ? pw[w + 13] : 0, + w + 14 < 1024u ? pw[w + 14] : 0, w + 15 < 1024u ? pw[w + 15] : 0); + } + } + } + } +} + +/* + * SESSION 153-CONTINUED-86: the fault-time register witness. The relayed + * errString names the fault; these reads add the machine state behind it at + * the earliest host observation point (the 0x1006 consume), before the + * FECS/GSP-RM clear the latches: MME_HWW_ESR (0x404490) + INFO (0x404494: + * PC_VALID/PC/MACRO_ADDRESS) + INFO2 (IR word), DS_HWW_ESR (0x405840) + + * ESR_2 (0x405848) + the SPH_INFO stage starts (0x405c00+si*4) that name the + * exact VA each failing pipeline stage fetched its shader program header + * from, the SPH program-region pair (0x405c1c/c20), and the trapped method + * address/data (0x400704/8). GR-class faults only (0xd graphics exception, + * 0x45 class error; PBDMA 0x20 has its own dump), bounded per boot so a fault + * storm cannot flood dmesg. + */ +static void gsp_phys_fault_regs_log(u32 except_type) +{ + static u32 frl_n; + void __iomem *bar0 = gsp_phys_gpu_bar0(); + u32 esr, info, info2, ds, ds2, trap_a, trap_d, rr_a, rr_b, sph[6], si; + + if (!bar0 || frl_n >= 48u) + return; + if (except_type != 0xdu && except_type != 0x45u) + return; + frl_n++; + esr = ioread32(bar0 + 0x00404490u); + info = ioread32(bar0 + 0x00404494u); + /* SESSION 153-CONTINUED-106: INFO2 is 0x40449c (0x404498 is CONFIG). */ + info2 = ioread32(bar0 + 0x0040449cu); + ds = ioread32(bar0 + 0x00405840u); + ds2 = ioread32(bar0 + 0x00405848u); + for (si = 0; si < 6u; si++) + sph[si] = ioread32(bar0 + 0x00405c00u + si * 4u); + rr_a = ioread32(bar0 + 0x00405c1cu); + rr_b = ioread32(bar0 + 0x00405c20u); + trap_a = ioread32(bar0 + 0x00400704u); + trap_d = ioread32(bar0 + 0x00400708u); + pr_info("vgsp-fault: regs except=0x%x mme_esr=%08x(pcv=%u ill=%u miss=%u extra=%u) mme_info=%08x(pc=%u mac=0x%x) mme_ir=%08x ds=%08x/%08x sph=%08x,%08x,%08x,%08x,%08x,%08x rr=%08x,%08x trap=%08x data=%08x\n", + except_type, esr, (info >> 28) & 1u, (esr >> 2) & 1u, + esr & 1u, (esr >> 1) & 1u, info, info & 0xfffu, + (info >> 12) & 0xffu, info2, ds, ds2, + sph[0], sph[1], sph[2], sph[3], sph[4], sph[5], + rr_a, rr_b, trap_a, trap_d); +} + +/* * Note every consumed 0x1006 in the bounded last-4 ring, advance the fault * epoch (the relay submit snapshot compares against it) and log the designed * observation line when the fault relay is armed. Decoding and counting are @@ -4926,10 +5811,14 @@ static void gsp_phys_fault_observe(const struct gsp_phys_rx_record *rec, GSP_PHYS_FAULT_NOTE_MAX; gsp_phys_fault_seen++; gsp_phys_fault_epoch++; - if (gsp_phys_fault_relay_enabled()) + if (gsp_phys_fault_relay_enabled()) { pr_info_ratelimited("vgsp-fault: func=0x1006 seq=%u exceptType=0x%x runlistId=%u chid=%u prevXid=0x%x errString=\"%s\"\n", rec->seq_num, out->except_type, out->runlist_id, out->chid, out->prev_xid, out->err); + gsp_phys_fault_regs_log(out->except_type); + } + if (out->except_type == 0x20u) + gsp_phys_pbdma_fault_dump(out->chid); } /* Note every consumed 0x1020 for the C3 corroboration ring. */ @@ -6827,6 +7716,35 @@ int gsp_phys_backend_static_wait(struct gsp_phys_backend_static_config *out, EXPORT_SYMBOL_GPL(gsp_phys_backend_static_wait); /* + * Mode-35 KGR static-data snapshots. The user-lane query set + * (gsp_phys_rmcall_user_lane_kgr_queries) is answered by the LIVE GSP with + * the device's true GR static data; the replies are kept here so the + * guest-side emulation can serve the identical bytes instead of fabricating + * GR values. The concrete consumer is the X driver's 3D-engine selection: + * nv3dAllocDevice maps GR_GET_INFO index SM_VERSION (0x0c) through a table + * and fails with "Failed to select a 3D engine" when it reads back as + * SM_VERSION_NONE (0) - the fabricated fill never set it, while the live + * GSP's KGR_GET_INFO reply carries the real value (0x75 for TU104) along + * with every other GR static field (warps, caps, topology, ctx buffers). + * + * One entry per captured cmd; the generation key ties the cache to the + * backend session exactly like gsp_phys_backend_static above, and the + * getter refuses entries once the session has moved on. + */ +#define GSP_PHYS_KGR_SNAP_ENTRIES 10 +#define GSP_PHYS_KGR_SNAP_MAX_SIZE 3712u /* KGR_GET_INFO: 8 engines x 58 x 8 */ + +struct gsp_phys_kgr_snap { + u32 cmd; + u32 size; + bool valid; + u32 generation; + u8 data[GSP_PHYS_KGR_SNAP_MAX_SIZE]; +}; + +static struct gsp_phys_kgr_snap gsp_phys_kgr_snaps[GSP_PHYS_KGR_SNAP_ENTRIES]; + +/* * Drop the cached static-config snapshot. Called when the backend session * leaves READY (or a new claim starts): the cache generation key is bumped * past the current backend generation so a concurrent getter cannot serve the @@ -6835,9 +7753,13 @@ EXPORT_SYMBOL_GPL(gsp_phys_backend_static_wait); */ void gsp_phys_backend_static_invalidate(void) { + unsigned int i; + mutex_lock(&gsp_phys_rm_lock); gsp_phys_backend_static.valid = false; gsp_phys_backend_static.generation = gsp_phys_backend_generation() + 1; + for (i = 0; i < ARRAY_SIZE(gsp_phys_kgr_snaps); i++) + gsp_phys_kgr_snaps[i].valid = false; mutex_unlock(&gsp_phys_rm_lock); } @@ -6891,6 +7813,71 @@ int gsp_phys_backend_static_get(struct gsp_phys_backend_static_config *out) } EXPORT_SYMBOL_GPL(gsp_phys_backend_static_get); +/* + * Copy the captured reply for cmd into dst. Returns 0 on success (with + * *out_len set), -ENODATA when nothing was captured for cmd (or the entry's + * generation no longer matches the backend session), -ENOSPC when the + * snapshot does not fit dst_len, -EAGAIN while the backend is not READY. + */ +int gsp_phys_kgr_snap_get(u32 cmd, void *dst, u32 dst_len, u32 *out_len) +{ + unsigned int i; + + if (!dst || !out_len) + return -EINVAL; + + if (gsp_phys_backend_state() != GSP_PHYS_BACKEND_READY) + return -EAGAIN; + + for (i = 0; i < ARRAY_SIZE(gsp_phys_kgr_snaps); i++) { + struct gsp_phys_kgr_snap *s = &gsp_phys_kgr_snaps[i]; + + if (s->cmd != cmd) + continue; + if (!s->valid || + s->generation != gsp_phys_backend_generation()) + return -ENODATA; + if (s->size > dst_len) + return -ENOSPC; + memcpy(dst, s->data, s->size); + *out_len = s->size; + return 0; + } + return -ENODATA; +} +EXPORT_SYMBOL_GPL(gsp_phys_kgr_snap_get); + +/* + * Record one mode-35 KGR query reply. Called under the boot thread only + * (same thread as the query itself), so no lock is needed beyond the + * generation key. + */ +static void gsp_phys_kgr_snap_store(u32 cmd, const void *data, u32 size) +{ + unsigned int i; + struct gsp_phys_kgr_snap *slot = NULL; + + if (size > GSP_PHYS_KGR_SNAP_MAX_SIZE) + return; + + for (i = 0; i < ARRAY_SIZE(gsp_phys_kgr_snaps); i++) { + if (gsp_phys_kgr_snaps[i].cmd == cmd) { + slot = &gsp_phys_kgr_snaps[i]; + break; + } + if (!slot && !gsp_phys_kgr_snaps[i].valid) + slot = &gsp_phys_kgr_snaps[i]; + } + if (!slot) + return; + + slot->cmd = cmd; + slot->size = size; + slot->generation = gsp_phys_backend_generation(); + memcpy(slot->data, data, size); + slot->valid = true; +} + /* ============================================================ * Host RM client ladder (R1): gsp-phys-rm-call * ============================================================ */ @@ -9626,14 +10613,94 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0) * (swref/published/pascal/gp100/dev_mmu.h:97-119,157; * uvm_pascal_mmu.c:64-141,235-303). */ +/* + * r431/r436 COHERENCY ROOT-CAUSE CANDIDATE: the leaf-PTE aperture for every + * host page this backend maps (the ladder windows, the win0 shadow pool, the + * bounce/data regions, the release/scratch shadows) was hardcoded to + * aperture 3 = SYSTEM_NON_COHERENT_MEMORY. The reference RM derives the + * pair from the allocation's CPU cache attribute (fabric_vaspace.c:1123-1134 + * for the aperture, mem_desc.c memdescGetVolatility for VOL: NV_MEMORY_CACHED + * -> SYSTEM_COHERENT (2) + volatile FALSE, NV_MEMORY_UNCACHED -> NON_COHERENT + * (3) + volatile TRUE). The r431 experiment used aperture 2 with VOL still 1 + * (0xd) - a pair the reference RM never produces - and it made the engine's + * truncated writes visible sooner but worsened the wedge on a fresh boot. + * r436 turns the knob into the full 2x2 matrix so each consistent encoding + * can be A/B'd: + * 0 = aperture 3 + VOL 1 (0xf) - historical, NV_MEMORY_UNCACHED + * 1 = aperture 2 + VOL 1 (0xd) - r431 experiment (inconsistent pair) + * 2 = aperture 2 + VOL 0 (0x5) - NV_MEMORY_CACHED (coherent, non-volatile) + * 3 = aperture 3 + VOL 0 (0x7) - non-coherent, non-volatile + * The PTEs are planted before the engine's first fetch, so the flag is + * boot-time only. + */ +static int gsp_phys_rmcall_pte_coh_param; +module_param_named(gsp_phys_rmcall_pte_coh, gsp_phys_rmcall_pte_coh_param, + int, 0444); +MODULE_PARM_DESC(gsp_phys_rmcall_pte_coh, + "r436: sysmem leaf-PTE aperture/VOL mode: 0=0xf (aperture3+VOL, historical), 1=0xd (aperture2+VOL, r431), 2=0x5 (aperture2, non-volatile), 3=0x7 (aperture3, non-volatile)"); + +static u64 gsp_phys_rmcall_sysmem_base_low(void) +{ + switch (gsp_phys_rmcall_pte_coh_param) { + case 1: + return 0xdull; /* aperture 2, VOL 1 */ + case 2: + return 0x5ull; /* aperture 2, VOL 0 */ + case 3: + return 0x7ull; /* aperture 3, VOL 0 */ + default: + return 0xfull; /* aperture 3, VOL 1 (historical) */ + } +} + +static u64 gsp_phys_rmcall_pte(u64 pa) +{ + /* + * Low nibble: VALID (1) | APERTURE<<1 | VOL (8). Aperture 2 = + * SYSTEM_COHERENT_MEMORY -> 1|4|8 = 0xd; aperture 3 = NON_COHERENT + * -> 1|6|8 = 0xf. r436: see gsp_phys_rmcall_pte_coh_param. + */ + u64 base = gsp_phys_rmcall_sysmem_base_low(); + + return base | ((pa >> 12) << 8); +} + +/* + * r436: the single PDE uses the same aperture/VOL pair as the leaf PTEs but + * with IS_PDE=0 in bit 0 (the low bit of the PTE base is VALID there), so the + * PDE base is the PTE base with bit 0 cleared: 0xe/0xc/0x4/0x6. + */ static u64 gsp_phys_rmcall_pde(u64 pa) { - return 0xeull | ((pa >> 12) << 8); + u64 base = gsp_phys_rmcall_sysmem_base_low() & ~1ull; + + return base | ((pa >> 12) << 8); } -static u64 gsp_phys_rmcall_pte(u64 pa) +/* + * SESSION 153-CONTINUED-92: leaf PTE for the engine-visible guest/scratch + * mappings with the configurable v2 KIND field (bits 63:56). + * + * m96 documented the symptom that motivates this: on Turing a kind=0 PTE is + * what the SM I-fetch refuses with MMU_NACK while the FE tolerates it, and + * `gsp_phys_rmcall_gr_pte_kind` was introduced to plant the win0/identity + * pool PTEs with a kind (UVM turing uses GENERIC_MEMORY 0x06). The knob was + * never applied to the scratch-PT pool mappings - every shadow VA the 3D + * window's program operands are relocated to (the VAs the DS fetches the + * SPH through) carried kind=0. The DS validates a program header, i.e. runs + * a program fetch; if that path is the SM-side one, kind=0 is a candidate + * cause for the SPH rejection that is independent of content (and explains + * why no content A/B ever moved the wall). With the knob 0 this is + * byte-identical to the previous PTE encoding; nonzero = experiment. + */ +static u64 gsp_phys_rmcall_gr_pte_leaf(u64 hpa) { - return 0xfull | ((pa >> 12) << 8); + u64 pte = gsp_phys_rmcall_pte(hpa); + int kind = gsp_phys_rmcall_gr_pte_kind_value(); + + if (kind) + pte |= (u64)(u32)kind << 56; + return pte; } /* @@ -9766,6 +10833,26 @@ static void gsp_phys_rmcall_pdb_build(void) (u8 *)gsp_phys_chunk_page_va(pt, 0) + (size_t)sem_idx * 8); /* + * SESSION 153-CONTINUED-96: secondary CE lane GPFIFO ring (slot 0x10d) + * and its one-page pushbuffer (slot 0x10e), planted here in the same + * 128 B PTE line as the ladder's own pages - stage 3, before any + * engine fetch (no invalidate exists). Always planted (the lane's + * page/PTE cost is two pages; a disabled lane simply never rings its + * doorbell), so pdb_build stays deterministic. + */ + put_unaligned_le64(gsp_phys_rmcall_pte( + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_chunks[ + GSP_PHYS_RMCALL_BUF_CE_GPFIFO], 0)), + (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)((GSP_PHYS_RMCALL_VA_CE_GPFIFO >> 12) & 0x1ffu) * 8); + put_unaligned_le64(gsp_phys_rmcall_pte( + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_chunks[ + GSP_PHYS_RMCALL_BUF_CE_PB], 0)), + (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)((GSP_PHYS_RMCALL_VA_CE_PB >> 12) & 0x1ffu) * 8); + /* * Rank-2 qmdwin: plant the dedicated module page for the forwarded * inline-QMD destination in this same small-PT line (slot 0x107, VA * 0x107000; slots 0x105/0x106 stay the P6a guest slots below). This @@ -9812,6 +10899,66 @@ static void gsp_phys_rmcall_pdb_build(void) } } /* + * SESSION 153-CONTINUED-66: plant page 2 of the module pushbuffer + * (VA_PB2, small-PT slot 0x102) and the dedicated warm-semaphore + * probe page (VA_GR_PROBE, slot 0x10a) in the same 128 B PTE line, + * at stage 3 before the engine's first fetch (no invalidate exists). + * Page 0 stays the registered BUF_PB chunk (stage 2 unchanged); this + * chunk is engine-reached purely through these PTEs, exactly like + * the qmdwin pair above. Without it the GR launch block keeps the + * single-page bound (<= 4 KB). + */ + if (!boot_stage.rmcall_gr_pb2.pages && boot_stage.pdev) { + if (!gsp_phys_chunk_alloc(boot_stage.pdev, + &boot_stage.rmcall_gr_pb2, 3u)) + gsp_phys_chunk_zero(&boot_stage.rmcall_gr_pb2); + else + pr_warn("vgsp-boot: rm-call: pushbuffer page2/3/4 allocation failed; >4 KB launches keep the single-page block\n"); + } + if (!boot_stage.rmcall_gr_probe.pages && boot_stage.pdev) { + if (!gsp_phys_chunk_alloc(boot_stage.pdev, + &boot_stage.rmcall_gr_probe, 1u)) + gsp_phys_chunk_zero(&boot_stage.rmcall_gr_probe); + else + pr_warn("vgsp-boot: rm-call: warm-sem probe page allocation failed; the probe keeps the pushbuffer upper half\n"); + } + if (boot_stage.rmcall_gr_pb2.pages && boot_stage.rmcall_gr_probe.pages) { + u32 pb2_idx = (u32)((GSP_PHYS_RMCALL_VA_PB2 >> 12) & 0x1ffu); + u32 pb3_idx = (u32)((GSP_PHYS_RMCALL_VA_PB3 >> 12) & 0x1ffu); + u32 pb4_idx = (u32)((GSP_PHYS_RMCALL_VA_PB4 >> 12) & 0x1ffu); + u32 pr_idx = (u32)((GSP_PHYS_RMCALL_VA_GR_PROBE >> 12) & + 0x1ffu); + + put_unaligned_le64(gsp_phys_rmcall_pte( + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_gr_pb2, 0)), + (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)pb2_idx * 8); + if (boot_stage.rmcall_gr_pb2.pages >= 2u) + put_unaligned_le64(gsp_phys_rmcall_pte( + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_gr_pb2, 1)), + (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)pb3_idx * 8); + if (boot_stage.rmcall_gr_pb2.pages >= 3u) + put_unaligned_le64(gsp_phys_rmcall_pte( + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_gr_pb2, 2)), + (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)pb4_idx * 8); + put_unaligned_le64(gsp_phys_rmcall_pte( + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_gr_probe, 0)), + (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)pr_idx * 8); + dma_wmb(); + boot_stage.rmcall_gr_pb2_planted = true; + pr_info("vgsp-boot: rm-call: GR launch pb page2/3/4 + warm-sem probe planted VA=0x%llx/0x%llx/0x%llx/0x%llx ptSlots=0x%x/0x%x/0x%x/0x%x (up to four-page block; before first fetch)\n", + GSP_PHYS_RMCALL_VA_PB2, GSP_PHYS_RMCALL_VA_PB3, + GSP_PHYS_RMCALL_VA_PB4, GSP_PHYS_RMCALL_VA_GR_PROBE, + pb2_idx, pb3_idx, pb4_idx, pr_idx); + } + /* * m69: plant the window-0 shadow pool PTEs (current win0 slot range) in the * same proven small-PT line, before the engine's first fetch. The * launch copies the QMD's program/param page content into these pages @@ -13551,6 +14698,12 @@ static int gsp_phys_rmcall_user_lane_userd(void __iomem *bar0) * (CONTEXT_BUFFERS_INFO in particular) may be prepared by the query. All are * zero-in best-effort getters at the capture's payload sizes; a failure * warns and the lane continues. + * + * Every successful reply is retained via gsp_phys_kgr_snap_store(): the + * guest-side emulation serves these exact bytes for the same commands, so + * the guest's GL/3D stack reads the live GSP's true GR static values + * (SM_VERSION in particular - see the snapshot comment near + * gsp_phys_backend_static_get). */ static void gsp_phys_rmcall_user_lane_kgr_queries(void __iomem *bar0) { @@ -13589,6 +14742,8 @@ static void gsp_phys_rmcall_user_lane_kgr_queries(void __iomem *bar0) q[i].name); pr_info("vgsp-boot: rm-call: mode 35 KGR query %s: size=%u status=0x%x rpc_result=0x%x ret=%d\n", q[i].name, q[i].size, status, rpc_result, ret); + if (ret == 0 && rpc_result == 0 && status == 0) + gsp_phys_kgr_snap_store(q[i].cmd, buf, q[i].size); kvfree(buf); } } @@ -13861,6 +15016,16 @@ static int gsp_phys_rmcall_user_lane_main_promote(void __iomem *bar0) status, rpc_result, GSP_PHYS_RMCALL_HANDLE_USER_LANE, boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_va); kvfree(p); + /* + * SESSION 153-CONTINUED-96: the F2 mapped-MAIN promote is the server's + * acceptance of the ladder's own context buffer (PA+VA, bInitialize=1), + * i.e. exactly the condition rmcall_ctx_valid records. Modes 0-34 set it + * at the end of their promote step (line ~13675); mode 35 promotes here, + * so without this the state file read valid=0 and the engine-ctx + * readback gate silently skipped. + */ + if (!ret) + boot_stage.rmcall_ctx_valid = true; return ret; } @@ -14581,6 +15746,35 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) boot_stage.rmcall_gr_object = GSP_PHYS_RMCALL_HANDLE_USER_LANE_GR_OBJ; } + /* + * Graphics-execution plan §F first 3D-lane experiment: + * construct the TURING_A (0xc597) class object on V7 + * after the accepted promote, through the same generic + * alloc helper the golden lanes use. The plan's open + * question is whether a 0xc597 construct on the + * execution lane is accepted without the wired-golden + * deadlock class; the result line below is the + * experiment's evidence either way, and the construct + * never gates the lane. + */ + { + u32 g3_rpc = 0, g3_status = 0; + int g3_ret = gsp_phys_rmcall_alloc_gr_object_class(bar0, + GSP_PHYS_RMCALL_CLASS_GR_3D, + GSP_PHYS_RMCALL_HANDLE_USER_LANE_GR3D, + GSP_PHYS_RMCALL_HANDLE_USER_LANE, + "mode 35 TURING_A on V7 (plan-F first 3D-lane experiment, after promote)", + &g3_rpc, &g3_status); + + pr_info("vgsp-boot: rm-call: mode 35 plan-F experiment: TURING_A (0xc597) on V7 hChannel=0x%x -> ret=%d hObject=0x%x status=0x%x rpc_result=0x%x (%s)\n", + GSP_PHYS_RMCALL_HANDLE_USER_LANE, + g3_ret, + g3_ret ? 0 : + GSP_PHYS_RMCALL_HANDLE_USER_LANE_GR3D, + g3_status, g3_rpc, + g3_ret ? "rejected/absent; lane continues" : + "accepted; V7 carries a 3D class object for the method-stream replay"); + } pr_info("vgsp-boot: rm-call: mode 35: execution channel switched from V6 0x%x to V7 0x%x (captured flags=0x%x internalFlags=0x%x hContextShare=0x%x hVASpace=0, gpFifoVa=0x%llx entries=%u, sysmem USERD, sysmem instance/ramfc (m62 readback), TURING_COMPUTE_A 0x%x); V6 stays allocated and unscheduled, stage 4 + forwarding run on V7\n", boot_stage.rmcall_channel, GSP_PHYS_RMCALL_HANDLE_USER_LANE, @@ -14898,23 +16092,31 @@ static u32 gsp_phys_rmcall_userd_ctl_off(void) * launch path publish byte-identically; the CE caller's semaphore clear and * method-block build happen before this call. The caller must hold * gsp_phys_rm_lock. - */ -static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, - u32 words, - struct gsp_phys_rmcall_ce_submit *out) + * + * SESSION 153-CONTINUED-96: the lane carries the channel's GPFIFO ring, + * hardware USERD chunk, out-fill chunks and work-submit token, so the + * secondary COPY0 lane publishes through the identical body on its own + * ring/USERD/doorbell token (a second channel must never advance the GR + * lane's GP_PUT or ring the GR channel's doorbell). + */ +struct gsp_phys_rmcall_lane { + struct gsp_phys_boot_chunk *ring; + struct gsp_phys_boot_chunk *userd; + struct gsp_phys_boot_chunk *pb; + struct gsp_phys_boot_chunk *sem; + struct gsp_phys_boot_chunk *src; + struct gsp_phys_boot_chunk *dst; + u32 token; + const char *tag; +}; + +static int gsp_phys_rmcall_publish_lane_locked(void __iomem *bar0, + u64 pb_gpu_va, u32 words, + const struct gsp_phys_rmcall_lane *lane, + struct gsp_phys_rmcall_ce_submit *out) { - struct gsp_phys_boot_chunk *ring = - &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO]; - struct gsp_phys_boot_chunk *userd_c = - &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD]; - struct gsp_phys_boot_chunk *pb = - &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB]; - struct gsp_phys_boot_chunk *sem = - &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM]; - struct gsp_phys_boot_chunk *src = - &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC]; - struct gsp_phys_boot_chunk *dst = - &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST]; + struct gsp_phys_boot_chunk *ring = lane->ring; + struct gsp_phys_boot_chunk *userd_c = lane->userd; u32 *ring_va, *userd; u32 gpput, gpget, slot, e0, e1; u32 runlist_id, ch_id, doorbell; @@ -14927,11 +16129,39 @@ static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, gpput = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); gpget = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPGET_OFF); if (gpput >= GSP_PHYS_RMCALL_GPFIFO_ENTRIES) { - pr_err("vgsp-boot: rm-call: USERD GP_PUT=%u is outside the %u-entry ring; refusing to submit\n", - gpput, GSP_PHYS_RMCALL_GPFIFO_ENTRIES); + pr_err("vgsp-boot: rm-call: %s USERD GP_PUT=%u is outside the %u-entry ring; refusing to submit\n", + lane->tag, gpput, GSP_PHYS_RMCALL_GPFIFO_ENTRIES); return -EIO; } - slot = gpput; + /* + * SESSION 153-CONTINUED-46: publish at the ENGINE's GP_GET, not at + * the raw GP_PUT. Observed on every stalled beef boot: the USERD + * drifts (GPGet=5 while GPPut cycles 1/2/4/7 - the engine's GET ran + * past the publisher's PUT, e.g. after a wrapped or otherwise + * deranged ring), and a ring is MODULO: PUT < GET numerically means + * (PUT-GET) mod ENTRIES pending entries - hundreds - so the PBDMA + * walks stale slots, executes garbage method blocks and the launch + * never retires (the trapped=0x8000022c / no-dispatch class). The + * forwarded channel is strictly single-inflight, so the engine's + * GET names exactly the next free slot: write our entry there and + * advance PUT to GET+1 so the PBDMA sees exactly one pending entry - + * ours. When GP_GET is out of range (uninitialised USERD), fall back + * to the old PUT-derived slot. + */ + slot = (gpget < GSP_PHYS_RMCALL_GPFIFO_ENTRIES) ? gpget : gpput; + { + static u32 slotfix_log; + + if (gpget != gpput && slotfix_log < 64) { + slotfix_log++; + pr_info("vgsp-boot: rm-call: %s publish at engine GP_GET=%u (raw GP_PUT=%u was %s; pending-would-be=%u)\n", + lane->tag, gpget, gpput, + gpput == gpget ? "equal" : + (gpput > gpget ? "ahead" : "behind"), + (gpput + GSP_PHYS_RMCALL_GPFIFO_ENTRIES - gpget) % + GSP_PHYS_RMCALL_GPFIFO_ENTRIES); + } + } /* * e0: bits 31:2 hold the GET field, which itself is (fetch address @@ -14948,8 +16178,8 @@ static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, */ e0 = (u32)pb_gpu_va & ~3u; if (unlikely(e0 != (u32)((pb_gpu_va >> 2) << 2))) - pr_err("vgsp-boot: rm-call: GPFIFO GET encode mismatch: e0=0x%08x decodes to fetch 0x%llx, expected pb VA 0x%llx\n", - e0, (u64)(e0 & ~3u), pb_gpu_va); + pr_err("vgsp-boot: rm-call: %s GPFIFO GET encode mismatch: e0=0x%08x decodes to fetch 0x%llx, expected pb VA 0x%llx\n", + lane->tag, e0, (u64)(e0 & ~3u), pb_gpu_va); e1 = (u32)((pb_gpu_va >> 32) & 0xffu) | (words << 10); put_unaligned_le32(e0, (u8 *)ring_va + slot * 8); put_unaligned_le32(e1, (u8 *)ring_va + slot * 8 + 4); @@ -14958,8 +16188,10 @@ static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, * GP_PUT advances on the client sysmem USERD chunk, which is the * channel's hardware USERD because the params carry that same chunk; * the host write is therefore what the engine polls (cycle 21). + * CONT-46: PUT = slot+1 (slot was chosen from GP_GET above), so the + * single new entry is the entire pending set. */ - put_unaligned_le32((gpput + 1) % GSP_PHYS_RMCALL_GPFIFO_ENTRIES, + put_unaligned_le32((slot + 1) % GSP_PHYS_RMCALL_GPFIFO_ENTRIES, (u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); /* * Doorbell value: workSubmitToken = ((runlistId << 16) & 0x7f00000) | @@ -14968,12 +16200,12 @@ static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, * write the formula's result (identical unless the token carries stray * high bits, which is logged). */ - runlist_id = (boot_stage.rmcall_token >> 16) & 0x7fu; - ch_id = boot_stage.rmcall_token & 0xfffu; + runlist_id = (lane->token >> 16) & 0x7fu; + ch_id = lane->token & 0xfffu; doorbell = ((runlist_id << 16) & 0x7f00000u) | (ch_id & 0xfffu); - if (doorbell != boot_stage.rmcall_token) - pr_warn("vgsp-boot: rm-call: token 0x%x carries bits outside runlistId[22:16]|chId[11:0]; writing the recomposed doorbell 0x%x\n", - boot_stage.rmcall_token, doorbell); + if (doorbell != lane->token) + pr_warn("vgsp-boot: rm-call: %s token 0x%x carries bits outside runlistId[22:16]|chId[11:0]; writing the recomposed doorbell 0x%x\n", + lane->tag, lane->token, doorbell); /* * C3: snapshot the fault epoch immediately before the doorbell, so a * 0x1006 consumed by this submission's timeout drain is accepted only @@ -14993,14 +16225,33 @@ static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, out->doorbell = doorbell; out->runlist_id = runlist_id; out->ch_id = ch_id; - out->pb_dma = (u64)gsp_phys_chunk_page_dma(pb, 0); - out->sem_dma = (u64)gsp_phys_chunk_page_dma(sem, 0); - out->src_dma = (u64)gsp_phys_chunk_page_dma(src, 0); - out->dst_dma = (u64)gsp_phys_chunk_page_dma(dst, 0); + out->pb_dma = (u64)gsp_phys_chunk_page_dma(lane->pb, 0); + out->sem_dma = (u64)gsp_phys_chunk_page_dma(lane->sem, 0); + out->src_dma = (u64)gsp_phys_chunk_page_dma(lane->src, 0); + out->dst_dma = (u64)gsp_phys_chunk_page_dma(lane->dst, 0); } return 0; } +static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, + u32 words, + struct gsp_phys_rmcall_ce_submit *out) +{ + const struct gsp_phys_rmcall_lane lane = { + .ring = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO], + .userd = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD], + .pb = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB], + .sem = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM], + .src = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC], + .dst = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST], + .token = boot_stage.rmcall_token, + .tag = "ladder", + }; + + return gsp_phys_rmcall_publish_lane_locked(bar0, pb_gpu_va, words, + &lane, out); +} + /* * One CE copy block on the ladder channel: clear the module semaphore, build * the methods for bytes into the module pushbuffer, publish the GPFIFO entry @@ -15022,10 +16273,30 @@ static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, * site passes false, so the default COPY0 ladder emits the same bytes as * before. */ -static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, - u64 dst_va, u64 sem_va, u32 bytes, - u32 sem_payload, bool gr, - struct gsp_phys_rmcall_ce_submit *out) +/* + * SESSION 153-CONTINUED-12: pb_off selects where in the module pushbuffer + * page the method block is written and fetched from. The warm-semaphore + * probe (gsp_phys_rmcall_gr_sem_push_locked) used the upper half so it + * could never overwrite a forwarded launch's pending method block: + * measured with the PBDMA dump, the probe's 10-dword block landing on + * words 0..9 of a pending 139-word 3D replay made the PBDMA walk the + * probe's SET_OBJECT/REPORT_SEM tail and then decode the replay window's + * operand 0x00230200 as an instruction -> PBENTRY on chid 4 and no + * retire. Every forwarded GR/CE block stays at offset 0. + * + * SESSION 153-CONTINUED-66: pb_alt/pb_alt_va point the block at a + * dedicated one-page module buffer (the probe's VA_GR_PROBE page) instead + * of the registered pushbuffer; a forwarded GR block may now span both + * pushbuffer pages, so no fixed offset of that page is safe for the probe. + */ +static int gsp_phys_rmcall_ce_submit_off_locked(void __iomem *bar0, + u64 src_va, u64 dst_va, + u64 sem_va, u32 bytes, + u32 sem_payload, bool gr, + u32 pb_off, + struct gsp_phys_boot_chunk *pb_alt, + u64 pb_alt_va, + struct gsp_phys_rmcall_ce_submit *out) { struct gsp_phys_boot_chunk *pb = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB]; @@ -15054,11 +16325,28 @@ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, if (!bar0 || !boot_stage.rmcall_token || !bytes || bytes > GSP_PHYS_RMCALL_COPY_SIZE) return -EINVAL; + if (pb_off & 3u) + return -EINVAL; pb_gpu_va = boot_stage.rmcall_va_pb; if (!pb_gpu_va || (pb_gpu_va & 3)) return -EIO; + /* + * SESSION 153-CONTINUED-66: pb_alt selects a dedicated one-page module + * buffer instead of the registered pushbuffer page (the warm-semaphore + * probe uses its own PTE-planted page now that a forwarded GR block may + * span BOTH pushbuffer pages). pb_off stays relative to that page. + */ + if (pb_alt && pb_alt->pages) { + pb = pb_alt; + pb_gpu_va = pb_alt_va; + if (!pb_gpu_va || (pb_gpu_va & 3)) + return -EIO; + } - pb_va = gsp_phys_chunk_page_va(pb, 0); + if (!gsp_phys_chunk_page_va(pb, 0) || pb_off > GSP_PHYS_PAGE_SIZE) + return -EIO; + pb_va = (u32 *)((u8 *)gsp_phys_chunk_page_va(pb, 0) + pb_off); + pb_gpu_va += pb_off; ring_va = gsp_phys_chunk_page_va(ring, 0); userd = gsp_phys_chunk_page_va(userd_c, 0); if (!pb_va || !ring_va || !userd || @@ -15079,9 +16367,82 @@ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, dst_va, bytes, sem_payload); + if (pb_off + words * 4 > GSP_PHYS_PAGE_SIZE) + return -EIO; + return gsp_phys_rmcall_publish_locked(bar0, pb_gpu_va, words, out); } +static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, + u64 dst_va, u64 sem_va, u32 bytes, + u32 sem_payload, bool gr, + struct gsp_phys_rmcall_ce_submit *out) +{ + return gsp_phys_rmcall_ce_submit_off_locked(bar0, src_va, dst_va, + sem_va, bytes, sem_payload, + gr, 0u, NULL, 0, out); +} + +/* + * SESSION 153-CONTINUED-96: one CE copy block on the secondary COPY0 lane. + * Byte-for-byte the CE submission body (same method block, same guards, + * same module-semaphore clear) but on the lane's own GPFIFO ring, USERD + * chunk, one-page pushbuffer and work-submit token. Only legal when the + * lane's bringup obtained a token (rmcall_ce_lane_valid). The caller must + * hold gsp_phys_rm_lock and polls the module semaphore separately, exactly + * like the readback's bounded page loop. + */ +static int gsp_phys_rmcall_ce_lane_submit_locked( + void __iomem *bar0, u64 src_va, u64 dst_va, u64 sem_va, + u32 bytes, u32 sem_payload, + struct gsp_phys_rmcall_ce_submit *out) +{ + const struct gsp_phys_rmcall_lane lane = { + .ring = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_CE_GPFIFO], + .userd = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_CE_USERD], + .pb = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_CE_PB], + .sem = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM], + .src = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC], + .dst = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST], + .token = boot_stage.rmcall_ce_token, + .tag = "CE lane", + }; + struct gsp_phys_boot_chunk *sem = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM]; + resource_size_t bar0_len; + u32 *pb_va; + u32 words; + + if (!boot_stage.rmcall_ce_lane_valid) + return -EOPNOTSUPP; + if (!boot_stage.pdev) + return -EINVAL; + bar0_len = pci_resource_len(boot_stage.pdev, 0); + if (bar0_len < GSP_PHYS_RMCALL_DOORBELL_OFF + 4) + return -EIO; + if (!bar0 || !boot_stage.rmcall_ce_token || !bytes || + bytes > GSP_PHYS_RMCALL_COPY_SIZE) + return -EINVAL; + pb_va = gsp_phys_chunk_page_va(lane.pb, 0); + if (!pb_va || !gsp_phys_chunk_page_va(lane.ring, 0) || + !gsp_phys_chunk_page_va(lane.userd, 0) || + !gsp_phys_chunk_page_va(sem, 0)) + return -EIO; + + /* A stale release from a previous submission must not be observed. */ + WRITE_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0), 0); + wmb(); + + words = gsp_phys_rmcall_build_methods(pb_va, sem_va, src_va, dst_va, + bytes, sem_payload); + if (words * 4 > GSP_PHYS_PAGE_SIZE) + return -EIO; + + return gsp_phys_rmcall_publish_lane_locked(bar0, + GSP_PHYS_RMCALL_VA_CE_PB, + words, &lane, out); +} + /* * H1 stage-4 GR verification: one GR semaphore release on the ladder channel, * shared by stage 4's GR branch and the debugfs trigger 9 re-run. The caller @@ -15094,6 +16455,39 @@ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, * semaphore. On timeout the existing status-queue drain runs and the helper * returns -ETIMEDOUT; it never retries. */ +static int gsp_phys_rmcall_gr_resched_locked(void __iomem *bar0); +static int gsp_phys_rmcall_gr_sem_push_locked(void __iomem *bar0); + +/* + * r445 wedge hard re-arm: the outcome-retry's warm semaphore probe proves the + * engine refused the channel's proven GR block (the sticky post-dispatch + * no-store wedge). One bounded recovery attempt instead of an immediate + * break: force a context save/unload with a bWait TSG preempt, re-run the + * mode-35 lane-enable tail + schedules (the m155 extended re-arm), then + * repeat the warm probe. Caller holds gsp_phys_rm_lock. Returns 0 when the + * engine executed the probe after the re-arm. + */ +static int gsp_phys_gr_wedge_hard_rearm_locked(void __iomem *bar0) +{ + u32 pre[2] = { 1u, 0u }; /* bWait=1, bManualTimeout=0 */ + u32 rpc_result = 0, status = 0; + int ret; + + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_tsg, + GSP_PHYS_RMCALL_CTRL_TSG_PREEMPT, + pre, sizeof(pre), sizeof(pre), NULL, 0, + &rpc_result, &status, + "TSG PREEMPT bWait=1 (r445 wedge hard re-arm)"); + pr_warn("vgsp-gr: WEDGE-REARM preempt status=0x%x rpc=0x%x ret=%d\n", + status, rpc_result, ret); + ret = gsp_phys_rmcall_gr_resched_locked(bar0); + pr_warn("vgsp-gr: WEDGE-REARM resched ret=%d\n", ret); + ret = gsp_phys_rmcall_gr_sem_push_locked(bar0); + pr_warn("vgsp-gr: WEDGE-REARM probe rc=%d (engine %s after re-arm)\n", + ret, ret ? "still wedged" : "ALIVE"); + return ret; +} + static int gsp_phys_rmcall_gr_sem_push_locked(void __iomem *bar0) { struct gsp_phys_boot_chunk *userd_c = @@ -15106,11 +16500,33 @@ static int gsp_phys_rmcall_gr_sem_push_locked(void __iomem *bar0) unsigned long deadline; int ret; - ret = gsp_phys_rmcall_ce_submit_locked(bar0, GSP_PHYS_RMCALL_VA_SRC, + /* + * SESSION 153-CONTINUED-12: this probe runs while a forwarded launch + * may still be pending in the same pushbuffer page. It must not land + * on words 0.. of that page: measured PB-FAULT-DUMP shows the probe's + * 10-dword block overwriting a pending 139-word 3D replay's head, so + * the PBDMA walks the probe then decodes a replay operand as an + * instruction (PBENTRY, no retire). + * + * SESSION 153-CONTINUED-66: the forwarded GR block now spans BOTH + * pushbuffer pages (up to 2048 dwords = 0x2000 bytes), so the old + * upper-half placement (0x800) would clobber it. The probe has its + * own PTE-planted page (VA_GR_PROBE, small-PT slot 0x10a); when that + * plant is unavailable the probe falls back to 0x800 as before. + */ + ret = gsp_phys_rmcall_ce_submit_off_locked(bar0, + GSP_PHYS_RMCALL_VA_SRC, GSP_PHYS_RMCALL_VA_DST, GSP_PHYS_RMCALL_VA_SEM, GSP_PHYS_RMCALL_COPY_SIZE, GSP_PHYS_RMCALL_SEM_PAYLOAD, true, + boot_stage.rmcall_gr_probe.pages ? + 0u : 0x800u, + boot_stage.rmcall_gr_probe.pages ? + &boot_stage.rmcall_gr_probe : + NULL, + boot_stage.rmcall_gr_probe.pages ? + GSP_PHYS_RMCALL_VA_GR_PROBE : 0, &sub); if (ret) { pr_err("vgsp-boot: rm-call: GR submission refused before the doorbell (%d)\n", @@ -15155,7 +16571,7 @@ static int gsp_phys_rmcall_gr_sem_push_locked(void __iomem *bar0) gsp_phys_status_drain(NULL); return -ETIMEDOUT; } - msleep(GSP_PHYS_RMCALL_POLL_MS); + gsp_phys_rmcall_wait_quantum(); } gpget = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPGET_OFF); @@ -15255,7 +16671,7 @@ static void gsp_phys_rmcall_ctx_image_probe(void __iomem *bar0) offs[k], poll); break; } - msleep(GSP_PHYS_RMCALL_POLL_MS); + gsp_phys_rmcall_wait_quantum(); } if (poll != GSP_PHYS_RMCALL_SEM_PAYLOAD) break; @@ -15356,7 +16772,7 @@ static int gsp_phys_rmcall_ce_copy_va(void __iomem *bar0, u64 src_va, *sem_out = poll; return -ETIMEDOUT; } - msleep(GSP_PHYS_RMCALL_POLL_MS); + gsp_phys_rmcall_wait_quantum(); } } @@ -15539,7 +16955,7 @@ int gsp_phys_backend_ce_copy_submit(u64 src_va, u64 dst_va, u64 sem_va, gsp_phys_work_timed_out); return -ETIMEDOUT; } - msleep(GSP_PHYS_RMCALL_POLL_MS); + gsp_phys_rmcall_wait_quantum(); } /* @@ -16058,7 +17474,7 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa, ret = -ETIMEDOUT; goto out; } - msleep(GSP_PHYS_RMCALL_POLL_MS); + gsp_phys_rmcall_wait_quantum(); } /* @@ -16280,14 +17696,42 @@ EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_ce_copy); #define GSP_PHYS_RMCALL_GR_CWD_REFCOUNT 1536u #define GSP_PHYS_RMCALL_GR_CWD_LEVELS 64u #define GSP_PHYS_RMCALL_GR_MAX_SM_COUNT 24u +/* + * r434 CWD-slot-count path: the falcon PRI write (CUDA's + * turingFalconPRIWriteMasked: flag/data/mask through SET_MME_SHADOW_SCRATCH + * 0/1/2, then SET_FALCON04 = register offset) and the gated method. Used only + * when gsp_phys_rmcall_gr_cwd_slot >= 0. + */ +#define GSP_PHYS_RMCALL_GR_MME_SCRATCH0 0x3400u +#define GSP_PHYS_RMCALL_GR_MME_SCRATCH1 0x3404u +#define GSP_PHYS_RMCALL_GR_MME_SCRATCH2 0x3408u +#define GSP_PHYS_RMCALL_GR_SET_FALCON04 0x0510u +#define GSP_PHYS_RMCALL_GR_SET_CWD_SLOT_COUNT 0x02b0u +#define GSP_PHYS_RMCALL_PRI_SKED_DEBUG_1 0x00407040u +#define GSP_PHYS_RMCALL_PRI_SKED_DEBUG_1_EN_CWD_SLOT_COUNT 0x800u /* Method block bound: init(152) + inline run(67) + PCAS(4) + REPORT_SEM(8) + - * the optional QMD-virt pair(2). */ -#define GSP_PHYS_RMCALL_GR_METHOD_WORDS 512u + * the optional QMD-virt pair(2). Raised 512 -> 1024 (SESSION 153- + * CONTINUED-53) so a 3D stream replay can carry the captured MME setup + * prelude (<=512 words) ahead of the largest chunked window (<=448) in one + * block. SESSION 153-CONTINUED-66: 1024 -> 2048 because the matured prelude + * reached 901 words and the deferral test (window=390 + prelude=901 + sa=47 + * vs the bound) dropped it on every launch; the block now spans the module + * pushbuffer's TWO 4 KB pages (VA_PB..VA_PB2) and is copied page-wise by + * gsp_phys_gr_pb_copy(). */ +#define GSP_PHYS_RMCALL_GR_METHOD_WORDS 4096u static u64 gsp_phys_gr_submitted; static u64 gsp_phys_gr_ok; static u64 gsp_phys_gr_refused; static u64 gsp_phys_gr_timed_out; +/* + * SESSION 153-CONTINUED-60: 3D-stream launches whose window was all-zero + * (the client's push page caught mid-rewrite). They are never handed to the + * engine: the zero method stream trips the PBDMA (trap 0x1168 -> priv-error + * poison, see the ZEROWIN-SKIP site). Counted so a boot's zero-window rate + * is visible without dmesg archaeology. + */ +static u64 gsp_phys_gr_zerowin_skips; static u64 gsp_phys_gr_retries; /* fresh-entry launch retries published */ static u64 gsp_phys_gr_pool_exhausted; /* m156 (E5): zero-write retries where the warm sem push proved the engine @@ -16347,6 +17791,50 @@ bool gsp_phys_gr_quiet_hold_active(void) } EXPORT_SYMBOL_GPL(gsp_phys_gr_quiet_hold_active); +/* + * r425 experiment (the second-launch wedge): the stock guest driver emits the + * per-channel compute init block ONCE at channel creation (turing_init.c), + * long before any launch, while this backend re-emits it as the preamble of + * every forwarded launch. The wedge strikes the SECOND launch of a channel, so + * test whether the re-emission is the trigger: with gsp_phys_rmcall_gr_init_once + * set, the init preamble is emitted only on the first forwarded launch after + * the module load (the engine's compute-class/CWD state persists in the + * context; every launch still carries its QMD + report semaphore). 0 keeps the + * historical per-launch emission (default; runtime-writable A/B). + */ +static bool gsp_phys_rmcall_gr_init_once_param; +module_param_named(gsp_phys_rmcall_gr_init_once, gsp_phys_rmcall_gr_init_once_param, + bool, 0644); +MODULE_PARM_DESC(gsp_phys_rmcall_gr_init_once, + "r425: emit the GR compute init preamble only on the first forwarded launch (0 = every launch, default)"); +static bool gsp_phys_rmcall_gr_init_emitted; + +/* + * r425 stale-QMD witness: the first engine-changed page of the LAST completed + * launch (bounce index + post-copy-out checksum). At a zero-write accept the + * page is re-scanned; a change proves a late store (a stuck/queued grid or a + * stale-QMD execution) landed after its launch was shipped. + */ +static u32 gsp_phys_gr_prev_out_bi; +static u64 gsp_phys_gr_prev_out_csum; +static bool gsp_phys_gr_prev_out_valid; + +/* r425: SKED/FECS/CWD register witness (one line; the healthy-vs-wedged + * comparison baseline lives in the same log). */ +static void gsp_phys_gr_sked_wit(void __iomem *bar0, const char *tag) +{ + if (!bar0) + return; + pr_info("vgsp-gr: SKED-WIT %s cwd0=%08x cwd1=%08x cwd2=%08x cwd3=%08x info1=%08x act=%08x grfifo=%08x fecs=%08x new=%08x ctxsw1=%08x qmdvirt0=%08x tpcm0=%08x\n", + tag, + readl(bar0 + 0x00407060u), readl(bar0 + 0x00407064u), + readl(bar0 + 0x00407068u), readl(bar0 + 0x0040706cu), + readl(bar0 + 0x00407028u), readl(bar0 + 0x00407054u), + readl(bar0 + 0x00400504u), readl(bar0 + 0x00409b00u), + readl(bar0 + 0x00409b04u), readl(bar0 + 0x00409400u), + readl(bar0 + 0x0040a500u), readl(bar0 + 0x0040a680u)); +} + /* m86: program-page override blob (see gsp-phys-gr-override). */ static u8 gsp_phys_gr_override[GSP_PHYS_PAGE_SIZE]; static u32 gsp_phys_gr_override_len; @@ -16456,32 +17944,3134 @@ static u32 *gsp_phys_rmcall_build_gr_init(u32 *p) GSP_PHYS_RMCALL_GR_MAX_SM_COUNT); p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_INIT_SM_DISP_CTRL, 0u); + /* + * r434 CWD-slot-count path (gsp_phys_rmcall_gr_cwd_slot >= 0): masked + * PRI write of SKED_DEBUG_1.ENABLE_CWD_SLOT_COUNT (bit 11) through the + * falcon scratch registers, then the gated SET_CWD_SLOT_COUNT method - + * the CUDA driver's own turingLaunchSetCwdSlotCount sequence. -1 (the + * default) emits nothing, so the historical preamble is unchanged. + */ + if (gsp_phys_rmcall_gr_cwd_slot_value() >= 0) { + p = gsp_phys_rmcall_push_gr( + p, GSP_PHYS_RMCALL_GR_MME_SCRATCH0, 0u); + p = gsp_phys_rmcall_push_gr( + p, GSP_PHYS_RMCALL_GR_MME_SCRATCH1, + GSP_PHYS_RMCALL_PRI_SKED_DEBUG_1_EN_CWD_SLOT_COUNT); + p = gsp_phys_rmcall_push_gr( + p, GSP_PHYS_RMCALL_GR_MME_SCRATCH2, + GSP_PHYS_RMCALL_PRI_SKED_DEBUG_1_EN_CWD_SLOT_COUNT); + p = gsp_phys_rmcall_push_gr( + p, GSP_PHYS_RMCALL_GR_SET_FALCON04, + GSP_PHYS_RMCALL_PRI_SKED_DEBUG_1); + p = gsp_phys_rmcall_push_gr( + p, GSP_PHYS_RMCALL_GR_SET_CWD_SLOT_COUNT, + (u32)gsp_phys_rmcall_gr_cwd_slot_value()); + } return p; } /* + * r597: extract the SEM_EXECUTE ACQUIRE groups from the captured guest + * prelude and copy them, verbatim, ahead of the rebuilt inline-QMD window. + * Two encodings are recognised: + * - C46F host-semaphore group: INC_METHOD addr 0x5c count>=5, words + * {LO, HI, PAYLOAD_LO, PAYLOAD_HI, EXECUTE}; EXECUTE & 7 != 1 is an + * acquire (the pump's own op decode). + * - C5C0 SET_REPORT_SEMAPHORE A..D: addr 0x1b00 count>=4, word D's low + * two bits select release (0) / acquire (non-zero). + * Only the wait methods are taken, so the proven window (init + inline QMD + * + PCAS + release) is untouched. Returns the number of words emitted. + */ +static u32 gsp_phys_rmcall_gr_emit_acquires(u32 *p, + const struct gsp_phys_gr_submit *req, + u32 budget) +{ + u32 off = 0, words = 0; + + if (!gsp_phys_rmcall_gr_acq_value() || !req || !req->raw) + return 0; + while (off < req->raw_qmd_hdr && words + 8 < budget) { + u32 hdr = req->raw[off]; + u32 op = (hdr >> 29) & 0x7; + u32 count = (hdr >> 16) & 0xfff; + u32 addr = (hdr & 0x1fff) << 2; + u32 n = 1 + count; + + if ((op != 1 && op != 3) || !count || + off + n > req->raw_qmd_hdr) { + off++; + continue; + } + if (addr == 0x5c && count >= 5) { + u32 v4 = req->raw[off + 5]; + u64 va = ((u64)(req->raw[off + 2] & 0xff) << 32) | + (u64)(req->raw[off + 1] & 0xfffffffc); + + if ((v4 & 0x7) != 1 && va) { + memcpy(p + words, &req->raw[off], n * 4); + words += n; + } + } else if (addr >= 0x1b00 && addr <= 0x1b0c && + count >= 4) { + u32 d = req->raw[off + 4] & 0x3; + + if (d != 0) { + memcpy(p + words, &req->raw[off], n * 4); + words += n; + } + } + off += n; + } + return words; +} + +/* + * r929 (GL 3D execution, Path A): replay a captured raw 3D method window. + * The sema service marks stream captures (the X server's C597 sections that + * carried their own operands) with a raw window (req->stream); when this is + * on, gsp_phys_rmcall_build_gr_launch emits SET_OBJECT TURING_A + the window + * verbatim + the channel release tail instead of the compute prelude/QMD, so + * the engine executes the REAL 3D methods (state binds, LINE_LENGTH_IN + + * LAUNCH_DMA uploads, program/tex binds) and, once draws are in the window, + * rasterizes. No effect on any submit without a stream window (all CUDA + * launches). Default on; A/B with gsp_phys_rmcall_gr_3d=0. + */ +static bool gsp_phys_rmcall_gr_3d = true; +module_param_named(gsp_phys_rmcall_gr_3d, gsp_phys_rmcall_gr_3d, bool, 0644); +MODULE_PARM_DESC(gsp_phys_rmcall_gr_3d, + "Replay a captured raw C597 method window verbatim instead of the compute prelude (default 1)"); + +/* + * SESSION 153-CONTINUED-59: drop single-word SDM-type entries (SEC_OP 0 + * SSDM / 2 STORE_SDM / 6 USE_SDM, tu102 dev_esched_pbdma.h type map) from + * every replayed stream: the 3D window patch, the MME prelude and the attr + * prelude emission. The r1102 PBENTRY first-sighting forensics localized + * the fault decode point to the MME prelude head (WINPROBE/PB-FAULT-DUMP + * P0 HDR_SHADOW=04666d14 = prelude[2], PB_DATA0=04646d00 = prelude[3], + * pbfetch=0x101ebc = block end after the whole 943-word block was + * prefetched, intr0 bit 18 in the guest relay): the client's setup push + * carries a 21-word run of `04666d00 04666d10 04666d14 04646d00 ...` + * (SDM-family single-word encodings, method fields 0x3400/0x3440/0x3450) + * that svc_mme_prelude_capture stored verbatim; the physical PBDMA refuses + * them ("invalid method encoding used in method", NV_PPBDMA_INTR_0 + * PBENTRY), which also explains why the whole block was fetched but the + * REPORT_SEMAPHORE never retired. The guest-vGPU parts (subdevice masks) + * are meaningless on the single-subdevice ladder channel - the same call + * the sub!=0 multi-word drop makes. Zero words are kept (benign padding). + * Default 1; A/B with 0. + */ +static bool gsp_phys_rmcall_gr_sdm_drop = true; +module_param_named(sema_gl_sdm_drop, gsp_phys_rmcall_gr_sdm_drop, bool, + 0644); +MODULE_PARM_DESC(sema_gl_sdm_drop, + "Drop single-word SDM-type entries (SEC_OP 0/2/6) from 3D window replays, the MME prelude and the attr prelude (default 1)"); + +/* + * SESSION 153-CONTINUED-53: MME context-setup prelude. The r1073-r1077 + * in-flight capture proved the draw-window replay raises a real MME + * ILLEGAL_OPCODE on CALL_MME_MACRO(2)=0x3810 (mme_pc=0, mme_ir=1, mme_mac + * =2) because the ladder context never received the client's macro + * program. The client does upload it (0x11c/0x114 pointer pairs, 0x118 + * code, 0x120 entry addresses, 0x124 shadow control; see + * svc_mme_prelude_capture); with this knob the launch builder preplays + * those captured runs verbatim - once per capture generation - right + * after the SET_OBJECT C597 bind and ahead of the window, so the macros + * the window calls are resident. Declared before build_gr_launch (its + * reader); the matching capture-side knob is sema_mme_prelude_cap. + * Default 1; A/B with 0. + */ +static bool gsp_phys_rmcall_gr_mme_prelude = true; +module_param_named(sema_gl_mme_prelude, gsp_phys_rmcall_gr_mme_prelude, bool, + 0644); +MODULE_PARM_DESC(sema_gl_mme_prelude, + "Preplay the captured client MME context-setup runs once per capture generation ahead of the first 3D window replay (default 1)"); + +/* + * SESSION 153-CONTINUED-63: MME START-ADDRESS RAM completion + stub knob. + * The captured SA table (svc_mme_sa_*) may be incomplete (the prelude + * buffer is capped at 512 words); every CALLED slot the table does not + * cover is loaded with a harmless end-noop stub at a fixed IR address so + * the call becomes a no-op instead of a jump to IR 0 with garbage state + * (the r1116 0x238c poison). Default 1; A/B with 0. + */ +#define GSP_PHYS_RMCALL_GR_MME_STUB_PC 0xC0u +/* + * SESSION 153-CONTINUED-69: the stub body, take 3. The CONT-68 body + * (20208c00 38c40046 038c4301) was copied from macro(0) instruction 3 - + * but that is a MID-MACRO instruction: the offline MME64 simulator shows + * it carries two output writes with mthd=NONE, so in isolation (method + * register still 0) it EMITS SET_OBJECT <- 0 twice - exactly the relayed + * trap "ChID 0001, Class 0000c597, Offset 00000000, Data 00000000" - and + * its end_next=1 requires the following instruction, so the macro also + * runs past the stub into whatever the IR holds there. + * + * The real minimal terminator is what Mesa's mme_tu104_builder_finish() + * emits: two default NOPs with end_next set on the FIRST (end_next - + * "end after the next instruction" - so the second NOP executes and then + * the macro returns). Both NOPs encode as w2=0x318c0300 (pred=ZERO, + * ALU0/1 dst=ZERO src=ZERO,ZERO, outs NONE) and emit NOTHING. The + * end_next bit is w2 bit 0. + */ +static const u32 gsp_phys_gr_mme_stub_code[6] = { + 0x00000000u, 0x318c0000u, 0x318c0301u, /* NOP, end_next=1 */ + 0x00000000u, 0x318c0000u, 0x318c0300u, /* NOP tail (executed) */ +}; +static bool gsp_phys_rmcall_gr_sa_stub = true; +module_param_named(sema_gl_mme_sa_stub, gsp_phys_rmcall_gr_sa_stub, bool, + 0644); +MODULE_PARM_DESC(sema_gl_mme_sa_stub, + "Load a harmless end-noop stub into every MME macro slot the window calls but the captured START-ADDRESS RAM table does not cover (default 1; the r1116 0x238c lane poison class)"); + +/* + * SESSION 153-CONTINUED-68: stub EVERY called slot, covered or not + * (attribution: with all calls no-op'd the window either still faults + * (the fault is in non-call content) or faults differently (the fault is + * macro execution). Default 0; A/B knob. + */ +static bool gsp_phys_rmcall_gr_sa_stub_all; +module_param_named(sema_gl_mme_sa_stub_all, gsp_phys_rmcall_gr_sa_stub_all, + bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_sa_stub_all, + "Stub every macro slot the window calls, including slots the captured SA table covers (attribution knob; default 0)"); + +static inline bool gsp_sema_gr_sa_stub_value(void) +{ + return READ_ONCE(gsp_phys_rmcall_gr_sa_stub); +} + +/* + * SESSION 153-CONTINUED-69: optional explicit stub IR address override. + * 0 = derive from the capture (ir_end, floor 0xC0). Nonzero pins the stub + * body/call target to this instruction address - the stub-landing A/B for + * the IR-boundary/ONEINC pointer-unit question. + */ +static u32 gsp_phys_rmcall_gr_mme_stub_pc; +module_param_named(sema_gl_mme_stub_pc, gsp_phys_rmcall_gr_mme_stub_pc, uint, + 0644); +MODULE_PARM_DESC(sema_gl_mme_stub_pc, + "Explicit MME stub IR instruction address (0 = derive from capture ir_end; try 0xf0/0x100 for the IR-boundary A/B)"); + +/* + * SESSION 153-CONTINUED-69: no-stall CALL drop. With the stub terminating + * macros as pure NOPs the macro consumes none of the CALL_MME_DATA + * parameter words the window pushes; the MME parameter port then + * backpressures on the window's larger calls - the r1142 stall parked the + * PBDMA mid-block with HDR_SHADOW = the 16-arg CALL_MME_MACRO(57) + * ONEINC run and the block never completed (while every <=4-arg window + * retired in ~100 us). Drop every CALL_MME_MACRO/CALL_MME_DATA run of a + * STUBBED slot from the emitted window instead: no call, no parameter + * words, no backpressure. Covered slots keep their calls (their real code + * consumes the parameters). Default 1; A/B with 0. + */ +static bool gsp_phys_rmcall_gr_call_drop = true; +module_param_named(sema_gl_mme_call_drop, gsp_phys_rmcall_gr_call_drop, bool, + 0644); +MODULE_PARM_DESC(sema_gl_mme_call_drop, + "Drop CALL_MME_MACRO/DATA runs of stubbed slots from emitted 3D windows (prevents the noop-stub parameter-port stall; default 1)"); + +/* + * SESSION 153-CONTINUED-70: per-slot macro re-enable mask, one slot at a + * time. The r1142-r1144 state drops EVERY stubbed-slot CALL run: the window + * retires on real HW (~94 us) but the macros that expand the driver's + * per-draw state (vertex stream binds, constant binds, ...) never execute, + * so no state the draw needs exists and every launch still stores 0 pages. + * A slot named in this mask is never treated as stubbed: its + * CALL_MME_MACRO/DATA words stay in the emitted window and the engine + * executes the macro against the replayed context (SA RAM + IR program as + * the client's own uploads left them; a slot the client never uploaded + * reads SA=0 exactly like silicon and lands in macro 0). A/B one bit per + * cycle; keep the proven-good set empty until the offline MME64 sim has + * shown the slot's macro emits sane methods for its caller args. + */ +static u64 gsp_phys_rmcall_gr_keep_mask; +module_param_named(sema_gl_mme_keep_mask, gsp_phys_rmcall_gr_keep_mask, + ullong, 0644); +MODULE_PARM_DESC(sema_gl_mme_keep_mask, + "Slots whose CALL_MME_MACRO/DATA runs are never dropped/stubbed (u64 bitmask; one slot at a time for macro re-enable)"); + +/* + * SESSION 153-CONTINUED-113: clamp the keep mask to capture-covered slots. + * The r1405 poison anatomy (traced live): the draw window calls + * CALL_MME_MACRO(34) and CALL_MME_MACRO(57); the capture holds 13 SA slots + * (0..12, IR <= 0xe9) while keep_mask=~0 exempted every slot from stubbing, + * so the calls ran, the MME fetched a byte the packed program does not hold + * (`info=badf1201 pc_valid=1 pc=513 mac=0x1f1`, `miss=1`, ir=badf1201) and + * the priv-error wedge wiped the context; the second recovery's hard re-arm + * left the engine unresponsive (probe rc=-110) and GP_GET frozen for the + * boot. A kept slot the capture cannot serve can only ever fetch + * non-resident code: the CONT-80 r1186 note already pairs "drop the runs + + * stub SA[slot]" as the safe combination for a nop_drop slot - apply the + * same combination to any kept slot outside the capture's coverage. The + * decision reads the live SA table per window, so a later capture + * generation that covers the slot releases it automatically. Default 1 + * (keep_clamp); 0 = the r1405 all-real behavior (A/B). + */ +static bool gsp_phys_rmcall_gr_keep_clamp = true; +module_param_named(sema_gl_mme_keep_clamp, gsp_phys_rmcall_gr_keep_clamp, + bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_keep_clamp, + "Clamp kept slots to capture-covered slots: a keep_mask slot with no captured SA entry (or uncovered IR) is stubbed + its CALL runs dropped instead of fetching non-resident code (default 1)"); + +static u32 gsp_phys_gr_keep_clamp_log; + +/* + * SESSION 153-CONTINUED-114: golden MME slot overlay (see the include + * comment). The capture is a per-boot lottery (r1407/r1385 got the full + * 55-slot program; r1405/6/10/11/12 only 13 slots), and a called slot + * outside the capture is stubbed with its CALL runs dropped - which is + * why the draw never executes on partial boots. The golden library is + * the same driver-emitted microcode word-for-word (validated against a + * live full capture), so a golden-filled slot is byte-shape-identical + * to a captured one downstream (pack image, 0x11c pairs, IRAM gate). + * + * gsp_phys_gr_golden_slot: returns 1 and the slot's instruction entry + + * word pointer when the library can serve slot j (and the knob is on). + * gsp_phys_gr_golden_active: the knob as a bool, for the cheap sites. + */ +static unsigned int gsp_phys_rmcall_gr_mme_golden; +module_param_named(sema_gl_mme_golden, gsp_phys_rmcall_gr_mme_golden, + uint, 0644); +MODULE_PARM_DESC(sema_gl_mme_golden, + "Golden MME slot overlay: serve called-but-uncaptured SA slots from the driver's own generated macro code (untracked kernel/golden-mme-tu104.h) instead of stubbing them, so CALL_MME_MACRO(DRAW_IAB etc.) runs deterministically (default 0; 1 = fill uncovered slots; 2 = force/overwrite all)"); +static unsigned int gsp_phys_rmcall_gr_mme_golden_force; +module_param_named(sema_gl_mme_golden_force, gsp_phys_rmcall_gr_mme_golden_force, + uint, 0644); +MODULE_PARM_DESC(sema_gl_mme_golden_force, + "Force the golden overlay over captured slots too (A/B for a stale capture; default 0 = fill uncovered only)"); + +static u32 gsp_phys_gr_golden_log; + +static int gsp_phys_gr_golden_slot(u32 j, u32 *entry, const u32 **words, + u32 *nwords) +{ + const struct golden_mme_tu104_slot *gs; + + if (!READ_ONCE(gsp_phys_rmcall_gr_mme_golden)) + return 0; + if (j >= GOLDEN_MME_TU104_SLOTS || j >= 64u) + return 0; + gs = &golden_mme_tu104_sa[j]; + if (!(gs->flags & GMME_F_PRESENT) || !gs->words || !gs->entry) + return 0; + if ((u64)gs->off + gs->words > GOLDEN_MME_TU104_WORDS) + return 0; + *entry = gs->entry; + *words = &golden_mme_tu104_words[gs->off]; + *nwords = gs->words; + return 1; +} + +/* + * SESSION 153-CONTINUED-79: per-launch method-block word cap + prelude drain. + * The r1172-r1175 A/B: every beef launch whose emitted block exceeded ~2000 + * words (3083-word full-SA prelude + window, 3315-3484 words) NEVER STARTED + * on the physical engine (MME-FAST events=0, grfifo frozen, ch_rel never + * written) while every block <= ~1300 words (1002-word partial capture + + * window) executed (retires in 90-170 us when the executed macro stayed + * inside the loaded IR). The failure class is fetch-side (a stuck GPFIFO + * entry), not content: once parked, every later entry also failed until a + * poison-latch TSG re-arm reset the PBDMA. The full-SA capture (55 slots, + * 3083 words, macros that chain past the first 15 slots e.g. macro 0 -> + * IR 0x201 = slot-25 entry) therefore cannot be replayed in ONE block. + * Emit at most gsp_phys_rmcall_gr_block_cap words per launch and drain the + * (SDM-sanitized, run-aligned) prelude across consecutive launches with a + * generation-keyed cursor; the emission latch is set only when the final + * part has been emitted, so a completed program is re-requested after the + * WEDGE-RECOVER latch clear. Default 2048; 0 = legacy 4096 module buffer. + */ +static u32 gsp_phys_rmcall_gr_block_cap = 2048u; +module_param_named(sema_gl_mme_block_cap, gsp_phys_rmcall_gr_block_cap, uint, + 0644); +MODULE_PARM_DESC(sema_gl_mme_block_cap, + "Max words emitted per GR launch method block (0 = 4096 module buffer; the >2048-word blocks never start on the physical engine)"); + +/* Generation-keyed drain cursor for the split MME prelude emission. */ +static u32 gsp_phys_gr_mme_pre_emit_gen; +static u32 gsp_phys_gr_mme_pre_emit_h; +static u32 gsp_phys_gr_mme_pre_emit_off; +/* + * CONT-105: generation-keyed cursor for the SPLIT packed MME prelude + * (see the sema_gl_mme_pack_split comment). pack_off counts IR words + * (always a multiple of 3 = whole instructions); pack_sa records that the + * SA table has gone out for this pass. The pair is reset together with the + * drain cursor above whenever the emission restarts. + */ +static u32 gsp_phys_gr_mme_pack_off; +static bool gsp_phys_gr_mme_pack_sa; + +/* + * CONT-103: prelude residency witness + calls-kept block shrink. + * The CONT-79 split-emission re-carries the whole prelude prefix on EVERY + * launch while keep_mask is set (idempotent re-emission, CONT-72), so the + * first calls-kept launch is 1613 prelude words + the 410-word window = + * 2126-2142 words - over the ~2050-word fetch wall (CONT-79: oversized + * blocks never start, PBDMA parks). The deadlock: the calls can only run + * after the program is resident, and the launch that would run them is too + * big to start. The re-emission is only needed while the program is NOT + * known resident; the launch that carries the final part is the witness + * (its poll break with GP_GET advanced proves the engine consumed it). + * live_gen records the generation whose carrier executed; while it equals + * the capture generation the drain restart is skipped and the calls-kept + * launch ships window-only (~530 words, well under the wall). + * sema_gl_mme_prelude_live_skip=0 restores the unconditional re-emission + * for a one-flip A/B. + */ +static u32 gsp_phys_gr_mme_pending_gen; /* final part carried by the last + * launch; consumed at its poll */ +static u32 gsp_phys_gr_mme_live_gen; /* generation whose carrier executed */ +static bool gsp_phys_rmcall_gr_mme_live_skip = true; +module_param_named(sema_gl_mme_prelude_live_skip, + gsp_phys_rmcall_gr_mme_live_skip, bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_prelude_live_skip, + "Skip the forced per-launch prelude re-emission once the generation's final part has been carried by a launch the engine executed (default 1; 0 = always re-emit, pre-CONT-103 behavior)"); + +/* + * CONT-103: elide the program-address pair runs (method space + * [0x2054, 0x2158]) from a calls-kept window. With the prelude-live skip the + * calls-kept launch starts, and the engine then traps 0x80002058 on the + * relocated (shadow-pool) pair operand and parks - the CONT-101 trap class. + * The binds persist on the channel from the earlier calls-held launches, so + * dropping the re-binds removes the trapping write; the calls and the MME + * program stay. Off by default: the A/B against the trapping state. + */ +static bool gsp_phys_rmcall_gr_mme_nop_pair; +module_param_named(sema_gl_mme_nop_pair, gsp_phys_rmcall_gr_mme_nop_pair, + bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_nop_pair, + "Elide program-address pair runs (0x2054/0x2094/0x2154 families) from a calls-kept 3D window (default 0; the engine traps 0x2058 on the relocated shadow operand)"); + +/* + * SESSION 153-CONTINUED-79: parameter-conserving NOP for kept parking slots. + * The r1177 bisect: keep={0,2,57} (full 3089-word program resident, split + * emission) parks the PBDMA on the emitted block (MME-FAST events=0, grfifo + * frozen, warm sem WEDGED) while keep={0,2} retires every launch - slot 57's + * CALL_MME_MACRO(57) run is the r1142 park class. Rather than drop the run + * (which also removes its parameter words from the stream), rewrite the call + * run's header to target the CALL_MME_DATA port (method+4) so the parameter + * words are still written (conserved) but no macro executes: the trigger + * word becomes a plain port write and the PBDMA never opens a macro context + * whose parameter drain can back up. A slot named in this mask must also be + * in sema_gl_mme_keep_mask (kept = retained in the window). Default 0. + */ +static u64 gsp_phys_rmcall_gr_nop_param_mask; +module_param_named(sema_gl_mme_nop_param_mask, gsp_phys_rmcall_gr_nop_param_mask, + ullong, 0644); +MODULE_PARM_DESC(sema_gl_mme_nop_param_mask, + "Kept slots whose CALL_MME_MACRO trigger is rewritten to a plain CALL_MME_DATA port write (params conserved, macro not executed; u64 bitmask)"); + +/* + * SESSION 153-CONTINUED-80: the r1181 live A/B validated the retarget above + * and it PARKS the PBDMA exactly like the raw kept call - with + * keep=5|1<<57 + nop=1<<57 the first post-change launch froze + * (MME-FAST begin grfifo=0x00000001, end events=0, no poison, warm sem + * WEDGED) while keep=5 (slot 57 call+params elided) retired in 94-106 us + * on the same boot (r1176/r1181 baseline). The bare DATA-port run without an + * open macro context is not a safe landing for the trigger. With this knob + * set, a nop-mask slot joins the drop path instead (the r1144-r1177 + * proven-safe behavior: whole run elided, retires clean, params NOT + * conserved). Use for the all-real-macro A/B: sa_stub_all=0 leaves + * 34/39/40/48/51 real while this drops only 57. Default 0 preserves the + * retarget behavior so the r1179 transform evidence stays reproducible. + */ +static u64 gsp_phys_rmcall_gr_nop_drop; +module_param_named(sema_gl_mme_nop_drop, gsp_phys_rmcall_gr_nop_drop, ullong, + 0644); +MODULE_PARM_DESC(sema_gl_mme_nop_drop, + "Slots whose call runs are dropped entirely (proven-safe retire, SA[slot] stubbed) instead of retargeted to the CALL_MME_DATA port (u64 bitmask; default 0)"); + +/* + * SESSION 153-CONTINUED-82: hold real CALL runs until the prelude latch. + * The r1220 (all-real, nop_drop=1<<57, sa_stub_all=N) wedge: the window's + * real CALL runs executed in the SAME launch that emitted only the first + * PART of a fresh capture generation (`MME-PRELUDE part gen=3 off=1776 + * words=1776/3083 ... (continuation next launch)`), so the called macros + * fetched IR code that was not resident yet - `MME-FAST launch ... + * pc_valid=1 pc=513`, priv-error poison, `Shader Program Header 18 Error`, + * `Class Error ... Offset 0000238c, Data 00020020` (the r1116 lane-poison + * class) and the boot wedged. The prelude latch (gsp_phys_gr_mme_prelude_* + * gen/h) is set only when the FINAL part has gone out; with this knob the + * launch builder holds (elides, the proven-safe drop shape) every remaining + * CALL_MME_MACRO/DATA run of the emitted window while EITHER the latch for + * this generation+channel is not set after this launch, OR this launch + * uploaded generation code for the first time (latch invalid before the + * emission). The r1222 all-real boot proved the second clause necessary: + * the drain-completing launch (part-2 IR uploads + window real calls in the + * SAME block, latch set by its own emission) still hit ILLEGAL_OPCODE - + * real calls resume on the next launch, when only the idempotent keep-forced + * re-emission of the already-resident prefix is in flight. Default 1 (the + * fix); A/B with 0 to reproduce the r1209/r1220/r1222 wedge. + */ +static bool gsp_phys_rmcall_gr_mme_hold_calls = true; +module_param_named(sema_gl_mme_hold_calls, gsp_phys_rmcall_gr_mme_hold_calls, + bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_hold_calls, + "Hold (elide) window CALL_MME_MACRO/DATA runs until the MME prelude latches for the current generation+channel (default 1; 0 = legacy, real calls run against a partial prelude)"); + +/* + * SESSION 153-CONTINUED-107: separately control the CONT-104 FE-idle clause + * of the hold (see the call site). With the IRAM gate proving residency the + * clause's only value is the r1359 race, and r1373 showed the FE wedged at + * 0x07071602 for minutes while every calls-kept window was held. Default 1 + * (pre-CONT-107 behavior); 0 releases the calls as soon as the prelude + * latch + IRAM gate allow. + */ +static bool gsp_phys_rmcall_gr_mme_hold_fe_idle = true; +module_param_named(sema_gl_mme_hold_fe_idle, + gsp_phys_rmcall_gr_mme_hold_fe_idle, bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_hold_fe_idle, + "Also hold the real CALL runs while the front-end is not settled (0x400504 != 1; default 1)"); + +/* + * SESSION 153-CONTINUED-84: emit NVC597_SET_MME_VERSION=1 (0x234) with the + * prelude step. The channel-init defaults the client pushes once per channel + * (Turing+) are never replayed, so the replayed context's MME stays in its + * reset/disabled state and every CALL_MME_MACRO run parks the front-end + * waiting for a macro that cannot start (r1245: call-bearing launches hang + * with all MME registers idle). Default 1; A/B with 0. + */ +static bool gsp_phys_rmcall_gr_mme_version = true; +module_param_named(sema_gl_mme_version, gsp_phys_rmcall_gr_mme_version, + bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_version, + "Emit SET_MME_VERSION=1 ahead of the replayed MME prelude (default 1; the Turing channel-init method the guest pushes once and the capture never carries)"); + +/* + * SESSION 153-CONTINUED-99: IR-only prelude drain. The call-bearing launch + * must carry the macros' SA pairs + IR words IN THE SAME BLOCK as the + * CALL_MME_MACRO runs: the split drain ships the full 3083-word capture + * across two launches (r1337 W1/W2), so the calls only ever ran on a launch + * whose block held part of the program - and every such launch faults MME + * ILLEGAL_OPCODE (relayed ESR 0x404490=0x80000004) the moment the first + * CALL executes (r1336/r1337/r1339/r1340, four boots; with {48,51,57} + * dropped in r1340 the remaining {0,2,34,39,40} calls fault identically). + * Shipping the FULL capture in one block does not work either: the 3441- + * and 3623-word blocks of r1341 never retired at all (MME-FAST begin + * grfifo=0d0d0001/00000001, no ch_rel, timeout ladder). The SA pairs + + * IR-load runs of the whole capture are only ~1277 of the 3083 words; the + * remainder is the setup family the macros do not fetch. With this knob the + * drain emits ONLY the runs targeting the MME SA/IR ports (0x114/0x118/ + * 0x11c/0x120), whole runs, so a single <=2048-word block can carry SA+IR + * (1277) + SA re-arm + window and the calls execute against the program + * the same launch uploaded. Default 0 (A/B; the r1337 drain behavior). + */ +static bool gsp_phys_rmcall_gr_mme_prelude_ir_only; +module_param_named(sema_gl_mme_prelude_ir_only, + gsp_phys_rmcall_gr_mme_prelude_ir_only, bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_prelude_ir_only, + "Emit only the MME SA/IR port runs (0x114/0x118/0x11c/0x120) from the prelude drain so SA+IR fit one working-size block (default 0)"); + +/* + * SESSION 153-CONTINUED-99b: packed (partitioned) prelude. The captured + * prelude buffer is the sum of the client's overlapping MME upload runs: + * each run writes its macro's body at its SA entry, and the runs overlap + * (runs for later slots rewrite shared tail regions). Replaying them + * verbatim in capture order therefore leaves the IR in a state where a + * macro's own body tail has been overwritten with the NEXT macro's bytes - + * executing it fetches non-code and the MME raises ILLEGAL_OPCODE + * (relayed ESR 0x404490=0x80000004; r1336/r1337/r1339/r1340/r1342, five + * boots). Verified against the driver's own generated macro source: the + * slot-34 run equals mmeCodeOGL_C597_DRAW_IAB word-for-word (75/75) but + * the capture-order final image differs at 50 of them. + * + * This knob rebuilds the program the way the engine must see it: each IR + * address takes the value from the run of the slot whose SA region owns it + * (run j clipped at the next SA entry), then emits the whole IR as ONE + * contiguous 0x114 ONE_INC run plus the full SA table as [slot, entry] + * pairs - about 1270 words, so a single <=2048-word block carries SA+IR + + * SA re-arm + window and the window's CALL_MME_MACRO runs execute against + * the program the same launch uploaded. Default 0 (capture-order drain). + */ +static bool gsp_phys_rmcall_gr_mme_prelude_pack; +module_param_named(sema_gl_mme_prelude_pack, + gsp_phys_rmcall_gr_mme_prelude_pack, bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_prelude_pack, + "Rebuild the MME IR as the partitioned program (each SA region from its own run) and emit it compactly with the SA table (default 0)"); + +/* Packed IR image; indexed by IR word address (<= 0x420 in captures). */ +static u32 gsp_phys_gr_mme_ir_pack[4096]; + +/* + * CONT-112: A/B IR-word override for the packed MME program. The offline + * replay of the captured SA stream (mmeref/, sim_run.py) shows slot 34 (the + * draw macro) emitting `0x1618<-0x61000008, 0xd74<-0, 0xd78<-count(0x3c), + * 0x1614<-0` from the CLIENT'S OWN call params - i.e. the emitted sequence + * is parameter-correct offline. This knob overrides ONE word of the packed + * image (the exact words the engine loads into its IRAM) so a live A/B can + * mutate the emitted method words one at a time (BEGIN data / SVS / DRAW + * count / END) and observe the engine's writes: the packed image is rebuilt + * per launch, so the value is live-tunable without a reload. Off by + * default (~0u); both params runtime-settable. + */ +static unsigned int gsp_phys_gr_mme_irpatch_off = ~0u; +module_param_named(sema_gl_mme_irpatch_off, gsp_phys_gr_mme_irpatch_off, + uint, 0644); +MODULE_PARM_DESC(sema_gl_mme_irpatch_off, + "IR word index in the packed MME program to override for A/B (default 4294967295 = off)"); +static unsigned int gsp_phys_gr_mme_irpatch_val; +module_param_named(sema_gl_mme_irpatch_val, gsp_phys_gr_mme_irpatch_val, + uint, 0644); +MODULE_PARM_DESC(sema_gl_mme_irpatch_val, + "Value written at sema_gl_mme_irpatch_off in the packed MME program (default 0)"); + +/* + * SESSION 153-CONTINUED-107: IRAM chronology + load-gate helpers. + * + * CONT-106 proved the trap-time fetch at PC 0x364 (macro 57 entry) reads + * ZERO IR words while the host-side pack is verbatim (CONT-105), and the + * IRAM debug port at 0x4044d0 also reads zero at 0x364 on that boot while + * the same port read the golden first words at pc=0 on r1364. The one + * unproven link is the LOAD: did the emitted 0x114 pack writes land in the + * MME's IRAM before the macro executed? These helpers read the live IRAM + * back and (a) log the timeline around each launch's fault-register + * transitions, (b) gate the real CALL runs on the readback matching the + * packed image at every covered SA entry. + * + * The probe reads the macro-57 entry group (0x364) under BOTH port address + * conventions - instruction base (write 0x364) and word base (write + * 3*0x364 = 0xA2C) - plus word 0, and prints the live words next to the + * pack-image expectation, so one line answers "is the program resident + * right now and which address unit does the port use". + */ +static void gsp_phys_mme_iram_probe(void __iomem *bar0, const char *tag, + u32 h_object) +{ + u32 a[3], bi[3], bw[3]; + + if (!bar0) + return; + gsp_phys_mme_iram_read3(bar0, 0u, a); + gsp_phys_mme_iram_read3(bar0, 0x364u, bi); + gsp_phys_mme_iram_read3(bar0, 3u * 0x364u, bw); + pr_info("vgsp-gr: MME-IRAM %s h=0x%x w0=%08x,%08x,%08x pack0=%08x,%08x,%08x i364=%08x,%08x,%08x w364=%08x,%08x,%08x pack364=%08x,%08x,%08x\n", + tag, h_object, a[0], a[1], a[2], + gsp_phys_gr_mme_ir_pack[0], + gsp_phys_gr_mme_ir_pack[1], + gsp_phys_gr_mme_ir_pack[2], + bi[0], bi[1], bi[2], + bw[0], bw[1], bw[2], + gsp_phys_gr_mme_ir_pack[3u * 0x364u], + gsp_phys_gr_mme_ir_pack[3u * 0x364u + 1u], + gsp_phys_gr_mme_ir_pack[3u * 0x364u + 2u]); +} + +/* + * SESSION 153-CONTINUED-107 gate knob (see the helper comment above). + * Default 0: A/B flip - with the gate on, a launch whose readback shows the + * IRAM not holding the packed image holds its real CALL runs and owes a + * full re-emission (prelude latch cleared; pack/drain cursors rewound), so + * the calls only ever execute against a verified-resident program. + */ +static bool gsp_phys_rmcall_gr_mme_iram_gate; +module_param_named(sema_gl_mme_iram_gate, gsp_phys_rmcall_gr_mme_iram_gate, + bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_iram_gate, + "Read the MME IRAM back (0x4044d0 port) and hold/re-emit the packed prelude until it reads back the packed image at the SA entries (default 0; the CONT-106 all-zero fetch)"); + +/* + * SESSION 153-CONTINUED-108: site control for the IRAM-gate readbacks. + * + * r1374 chronology (first gate boot that released calls): the carrier + * launch (window dropped, no calls) read grfifo=01010001 (empty) at +5 ms; + * the poll-break gate then wrote the MME debug port (0x4044d0) DURING that + * live launch; the first sample of the next launch 2.9 s later read + * grfifo=07071602 (not-empty + FULL, COUNT=22) with NO launch active in + * between - the front end never drained again for the rest of the boot + * (26 launches, all beginning FULL). CONT-107 already recorded the sibling + * evidence: every probe-bearing cycle (r1369..r1371) parked the engine on + * the first X-channel launches while the readback-free cycle (r1368) ran + * the full lane. Both gate sites write the same port, so each gets its own + * knob for the A/B: + * - break (default 0): the poll-break readback after the carrier + * release. Off by default - it is the r1374 suspect. Skipping it does + * not skip the live-gen promotion (that keys on the engine GP_GET + * fetch witness); the build site still gates the calls. + * - build (default 1): the pre-doorbell readback that authorizes the + * calls-bearing launch. Kept on; flip to 0 at runtime to isolate it. + */ +static bool gsp_phys_rmcall_gr_mme_iram_gate_break; +module_param_named(sema_gl_mme_iram_gate_break, + gsp_phys_rmcall_gr_mme_iram_gate_break, bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_iram_gate_break, + "Do the IRAM-gate readback at the launch poll break (writes the debug port while the carrier is finishing; default 0 - the r1374 FIFO-fill suspect)"); + +static bool gsp_phys_rmcall_gr_mme_iram_gate_build = true; +module_param_named(sema_gl_mme_iram_gate_build, + gsp_phys_rmcall_gr_mme_iram_gate_build, bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_iram_gate_build, + "Do the IRAM-gate readback in the launch builder before the doorbell (default 1; the calls-authorizing check)"); + +/* + * SESSION 153-CONTINUED-107: separate knob for the per-launch chronology + * probe lines. They write the MME debug port DURING a live launch, and the + * first three probe-bearing cycles (r1369..r1371) all parked the engine on + * the first X-channel launches while the last probe-free cycle (r1368) ran + * the full lane; until the port's side effects are cleared, keep the probe + * off by default and enable it only on a known-good boot (A/B). + */ +module_param_named(sema_gl_mme_iram_probe, gsp_phys_rmcall_gr_mme_iram_probe, + bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_iram_probe, + "Per-launch MME-IRAM chronology probe lines in the post-doorbell spin (writes the debug port during a live launch; default 0)"); +MODULE_PARM_DESC(sema_gl_mme_iram_probe, + "Per-launch MME-IRAM chronology probe lines in the post-doorbell spin (writes the debug port during a live launch; default 0)"); + +/* + * Read every covered SA entry from the live IRAM and compare with the packed + * image. Auto-calibrates the port's address unit once (word base 3*entry vs + * instruction base entry - the r1364 validation could only settle address 0). + * Returns true when every checked entry matches (or there is nothing to + * check); false on any mismatch (all-zero included). Bounded logging. + */ +static bool gsp_phys_gr_mme_iram_gate(void __iomem *bar0, u32 h_object, + u32 gen, const char *tag) +{ + static u32 gate_run_n, gate_fail_n, gate_pass_n; + static int base_sel = -1; /* -1 unresolved; 0 word; 1 instruction */ + u32 sa_ent[64], i; + u32 checked = 0, match = 0, zero = 0; + u32 bad_slot = 0xffffffffu, bad_entry = 0, checked_to_bad = 0; + u32 got0 = 0, got1 = 0, got2 = 0, exp0 = 0, exp1 = 0, exp2 = 0; + u64 sa_mask; + + if (!bar0) + return true; + sa_mask = gsp_sema_mme_sa_get(sa_ent, 64); + for (i = 0; i < 64u; i++) { + u32 entry, base, g0, g1, g2, got[3]; + + if (!(sa_mask & (1ull << i)) || !sa_ent[i]) + continue; + entry = sa_ent[i]; + if (entry * 3u + 2u >= 4096u) + continue; + g0 = gsp_phys_gr_mme_ir_pack[entry * 3u]; + g1 = gsp_phys_gr_mme_ir_pack[entry * 3u + 1u]; + g2 = gsp_phys_gr_mme_ir_pack[entry * 3u + 2u]; + /* + * Zero-gold entries are stub-covered (no captured code) or + * not part of the packed image; they carry nothing to gate. + */ + if (!g0 && !g1 && !g2) + continue; + if (base_sel < 0 && entry) { + /* + * One-time unit calibration on the first non-zero + * entry: try the word base, then the instruction + * base. Leaving it unresolved simply keeps the word + * base (the documented form). + */ + gsp_phys_mme_iram_read3(bar0, 3u * entry, got); + if (got[0] == g0 && got[1] == g1 && got[2] == g2) { + base_sel = 0; + pr_info("vgsp-gr: MME-IRAM-BASE resolved=word (3*entry) at entry=0x%x\n", + entry); + } else { + gsp_phys_mme_iram_read3(bar0, entry, got); + if (got[0] == g0 && got[1] == g1 && + got[2] == g2) { + base_sel = 1; + pr_info("vgsp-gr: MME-IRAM-BASE resolved=instruction (entry) at entry=0x%x\n", + entry); + } + } + } + base = (base_sel == 1) ? entry : 3u * entry; + gsp_phys_mme_iram_read3(bar0, base, got); + checked++; + if (got[0] == g0 && got[1] == g1 && got[2] == g2) { + match++; + continue; + } + if (!got[0] && !got[1] && !got[2]) + zero++; + if (bad_slot == 0xffffffffu) { + bad_slot = i; + bad_entry = entry; + checked_to_bad = checked; + got0 = got[0]; got1 = got[1]; got2 = got[2]; + exp0 = g0; exp1 = g1; exp2 = g2; + } + } + if (!checked || match == checked) { + if (gate_pass_n < 8u) { + gate_pass_n++; + pr_info("vgsp-gr: MME-IRAM-GATE pass %s h=0x%x gen=%u base=%d checked=%u run=%u\n", + tag, h_object, gen, base_sel, checked, + gate_run_n); + } + gate_run_n++; + return true; + } + gate_run_n++; + if (gate_fail_n < 24u) { + gate_fail_n++; + pr_warn("vgsp-gr: MME-IRAM-GATE FAIL %s h=0x%x gen=%u base=%d checked=%u match=%u zero=%u bad slot=%u(entry=0x%x @%u) got=%08x,%08x,%08x want=%08x,%08x,%08x run=%u\n", + tag, h_object, gen, base_sel, checked, match, zero, + bad_slot, bad_entry, checked_to_bad, + got0, got1, got2, exp0, exp1, exp2, gate_run_n); + } + return false; +} + +/* + * SESSION 153-CONTINUED-86: the missing SPH (shader-program-header) state. + * + * The executed beef launches fault `Graphics Exception: Shader Program + * Header 9/11/18 Error` + `DS_HWW_ESR 0x405840=0xa2040a00` (r1272): the DS + * unit validates the header of every pipeline program bound when a draw + * issues, including the SPH VERSION against the channel's programmed + * window. The Turing channel-init (the 3D driver's own + * _nv3dInitChannelTuring) writes the SPH version pair ONCE per channel: + * + * INC run @0x16a4, count 2: + * METHOD 0x16a4 (SET_SPH_VERSION) <- 4 | (4 << 16) = 0x00040004 + * METHOD 0x16a8 (CHECK_SPH_VERSION) <- 0x00040004 + * + * (fields: CURRENT 15:0, OLDEST_SUPPORTED 31:16; the value 4 is the SPH V4 + * the Turing shader images carry). The replayed context never receives it: + * the capture families (MME prelude 0x114..0x124/0x234/0x550..0x568/ + * 0x3400..0x34fc, attr prelude 0x1100..0x11fc/0x2000..0x205c) do not cover + * 0x16a4, and the captured 3D windows contain no such write (verified in the + * emitted-window decode and the raw guest-RAM stream). With the register + * unset (0/0) every fetched SPH fails its version window -> per-SPH error + * bits -> 0 GR-STATE retires and no stores. The client's own emitted init + * pair was located verbatim in the guest pushbuffer image (guest PA + * 0x8672d5b4: `200205a9 00040004 00040004`), so the launch builder emits + * exactly that run ahead of the window (right after SET_OBJECT, beside the + * MME version write). 3 words, idempotent; emitted on every replayed 3D + * window. Default 1; A/B with 0 reproduces the SPH fault. + */ +static bool gsp_phys_rmcall_gr_sph_version = true; +module_param_named(sema_gl_sph_version, gsp_phys_rmcall_gr_sph_version, + bool, 0644); +MODULE_PARM_DESC(sema_gl_sph_version, + "Emit the Turing channel-init SET_SPH_VERSION/CHECK_SPH_VERSION pair (0x16a4/0x16a8 <- 0x00040004) ahead of each replayed 3D window (default 1; A/B with 0 reproduces the Shader Program Header 9/11/18 fault)"); + +/* + * SESSION 153-CONTINUED-93: forced SPH version-window value. 0 = auto (the + * blob-derived window, see the emission site); nonzero = emit exactly this + * 32-bit value for both 0x16a4/0x16a8. Live A/B for the DS version-window + * hypothesis: the emitted 4/4 window may reject the fetched SPHs while a + * wider/older window (e.g. 0x00040003) accepts them. + */ +static u64 gsp_phys_rmcall_gr_sph_version_val; +module_param_named(sema_sph_version_val, gsp_phys_rmcall_gr_sph_version_val, + ullong, 0644); +MODULE_PARM_DESC(sema_sph_version_val, + "Forced SET_SPH_VERSION/CHECK_SPH_VERSION value (0 = auto from the loaded blobs)"); + +/* + * SESSION 153-CONTINUED-90: SET_SHADER_EXCEPTIONS (0x1528 <- 0). + * + * NVIDIA's own Turing 3D channel init (_nv3dInitChannelTuring) pushes this + * FIRST, with the comment "Disable shader exceptions. This matches OpenGL + * driver behavior." The client's captured init stream carries the write too + * (sema-pushraw `off=0x5a4 op=4 addr=0x1528 v0=0`). The replayed context + * never receives it: the emitted push (PB-FAULT-DUMP) goes SET_OBJECT -> + * SPH version pair -> MME version with no 0x1528 run, so the DS keeps + * raising "Graphics Exception: Shader Program Header 5/9/10/11 Error" on + * every draw whose fetched SPH it would tolerate with exceptions disabled - + * and the exception poisons the channel before any store retires. An + * executing r1294 launch names the consequence: the first post-exception + * method write reports `ChID 0001, Class 0000c597, Offset 0000022c, Data + * 00000800` (SET_GO_IDLE_TIMEOUT poisoned by the exception state). + * Emitted right after SET_OBJECT, exactly like the reference init order. + * Default 1; A/B with 0 restores the old behavior. + */ +static bool gsp_phys_rmcall_gr_shader_exc = true; +module_param_named(sema_gl_shader_exc, gsp_phys_rmcall_gr_shader_exc, + bool, 0644); +MODULE_PARM_DESC(sema_gl_shader_exc, + "Emit SET_SHADER_EXCEPTIONS (0x1528) <- 0 after SET_OBJECT like _nv3dInitChannelTuring (default 1)"); + +/* + * SESSION 153-CONTINUED-91: clear DS_HWW_REPORT_MASK/_2 (0x405844/0x40584c) + * ahead of every replayed 3D window (host write + in-stream MME PRI R/W + * macro 5) so a DS SPH validation miss latches without poisoning the PRI + * block and the GSP-RM exception handler killing the channel. See the + * emission-site comment. Default 1; A/B with 0 restores the old behavior + * (every SPH miss = priv-error poison + channel wedge). + */ +static bool gsp_phys_rmcall_gr_ds_mask_clear = true; +module_param_named(sema_ds_mask_clear, gsp_phys_rmcall_gr_ds_mask_clear, + bool, 0644); +MODULE_PARM_DESC(sema_ds_mask_clear, + "Clear DS_HWW_REPORT_MASK/_2 ahead of replayed 3D windows (default 1; 0 = leave the all-report mask)"); + +/* + * SESSION 153-CONTINUED-91: emit the Turing program-region pair (the MME + * shadow scratch 0x78/0x79 = NVC397_SET_MME_PROGRAM_REGION) with the + * launch's own program family page as the shader-heap base, so the DS's + * SPH validation has a region to accept the fetched SPHs against. See the + * emission-site comment. Default 1; A/B with 0. + */ +static bool gsp_phys_rmcall_gr_sph_region = true; +module_param_named(sema_gl_sph_region, gsp_phys_rmcall_gr_sph_region, + bool, 0644); +MODULE_PARM_DESC(sema_gl_sph_region, + "Emit the Volta+ program-region pair (SET_MME_SHADOW_SCRATCH 0x78/0x79 <- program family page) ahead of replayed 3D windows (default 1)"); + +/* + * Runtime-selectable program-region base. 0 = auto (the launch's own + * program family page, the guest-VA heap base the real driver would use); + * a nonzero value forces that base - the engine-side shadow-VA space needs + * its own experiment (the SPH_INFO snapshot shows the DS fetched through + * shadow VAs 0x620900/0x620200/0x620000, so a guest-VA base may be the + * mismatch). Writable at runtime for a live A/B. + */ +static u64 gsp_phys_rmcall_gr_sph_region_base; +module_param_named(sema_sph_region_base, gsp_phys_rmcall_gr_sph_region_base, + ullong, 0644); +MODULE_PARM_DESC(sema_sph_region_base, + "Program-region base override (0 = launch program family page; nonzero = explicit engine-side base)"); + +/* + * SESSION 153-CONTINUED-101: MME fixed-constant-buffer scratch seed + * (SET_MME_SHADOW_SCRATCH(0x18), method 0x3460). + * + * mmeCodeOGL_C597_SETUP_FIXED_CONST (macro slot 0) reads shadow scratch + * 0x18/0xd18 via STATE and unconditionally emits its high/low halves as + * SET_CONSTANT_BUFFER_SELECTOR_B/C (the compiled C597 form carries NO + * bug2548370 branch: the config table says no972630). With the replay + * context's scratch at 0 the macro emits selector B/C = 0 and the draw's + * constant fetch has no address. The client's own write (gf100Init- + * ConstantBuffers: (u32)(fixed CB VA >> 8)) is a once-per-channel event + * the ladder never receives, so the builder seeds it ahead of the + * window. Value source: sema_gl_mme_seed_val when nonzero, else the + * client's own captured write (gsp_sema_mme_scratch18_get). With + * sema_gl_mme_seed_xlat (default 1) the page is translated through the + * launch's own page set / shadow mapping first (the same relocator the + * window operand pairs use) so the emitted selector resolves in the + * engine's address space; on MISS the raw value is emitted (the A/B + * control). Default off; live-tunable. + */ +static bool gsp_phys_rmcall_gr_mme_seed; +module_param_named(sema_gl_mme_seed, gsp_phys_rmcall_gr_mme_seed, bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_seed, + "Seed MME shadow scratch 0x18 (method 0x3460) with the fixed-CB base>>8 ahead of replayed 3D windows (default 0)"); +static u32 gsp_phys_rmcall_gr_mme_seed_val; +module_param_named(sema_gl_mme_seed_val, gsp_phys_rmcall_gr_mme_seed_val, + uint, 0644); +MODULE_PARM_DESC(sema_gl_mme_seed_val, + "Explicit scratch 0x18 value (fixed CB VA>>8); 0 = use the client's captured 0x3460 write"); +static bool gsp_phys_rmcall_gr_mme_seed_xlat = true; +module_param_named(sema_gl_mme_seed_xlat, gsp_phys_rmcall_gr_mme_seed_xlat, + bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_seed_xlat, + "Translate the seed value's page to the engine-side shadow/identity VA before emitting (default 1; 0 = raw)"); + +/* + * SESSION 153-CONTINUED-105: golden MME runtime state seed, STATE + * 0xD26..0xD2A (SET_MME_SHADOW_SCRATCH 0x26..0x2a, methods 0x3498/0x349c/ + * 0x34a0/0x34a4/0x34a8). The bug2548370 loadConstantBuffer macro (slot 57 / + * macro address 0x39) branches at its entry on these five words: + * + * 0xD26 BUG2548370_PAGE_ALLOCATION_COUNT + * 0xD27 BUG2548370_ZERO_VIEW_OFFSET + * 0xD28 BUG2548370_LAST_UPDATED_CONSTANT_BUFFER_SELECTOR_B_C + * 0xD29 BUG2548370_LOAD_CONSTANT_BUFFER_OFFSET + * 0xD2A BUG2548370_LAST_UPDATED_LOAD_CONSTANT_BUFFER_OFFSET + * + * (verified against the driver's generated macro source: the entry reads + * exactly 0xd28/0xd19/0xd29/0xd2a/0xd26/0xd27 and the WAR path compares + * 0xd26 against 30 and 0xd27 against zero). The client's own prelude + * writes only 0xd2a <- 0x80000000 / 0xd29 <- 0 ("reset tracking after each + * draw") and the per-channel 0x3464/0x3460 writes; 0xd26/0xd27/0xd28 carry + * over from the client's own context and never appear in the captured + * stream. The replayed context therefore reads whatever the replay left + * there. Emit the five writes ahead of every replayed 3D window (values + * from sema_mme_state_d26..d2a; all-zero default = the zero-valid form). + * Default off; live-tunable A/B. + */ +static bool gsp_phys_rmcall_gr_mme_state_seed; +module_param_named(sema_gl_mme_state_seed, gsp_phys_rmcall_gr_mme_state_seed, + bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_state_seed, + "Seed MME runtime state 0xD26..0xD2A (methods 0x3498..0x34a8) ahead of replayed 3D windows (default 0)"); +static u32 gsp_phys_rmcall_gr_mme_state_d26; +module_param_named(sema_mme_state_d26, gsp_phys_rmcall_gr_mme_state_d26, + uint, 0644); +MODULE_PARM_DESC(sema_mme_state_d26, "Seed value for STATE 0xD26 PAGE_ALLOCATION_COUNT (method 0x3498)"); +static u32 gsp_phys_rmcall_gr_mme_state_d27; +module_param_named(sema_mme_state_d27, gsp_phys_rmcall_gr_mme_state_d27, + uint, 0644); +MODULE_PARM_DESC(sema_mme_state_d27, "Seed value for STATE 0xD27 ZERO_VIEW_OFFSET (method 0x349c)"); +static u32 gsp_phys_rmcall_gr_mme_state_d28; +module_param_named(sema_mme_state_d28, gsp_phys_rmcall_gr_mme_state_d28, + uint, 0644); +MODULE_PARM_DESC(sema_mme_state_d28, "Seed value for STATE 0xD28 LAST_UPDATED_CB_SELECTOR_B_C (method 0x34a0)"); +static u32 gsp_phys_rmcall_gr_mme_state_d29; +module_param_named(sema_mme_state_d29, gsp_phys_rmcall_gr_mme_state_d29, + uint, 0644); +MODULE_PARM_DESC(sema_mme_state_d29, "Seed value for STATE 0xD29 LOAD_CONSTANT_BUFFER_OFFSET (method 0x34a4)"); +static u32 gsp_phys_rmcall_gr_mme_state_d2a; +module_param_named(sema_mme_state_d2a, gsp_phys_rmcall_gr_mme_state_d2a, + uint, 0644); +MODULE_PARM_DESC(sema_mme_state_d2a, "Seed value for STATE 0xD2A LAST_UPDATED_LOAD_CONSTANT_BUFFER_OFFSET (method 0x34a8)"); + +/* + * SESSION 153-CONTINUED-105: split the packed MME prelude emission across + * launches. The CONT-99b pack rebuilds the partitioned IR and emits it as + * ONE contiguous ONEINC run; for the current 55-slot capture that stream is + * 2 + maxa + 3*npair = ~2966 words (maxa = 2799, npair = 55), over the + * 2048-word per-launch block cap, so the pack silently falls back to the + * raw capture-order drain (which is only correct when the capture's runs + * do not overlap). With this knob the packed stream is drained with a + * generation-keyed cursor: the SA table goes out once, the IR goes out as + * whole-instruction chunks in consecutive launches (each chunk = one + * ONEINC 0x114 run with its own instruction-address pointer), and the + * prelude latch is set only when the last chunk is emitted. Calls stay + * held across the chunks (hold_calls + the FE-idle witness). Default 0. + */ +static bool gsp_phys_rmcall_gr_mme_pack_split = true; +module_param_named(sema_gl_mme_pack_split, gsp_phys_rmcall_gr_mme_pack_split, + bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_pack_split, + "Split the packed MME prelude (SA table + IR chunks) across launches when it exceeds the block cap (default 1; 0 = all-or-nothing CONT-99b pack)"); + +/* + * SESSION 153-CONTINUED-93: additionally emit the legacy program-region + * pair (0x1608/0x160c) beside the MME shadow pair (see the emission site). + * Default 0 (A/B; the DS snapshot registers share the legacy pair's names). + */ +static bool gsp_phys_rmcall_gr_sph_region_legacy; +module_param_named(sema_sph_region_legacy, gsp_phys_rmcall_gr_sph_region_legacy, + bool, 0644); +MODULE_PARM_DESC(sema_sph_region_legacy, + "Also emit the legacy SET_PROGRAM_REGION_A/B (0x1608/0x160c) pair with the same base (default 0)"); + +/* + * SESSION 153-CONTINUED-87: the missing TESS_CONTROL/PIXEL program binds. + * The client's real channel state binds every pipeline stage once per + * context; the captured init-push image (guest RAM scrape) carries + * `20020825 [0, 04400200]` (j=2 TESS_CONTROL) and + * `20020855 [0, 04400000]` (j=5 PIXEL) beside the SPH version pair - + * but the captured per-frame window carries only the j=1 (VTG) pair + * (0x4400900/0x4400300). The DS validates the SPH of every pipeline + * stage the draw activates, so the unbound stages latch their header + * errors (DS_HWW_ESR 0xa2040a00 = SPH9|SPH11|SPH18) while the V3 blob + * passes wherever a stage IS bound. Emit both binds ahead of the + * window at the window's own program-family page (all four VAs share + * one 4K page: 0x4400000 j=5, 0x4400200 j=2, 0x4400300/0x4400900 j=1), + * pushed through the same relocation pass as the window - a MISS skips + * the bind entirely (never a raw client VA). Beef client channels only + * (the family layout is the client's). Default 1; A/B with 0. + */ +static bool gsp_phys_rmcall_gr_sph_bind = true; +module_param_named(sema_gl_sph_bind, gsp_phys_rmcall_gr_sph_bind, bool, 0644); +MODULE_PARM_DESC(sema_gl_sph_bind, + "Emit the client's missing TESS_CONTROL (j=2) and PIXEL (j=5) pipeline program binds ahead of replayed 3D windows (program-family page of the window's prog VA; default 1)"); + +bool gsp_phys_rmcall_gr_sph_bind_value(void) +{ + return READ_ONCE(gsp_phys_rmcall_gr_sph_bind); +} + +/* + * SESSION 153-CONTINUED-92: leave program-family operands at their ORIGINAL + * guest VAs when the operand's page is resolved and identity-mappable. + * + * The DS I-fetch reaches the delivered images through the shadow VAs the + * window pass picks (r1298-r1306: stages fetched at 0x620900/0x620200/ + * 0x620000) and rejects them with the bytes provably correct at those VAs + * (INLINE-RB rb=020a0481; canary proves the fetch follows the mapped page's + * content). The one variable left is the address space the DS validates in: + * the client programmed its own guest-VA program heap, and the shadow VAs + * (0x2xxxxx..0x6xxxxx pools) sit outside any region the client's state + * describes - a region/space mismatch rejects all bound stages regardless + * of content (rr=0/0, ds=a400e20 at every retire). With this knob the + * window pass first tries the Tier-1 identity mapper on the operand's own + * guest VA (launch page set first, the sema resolver else): when the guest + * VA is mapped to the resolved HPA, the operand is left verbatim, so the + * DS fetches and validates the SPH at the same VA space the client's heap + * uses. A page without an identity mapping keeps the shadow rewrite exactly + * as before (fail-open), and the identity-kept pairs still count toward the + * pass's relocated census so the SPH-BIND guard (orel == nrun) is stable. + * r1307 RESULT (experiment executed, negative): with the knob on, the + * beef lane's program pairs stayed at their guest VAs (0x4350900/0x4350f00/ + * 0x4350200/0x4350000, PROG-GUEST-VA receipts) and the engine faulted + * `MMU Fault: ENGINE GRAPHICS HUBCLIENT_PD faulted @ 0x0_04350000. FAULT_PTE + * ACCESS_TYPE_VIRT_READ` within 2.5 s of every such launch - the Tier-1 + * identity map installs (the PDE window is walked: FAULT_PTE, not + * FAULT_PDE) yet the engine still cannot fetch at the client's own VA. + * This reconfirms the r929 v2 lesson: guest VAs are not engine-fetchable on + * this replay channel, so the "validate in guest VA space" hypothesis is + * not testable by operand rewriting (the fetch dies before any DS check). + * Default 0 (baseline shadow rewrite); 1 reproduces the FAULT_PTE experiment. + */ +static bool gsp_phys_rmcall_gr_prog_guest_va; +module_param_named(sema_gl_prog_guest_va, gsp_phys_rmcall_gr_prog_guest_va, + bool, 0644); +MODULE_PARM_DESC(sema_gl_prog_guest_va, + "Leave program-family (0x2014-family) operands at their original guest VAs when identity-mappable (default 0; 1 = the r1307 FAULT_PTE experiment - guest VAs are not engine-fetchable, do not keep)"); + +/* + * SESSION 153-CONTINUED-84: SET_MME_SHADOW_RAM_CONTROL mode before the + * window. The vendor comment (mesh shader bind path) is explicit: + * "MME filtering swallows them all if old and new state values are the + * same" - METHOD_TRACK_WITH_FILTER (1) suppresses every method write whose + * value equals the shadowed one. Our replay repeats the same window every + * launch, so after the first replay every redundant write - including the + * CALL_MME_MACRO/CALL_MME_DATA parameter writes - matches the shadow and + * is swallowed; the macro call never reaches the MME (r1236-r1250: calls + * in the emitted window, all MME registers idle, some launches park the + * front-end). The mode-bearing writes live in the captured PRELUDE + * (0x124 x4 in the client's setup push), which the win_patch window sweep + * does not touch; emit METHOD_TRACK (0 = filtering OFF, semantics + * unchanged: the write simply applies) right before the SA re-arm and the + * window. Value = sema_gl_mme_track (0 = TRACK; 1/2/3 = other modes; -1 = + * do not emit). + */ +static int gsp_phys_rmcall_gr_mme_track; +module_param_named(sema_gl_mme_track, gsp_phys_rmcall_gr_mme_track, + int, 0644); +MODULE_PARM_DESC(sema_gl_mme_track, + "SET_MME_SHADOW_RAM_CONTROL mode emitted before each replayed window (0=TRACK filtering off (default); 1=TRACK_WITH_FILTER; 2=PASSTHROUGH; 3=REPLAY; -1=off)"); + +/* + * SESSION 153-CONTINUED-85: A/B knob for the SA re-arm (the r1237 fix). + * Before CONT-84 the calls in the replayed window resolved to the SA state + * the prelude left behind (its last captured SA write re-clobbers the slot; + * r1231: SA[34]=0 lands last, every draw call resolved to macro 0 and + * emitted nothing - and the r1228-phase launches retired). With the re-arm + * the calls resolve to the captured IR entries; every call-bearing launch + * since (r1245/r1250/r1265/r1270) parks or poisons the front-end. This knob + * restores the pre-arm resolution in one boot so the two states can be A/B'd + * without a rebuild (0 = do not re-point; default 1 = re-arm). + */ +static bool gsp_phys_rmcall_gr_mme_sa_rearm = true; +module_param_named(sema_gl_mme_sa_rearm, gsp_phys_rmcall_gr_mme_sa_rearm, + bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_sa_rearm, + "Re-emit LOAD_MME_START_ADDRESS_RAM pairs before each replayed window so CALL_MME_MACRO resolves to the captured SA entries (default 1; 0 = pre-arm resolution, A/B)"); + +/* + * The same stubbed-slot predicate the launch builder applies (see the + * sa_stub emit decision): a slot is stubbed when stubbing is on and either + * the all-slots knob is set, the capture holds no SA entry, or the IR + * instruction the entry names never entered the replay. A kept slot + * (sema_gl_mme_keep_mask) is never stubbed - unless the capture cannot + * serve it at all (keep_clamp, see the CONT-113 param comment). + */ +static bool gsp_phys_gr_slot_kept(u32 j, u64 sa_mask, const u32 *sa_ent) +{ + if (!(READ_ONCE(gsp_phys_rmcall_gr_keep_mask) & (1ull << j))) + return false; + if (READ_ONCE(gsp_phys_rmcall_gr_nop_drop) & (1ull << j)) + return false; + if (!READ_ONCE(gsp_phys_rmcall_gr_keep_clamp)) + return true; + if ((sa_mask & (1ull << j)) && gsp_sema_mme_ir_covered(sa_ent[j])) + return true; + /* + * SESSION 153-CONTINUED-114: a keep_mask slot the golden library + * serves is kept without capture coverage - the pack overlay + * carries its code (see the include comment). + */ + { + u32 gent, gn; + const u32 *gw; + + if (gsp_phys_gr_golden_slot(j, &gent, &gw, &gn)) + return true; + } + if (gsp_phys_gr_keep_clamp_log < 16u) { + gsp_phys_gr_keep_clamp_log++; + pr_warn("vgsp-gr: MME-KEEP-CLAMP slot=%u (keep_mask set but capture has no SA entry/coverage; stubbed + calls dropped, never run) entry=0x%x covered=%u\n", + j, sa_ent[j], + (unsigned)gsp_sema_mme_ir_covered(sa_ent[j])); + } + return false; +} + +static bool gsp_phys_gr_slot_stubbed(u32 j, u64 sa_mask, const u32 *sa_ent) +{ + if (j >= 64u || !gsp_sema_gr_sa_stub_value()) + return false; + /* + * SESSION 153-CONTINUED-80: a nop-drop slot is force-stubbed even + * when it is also kept or covered - the r1186 evidence shows a kept + * slot 57 loses its SA end-noop stub (the stub builder skips kept + * slots) and any call-run encoding the drop filter misses then + * executes the real macro 57 (the park class). Dropping the runs and + * stubbing SA[57] together is the safe combination. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_nop_drop) & (1ull << j)) + return true; + if (gsp_phys_gr_slot_kept(j, sa_mask, sa_ent)) + return false; + /* + * SESSION 153-CONTINUED-114: a slot the golden library serves is + * never stubbed (its CALL runs stay in the window and the pack + * carries its code; see the include comment). + */ + { + u32 gent, gn; + const u32 *gw; + + if (gsp_phys_gr_golden_slot(j, &gent, &gw, &gn)) + return false; + } + if (READ_ONCE(gsp_phys_rmcall_gr_sa_stub_all)) + return true; + if (!(sa_mask & (1ull << j))) + return true; + return !gsp_sema_mme_ir_covered(sa_ent[j]); +} + +/* + * In-place filter for the relocated 3D window: remove every run that + * writes the CALL_MME_MACRO(j)/CALL_MME_DATA(j) method pair of a stubbed + * slot. Pure run-level removal (runs are self-contained: op 1/3/5 carry + * 1+cnt data words; 0/2/4/6 are single words), so the surviving method + * sequence is unchanged and both the F2A window walk and the engine see + * the same stream shape with the calls elided. + */ +static u32 gsp_phys_gr_mme_drop_calls(u32 *win, u32 n, u64 sa_mask, + const u32 *sa_ent, u64 *dropped_mask) +{ + u32 r = 0, w = 0, dropped = 0, transform = 0; + u64 dmask = 0; + + while (r < n) { + u32 h = win[r]; + u32 op = (h >> 29) & 7u; + u32 cnt = (h >> 16) & 0xfffu; + u32 span = 1u, k, m; + bool iscall = false; + + if ((op == 1u || op == 3u || op == 5u) && cnt && + r + 1u + cnt <= n) + span = 1u + cnt; + /* + * Decode the touched methods the way the engine walks them: + * multi-word runs land on m + 4k; single-word entries carry + * their method in the low bits (IMMD and the SDM family both + * put it at 11:0 for our purposes). + */ + for (k = 0; k < span; k++) { + m = ((h & 0xfffu) << 2) + 4u * k; + if (k && op != 1u && op != 3u) + break; + if (m >= 0x3800u && m <= 0x39fcu) { + u32 j = (m - 0x3800u) / 8u; + + if ((READ_ONCE(gsp_phys_rmcall_gr_keep_mask) & + (1ull << j)) && + (READ_ONCE(gsp_phys_rmcall_gr_nop_param_mask) & + (1ull << j))) { + if (READ_ONCE(gsp_phys_rmcall_gr_nop_drop)) { + /* + * SESSION 153-CONTINUED-80: the + * retarget parks the PBDMA + * live (see the + * sema_gl_mme_nop_drop + * comment) - drop the whole + * run instead. + */ + iscall = true; + dmask |= 1ull << j; + break; + } + /* + * SESSION 153-CONTINUED-79: kept + + * nop-param slot - keep the run but + * retarget its header to the DATA + * port so the params still land + * (see the sema_gl_mme_nop_param_mask + * comment). + */ + u32 nm = 0x3804u + 8u * j; + + win[r] = (h & ~0xfffu) | + ((nm >> 2) & 0xfffu); + dmask |= 1ull << j; + transform++; + } else if (gsp_phys_gr_slot_stubbed(j, sa_mask, + sa_ent)) { + iscall = true; + dmask |= 1ull << j; + break; + } + } + } + if (iscall) { + dropped += span; + r += span; + continue; + } + if (w != r) + memmove(&win[w], &win[r], (size_t)span * 4u); + w += span; + r += span; + } + if (transform) + pr_info("vgsp-gr: MME-CALL-NOP %u call run(s) retargeted to the CALL_MME_DATA port (params conserved)\n", + transform); + if (dropped_mask) + *dropped_mask = dmask; + return dropped; +} + +/* + * SESSION 153-CONTINUED-61: generation of the MME prelude already emitted + * into the engine context (0 = none). The r1110 evidence: the prelude was + * emitted once on the first beef launch (gen=1, 512 words), that launch + * priv-errored (grfifo=0xbadf1201) and WEDGE-RECOVER ran TSG preempt + + * resched - which resets the engine context and with it the resident MME + * program - yet the latch kept suppressing re-emission, so every later + * window replay ran its macros against an unloaded MME (mme_pc=513 class + * stall, no retire). The recovery clears the latch so the next launch + * re-preloads the context. + * + * SESSION 153-CONTINUED-68: the latch is keyed by (generation, channel). + * A bare generation latch is SHARED across channels: the X channel's + * first post-maturity launch consumed the emission for the whole system + * and every beef launch then shipped without the program (r1140: only + * h=0x1000013 logged "MME-PRELUDE emitted gen=3", every beef launch + * words=400 prelude-less). Re-emitting per channel is idempotent. + */ +static u32 gsp_phys_gr_mme_prelude_gen; +static u32 gsp_phys_gr_mme_prelude_h; +static u32 gsp_phys_gr_mme_pre_full_dump; + +/* + * SESSION 153-CONTINUED-107: readback-failure counter + give-up bound for + * the load-only starvation (see the mme_load_only comment). 0 = unlimited; + * default 24. + */ +static u32 gsp_phys_gr_mme_iram_gate_fails; +static u32 gsp_phys_rmcall_gr_mme_iram_gate_max = 24u; + +/* + * SESSION 153-CONTINUED-107: gate decision shared by both IRAM-gate call + * sites (see gsp_phys_gr_mme_iram_gate). The readback failed, so hold the + * calls and owe a full re-emission. Clearing the prelude latch makes the + * next launch emit again (the builder's emit condition); rewinding the pack + * and drain cursors makes that re-emission restart from the first word. The + * pending-gen latch is cleared so the poll break cannot promote the + * generation to "resident". + */ +static void gsp_phys_gr_mme_iram_gate_fail(void) +{ + gsp_phys_gr_mme_iram_gate_fails++; + WRITE_ONCE(gsp_phys_gr_mme_pending_gen, 0); + WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, 0); + WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_sa, false); +} + +module_param_named(sema_gl_mme_iram_gate_max, + gsp_phys_rmcall_gr_mme_iram_gate_max, uint, 0644); +MODULE_PARM_DESC(sema_gl_mme_iram_gate_max, + "Give up the IRAM-gate load-only starvation after this many readback failures (0 = never; default 24)"); + +static bool gsp_phys_gr_mme_iram_gate_giveup(void) +{ + u32 m = READ_ONCE(gsp_phys_rmcall_gr_mme_iram_gate_max); + + return m && READ_ONCE(gsp_phys_gr_mme_iram_gate_fails) >= m; +} + +/* + * SESSION 153-CONTINUED-57: attr/pipeline state prelude. The r1091 + * full-MME-prelude boot cleared MME (mme_esr=80000000) but the first beef + * window then latched a draw-shader program-header error + * (NV_PGRAPH_PRI_DS_HWW_ESR 0x405840 = 0xa2040a00: SPH9|SPH11|SPH18, + * M_PIPE_STAGE = VSB|PS) and a class error at SET_VERTEX_ATTRIBUTE_A(2) + * (0x1168, INVALID_ENUM): the replayed window is a mid-stream state cut - + * the attribute/pipeline record the client set in EARLIER windows (the + * missing stages' SET_PIPELINE_PROGRAM_ADDRESS binds and the attribute + * format state) was never replayed, so the DS validated program headers + * that were never bound. With this knob the launch builder preplays the + * captured runs from the service's attr prelude buffer (see + * svc_attr_prelude_capture), relocated exactly like the window itself + * (the patched copy is computed by the caller - it needs the launch page + * set and the live shadow counter), run-aligned, drained across the first + * launches of a capture generation. Declared before build_gr_launch (its + * reader); the matching capture-side knob is sema_attr_prelude_cap. + * Default 1; A/B with 0. + */ +static bool gsp_phys_rmcall_gr_attr_prelude = true; +module_param_named(sema_gl_attr_prelude, gsp_phys_rmcall_gr_attr_prelude, + bool, 0644); +MODULE_PARM_DESC(sema_gl_attr_prelude, + "Preplay the captured client attr/pipeline runs (0x1100..0x11fc, 0x2000..0x205c, relocation applied) ahead of the first 3D window replays of a capture generation (default 1)"); + +/* + * The patched (relocation-applied) attr prelude for the current capture + * generation, built by the submit path just before build_gr_launch. Single + * caller (the ordered single-inflight sema work item), so plain statics. + */ +#define GSP_PHYS_GR_ATTR_PRE_WORDS 512u +static u32 gsp_phys_gr_attr_pre[GSP_PHYS_GR_ATTR_PRE_WORDS]; +static u32 gsp_phys_gr_attr_pre_n; +static u32 gsp_phys_gr_attr_pre_gen; /* capture gen the patch is from */ +static u32 gsp_phys_gr_attr_emit_gen; /* gen whose runs are draining */ +static u32 gsp_phys_gr_attr_emit_off; /* word cursor into attr_pre */ + +/* + * SESSION 153-CONTINUED-62: surface (render-target) prelude. The beef + * client's per-frame draw windows are mid-stream cuts: the colour-target + * bind is sticky context state (the client and the X server write it once + * per context), so the replayed windows carry none - and every replayed + * draw rasterized into an unbound target and stored nothing (the observed + * 0/N retire). The service captures the last C597 colour-target run from + * any trusted push (svc_surf_prelude_capture; the observed carrier is the + * X server's 1280x720 screen bind) and this site replays it - aliased and + * relocated exactly like the window - ahead of any window that carries no + * usable target bind of its own. Emitted per capture generation (the + * patched buffer is reused while the captured run is unchanged, and the + * alias VA is stable per boot); a fresh generation re-patches. A/B via + * the sema_gl_surf_prelude knob (default 1). + */ +#define GSP_PHYS_GR_SURF_PRE_WORDS 32u +static u32 gsp_phys_gr_surf_pre[GSP_PHYS_GR_SURF_PRE_WORDS]; /* patched */ +static u32 gsp_phys_gr_surf_pre_in[GSP_PHYS_GR_SURF_PRE_WORDS]; +static u32 gsp_phys_gr_surf_pre_n; /* patched words, 0 = none */ +static u32 gsp_phys_gr_surf_pre_gen; /* capture gen the patch is from */ +static bool gsp_phys_rmcall_gr_surf_prelude = true; +module_param_named(sema_gl_surf_prelude, gsp_phys_rmcall_gr_surf_prelude, + bool, 0644); +MODULE_PARM_DESC(sema_gl_surf_prelude, + "Replay the captured C597 colour-target bind ahead of 3D windows that carry no render target of their own (default 1)"); + +/* + * SESSION 153-CONTINUED-62: does this window already carry a usable + * colour-target bind? A run with va == 0 (the all-zero degenerate bind + * seen on the client's 0x402094c push) does not count - the surface + * prelude is injected for it too. + */ +static bool gsp_phys_gr_window_has_surface(const u32 *win, u32 n){ + u32 off = 0; + + while (off < n) { + u32 hdr = win[off]; + u32 op = (hdr >> 29) & 7u; + u32 cnt = (hdr >> 16) & 0xfffu; + u32 addr = (hdr & 0x1fffu) << 2; + u32 span = 1u; + + if ((op == 1u || op == 3u || op == 5u) && cnt && + off + 1u + cnt <= n) + span = 1u + cnt; + else { + off++; + continue; + } + if (op == 1u && addr >= 0x800u && addr < 0xc00u && + (addr & 0x3fu) == 0u && cnt >= 2u) { + u64 va = ((u64)(win[off + 1] & 0xffu) << 32) | + (u64)(win[off + 2] & 0xfffff000u); + + if (va) + return true; + } + off += span; + } + return false; +} + + +/* * GR launch method block: the inline QMD run at its original shape and the * channel REPORT_SEMAPHORE release. The stock guest push does NOT emit * SET_OBJECT before the inline run (the compute class is bound on the * channel long before the launch), so the replay must not either. * Returns the block size in dwords. */ +/* SESSION 153-CONTINUED-101: forward declaration - the seed emission in + * gsp_phys_rmcall_build_gr_launch (below) translates the seed page through + * the same relocator the window operand pairs use (defined further down). */ +static bool gsp_phys_rmcall_gr_3d_reloc(u32 *up_w, u32 *lo_w, + const struct gsp_phys_gr_submit *req, + u32 *n_shadow); + +/* + * SESSION 153-CONTINUED-104: FE idle witness for the calls-kept gate. + * The r1358/r1359 A/B (identical config; only the fixed-CB seed value + * differed) exposes the r1222-class race directly: + * - r1358: the calls-kept launch was submitted with the FE settled + * (MME-FAST begin grfifo=0x00000001) and the whole real-call window + * executed - the macro emissions landed (the wall became the CB read + * FAULT_PTE at fixed-CB+0x4000, i.e. the raw constant path now works); + * - r1359: the same launch was submitted while the FE still showed + * grfifo=0x0d0d1602 (the previous prelude-carrying launch's SA/IR + * method runs still draining) and the first macro execution poisoned + * with a REAL MME ILLEGAL_OPCODE (ESR 0x404490=0x80000004). + * The MME consumes the prelude's SA/IR port runs asynchronously, so a + * CALL_MME_MACRO decoded while those writes are in flight fetches + * half-loaded IR. Hold the real CALL runs until 0x400504 reads the idle + * value (0x00000001): the held launches still retire (the proven-safe + * drop shape) and a later window retries; once idle the calls run + * against a settled program. Fail-open when BAR0 is unavailable. + */ +static bool gsp_phys_gr_fe_idle(void) +{ + void __iomem *bar0 = gsp_phys_gpu_bar0(); + + return !bar0 || ioread32(bar0 + 0x00400504u) == 0x00000001u; +} + static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, const struct gsp_phys_gr_submit *req, const u32 *qmd, u64 qmd_va, - u64 sem_va, u32 sem_payload) + u64 sem_va, u32 sem_payload, + const u32 *win3d, u32 win3d_n, + const u32 *attr_pre, + u32 attr_pre_n) { u32 *p = pb; int prelude = gsp_phys_rmcall_gr_prelude_value(); int qmdvirt = gsp_phys_rmcall_gr_qmdvirt_value(); int two_step = gsp_phys_rmcall_gr_pcas_two_step_value(); - p = gsp_phys_rmcall_build_gr_init(p); + /* + * r929 (Path A): a captured 3D method window replays verbatim. Bind + * TURING_A (0xC597) on the channel subchannel (the same bind the + * window itself carries - belt and braces, a rebind is harmless), + * then the window words (address operands relocated to shadow-pool + * VAs by gsp_phys_rmcall_gr_3d_patch_window), then the channel + * release tail. No compute init, no QMD, no PCAS: the engine walks + * real 3D methods (SET_OBJECT C597, state binds, LINE_LENGTH_IN + + * LAUNCH_DMA inline uploads, program/tex binds). Completion is the + * trailing REPORT_SEMAPHORE alone (the submit path leaves qmd_rel + * off for this mode). + */ + if (gsp_phys_rmcall_gr_3d && req && win3d && win3d_n && + win3d_n + 2u + 3u + 8u <= GSP_PHYS_RMCALL_GR_METHOD_WORDS) { + /* + * SESSION 153-CONTINUED-53: MME context-setup prelude (see the + * sema_gl_mme_prelude knob above). Preplay the captured client + * setup runs once per capture generation, right after the + * SET_OBJECT C597 bind and ahead of the window, so + * CALL_MME_MACRO() runs execute against a loaded macro + * program instead of pc=0 -> ILLEGAL_OPCODE. The read is + * seqchecked (generation before/after): a concurrent append + * only defers emission to the next launch. A window too large + * to also carry the prelude replays without it (logged) - the + * emission then happens on the next launch that fits. + */ + bool emit_prelude = false; + bool pre_done = false; + bool mme_hold_calls = false; + bool mme_hold_fe_busy = false; + bool mme_load_only = false; + bool latch_before = false; + u32 pre_words_out = 0; + const u32 *pl = NULL; + u32 pl_n = 0, pl_gen = 0, pl_room = 0; + u32 block_cap = READ_ONCE(gsp_phys_rmcall_gr_block_cap); + + if (block_cap < 64u || + block_cap > GSP_PHYS_RMCALL_GR_METHOD_WORDS) + block_cap = GSP_PHYS_RMCALL_GR_METHOD_WORDS; + /* + * SESSION 153-CONTINUED-86: harden the block cap against the + * emissions the block-cap arithmetic below does not fully + * count (the SA re-arm, 3 words per slot, up to 32 slots - + * see the MME-SA-REARM site). Clamping the cap keeps every + * combination inside the 4096-word method buffer; with the + * default 2048 cap this changes nothing. + */ + if (block_cap > GSP_PHYS_RMCALL_GR_METHOD_WORDS - 128u) + block_cap = GSP_PHYS_RMCALL_GR_METHOD_WORDS - 128u; + /* + * SESSION 153-CONTINUED-63: MME START-ADDRESS RAM completion. + * The prelude capture is capped at 512 words, so the client's + * SA table (macro slot -> IR address) may be incomplete; on a + * fresh context the missing slots read 0 and CALL_MME_MACRO(j) + * jumps to IR 0 - the slot-0 macro - which with the caller's + * args emitted LOAD_CONSTANT_BUFFER_OFFSET <- 0x20020 and + * poisoned the lane (r1116; verified offline with the MME64 + * simulator). Stubbing every CALLED slot the capture's SA + * table does not cover with a harmless end-noop makes the + * unknown call a no-op instead of a lane-killer. The called + * slots come from the window being emitted (IMMD writes to + * 0x3800+j*8 and INC/ONEINC runs writing any 0x3800..0x39f8 + * method, the .Macro half of each pair). + */ + u32 sa_ent[64]; + u64 sa_mask = 0; + u32 sa_stub[40]; + u32 sa_stub_ent[40]; + u32 sa_cov[40]; + u32 sa_cov_ent[40]; + u32 sa_nstub = 0, sa_ncov = 0, sa_block = 0; + + if (gsp_sema_gr_sa_stub_value() && win3d && win3d_n) { + u32 so = 0; + + sa_mask = gsp_sema_mme_sa_get(sa_ent, 64); + while (so < win3d_n && (sa_nstub < 40u || sa_ncov < 40u)) { + u32 h = win3d[so]; + u32 sop = (h >> 29) & 7u; + u32 scnt = (h >> 16) & 0x1fffu; + u32 sspan = 1u; + u32 k, slo, shi; + + if ((sop == 1u || sop == 3u || sop == 5u) && + scnt && so + 1u + scnt <= win3d_n) { + sspan = 1u + scnt; + slo = 0; + /* + * SESSION 153-CONTINUED-68: ONE_INC + * (op 5) writes only data[0] to the + * header method; the rest go to + * method+4 (the data port, see + * nv3dLoadConstantsHeader). Walking + * mthd+4k for op=5 invented ghost + * CALL_MME_MACRO slots (r1134/1135 + * list). + */ + shi = (sop == 5u) ? 0u : scnt - 1u; + } else if (sop == 4u) { + slo = 0; + shi = 0; + } else { + so++; + continue; + } + for (k = slo; k <= shi; k++) { + u32 mthd = ((h & 0xfffu) << 2) + 4u * k; + u32 j, t; + bool have = false; + + if (mthd < 0x3800u || mthd >= 0x3a00u || + (mthd & 7u)) + continue; + j = (mthd - 0x3800u) / 8u; + if (j >= 64u) + continue; + /* + * SESSION 153-CONTINUED-70: a kept slot + * (sema_gl_mme_keep_mask) is never + * stubbed - its real macro must run + * against the context exactly as the + * client's own SA/IR uploads left it. + * Logged in the covered list for the + * census; no stub pair is emitted. + * SESSION 153-CONTINUED-113: unless + * the capture cannot serve it at all + * (keep_clamp) - then it falls into + * the stub + call-drop path below. + */ +if (gsp_phys_gr_slot_kept(j, sa_mask, + sa_ent)) { + for (t = 0; t < sa_ncov; t++) + if (sa_cov[t] == j) + have = true; + if (!have && sa_ncov < 40u) { + sa_cov[sa_ncov] = j; + sa_cov_ent[sa_ncov] = + sa_ent[j]; + sa_ncov++; + } + continue; + } + /* + * SESSION 153-CONTINUED-114: a called + * slot outside the capture that the + * golden library serves counts as + * covered - the pack overlay emits its + * SA pair + code, so no stub pair is + * emitted and the CALL runs survive + * the drop pass (see slot_stubbed). + */ + { + u32 gent, gn; + const u32 *gw; + + if (gsp_phys_gr_golden_slot( + j, &gent, &gw, + &gn)) { + for (t = 0; t < sa_ncov; + t++) + if (sa_cov[t] == j) + have = true; + if (!have && + sa_ncov < 40u) { + sa_cov[sa_ncov] = j; + sa_cov_ent[sa_ncov] = + gent; + sa_ncov++; + } + continue; + } + } + /* + * SESSION 153-CONTINUED-68: a slot is only + * usable when the capture holds its SA entry + * AND the IR instruction that entry names + * actually entered the replay (see + * gsp_sema_mme_ir_covered); a last-wins/ + * truncated mismatch makes the engine fetch + * macro data that is not there + * (MISSING_MACRO_DATA). Called-and-covered + * slots are logged too (r1134 ILLEGAL_OPCODE + * attribution: which macros actually run). + */ + if (sa_mask & (1ull << j)) { + if (!READ_ONCE(gsp_phys_rmcall_gr_sa_stub_all) && + gsp_sema_mme_ir_covered(sa_ent[j]) && + !(READ_ONCE(gsp_phys_rmcall_gr_nop_drop) & + (1ull << j))) { + for (t = 0; t < sa_ncov; t++) + if (sa_cov[t] == j) + have = true; + if (!have && sa_ncov < 40u) { + sa_cov[sa_ncov] = j; + sa_cov_ent[sa_ncov] = + sa_ent[j]; + sa_ncov++; + } + continue; + } + } + for (t = 0; t < sa_nstub; t++) + if (sa_stub[t] == j) + have = true; + if (!have && sa_nstub < 40u) { + sa_stub[sa_nstub] = j; + sa_stub_ent[sa_nstub] = + (sa_mask & (1ull << j)) ? + sa_ent[j] : 0u; + sa_nstub++; + } + } + so += sspan; + } + if (sa_nstub) + sa_block = 8u + 3u * sa_nstub; + } + + /* + * SESSION 153-CONTINUED-68: full run-header census of the 3D + * window being replayed (bounded; first 4 windows). The + * r1134 ILLEGAL_OPCODE attribution needs the real call + * structure: which methods the window writes, how many data + * words each run carries, and the CALL_MME_MACRO/DATA + * pattern (op 1 INC vs op 5 ONE_INC) around them. + */ + { + static u32 win_dump_n; + static u32 win_dump_beef; + + /* + * SESSION 153-CONTINUED-69: the first-4 budget was + * consumed by the X channel's early windows in every + * boot, so the beef draw-class window (the one that + * faults) never got its run-header census. Two extra + * dumps are reserved for beef-family windows. + */ + if (win3d && win3d_n && + (win_dump_n < 4u || + (((req->h_object & 0xfffff000u) == 0xbeef0000u) && + win_dump_beef < 2u))) { + u32 wo = 0, wr = 0; + + win_dump_n++; + if ((req->h_object & 0xfffff000u) == 0xbeef0000u) + win_dump_beef++; + pr_info("vgsp-gr: MME-WINRUN dump h=0x%x push=0x%llx window=%u\n", + req->h_object, req->push_va, win3d_n); + while (wo < win3d_n && wr < 200u) { + u32 h = win3d[wo]; + u32 op = (h >> 29) & 7u; + u32 cnt = (h >> 16) & 0x1fffu; + u32 addr = (h & 0xfffu) << 2; + u32 span = 1u; + u32 d0 = (wo + 1u < win3d_n) ? + win3d[wo + 1u] : 0u; + u32 d1 = (wo + 2u < win3d_n) ? + win3d[wo + 2u] : 0u; + + if ((op == 1u || op == 3u || op == 5u) && + cnt && wo + 1u + cnt <= win3d_n) + span = 1u + cnt; + pr_info("vgsp-gr: MME-WINRUN off=%x op=%u cnt=%u mthd=%x d0=%x d1=%x\n", + wo * 4u, op, cnt, addr, d0, d1); + wo += span; + wr++; + } + } + } + + if (READ_ONCE(gsp_phys_rmcall_gr_mme_prelude)) { + pl = gsp_sema_mme_prelude_get(&pl_n, &pl_gen); + /* + * SESSION 153-CONTINUED-72: with keep_mask bits set a + * kept macro can execute, and it needs the MME program + * resident. The r1152 evidence: the prelude was carried + * by one launch of the generation only; when that launch + * did not complete (engine fetch NONE / timeout), the + * later kept-macro launches ran an UNLOADED program and + * priv-errored (ILLEGAL_OPCODE -> poison). Force + * re-emission on every launch while any slot is kept + * (idempotent per the emission contract). + * + * SESSION 153-CONTINUED-79: the emission is drained in + * per-launch parts capped at block_cap words (see the + * sema_gl_mme_block_cap comment; oversized blocks never + * start). The latch (gsp_phys_gr_mme_prelude_gen/h) is + * set only when the final part has gone out. + */ + if (pl && pl_n && pl_gen) { + /* + * Fixed overhead: SET_OBJECT bind (2), SA-stub + * block, window, and the 4x2-word release tail. + */ + u32 fixed = 2u + sa_block + win3d_n + 8u; + + /* + * SESSION 153-CONTINUED-82: latch state BEFORE + * this launch's emission (see the hold decision + * below). A launch that first uploads a + * generation's MME code must not run real CALL + * runs in the same block (the r1222 evidence: + * the drain-completing launch bundled part-2's + * IR uploads with the window's real calls and + * hit ILLEGAL_OPCODE even though the latch was + * set by its own emission step). + */ + latch_before = + (pl_gen == READ_ONCE( + gsp_phys_gr_mme_prelude_gen) && + req->h_object == READ_ONCE( + gsp_phys_gr_mme_prelude_h)); + + pl_room = (fixed + 24u <= block_cap) ? + block_cap - fixed : 0u; + if (pl_room > pl_n) + pl_room = pl_n; + /* + * (Re)start the drain on a new generation/ + * channel, or when the request is forced (kept + * slots; a context-wiping wedge recovery clears + * the latch) and nothing is left to drain. + */ + if (pl_gen != + READ_ONCE(gsp_phys_gr_mme_pre_emit_gen) || + req->h_object != + READ_ONCE(gsp_phys_gr_mme_pre_emit_h) || + (READ_ONCE(gsp_phys_gr_mme_pre_emit_off) >= + pl_n && + ((READ_ONCE(gsp_phys_rmcall_gr_keep_mask) && + !(READ_ONCE( + gsp_phys_rmcall_gr_mme_live_skip) && + READ_ONCE(gsp_phys_gr_mme_live_gen) == + pl_gen)) || + pl_gen != READ_ONCE( + gsp_phys_gr_mme_prelude_gen)))) { + WRITE_ONCE(gsp_phys_gr_mme_pre_emit_gen, + pl_gen); + WRITE_ONCE(gsp_phys_gr_mme_pre_emit_h, + req->h_object); + WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0); + /* + * CONT-105: the packed split shares this + * restart edge; reset its cursor too so a + * wedged-recover latch clear replays the + * whole packed stream. + */ + WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_sa, + false); + } + if (pl_room >= 24u) { + emit_prelude = + (pl_gen != READ_ONCE( + gsp_phys_gr_mme_prelude_gen) || + req->h_object != READ_ONCE( + gsp_phys_gr_mme_prelude_h) || + READ_ONCE( + gsp_phys_gr_mme_pre_emit_off) < + pl_n); + } else { + static u32 pre_defer_log; + + if (pre_defer_log < 8) { + pre_defer_log++; + pr_info("vgsp-gr: MME-PRELUDE deferred (window=%u prelude=%u sa=%u cap=%u) h=0x%x\n", + win3d_n, pl_n, sa_block, + block_cap, + req->h_object); + } + } + } + } + + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | (1u << 16) | + (0u << 13) | + (GSP_PHYS_RMCALL_GR_SET_OBJECT >> 2); + *p++ = GSP_PHYS_RMCALL_CLASS_GR_3D; + /* + * SESSION 153-CONTINUED-90: SET_SHADER_EXCEPTIONS (0x1528) + * <- 0, the first write of NVIDIA's own Turing 3D channel + * init (see the sema_gl_shader_exc knob): with shader + * exceptions enabled the DS raises "Shader Program Header + * 5/9/10/11" on the draws whose SPHs it would otherwise + * tolerate, and the raised exception poisons the channel + * before any store retires. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_shader_exc)) { + static u32 shader_exc_log; + + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + (1u << 16) | (0x1528u >> 2); + *p++ = 0u; + if (shader_exc_log < 8u) { + shader_exc_log++; + pr_info("vgsp-gr: SHADER-EXC emitted 0x1528 <- 0 (2 words; pos=%u h=0x%x push=0x%llx)\n", + (u32)(p - pb), req->h_object, + req->push_va); + } + } + /* + * SESSION 153-CONTINUED-91: DS HWW report suppression. + * + * The DS validates the SPH of every bound pipeline program at + * draw time and reports failures through DS_HWW_REPORT_MASK + * (0x405844, observed live = 0x00ffffff = every SPH<n>_ERR + * reported; mask_2 = 0x77). A reported failure poisons the + * whole PRI block (reads return 0xbadf1201 - the priv-error + * marker already documented for the MME latch) and the GSP-RM + * exception handler kills the channel: every draw attempt + * wedges at the first SPH error storm even when the SPH bytes + * at the bound VAs are the client's own real V4 images (r1298: + * INLINE-RB rb=020a0481 at the bind VA, DS still 5/9/10/11). + * The reference driver's own diagnostic path (Aftermath) + * explicitly ENABLES these reports to capture them, i.e. the + * reports are a diagnostic tap, not the shader-execute gate. + * Clear the mask so a SPH miss latches (readable between + * faults) without poisoning the channel. Host write only: + * an in-stream CALL_MME_MACRO(5) (the NVN PRI R/W macro) + * faulted with MME ILLEGAL_OPCODE (pc=513 mac=0xf1) on + * r1300 - the Turing MME's loaded macro program does not + * carry that GF100-era slot - so the register is written + * directly (it persists: the next beef launch reads 0/0). + * A/B with sema_ds_mask_clear=0. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_ds_mask_clear)) { + static u32 ds_mask_log; + void __iomem *ds_bar0 = gsp_phys_gpu_bar0(); + u32 m0 = 0, m2 = 0; + + if (ds_bar0) { + m0 = ioread32(ds_bar0 + 0x00405844u); + m2 = ioread32(ds_bar0 + 0x0040584cu); + if (m0 || m2) { + iowrite32(0u, ds_bar0 + 0x00405844u); + iowrite32(0u, ds_bar0 + 0x0040584cu); + } + /* + * SESSION 153-CONTINUED-92: clear the DS HWW + * ESR latch (W1C) ahead of every window so the + * retire readback names EXACTLY this launch's + * failed SPH checks instead of the union of + * every prior draw's. The latch was observed + * sticky across launches (r1308: a6000e20 + * constant over 25 s of draws), which made + * "did this window's checks change?" unreadable. + * Log the value being cleared (the previous + * window's set) so the A/B sequence is in the + * ring either way. + */ + { + u32 esr = ioread32(ds_bar0 + 0x00405840u); + + if (esr) + iowrite32(esr, ds_bar0 + 0x00405840u); + if (ds_mask_log < 24u) { + ds_mask_log++; + pr_info("vgsp-gr: DS-ESR pre-clear=%08x (prev window's latch; this window starts clean) h=0x%x push=0x%llx\n", + esr, req->h_object, + req->push_va); + } + } + } + if (ds_mask_log < 24u) { + ds_mask_log++; + pr_info("vgsp-gr: DS-MASK clear mask=%08x/%08x rb=%08x/%08x h=0x%x push=0x%llx\n", + m0, m2, + ds_bar0 ? ioread32(ds_bar0 + 0x00405844u) : 0u, + ds_bar0 ? ioread32(ds_bar0 + 0x0040584cu) : 0u, + req->h_object, req->push_va); + } + } + /* + * SESSION 153-CONTINUED-86: the Turing channel-init SPH version + * pair (see the sema_gl_sph_version knob). Emitted on EVERY + * replayed 3D window (3 words, idempotent): the DS validates + * each bound program's SPH version against this window at draw + * time, and the replay context never received the client's + * once-per-channel write. Emitted before the prelude-room + * computations below so the live (p - pb) accounting covers + * it exactly. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_sph_version)) { + /* + * SESSION 153-CONTINUED-87: the window's accepted + * version range must COVER the images the engine + * will fetch. The client's own init push writes + * 4|(4<<16), but our authored (V3) images are + * rejected by a 4/4 window (SPH9|11|18) while the + * r950-era acceptance - same V3 image - ran with + * NO version window. Emit oldest=V(blob), + * current=max(V(blob),4) so both the V3 image and + * a future V4 re-tag stay inside the range. + */ + u32 bv = (u32)gsp_sema_gl_sph_blob_version(); + u32 val = 0x00040004u; + u64 ov = READ_ONCE(gsp_phys_rmcall_gr_sph_version_val); + + if (ov) + val = (u32)ov; + else if (bv >= 1u && bv <= 15u) + val = ((bv < 4u ? 4u : bv) << 16) | bv; + + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + (2u << 16) | (0x16a4u >> 2); + *p++ = val; + *p++ = val; + } + /* + * SESSION 153-CONTINUED-91: the Turing program region. + * + * The DS's SPH validation checks each fetched SPH against the + * channel's PROGRAM REGION. On pre-Volta that was + * SET_PROGRAM_REGION_A/B (0x1608/0x160c); on Volta+ those + * methods are obsolete and the driver writes the same 64-bit + * shader-code-heap base through the MME shadow scratch pair + * (nvTuring: NVC397_SET_MME_PROGRAM_REGION = SET_MME_SHADOW_ + * SCRATCH(MME_SCRATCH_PROGRAM_REGION_A=0x78/B=0x79), + * gv100ResendProgramAddresses/`gf100Init`; method 0x3400 + + * i*4 -> 0x35e0/0x35e4). Observed live at the faulting draw: + * DS_HWW_ESR_SPH_INFO_PROGRAM_REGION_A/B = 0/0 and DS latches + * SPH errors for every bound stage whose bytes are provably + * correct (r1298 INLINE-RB rb=020a0481 at the bind VA; + * r1303 GR-STATE sph stages 1/2/5 = 0x620900/0x620200/0x620000 + * with ds=a4000e20 and rr=0/0) - a region mismatch rejects all + * of them regardless of content. Emit the pair (A = upper + * 7:0, B = lower 31:0) with the launch's own program family + * page as the heap base. A/B with sema_gl_sph_region=0. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_sph_region) && req->prog_va) { + static u32 sph_reg_log; + u64 base = READ_ONCE(gsp_phys_rmcall_gr_sph_region_base); + + if (!base) + base = req->prog_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + (1u << 16) | (0x35e0u >> 2); + *p++ = (u32)(base >> 32) & 0xffu; + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + (1u << 16) | (0x35e4u >> 2); + *p++ = (u32)base; + if (sph_reg_log < 16u) { + sph_reg_log++; + pr_info("vgsp-gr: SPH-REGION emitted base=0x%llx (scratch 0x78/0x79) h=0x%x push=0x%llx\n", + base, req->h_object, req->push_va); + } + } + /* + * SESSION 153-CONTINUED-101: MME fixed-constant-buffer scratch + * seed (see the sema_gl_mme_seed* knobs). mmeCodeOGL_C597_ + * SETUP_FIXED_CONST (slot 0) emits SET_CONSTANT_BUFFER_ + * SELECTOR_B/C from shadow scratch 0x18/0xd18; the replay + * context never received the client's once-per-channel write + * (gf100InitConstantBuffers pushes method 0x3460), so the + * macro read 0 and the selector went empty. Emit the seed + * ahead of the window; the value is the fixed CB address>>8, + * page-translated to the launch's shadow/identity mapping when + * sema_gl_mme_seed_xlat is set (see gsp_phys_rmcall_gr_3d_ + * reloc - same machinery as the operand pairs). + */ + if (READ_ONCE(gsp_phys_rmcall_gr_mme_seed)) { + static u32 mme_seed_log; + u32 sv = READ_ONCE(gsp_phys_rmcall_gr_mme_seed_val); + bool have = sv != 0u; + + if (!have) + have = gsp_sema_mme_scratch18_get(&sv); + if (have && sv && + (u32)(p - pb) + 8u < GSP_PHYS_RMCALL_GR_METHOD_WORDS) { + u64 va = (u64)sv << 8; + bool xl = false; + + if (READ_ONCE(gsp_phys_rmcall_gr_mme_seed_xlat) + && va) { + u32 up_w = (u32)((va >> 32) & 0xffu); + u32 lo_w = (u32)va & 0xfffffffcu; + u32 n_sh = 0; + + if (gsp_phys_rmcall_gr_3d_reloc( + &up_w, &lo_w, req, &n_sh)) { + va = ((u64)(up_w & 0xffu) + << 32) | + (u64)(lo_w & 0xfffffffcu); + sv = (u32)(va >> 8); + xl = true; + } + } + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + (1u << 16) | (0x3460u >> 2); + *p++ = sv; + if (mme_seed_log < 16u) { + mme_seed_log++; + pr_info("vgsp-gr: MME-SEED scratch 0x18 <- 0x%08x (va=0x%llx xlat=%u) h=0x%x push=0x%llx\n", + sv, va, xl, req->h_object, + req->push_va); + } + } else if (mme_seed_log < 16u) { + mme_seed_log++; + pr_info("vgsp-gr: MME-SEED no value (client 0x3460 not seen yet; set sema_gl_mme_seed_val) h=0x%x\n", + req->h_object); + } + } + /* + * SESSION 153-CONTINUED-105: golden MME runtime state seed + * (see the sema_gl_mme_state_seed comment). Five idempotent + * shadow-scratch writes ahead of the window so the + * bug2548370 loadConstantBuffer macro (slot 57) branches on + * the seeded values instead of whatever the replay context + * happens to hold (the captured stream never carries + * 0xd26/0xd27/0xd28). + */ + if (READ_ONCE(gsp_phys_rmcall_gr_mme_state_seed)) { + static u32 mme_state_seed_log; + u32 mv[5]; + u32 mi; + + mv[0] = READ_ONCE(gsp_phys_rmcall_gr_mme_state_d26); + mv[1] = READ_ONCE(gsp_phys_rmcall_gr_mme_state_d27); + mv[2] = READ_ONCE(gsp_phys_rmcall_gr_mme_state_d28); + mv[3] = READ_ONCE(gsp_phys_rmcall_gr_mme_state_d29); + mv[4] = READ_ONCE(gsp_phys_rmcall_gr_mme_state_d2a); + if ((u32)(p - pb) + 10u < + GSP_PHYS_RMCALL_GR_METHOD_WORDS) { + for (mi = 0; mi < 5u; mi++) { + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM + << 29) | + (1u << 16) | + ((0x3498u + 4u * mi) >> 2); + *p++ = mv[mi]; + } + if (mme_state_seed_log < 16u) { + mme_state_seed_log++; + pr_info("vgsp-gr: MME-STATE-SEED d26=%08x d27=%08x d28=%08x d29=%08x d2a=%08x h=0x%x push=0x%llx\n", + mv[0], mv[1], mv[2], mv[3], + mv[4], req->h_object, + req->push_va); + } + } + } + /* + * SESSION 153-CONTINUED-93: the LEGACY program-region pair + * (0x1608/0x160c = the pre-Volta SET_PROGRAM_REGION_A/B whose + * names the DS's own snapshot registers + * DS_HWW_ESR_SPH_INFO_PROGRAM_REGION_A/B carry). Those + * snapshots read 0/0 on every SPH-faulting window while the + * MME shadow pair (0x78/0x79) is provably emitted, so the + * DS's validation region may be the legacy pair's state, not + * the MME scratch. The methods are "obsolete" on Volta+ (not + * in the c597 class table), so this is an A/B: a PBDMA that + * refuses them faults the launch visibly, an accepting one + * tells us via the latch. Knob sema_sph_region_legacy + * (default 0). Emitted beside the MME pair with the same + * base. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_sph_region) && req->prog_va && + READ_ONCE(gsp_phys_rmcall_gr_sph_region_legacy)) { + static u32 sph_reg_leg_log; + u64 base = READ_ONCE(gsp_phys_rmcall_gr_sph_region_base); + + if (!base) + base = req->prog_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + (1u << 16) | (0x1608u >> 2); + *p++ = (u32)(base >> 32) & 0xffu; + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + (1u << 16) | (0x160cu >> 2); + *p++ = (u32)base; + if (sph_reg_leg_log < 16u) { + sph_reg_leg_log++; + pr_info("vgsp-gr: SPH-REGION-LEGACY emitted base=0x%llx (0x1608/0x160c) h=0x%x push=0x%llx\n", + base, req->h_object, req->push_va); + } + } + /* + * SESSION 153-CONTINUED-84: SET_MME_VERSION=1 (0x234). + * + * The Turing channel-init defaults push + * `nvcMethodPusher: NVC597_SET_MME_VERSION <- 1` once per + * channel (class c597, TURING_OR_NEWER), and the replayed + * context otherwise never sees it: the captured MME setup + * families carry IR/SA/shadow-scratch loads but no 0x234 run + * (verified in the injected full-SA stream too). With the + * version register unset the MME never starts a macro - the + * r1245 evidence: call-bearing launches (real {0,2,34,39,40}, + * full 55-slot prelude, SA re-armed before the window) hang + * the front-end (`MME-FAST ... events=0` -> `GR LAUNCH FAILED: + * timeout after 2500 ms` with qmd_rel=0/ch_rel=0) while every + * MME register stays idle (`pc_valid=0`, `gen0=0`, + * `mme_esr=80000000`): the FE waits forever for a macro the + * halted MME cannot run. Emitted with the prelude step (the + * resident program is (re)loaded right after), ahead of the + * window, so the version select has landed before the first + * CALL_MME_MACRO run. Knob: sema_gl_mme_version (default 1; + * A/B with 0 reproduces the parked launches). + */ + if (emit_prelude && READ_ONCE(gsp_phys_rmcall_gr_mme_version)) { + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + (1u << 16) | + (0x234u >> 2); + *p++ = 1u; + } + if (emit_prelude) { + u32 g0 = pl_gen, g1 = 0, sdrop = 0, pwords = 0; + u32 doff = READ_ONCE(gsp_phys_gr_mme_pre_emit_off); + u32 room = (block_cap > (u32)(p - pb) + win3d_n + 8u + + sa_block) ? + block_cap - (u32)(p - pb) - win3d_n - 8u - + sa_block : 0u; + + /* + * SESSION 153-CONTINUED-79: run-aligned drain at the + * generation cursor; whole runs only, SDM-family + * single-word entries skipped exactly like the legacy + * sanitize (see gsp_phys_rmcall_gr_sdm_drop). + */ + /* + * SESSION 153-CONTINUED-99b: packed emission (see the + * sema_gl_mme_prelude_pack comment). Rebuild the + * partitioned IR, then emit the SA table + one + * contiguous 0x114 run; the drain loop is skipped. + */ + bool packed = false; + + if (READ_ONCE(gsp_phys_rmcall_gr_mme_prelude_pack)) { + u32 sa_ent[64]; + u64 sa_mask = gsp_sema_mme_sa_get(sa_ent, 64); + u32 maxa = 0, r = 0, nruns = 0, npair = 0; + u32 need, kk; + u32 *pp; + + memset(gsp_phys_gr_mme_ir_pack, 0, + sizeof(gsp_phys_gr_mme_ir_pack)); + while (r < pl_n) { + u32 h = pl[r]; + u32 op = (h >> 29) & 7u; + u32 cnt = (h >> 16) & 0xfffu; + u32 span = ((op == 1u || op == 3u || + op == 5u) && cnt && + r + 1u + cnt <= pl_n) ? + 1u + cnt : 1u; + + if (r + span > pl_n) + span = pl_n - r; + if (op == 5u && + ((h & 0x1fffu) << 2) == 0x114u && + cnt >= 2u) { + u32 ptr = pl[r + 1u]; + u32 blen = cnt - 1u; + u32 nxt = 0, k; + + for (kk = 0; kk < 64u; kk++) + if ((sa_mask & (1ull << kk)) && + sa_ent[kk] > ptr && + (!nxt || + sa_ent[kk] < nxt)) + nxt = sa_ent[kk]; + /* + * CONT-100 units fix: `ptr` is an + * MME INSTRUCTION address (the + * driver advances codeOffset by + * codeSize/3, mmeDownload.cpp), + * and each instruction is 3 IR + * words stored at 3*ptr..3*ptr+2. + * Indexing the contiguous word + * image at ptr+k (the pre-CONT-100 + * form) misplaced every macro and + * left 2/3 of the program + * unreachable (r1343/r1344 poison, + * only ~312 of 933 instructions + * covered). Clamp the tail at the + * next SA entry in INSTRUCTIONS. + */ + if (nxt && + ptr + (blen + 2u) / 3u > nxt) + blen = 3u * (nxt - ptr); + for (k = 0; k < blen && + ptr * 3u + k < 4096u; k++) + gsp_phys_gr_mme_ir_pack[ptr * 3u + k] = + pl[r + 2u + k]; + if (ptr * 3u + blen > maxa) + maxa = ptr * 3u + blen; + nruns++; + } + r += span; + } + /* + * SESSION 153-CONTINUED-114: golden overlay. + * After the capture fill, merge the golden + * library's words into the packed image and + * the local SA pair table for every slot the + * capture cannot serve (mode 1), or every + * slot (mode 2 / force). Written after the + * capture fill so the golden words win any + * overlap; a complete capture is untouched + * in mode 1 (the overlay then only logs). + */ + if (READ_ONCE(gsp_phys_rmcall_gr_mme_golden)) { + u32 gj, goldn = 0; + + for (gj = 0; gj < 64u; gj++) { + u32 gent, gn, q; + const u32 *gw; + + if (!gsp_phys_gr_golden_slot( + gj, &gent, &gw, + &gn)) + continue; + if (!READ_ONCE(gsp_phys_rmcall_gr_mme_golden_force)) { + /* + * Mode 1: only slots the + * capture cannot serve + * (no SA entry, or the + * entry's IR never + * entered the replay). + */ + if ((sa_mask & + (1ull << gj)) && + sa_ent[gj] && + gsp_sema_mme_ir_covered( + sa_ent[gj])) + continue; + } + for (q = 0; + q < gn && + gent * 3u + q < 4096u; + q++) + gsp_phys_gr_mme_ir_pack[gent * 3u + q] = + gw[q]; + if (gent * 3u + gn > maxa) + maxa = gent * 3u + gn; + sa_mask |= 1ull << gj; + sa_ent[gj] = gent; + goldn++; + } + if (goldn && gsp_phys_gr_golden_log < 8u) { + gsp_phys_gr_golden_log++; + pr_info("vgsp-gr: MME-GOLDEN overlay %u slot(s) maxa=%u h=0x%x push=0x%llx\n", + goldn, maxa, + req->h_object, + req->push_va); + } + } + /* + * CONT-112: single-word A/B override of the + * packed program (see the sema_gl_mme_irpatch + * comment). Applied after the image is built so + * the override is what the macro executes and + * what the IRAM gate compares against. + */ + if (gsp_phys_gr_mme_irpatch_off < maxa) { + pr_info("vgsp-gr: MME-IRPATCH word[%u] = 0x%08x (was 0x%08x, maxa=%u)\n", + gsp_phys_gr_mme_irpatch_off, + gsp_phys_gr_mme_irpatch_val, + gsp_phys_gr_mme_ir_pack[ + gsp_phys_gr_mme_irpatch_off], + maxa); + gsp_phys_gr_mme_ir_pack[ + gsp_phys_gr_mme_irpatch_off] = + gsp_phys_gr_mme_irpatch_val; + } + for (kk = 0; kk < 64u; kk++) + if ((sa_mask & (1ull << kk)) && + sa_ent[kk]) + npair++; + need = 2u + maxa + 3u * npair; + /* + * CONT-105: chunked emission. The CONT-99b + * all-or-nothing form only fires when the + * whole packed stream fits one block (need <= + * room); the current 55-slot capture needs + * ~2966 words against ~1600 available, so it + * silently fell back to the raw drain. The + * split form drains SA (once) + whole- + * instruction IR chunks across consecutive + * launches with the generation cursor reset + * alongside the drain cursor (see + * gsp_phys_gr_mme_pack_off). + */ + { + bool split = READ_ONCE( + gsp_phys_rmcall_gr_mme_pack_split); + u32 poff = READ_ONCE( + gsp_phys_gr_mme_pack_off); + bool sa_done = READ_ONCE( + gsp_phys_gr_mme_pack_sa); + u32 left, chunk; + bool can; + + /* + * Reserve the window's SA re-arm + * budget (<= 32 entries * 3 words) so + * the block stays under block_cap. + */ + left = (split && room > 96u) ? + room - 96u : (split ? 0u : room); + can = maxa >= 3u && poff < maxa && + (split ? left >= 5u : + need <= room); + if (can && split && !sa_done && + 3u * npair > left) + can = false; + if (can) { + pp = p; + if (!sa_done) { + for (kk = 0; kk < 64u; + kk++) { + if (!(sa_mask & + (1ull << + kk)) || + !sa_ent[kk]) + continue; + *pp++ = ((u32) + GSP_PHYS_RMCALL_METHOD_IMM + << 29) | + (2u << 16) | + (0x11cu >> 2); + *pp++ = kk; + *pp++ = + sa_ent[kk]; + } + left -= 3u * npair; + WRITE_ONCE( + gsp_phys_gr_mme_pack_sa, + true); + } + chunk = maxa - poff; + if (chunk > left - 2u) + chunk = left - 2u; + chunk -= chunk % 3u; + if (chunk >= 3u) { + *pp++ = ((u32) + GSP_PHYS_RMCALL_METHOD_ONEINC + << 29) | + ((chunk + 1u) << 16) | + (0x114u >> 2); + *pp++ = poff / 3u; + memcpy(pp, + &gsp_phys_gr_mme_ir_pack[poff], + (size_t)chunk * 4u); + pp += chunk; + poff += chunk; + WRITE_ONCE( + gsp_phys_gr_mme_pack_off, + poff); + } + pwords = (u32)(pp - p); + p = pp; + packed = true; + if (poff >= maxa) { + doff = pl_n; + pr_info("vgsp-gr: MME-PRELUDE pack ir_words=%u runs=%u sa=%u h=0x%x push=0x%llx\n", + maxa, nruns, + npair, + req->h_object, + req->push_va); + } else { + pr_info("vgsp-gr: MME-PRELUDE pack-part ir_words=%u/%u sa=%u split=%u h=0x%x push=0x%llx\n", + poff, maxa, + npair, + split ? 1u : 0u, + req->h_object, + req->push_va); + } + } else if (split && maxa >= 3u) { + /* + * Split mode never falls back + * to the raw drain: an + * un-emittable chunk simply + * waits for a launch with + * room (the cursor is + * untouched; the prelude + * latch stays unset so the + * calls stay held). + */ + packed = true; + } + } + } + while (!packed && doff < pl_n && + pwords + 1u <= room) { + u32 h = pl[doff]; + u32 op = (h >> 29) & 7u; + u32 cnt = (h >> 16) & 0xfffu; + u32 span = ((op == 1u || op == 3u || + op == 5u) && cnt && + doff + 1u + cnt <= pl_n) ? + 1u + cnt : 1u; + + if (doff + span > pl_n) + span = pl_n - doff; + if (READ_ONCE(gsp_phys_rmcall_gr_sdm_drop) && + h && (op == 0u || op == 2u || op == 6u)) { + sdrop += span; + doff += span; + continue; + } + /* + * SESSION 153-CONTINUED-99: IR-only drain (see + * the sema_gl_mme_prelude_ir_only comment). + * Keep only the runs aimed at the MME SA/IR + * ports; skip everything else with the same + * run-aligned accounting as the SDM drop. + */ + if (READ_ONCE( + gsp_phys_rmcall_gr_mme_prelude_ir_only)) { + u32 ma = (h & 0x1fffu) << 2; + + if (ma != 0x114u && ma != 0x118u && + ma != 0x11cu && ma != 0x120u) { + sdrop += span; + doff += span; + continue; + } + } + if (pwords + span > room) + break; + memcpy(p, &pl[doff], (size_t)span * 4u); + p += span; + pwords += span; + doff += span; + } + WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, doff); + pre_done = (doff >= pl_n); + pre_words_out = pwords; + /* seqcheck: a mid-copy append must not be recorded as + * emitted (the copy may be a torn prefix). */ + gsp_sema_mme_prelude_get(NULL, &g1); + if (pre_done && g1 == g0) { + WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, g0); + WRITE_ONCE(gsp_phys_gr_mme_prelude_h, + req->h_object); + /* CONT-103: this launch carries the final + * part; its poll outcome (executed or not) + * decides whether the program is live. */ + WRITE_ONCE(gsp_phys_gr_mme_pending_gen, g0); + pr_info("vgsp-gr: MME-PRELUDE emitted gen=%u words=%u window=%u h=0x%x push=0x%llx sdm_drop=%u cap=%u\n", + g0, pl_n, win3d_n, req->h_object, + req->push_va, sdrop, block_cap); + /* + * SESSION 153-CONTINUED-70: one full dump of the + * replayed MME context-setup program per boot + * (the offline MME64 sim needs the same words the + * engine will execute). Also list the SA table + * with per-entry IR coverage so the slot-by-slot + * macro re-enable has its proof in one place. + */ + if (!gsp_phys_gr_mme_pre_full_dump) { + u32 k, sj; + u32 sa_ent[64]; + u64 sa_mask; + u32 cov_n = 0; + + gsp_phys_gr_mme_pre_full_dump = 1; + sa_mask = gsp_sema_mme_sa_get(sa_ent, 64); + for (sj = 0; sj < 64u; sj++) + if ((sa_mask & (1ull << sj)) && + gsp_sema_mme_ir_covered(sa_ent[sj])) + cov_n++; + pr_info("vgsp-gr: MME-PRE-DUMP words=%u gen=%u h=0x%x ir_end=0x%x sa_mask=0x%llx covered=%u code_words=%u keep_mask=0x%llx\n", + pl_n, g0, req->h_object, + gsp_sema_mme_ir_end_get(), + (unsigned long long)sa_mask, + cov_n, gsp_sema_mme_code_words_get(), + (unsigned long long)READ_ONCE(gsp_phys_rmcall_gr_keep_mask)); + for (k = 0; k + 4u <= pl_n; k += 4u) + pr_info("vgsp-gr: MME-PRE[%03u] %08x %08x %08x %08x\n", + k, pl[k], pl[k + 1], + pl[k + 2], pl[k + 3]); + for (sj = 0; sj < 64u; sj++) + if (sa_mask & (1ull << sj)) + pr_info("vgsp-gr: MME-PRE-SA[%u] entry=0x%x cov=%u\n", + sj, sa_ent[sj], + gsp_sema_mme_ir_covered(sa_ent[sj])); + } + } else if (pre_done) { + pr_info("vgsp-gr: MME-PRELUDE copy raced an append (gen %u->%u); will re-emit next launch\n", + g0, g1); + } else { + pr_info("vgsp-gr: MME-PRELUDE part gen=%u off=%u words=%u/%u window=%u h=0x%x push=0x%llx sdm_drop=%u cap=%u (continuation next launch)\n", + g0, doff, pwords, pl_n, win3d_n, + req->h_object, req->push_va, sdrop, + block_cap); + } + } + /* + * SESSION 153-CONTINUED-107: prelude-only LOAD carriers while + * the IRAM gate has not verified the packed program. + * + * r1372's first instruction (decisive): the calls-held pack + * carrier still poisoned (`DS-FAST ... ds a2040a00->badf1201 + * ... MME-FAST end launch events=1 (poison)` -> WEDGE-RECOVER + * wipes the context) because the block also carried the WINDOW + * - the window's own SPH state trips the DS validation and the + * FECS poison episode interrupts the 0x114/0x118 IRAM load + * mid-flight. Holding the calls is not enough; the LOAD must + * ride a carrier whose methods cannot fault, so while the + * generation is not verified live (live_gen != pl_gen) emit + * the pack/SA/release WITHOUT the window. The carrier retires, + * the poll-break gate reads the IRAM back; on a pass live_gen + * is promoted and the very next launch carries the window + * again (latch && live). Bounded by + * sema_gl_mme_iram_gate_max so a broken readback cannot starve + * the lane forever. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_mme_iram_gate) && + READ_ONCE(gsp_phys_rmcall_gr_mme_prelude_pack) && + READ_ONCE(gsp_phys_rmcall_gr_mme_hold_calls) && + pl && pl_n && pl_gen && + !gsp_phys_gr_mme_iram_gate_giveup()) { + bool latch_now = + (pl_gen == + READ_ONCE(gsp_phys_gr_mme_prelude_gen) && + req->h_object == + READ_ONCE(gsp_phys_gr_mme_prelude_h)); + bool live_now = + (READ_ONCE(gsp_phys_rmcall_gr_mme_live_skip) && + READ_ONCE(gsp_phys_gr_mme_live_gen) == pl_gen); + + if (!(latch_now && live_now)) + mme_load_only = true; + } + /* + * SESSION 153-CONTINUED-82: hold real CALL runs until the + * prelude latch for this generation+channel is set (see the + * sema_gl_mme_hold_calls comment). Evaluated AFTER the + * emission step above, so a launch that completed the drain + * in this same block has just written the latch. Real calls + * stay held when EITHER + * - the latch for this generation+channel is not valid after + * this launch (partial drain, raced append, or a fresh + * generation whose code is not fully resident), or + * - this launch uploaded generation code for the FIRST time + * (emitted words with the latch invalid before the step) - + * the r1222 evidence: the drain-completing launch bundled + * part-2's IR uploads with the window's real calls and hit + * ILLEGAL_OPCODE even though its own emission set the + * latch. A later launch (latch already valid, only the + * idempotent keep-forced re-emission of the prefix) runs + * the real calls against the now-resident program. + */ + if (pl && pl_n && pl_gen && + READ_ONCE(gsp_phys_rmcall_gr_mme_prelude) && + READ_ONCE(gsp_phys_rmcall_gr_mme_hold_calls)) { + u32 lg = READ_ONCE(gsp_phys_gr_mme_prelude_gen); + u32 lh = READ_ONCE(gsp_phys_gr_mme_prelude_h); + bool latch_after = (pl_gen == lg && + req->h_object == lh); + + mme_hold_calls = !latch_after || + (pre_words_out && !latch_before); + /* + * CONT-104: also hold while the front-end is not + * settled (the r1359 MME ILLEGAL_OPCODE race; see + * gsp_phys_gr_fe_idle). The held launch retires and + * a later window retries once 0x400504 reads idle. + * SESSION 153-CONTINUED-107: separately knob-gated - + * with the IRAM gate proving residency, the residual + * value of this clause is only the r1359 race; r1373 + * left the FE at 0x07071602 for minutes and starved + * every calls-kept window, so allow an explicit A/B + * (sema_gl_mme_hold_fe_idle=0). + */ + if (!mme_hold_calls && + READ_ONCE(gsp_phys_rmcall_gr_mme_hold_fe_idle) && + !gsp_phys_gr_fe_idle()) { + mme_hold_calls = true; + mme_hold_fe_busy = true; + } + } + /* + * SESSION 153-CONTINUED-107: IRAM readback gate (see the + * gsp_phys_gr_mme_iram_gate comment). The calls would run this + * launch; verify the live IRAM actually holds the packed image + * at the SA entries first. On a mismatch (the CONT-106 + * all-zero fetch) hold the calls and owe a full re-emission - + * the r1368 evidence says the real CALL runs against a + * non-resident program poison the lane (ILLEGAL_OPCODE at the + * macro entry) instead of drawing. Only meaningful on the + * packed path (the pack image is the comparator). + */ + if (!mme_hold_calls && pl && pl_n && pl_gen && + READ_ONCE(gsp_phys_rmcall_gr_mme_prelude) && + READ_ONCE(gsp_phys_rmcall_gr_mme_hold_calls) && + READ_ONCE(gsp_phys_rmcall_gr_mme_prelude_pack) && + READ_ONCE(gsp_phys_rmcall_gr_mme_iram_gate) && + READ_ONCE(gsp_phys_rmcall_gr_mme_iram_gate_build)) { + void __iomem *gbar0 = gsp_phys_gpu_bar0(); + + if (gbar0 && + !gsp_phys_gr_mme_iram_gate(gbar0, req->h_object, + pl_gen, "build")) { + mme_hold_calls = true; + gsp_phys_gr_mme_iram_gate_fail(); + } + } + /* + * SESSION 153-CONTINUED-63: emit the SA-RAM completion computed + * above - a one-instruction end-noop stub in the IR plus one + * ONE_INC 0x11c run loading [slot, stub] for every called slot + * the capture's SA table does not cover (see the sa_stub + * comment at the emit decision). Emitted with the prelude (same + * generation latch), so it lands once per capture generation + * and the context keeps it for every later window. + */ + if (emit_prelude && sa_block) { + u32 t, stub_pc = gsp_sema_mme_ir_end_get(); + + /* + * SESSION 153-CONTINUED-68: the stub PC must be past the + * replayed program's last instruction - the window also + * calls COVERED slots, and a fixed PC inside a captured + * macro body would clobber its code. Fall back to 0xC0 + * when the capture reports nothing yet. + */ + if (stub_pc < GSP_PHYS_RMCALL_GR_MME_STUB_PC) + stub_pc = GSP_PHYS_RMCALL_GR_MME_STUB_PC; + /* + * SESSION 153-CONTINUED-69: explicit stub-PC override + * (sema_gl_mme_stub_pc). The ir_end-derived PC sits at + * the first instruction past the replayed program; if + * the MME's IR allocation/ONEINC pointer semantics + * differ (instruction vs word units, allocation + * boundary), the stub body may land where the engine + * does not look. This knob moves the stub (e.g. to + * 0xf0/0x100, or back to the 0xC0 floor) so the + * landing can be A/B-tested without a rebuild. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_mme_stub_pc)) + stub_pc = READ_ONCE(gsp_phys_rmcall_gr_mme_stub_pc); + /* + * The IR upload MUST use the client's own SEC_OP (5, + * ONE_INC): with op=1 the method auto-increment walked + * the data words onto 0x11c/0x120 and clobbered SA[0] + * while writing no usable stub code (r1131). count = 7 + * = 1 pointer + 6 code words (two 3-word NOPs; see the + * gsp_phys_gr_mme_stub_code comment). + */ + *p++ = (GSP_PHYS_RMCALL_METHOD_ONEINC << 29) | + (7u << 16) | (0x114u >> 2); + *p++ = stub_pc; + *p++ = gsp_phys_gr_mme_stub_code[0]; + *p++ = gsp_phys_gr_mme_stub_code[1]; + *p++ = gsp_phys_gr_mme_stub_code[2]; + *p++ = gsp_phys_gr_mme_stub_code[3]; + *p++ = gsp_phys_gr_mme_stub_code[4]; + *p++ = gsp_phys_gr_mme_stub_code[5]; + /* + * Each [slot, stub] pair is its own INC 0x11c run: the + * pair methods are re-written per slot (a single longer + * run would walk onto 0x124 SetMmeShadowRamControl and + * beyond). + */ + for (t = 0; t < sa_nstub; t++) { + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + (2u << 16) | (0x11cu >> 2); + *p++ = sa_stub[t]; + *p++ = stub_pc; + } + pr_info("vgsp-gr: MME-SA-STUB slots=%u words=%u pc=0x%x h=0x%x push=0x%llx\n", + sa_nstub, sa_block, stub_pc, req->h_object, + req->push_va); + { + char sb[192]; + u32 k, o = 0; + + sb[0] = '\0'; + for (k = 0; k < sa_nstub && o + 8u < sizeof(sb); k++) + o += scnprintf(sb + o, sizeof(sb) - o, + "%s%u:%x", k ? "," : "", + sa_stub[k], + sa_stub_ent[k]); + pr_info("vgsp-gr: MME-SA-STUB list=[%s]\n", sb); + } + { + char sb[192]; + u32 k, o = 0; + + sb[0] = '\0'; + for (k = 0; k < sa_ncov && o + 8u < sizeof(sb); k++) + o += scnprintf(sb + o, sizeof(sb) - o, + "%s%u:%x", k ? "," : "", + sa_cov[k], + sa_cov_ent[k]); + pr_info("vgsp-gr: MME-SA-COVERED list=[%s]\n", + sb); + } + } + /* + * SESSION 153-CONTINUED-57: attr/pipeline state prelude (see the + * sema_gl_attr_prelude knob above). Emit the patched captured + * runs after the MME prelude and ahead of the window, + * run-aligned, draining across launches when the remaining + * block budget cannot take the whole remainder. The cursor + * resets when a new capture generation is patched. A window + * with no room at all defers the whole step to the next launch + * (logged by the patch site). + */ + if (attr_pre && attr_pre_n && + READ_ONCE(gsp_phys_rmcall_gr_attr_prelude)) { + u32 avail = block_cap - + (u32)(p - pb) - (win3d_n + 2u + 8u); + u32 off, copied = 0; + + /* + * SESSION 153-CONTINUED-79: the attr prelude shares + * the same per-launch block cap (see + * sema_gl_mme_block_cap); unsigned underflow would + * hand it the whole module buffer on an oversized + * window. + */ + if ((u32)(p - pb) + win3d_n + 2u + 8u >= block_cap) + avail = 0; + + if (gsp_phys_gr_attr_emit_gen != + READ_ONCE(gsp_phys_gr_attr_pre_gen)) { + WRITE_ONCE(gsp_phys_gr_attr_emit_gen, + READ_ONCE(gsp_phys_gr_attr_pre_gen)); + WRITE_ONCE(gsp_phys_gr_attr_emit_off, 0); + } + off = READ_ONCE(gsp_phys_gr_attr_emit_off); + while (off < attr_pre_n) { + u32 hdr = attr_pre[off]; + u32 op = (hdr >> 29) & 7u; + u32 cnt = (hdr >> 16) & 0xfffu; + u32 span = ((op == 1u || op == 3u || + op == 5u) && cnt) ? + 1u + cnt : 1u; + + if (off + span > attr_pre_n) + span = attr_pre_n - off; + /* + * SESSION 153-CONTINUED-59: the same SDM + * sanitize as the MME prelude (see + * gsp_phys_rmcall_gr_sdm_drop); the cursor + * still advances so the drop is permanent. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_sdm_drop) && + hdr && (op == 0u || op == 2u || + op == 6u)) { + off += span; + continue; + } + if (copied + span > avail) + break; + memcpy(&p[copied], &attr_pre[off], + (size_t)span * 4u); + copied += span; + off += span; + } + if (copied) { + p += copied; + WRITE_ONCE(gsp_phys_gr_attr_emit_off, off); + pr_info("vgsp-gr: ATTR-PRELUDE emitted gen=%u words=%u/%u window=%u h=0x%x push=0x%llx%s\n", + READ_ONCE(gsp_phys_gr_attr_pre_gen), + off, attr_pre_n, win3d_n, + req->h_object, req->push_va, + off < attr_pre_n ? + " (continuation next launch)" : ""); + } else if (off < attr_pre_n) { + static u32 attr_defer_log; + + if (attr_defer_log < 8) { + attr_defer_log++; + pr_info("vgsp-gr: ATTR-PRELUDE deferred (no room: window=%u avail=%u) h=0x%x\n", + win3d_n, avail, + req->h_object); + } + } + } + /* + * SESSION 153-CONTINUED-84: shadow-RAM mode before the window + * (see the sema_gl_mme_track comment). METHOD_TRACK (0) is + * emitted by default so the repeated CALL/DATA parameter + * writes are not swallowed by the redundant-method filter. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_mme_track) >= 0 && + p + 2u <= pb + GSP_PHYS_RMCALL_GR_METHOD_WORDS) { + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + (1u << 16) | (0x124u >> 2); + *p++ = (u32)READ_ONCE(gsp_phys_rmcall_gr_mme_track); + } + /* + * SESSION 153-CONTINUED-84: SA re-arm BEFORE the window. + * The r1236 evidence: this block was emitted AFTER the + * window copy (contradicting its own "before the window" + * log), so the window's CALL_MME_MACRO(j) runs executed + * against the SA table state left by the prelude - whose + * last captured SA write re-clobbers the slot (r1231: + * SA[34]=0 lands last, every draw call resolved to macro + * 0 and emitted nothing). Because the prelude is + * re-emitted on every launch while any slot is kept, the + * late re-arm was re-clobbered before the next window + * anyway; the re-point only ever held between a window + * and the next prelude re-emission. Emitting the re-arm + * ahead of the window makes every launch's calls resolve + * to the captured SA entry. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_mme_sa_rearm)) { + u32 sa_ent[64]; + u64 sa_mask = gsp_sema_mme_sa_get(sa_ent, 64); + u32 r = 0, rearm = 0; + + while (r < win3d_n && rearm < 32u) { + u32 h = win3d[r]; + u32 op = (h >> 29) & 7u; + u32 cnt = (h >> 16) & 0xfffu; + u32 span = ((op == 1u || op == 3u || + op == 5u) && cnt && + r + 1u + cnt <= win3d_n) ? + 1u + cnt : 1u; + u32 mb = (h & 0xfffu) << 2; + + if (mb >= 0x3800u && mb <= 0x39fcu) { + u32 j = (mb - 0x3800u) / 8u; + + if (j < 64u && + (sa_mask & (1ull << j)) && + sa_ent[j]) { + *p++ = ((u32) + GSP_PHYS_RMCALL_METHOD_IMM + << 29) | + (2u << 16) | + (0x11cu >> 2); + *p++ = j; + *p++ = sa_ent[j]; + rearm++; + } + } + r += span; + } + if (rearm) + pr_info("vgsp-gr: MME-SA-REARM %u slot(s) re-pointed before the window (h=0x%x push=0x%llx)\n", + rearm, req->h_object, req->push_va); + } + if (mme_load_only) { + /* + * SESSION 153-CONTINUED-107: no window on a LOAD + * carrier (see the mme_load_only comment). The block + * is SET_OBJECT + prelude/SA/pack + release only; + * the engine can retire it without any DS/shader + * state to validate, so the IRAM load cannot be + * interrupted by the SPH poison. + */ + static u32 load_only_log; + + load_only_log++; + if (load_only_log <= 24u) + pr_info("vgsp-gr: MME-LOAD-ONLY carrier h=0x%x gen=%u window=%u dropped until the IRAM gate verifies (log %u) push=0x%llx\n", + req->h_object, pl_gen, win3d_n, + load_only_log, req->push_va); + } else if (mme_hold_calls) { + /* + * SESSION 153-CONTINUED-82: the prelude latch for + * this generation+channel is not set - elide every + * remaining CALL_MME_MACRO/DATA run of the window + * (the proven-safe drop shape; see the + * sema_gl_mme_hold_calls comment). Runs touching + * the MME macro method space 0x3800..0x39fc are the + * ones that execute a macro; the drop decision is + * the same walk gsp_phys_gr_mme_drop_calls uses. + */ + u32 r = 0, w = 0, held = 0; + + while (r < win3d_n) { + u32 h = win3d[r]; + u32 op = (h >> 29) & 7u; + u32 cnt = (h >> 16) & 0xfffu; + u32 span = 1u; + bool iscall = false; + u32 k; + + if ((op == 1u || op == 3u || op == 5u) && + cnt && r + 1u + cnt <= win3d_n) + span = 1u + cnt; + for (k = 0; k < span; k++) { + u32 m = ((h & 0xfffu) << 2) + + 4u * k; + + if (k && op != 1u && op != 3u) + break; + if (m >= 0x3800u && m <= 0x39fcu) { + iscall = true; + break; + } + } + if (iscall) { + held += span; + r += span; + continue; + } + memcpy(&p[w], &win3d[r], (size_t)span * 4u); + w += span; + r += span; + } + if (held) + pr_info("vgsp-gr: MME-CALL-HOLD %u word(s) of real CALL runs held (prelude gen=%u not latched for h=0x%x; window=%u->%u) push=0x%llx%s\n", + held, pl_gen, req->h_object, + win3d_n, w, req->push_va, + mme_hold_fe_busy ? + " [FE-not-idle (CONT-104)]" : ""); + p += w; + } else if (READ_ONCE(gsp_phys_rmcall_gr_mme_nop_pair)) { + /* + * CONT-103: the calls-kept launch now starts (the + * prelude-live skip shrank it under the ~2050-word + * wall) and the engine decodes the window, but the + * FE then TRAPS on the relocated program-address + * pair write (trap=0x80002058 data=<shadow VA>, + * CONT-101: the relocated 0x2054-pair operand) and + * parks the front-end (grfifo 00000001). The binds + * persist on the channel from the earlier + * calls-held launches of the same capture, so + * eliding the pair re-binds removes the trapping + * write while keeping the program bound. Drops any + * method run overlapping the program-pair space + * [0x2054, 0x2158] (0x2054/0x2058, 0x2094/0x2098, + * 0x2154/0x2158 - the client window's own pairs and + * the synthesized SPH binds alike; this walk runs on + * the final window, prepends included). + */ + u32 r = 0, w = 0, elided = 0; + + while (r < win3d_n) { + u32 h = win3d[r]; + u32 op = (h >> 29) & 7u; + u32 cnt = (h >> 16) & 0xfffu; + u32 span = 1u, m0, mend; + + if ((op == 1u || op == 3u || op == 5u) && + cnt && r + 1u + cnt <= win3d_n) + span = 1u + cnt; + m0 = (h & 0xfffu) << 2; + mend = m0 + 4u * (span - 1u); + if ((op == 0u || op == 1u || op == 3u || + op == 4u || op == 5u) && + m0 <= 0x2158u && mend >= 0x2054u) { + elided += span; + r += span; + continue; + } + memcpy(&p[w], &win3d[r], (size_t)span * 4u); + w += span; + r += span; + } + if (elided) + pr_info("vgsp-gr: MME-PAIR-DROP %u word(s) of program-pair runs elided (calls-kept launch; binds persist from the held-call launches) h=0x%x push=0x%llx window=%u->%u\n", + elided, req->h_object, req->push_va, + win3d_n, w); + p += w; + } else { + memcpy(p, win3d, (size_t)win3d_n * 4); + p += win3d_n; + } + /* + * SESSION 153-CONTINUED-84: the SA re-arm block used to be + * emitted HERE (after the window). It was moved ahead of the + * window above; see that block's comment for the r1236 + * evidence. An SA[slot]=entry re-point behind the call that + * needs it cannot help that call, and the per-launch + * prelude re-emission re-clobbers it before the next window. + */ + p = gsp_phys_rmcall_push_gr(p, + GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_A, + (u32)(sem_va >> 32)); + p = gsp_phys_rmcall_push_gr(p, + GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_B, + (u32)sem_va); + p = gsp_phys_rmcall_push_gr(p, + GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_C, + sem_payload); + p = gsp_phys_rmcall_push_gr(p, + GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_D, + GSP_PHYS_RMCALL_GR_SEM_RELEASE_D); + { + static u32 gr3d_log; + + if (gr3d_log < 16) { + gr3d_log++; + pr_info("vgsp-gr: 3D STREAM replay h=0x%x push=0x%llx class=0xc597 window=%u words=%u verbatim (no prelude/QMD)\n", + req->h_object, req->push_va, + win3d_n, (u32)(p - pb)); + } + /* + * SESSION 153-CONTINUED-45 diag: the exact block the + * PBDMA fetches (first/last window words). The beef + * family's replay schedule-stalls (grfifo frozen) while + * the X family's retires; this dump is the content A/B + * behind that divergence. Bounded per boot. + */ + if (gr3d_log < 16) { + u32 wi; + + pr_info("vgsp-gr: 3DSTREAM hdr=%08x %08x win0..9=", + pb[0], pb[1]); + for (wi = 0; wi < 10 && wi < win3d_n; wi++) + pr_cont("%08x ", win3d[wi]); + pr_cont("| tail="); + for (wi = (win3d_n > 10 ? win3d_n - 10 : 0); + wi < win3d_n; wi++) + pr_cont("%08x ", win3d[wi]); + pr_cont("\n"); + } + } + return (u32)(p - pb); + } + + /* + * r425 A/B: with gsp_phys_rmcall_gr_init_once the compute init preamble + * is emitted only on the first forwarded launch (the stock driver's + * per-channel shape); subsequent launches start at the inline-QMD + * window. + */ + if (!READ_ONCE(gsp_phys_rmcall_gr_init_once_param) || + !READ_ONCE(gsp_phys_rmcall_gr_init_emitted)) { + p = gsp_phys_rmcall_build_gr_init(p); + WRITE_ONCE(gsp_phys_rmcall_gr_init_emitted, true); + } if (qmdvirt) p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_QMD_VIRT_CONTROL, GSP_PHYS_RMCALL_GR_QMD_VIRT_IQ2M_SEND_PCAS); /* + * r597: the prelude's acquire groups (if any) go before the QMD run + * so the engine cannot read the launch's data pages until the copy + * channel's semaphore has advanced. + */ + { + u32 w = gsp_phys_rmcall_gr_emit_acquires(p, req, + 128u); + static u32 acq_log; + + if (w) { + p += w; + if (acq_log < 16) { + acq_log++; + pr_info("vgsp-gr: prelude acquire groups emitted words=%u h=0x%x push=0x%llx\n", + w, req->h_object, req->push_va); + } + } + } + /* * Verbatim replay: copy the captured GRCE prelude and the guest's * own inline-QMD run, keeping the method headers, and relocate only * the ADDRESS_A/B data words and the 64 QMD words exactly like the @@ -16656,7 +21246,7 @@ static int gsp_phys_rmcall_gr_map_va_locked(void __iomem *bar0, u64 guest_va, pt_va = gsp_phys_chunk_page_va(pt, 0); if (!pt_va) return -EIO; - put_unaligned_le64(gsp_phys_rmcall_pte(hpa), + put_unaligned_le64(gsp_phys_rmcall_gr_pte_leaf(hpa), pt_va + (size_t)pte_idx * 8); dma_wmb(); boot_stage.rmcall_gr_win[slot].pte[i].idx = pte_idx; @@ -16824,7 +21414,7 @@ static int gsp_phys_rmcall_gr_map_shadow_locked(u32 slot, u64 hpa, pt = gsp_phys_rmcall_guest_pt_page_va(page); if (!pt) return -EIO; - put_unaligned_le64(gsp_phys_rmcall_pte(hpa), + put_unaligned_le64(gsp_phys_rmcall_gr_pte_leaf(hpa), pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * line_in_page) * 8); dma_wmb(); @@ -16871,7 +21461,7 @@ static int gsp_phys_rmcall_gr_map_shadow_contig_locked(u32 line, u32 pte, pt = gsp_phys_rmcall_guest_pt_page_va(page); if (!pt) return -EIO; - put_unaligned_le64(gsp_phys_rmcall_pte(hpa), + put_unaligned_le64(gsp_phys_rmcall_gr_pte_leaf(hpa), pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * line_in_page + pte) * 8); dma_wmb(); @@ -17212,6 +21802,54 @@ static struct gsp_phys_boot_chunk *gsp_phys_gr_data_chunk(void) } /* + * r452: scan the whole active GR data-bounce chunk for an 8-byte pattern + * (write "0x<pattern>" to /sys/module/vgsp/parameters/gsp_phys_gr_scan). + * Verifies by content where a lost engine store actually landed: the chunk + * is the only bulk memory the operand window PTEs map, so a store that + * executed but is absent from the writer's tracked pages must be findable + * here (a stale-PTE landing) or nowhere (a dropped store). + */ +static int gsp_phys_gr_scan_set(const char *val, + const struct kernel_param *kp) +{ + struct gsp_phys_boot_chunk *c = gsp_phys_gr_data_chunk(); + u64 pattern = 0; + u32 i, hits = 0; + + if (kstrtoull(val, 0, &pattern)) + return -EINVAL; + if (!c->pages) + return -ENODEV; + for (i = 0; i < c->pages; i++) { + const u8 *p = gsp_phys_chunk_page_va(c, i); + u32 off; + + if (!p) + continue; + for (off = 0; off + 8 <= GSP_PHYS_PAGE_SIZE; off += 8) { + if (get_unaligned_le64(p + off) == pattern) { + if (hits < 32) + pr_info("vgsp-gr: CHUNK-SCAN found 0x%llx at bi=%u off=0x%x (va=0x%llx)\n", + pattern, i, off, + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 + + ((u64)i << 12) + off); + hits++; + } + } + } + pr_info("vgsp-gr: CHUNK-SCAN 0x%llx done: %u hit(s) over %u pages\n", + pattern, hits, c->pages); + return 0; +} + +static const struct kernel_param_ops gsp_phys_gr_scan_ops = { + .set = gsp_phys_gr_scan_set, +}; +module_param_cb(gsp_phys_gr_scan, &gsp_phys_gr_scan_ops, NULL, 0200); +MODULE_PARM_DESC(gsp_phys_gr_scan, + "Scan the GR data-bounce chunk for an 8-byte pattern (hex)"); + +/* * B-2: page index of a relocated operand VA inside the engine-visible data * region, replacing the five inlined (reloc_va - 0x110000) >> 12 derivations * and their MAIL_SLOT bounds. Grown region: VAs from @@ -17257,6 +21895,45 @@ static u32 gsp_phys_gr_data_bi(u64 reloc_va) * gsp-phys-work item is the only caller, reset per submission exactly like * bounce_csum_in. */ +/* + * r564: minimum data records for a forwardable launch. Healthy vadd-class + * launches resolve 968-969 records (the client's whole arena); a launch + * with only a handful after the fill's drops has an unresolved argument + * set and must not be handed to the engine (see the submit guard). + * + * SESSION 127: the Vulkan inline-QMD launch (the C5C0 translator) is a + * compute dispatch whose operand set is one descriptor page and one + * parameter bank - a handful of records is healthy for it. Tunable module + * parameter (default 16 preserves the CUDA-era threshold); the translator + * cycles pass gsp_phys_gr_min_data=2. + */ +#define GSP_PHYS_GR_MIN_DATA_RECS 16u +static unsigned int gsp_phys_gr_min_data_param = GSP_PHYS_GR_MIN_DATA_RECS; +module_param_named(gsp_phys_gr_min_data, gsp_phys_gr_min_data_param, uint, 0644); +MODULE_PARM_DESC(gsp_phys_gr_min_data, + "GR submit minimum data records (degenerate-set refusal threshold, 0..4096; 0 = explicit bypass for the no-external-data launch class; default 16)"); + +/* + * r801 (SESSION 143): 0 is an explicit BYPASS, not a clamp to 1. The + * Vulkan launch's single QMD pointer pair names its own QMD window and its + * params are scalars, so the launch has no external data by construction; + * the bare n_data>=min gate wrongly classifies it degenerate. The bypass is + * safe now that gsp_phys_chunk_page_va answers NULL for an unallocated + * chunk (r800): before that guard the empty-set post-mortem print + * dereferenced a garbage pointer and oopsed at + * gsp_phys_backend_guest_gr_submit+0x4394. The proper predicate (all + * candidates launch-internal-filtered, none unresolved => not degenerate) + * replaces the bare threshold next; until then 0 stays an explicit, + * opt-in debug/test value and the default remains 16. + */ +static unsigned int gsp_phys_gr_min_data_value(void) +{ + unsigned int v = READ_ONCE(gsp_phys_gr_min_data_param); + + if (v > 4096) + v = 4096; + return v; +} #define GSP_PHYS_GR_GAP_MAX 4096u /* total late pages per launch */ /* * relfix PART 3 (r217 span containment): the late path authors ONLY the @@ -17661,7 +22338,7 @@ static bool gsp_phys_gr_reloc_fix_word(const struct gsp_phys_gr_submit *req, */ static noinline u32 gsp_phys_gr_scan_pages(const struct gsp_phys_gr_submit *req, - const u32 *csum_in, u32 *n_tracked, u64 *hash_out) + const u64 *csum_in, u32 *n_tracked, u64 *hash_out) { static u32 checked[GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES]; struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); @@ -17707,15 +22384,24 @@ gsp_phys_gr_scan_pages(const struct gsp_phys_gr_submit *req, *n_tracked = n_all; for (j = 0; j < n_all; j++) { const u8 *src = gsp_phys_chunk_page_va(dchunk, checked[j]); - u32 ci, csum = 0; + u64 fp; if (!src) continue; - for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) - csum += get_unaligned_le32(src + ci); - if (csum != csum_in[checked[j]]) + /* + * r452: FNV-1a per page, not a 32-bit additive sum. The + * add-sum was PROVABLY blind to a power-of-two in-place + * scale: doubling every word of a 4 KB page shifts each + * exponent by +1, i.e. +2^23 per word, and 1024 words give + * 2^33 == 0 (mod 2^32) - so c[i] *= 2.0f left the checksum + * bit-identical, the page classified "unchanged", and the + * client read the pre-scale values (the deterministic + * streams2/scale class). FNV-1a has no such structure. + */ + fp = gsp_phys_gr_fnv1a64(src, GSP_PHYS_PAGE_SIZE); + if (fp != csum_in[checked[j]]) changed++; - hash ^= csum; + hash ^= fp; hash *= 1099511628211ULL; } if (hash_out) @@ -17723,9 +22409,35 @@ gsp_phys_gr_scan_pages(const struct gsp_phys_gr_submit *req, return changed; } +/* + * r434: post-launch deposit dump (hardware-only words). The r430 QMD64 dump + * runs at submission, before the engine touches the deposit, so the + * hardware-only fields (w50 INNER_GET, w51 INNER_PUT, w52 span-list head, + * w53 next-QMD) are always zero there. Reading the same page AFTER the + * retire wait shows what the SKED actually consumed: w50/w51 are the CTA + * (INNER) get/put counters, so a wedged launch whose grid "completed" with + * zero stores is distinguishable from a genuinely dispatched grid by + * whether INNER_PUT advanced to the CTA count. One line per launch, gated + * by the skedrec param. + */ +static void gsp_phys_gr_qmd64_post(const u8 *deposit, const char *tag) +{ + char qb[64 * 9 + 1]; + int qo = 0; + u32 i; + + if (!deposit || !gsp_phys_rmcall_gr_skedrec_value()) + return; + for (i = 0; i < GSP_PHYS_GR_QMD_WORDS && qo < (int)sizeof(qb) - 10; + i++) + qo += scnprintf(qb + qo, sizeof(qb) - qo, "%08x ", + get_unaligned_le32(deposit + i * 4)); + pr_info("vgsp-gr: QMD64-POST %s %s\n", tag, qb); +} + static noinline u32 gsp_phys_gr_output_retire_wait(const struct gsp_phys_gr_submit *req, - const u32 *csum_in, u32 *n_tracked) + const u64 *csum_in, u32 *n_tracked) { u32 n_all = 0, iter; u32 changed = 0; @@ -17859,6 +22571,82 @@ static bool gsp_phys_gr_qmd_fetched(const u8 *qmd_mem) } /* + * r427 idempotent fast-accept (CONT.86 §6.1). A zero-write launch is either + * the no-execute wedge OR a legitimate idempotent re-launch (the same QMD + * content with the same staged inputs, whose result is already in place) - + * the host cannot tell them apart, and both burn the 40 s + 80 s quiet holds. + * The launch identity below (QMD minus the per-launch release0 address and + * payload, folded with the staged content checksums of every tracked page) is + * recorded for every launch that WROTE; a zero-write launch whose identity + * matches an entry accepts immediately, with no retries and no quiet window. + * Safe by construction: identical code + identical staged content + a prior + * execution that produced this launch's own footprint means the correct + * result is already in the buffers; a side-effecting kernel's re-launch is + * equally wrong under the current accept (the client sees the stale result + * either way) - this only removes the stall. + */ +#define GSP_PHYS_GR_IDEM_SLOTS 64u +static u64 gsp_phys_gr_idem_fp[GSP_PHYS_GR_IDEM_SLOTS]; +static u32 gsp_phys_gr_idem_n; + +static u64 gsp_phys_gr_launch_fp(const u32 *qmd, + const struct gsp_phys_gr_submit *req, + const u64 *csum_in) +{ + u64 h = 1469598103934665603ULL; + u32 i; + + for (i = 0; i < GSP_PHYS_GR_QMD_WORDS; i++) { + /* Skip release0's address (w23/w24) and payload (w25): the + * driver allocates a fresh release slot per launch, so they + * differ even for an identical re-launch. */ + u32 w; + + if (i == 23u || i == 24u || i == 25u) + continue; + w = qmd[i]; + h ^= w; + h *= 1099511628211ULL; + } + for (i = 0; i < req->n_data; i++) { + u32 bi = gsp_phys_gr_data_bi(req->data[i].reloc_va); + + if (bi == GSP_PHYS_GR_DATA_BI_NONE) + continue; + h ^= csum_in[bi]; + h *= 1099511628211ULL; + } + return h; +} + +static bool gsp_phys_gr_idem_match(u64 fp) +{ + u32 i; + + for (i = 0; i < gsp_phys_gr_idem_n; i++) + if (gsp_phys_gr_idem_fp[i] == fp) + return true; + return false; +} + +static void gsp_phys_gr_idem_record(u64 fp) +{ + u32 i; + + for (i = 0; i < gsp_phys_gr_idem_n; i++) + if (gsp_phys_gr_idem_fp[i] == fp) + return; + if (gsp_phys_gr_idem_n < GSP_PHYS_GR_IDEM_SLOTS) { + gsp_phys_gr_idem_fp[gsp_phys_gr_idem_n++] = fp; + } else { + /* ring: drop the oldest */ + memmove(&gsp_phys_gr_idem_fp[0], &gsp_phys_gr_idem_fp[1], + (GSP_PHYS_GR_IDEM_SLOTS - 1u) * sizeof(u64)); + gsp_phys_gr_idem_fp[GSP_PHYS_GR_IDEM_SLOTS - 1u] = fp; + } +} + +/* * COPYAUDIT (2026-10-18 copy-completeness audit): per-copy sighting of the * GR operand staging copies. Every staged page is spot-checked (16 bytes * at the start and end of the page, source vs destination, right after the @@ -17881,6 +22669,22 @@ static void gsp_phys_gr_page_spot_check(const u8 *dst, const u8 *src, } /* + * r452: the legacy 32-bit additive page sum, kept ONLY for the all-zero + * filter. It is never a change-detection fingerprint any more: doubling every + * word of a 4 KB page leaves the sum bit-identical (1024 x 2^23 == 2^33 == 0 + * mod 2^32), which silently classified an in-place x2 scale as "no engine + * store" and shipped stale values (the deterministic streams2 class). + */ +static inline u32 gsp_phys_gr_page_sum32(const u8 *p) +{ + u32 ci, csum = 0; + + for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) + csum += get_unaligned_le32(p + ci); + return csum; +} + +/* * COPYWIT r226 zero filter: an all-zero page is never a meaningful alias * partner. A fresh client's fill stages long runs of zero pages * (uninitialized cudaMalloc buffers), and consecutive zero pages produced @@ -17926,7 +22730,7 @@ static u32 gsp_phys_gr_in_census_log; * again). Returns the number of pages staged. */ static u32 gsp_phys_gr_fill_bounce(const struct gsp_phys_gr_submit *req, - u32 *csum_in, u64 *staged_hpa) + u64 *csum_in, u64 *staged_hpa) { struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); struct { @@ -17963,7 +22767,9 @@ static u32 gsp_phys_gr_fill_bounce(const struct gsp_phys_gr_submit *req, "gr-bounce-copy-in"); for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) csum += get_unaligned_le32(dst + ci); - csum_in[bi] = csum; + /* r452: the change baseline is the FNV-1a fingerprint (the + * add-sum above survives only as the all-zero test). */ + csum_in[bi] = gsp_phys_gr_fnv1a64(dst, GSP_PHYS_PAGE_SIZE); zero = gsp_phys_gr_page_all_zero(csum, dst); if (!zero && gsp_phys_gr_copywit_enabled()) { u32 j; @@ -18012,7 +22818,7 @@ static u32 gsp_phys_gr_fill_bounce(const struct gsp_phys_gr_submit *req, * retry, so a retried launch starts from the guest inputs in both sets. * Returns the number of pages staged. */ -static u32 gsp_phys_gr_fill_gap_bounce(u32 *csum_in, u64 *staged_hpa) +static u32 gsp_phys_gr_fill_gap_bounce(u64 *csum_in, u64 *staged_hpa) { struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); u32 gi, copied = 0; @@ -18020,7 +22826,6 @@ static u32 gsp_phys_gr_fill_gap_bounce(u32 *csum_in, u64 *staged_hpa) for (gi = 0; gi < gsp_phys_gr_gap_n; gi++) { const u8 *src = (const u8 *)__va(gsp_phys_gr_gap[gi].hpa); u8 *dst = gsp_phys_chunk_page_va(dchunk, gsp_phys_gr_gap[gi].bi); - u32 ci, csum = 0; if (!src || !dst) continue; @@ -18030,9 +22835,9 @@ static u32 gsp_phys_gr_fill_gap_bounce(u32 *csum_in, u64 *staged_hpa) staged_hpa[gsp_phys_gr_gap[gi].bi] = gsp_phys_gr_gap[gi].hpa; gsp_phys_gr_page_spot_check(dst, src, gsp_phys_gr_gap[gi].va, "gr-bounce-copy-in"); - for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) - csum += get_unaligned_le32(dst + ci); - csum_in[gsp_phys_gr_gap[gi].bi] = csum; + /* r452: FNV-1a baseline (see gsp_phys_gr_scan_pages). */ + csum_in[gsp_phys_gr_gap[gi].bi] = + gsp_phys_gr_fnv1a64(dst, GSP_PHYS_PAGE_SIZE); copied++; } dma_wmb(); @@ -18129,7 +22934,8 @@ static bool gsp_phys_gr_win0_retry_rearm(u32 *qmd, */ static noinline void gsp_phys_gr_poll_break_log(u32 w0, u32 want_qmd, u32 w1, u32 since_us, - const struct gsp_phys_rmcall_ce_submit *sub) + const struct gsp_phys_rmcall_ce_submit *sub, + u32 h_object) { struct gsp_phys_boot_chunk *ud_c = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD]; @@ -18137,11 +22943,177 @@ gsp_phys_gr_poll_break_log(u32 w0, u32 want_qmd, u32 w1, u32 since_us, u32 usr_gpget = ud ? get_unaligned_le32(ud + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPGET_OFF) : 0; + /* + * SESSION 153-CONTINUED-47: re-read the ring slot the launch + * published (the entry AS THE ENGINE SEES IT) after the wait, so an + * X-vs-beef diff on one boot compares the exact e0/e1 the PBDMA + * fetched, plus the doorbell-visible GP_GET/GP_PUT movement. + */ + struct gsp_phys_boot_chunk *r_c = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO]; + u8 *ring = r_c->pages ? gsp_phys_chunk_page_va(r_c, 0) : NULL; + u32 re0 = 0, re1 = 0; - pr_info("vgsp-gr: poll break w0=0x%x want=0x%x ch=0x%x after %u us usr_gpget=0x%x gpget=0x%x gpput=0x%x\n", + if (ring && sub->slot < GSP_PHYS_RMCALL_GPFIFO_ENTRIES) { + re0 = get_unaligned_le32(ring + sub->slot * 8); + re1 = get_unaligned_le32(ring + sub->slot * 8 + 4); + } + + pr_info("vgsp-gr: poll break w0=0x%x want=0x%x ch=0x%x after %u us usr_gpget=0x%x gpget=0x%x gpput=0x%x ring[%u]=%08x %08x (pub %08x %08x)\n", w0, want_qmd, w1, (u32)ktime_to_us(ktime_get()) - since_us, - usr_gpget, sub->gpget, sub->gpput); + usr_gpget, sub->gpget, sub->gpput, + sub->slot, re0, re1, sub->e0, sub->e1); + /* + * CONT-103: engine-fetch witness + prelude-liveness bookkeeping. + * GP_GET advancing past the published slot is the PBDMA consuming + * this launch's GPFIFO entry - the fetch witness the QMD sentinel + * cannot provide on the stream3d lane (no QMD deposit there). A + * break with GP_GET frozen is a stale/software-authored release. + */ + pr_info("vgsp-gr: FETCH-WIT h=0x%x slot=%u gpget %u->%u %s\n", + h_object, sub->slot, sub->gpget, usr_gpget, + usr_gpget != sub->gpget ? "engine-fetched" : + "GP_GET-frozen (software ack)"); + if (usr_gpget != sub->gpget && + READ_ONCE(gsp_phys_gr_mme_pending_gen)) { + WRITE_ONCE(gsp_phys_gr_mme_live_gen, + READ_ONCE(gsp_phys_gr_mme_pending_gen)); + WRITE_ONCE(gsp_phys_gr_mme_pending_gen, 0); + } + /* r425: the healthy-launch SKED/FECS baseline for the wedge compare. */ + gsp_phys_gr_sked_wit(gsp_phys_gpu_bar0(), "post-poll"); +} + +/* + * CONT-103: fetch-witness for a launch that did NOT break - GP_GET frozen + * means the PBDMA never consumed the entry (the >2050-word fetch wall / + * parked front-end class), so the carrier's generation is not live and the + * next launch must re-emit the prelude. noinline: keeps the submit frame. + */ +static noinline void +gsp_phys_gr_fetch_miss_log(u32 h_object, + const struct gsp_phys_rmcall_ce_submit *sub, + const char *tag) +{ + struct gsp_phys_boot_chunk *ud_c = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD]; + u8 *ud = ud_c->pages ? gsp_phys_chunk_page_va(ud_c, 0) : NULL; + u32 usr_gpget = ud ? + get_unaligned_le32(ud + gsp_phys_rmcall_userd_ctl_off() + + GSP_PHYS_RMCALL_USERD_GPGET_OFF) : 0; + + pr_info("vgsp-gr: FETCH-WIT %s h=0x%x slot=%u gpget %u->%u %s\n", + tag, h_object, sub->slot, sub->gpget, usr_gpget, + usr_gpget != sub->gpget ? "engine-fetched" : + "GP_GET-frozen (NOT fetched: parked or over-cap block)"); + /* + * SESSION 153-CONTINUED-109: readback-free live promotion on the + * timeout path. The poll-break path promotes the generation whose + * final prelude part was carried when GP_GET advanced (the engine + * consumed the entry). A launch can also time out the w0/w1 release + * wait AFTER the engine fetched it (r1377: `FETCH-WIT timeout ... + * gpget 5->6 engine-fetched` on the load-only carriers - their + * releases only matched ~10 s later through the ladder), and the + * historical code cleared pending_gen here, permanently losing the + * promotion: live_gen never caught pl_gen, every later launch stayed + * window-elided, and the MME never got a calls-bearing launch + * (`pc_valid=0` on all samples). Promote on the same GP_GET witness + * the break path trusts; only a genuinely unfetched carrier clears. + */ + if (usr_gpget != sub->gpget && + READ_ONCE(gsp_phys_gr_mme_pending_gen)) { + pr_info("vgsp-gr: MME-LIVE promote via GP_GET advance (%s) gen=%u h=0x%x\n", + tag, READ_ONCE(gsp_phys_gr_mme_pending_gen), + h_object); + WRITE_ONCE(gsp_phys_gr_mme_live_gen, + READ_ONCE(gsp_phys_gr_mme_pending_gen)); + WRITE_ONCE(gsp_phys_gr_mme_pending_gen, 0); + } else { + WRITE_ONCE(gsp_phys_gr_mme_pending_gen, 0); + } +} + +/* + * SESSION 153-CONTINUED-72: GR-STATE retire readback. The keep-mask A/B + * must answer "does the kept macro execute at all" before chasing the + * 0/903 stores. The FE decode-state EX records (0x40a630..3c) name the + * last MME-GENERATED method write: bit31 GENERATED, 21:12 macro address, + * 11:0 pc - gen0=1 with the kept slot's macro address at the retire is + * the live witness that the engine fetched and ran the kept macro. The + * MME HWW latch (0x404490 + INFO/IR), the trapped method (0x40704 offset + * + bit20 mmegen, data 0x40708) and the DS latch catch the write when it + * faulted. One line per retire; bounded per boot (beef up to 48, others + * 4) so a battery boot cannot flood. + */ +static noinline void gsp_phys_gr_state_rb_log(u32 h_object) +{ + static u32 gs_beef_n, gs_other_n; + void __iomem *bar0 = gsp_phys_gpu_bar0(); + bool beef = (h_object & 0xfffff000u) == 0xbeef0000u; + u32 fed0, fed1, fed2, fed3, esr, info, ir2, trap, data, ds, gf; + /* + * SESSION 153-CONTINUED-91: with the DS HWW report mask cleared + * (sema_ds_mask_clear) a SPH validation miss latches in DS_HWW_ESR + * without the GSP-RM fault handler poisoning the PRI block, so the + * per-stage SPH_INFO snapshot is finally readable: STAGE0..5 START + * (0x405c00+si*4) / UPPER (0x405c24+si*4) name the exact SPH VA each + * pipeline stage fetched, and PROGRAM_REGION_A/B (0x405c1c/c20) the + * region the DS validated against (the r1277+ wall's missing datum). + */ + u32 sph[6], sphu[6], rr_a, rr_b; + u32 si; + + if (!bar0) + return; + if (beef ? gs_beef_n >= 48u : gs_other_n >= 4u) + return; + if (beef) + gs_beef_n++; + else + gs_other_n++; + for (si = 0; si < 6u; si++) { + sph[si] = ioread32(bar0 + 0x00405c00u + si * 4u); + sphu[si] = ioread32(bar0 + 0x00405c24u + si * 4u); + } + rr_a = ioread32(bar0 + 0x00405c1cu); + rr_b = ioread32(bar0 + 0x00405c20u); + fed0 = ioread32(bar0 + 0x0040a630u); + fed1 = ioread32(bar0 + 0x0040a634u); + fed2 = ioread32(bar0 + 0x0040a638u); + fed3 = ioread32(bar0 + 0x0040a63cu); + esr = ioread32(bar0 + 0x00404490u); + info = ioread32(bar0 + 0x00404494u); + /* SESSION 153-CONTINUED-106: INFO2 is 0x40449c (0x404498 is CONFIG). */ + ir2 = ioread32(bar0 + 0x0040449cu); + trap = ioread32(bar0 + 0x00400704u); + data = ioread32(bar0 + 0x00400708u); + ds = ioread32(bar0 + 0x00405840u); + gf = ioread32(bar0 + 0x00400504u); + pr_info("vgsp-gr: GR-STATE retire h=0x%x fed=%08x,%08x,%08x,%08x (pc0=%u mac0=0x%x gen0=%u) mme_esr=%08x info=%08x ir=%08x trap=%08x(mthd=0x%x mmegen=%u) data=%08x ds=%08x grfifo=%08x\n", + h_object, fed0, fed1, fed2, fed3, + fed0 & 0xfffu, (fed0 >> 12) & 0x3ffu, (fed0 >> 31) & 1u, + esr, info, ir2, trap, trap & 0x3ffcu, (trap >> 20) & 1u, + data, ds, gf); + pr_info("vgsp-gr: GR-STATE sph h=0x%x ds2=%08x sph=%08x,%08x,%08x,%08x,%08x,%08x sphU=%02x,%02x,%02x,%02x,%02x,%02x rr=%08x,%08x\n", + h_object, ioread32(bar0 + 0x00405848u), + sph[0], sph[1], sph[2], sph[3], sph[4], sph[5], + sphu[0] & 0xffu, sphu[1] & 0xffu, sphu[2] & 0xffu, + sphu[3] & 0xffu, sphu[4] & 0xffu, sphu[5] & 0xffu, + rr_a, rr_b); + /* + * SESSION 153-CONTINUED-91: did the emitted program-region pair + * (SET_MME_SHADOW_SCRATCH 0x78/0x79 = method 0x35e0/0x35e4 through + * class 0xc597) actually land in the MME shadow? The shadow read + * port is the same one the store-side probe uses; a zero readback + * means the write never took effect and every region experiment so + * far was a no-op. + */ + if (beef) + pr_info("vgsp-gr: GR-STATE region-shadow h=0x%x scratch[0x78]=%08x scratch[0x79]=%08x\n", + h_object, + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x35e0u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x35e4u)); } /* @@ -18293,6 +23265,8 @@ static u32 gsp_phys_m91_fault_start; static u32 gsp_phys_m91_cap_used; static bool gsp_phys_m91_cap_suppressed; static bool gsp_phys_m91_icache_log; +static bool gsp_phys_r449_ccache_log; +static bool gsp_phys_gr_rel_membar_log; static noinline void gsp_phys_gr_m91_fault_cap(const struct gsp_phys_gr_submit *req, const u32 *qmd, u64 qmd_dst, const u8 *qmd_dst_mem, @@ -18381,7 +23355,7 @@ gsp_phys_gr_m91_fault_cap(const struct gsp_phys_gr_submit *req, fn = &gsp_phys_fault_notes[(gsp_phys_fault_note_next + GSP_PHYS_FAULT_NOTE_MAX - 1u) % GSP_PHYS_FAULT_NOTE_MAX]; - pr_info("vgsp-m91: FAULT-CAP ok=%u faults=%u retry=%s attempts=%u qmd=0x%llx qmd64=%s prog_w48=0x%08x w49=0x%08x prog_va=0x%llx prog_pa=0x%llx code_src=%s off=0x%x code64=%s fault_et=0x%x fault_chid=%u fault_err=\"%.48s\"\n", + pr_info("vgsp-m91: FAULT-CAP ok=%u faults=%u retry=%s attempts=%u qmd=0x%llx qmd64=%s prog_w48=0x%08x w49=0x%08x prog_va=0x%llx prog_pa=0x%llx code_src=%s off=0x%x code64=%s fault_et=0x%x fault_chid=%u fault_err=\"%.200s\"\n", ok ? 1u : 0u, faults, fast ? "fast-rearm" : (rearmed ? "resched/fresh-entry" : "none"), attempts, qmd_dst, qhex, qmd[48], qmd[49], pva, ppa, src, off, @@ -18575,6 +23549,2151 @@ static void gsp_phys_gr_stall_log(const char *reason, u64 push_va, reason, push_va, jiffies_to_msecs(jiffies - start), action); } +/* + * SESSION 153-CONTINUED-27: big-surface alias bookkeeping. A replayed + * 3D window's colour target (C597 INC 0x800+j*0x40 [upper, lower, w, h, + * fmt] and its TWOD twin INC 0x218 [w, h, upper, lower]) can be a surface + * far larger than the launch's 16-page set (the X screen 0x120400000, + * 1280x720x4 = 900 pages). Two earlier routes are dead ends for it: + * - one-page shadow relocation (r992: `pair addr=0x840 va=0x120400000 + * -> MISS` then FAULT_PDE @ +0xa000 - the raster walk crosses the + * single alias page); + * - Tier-1 windows at the surface's own VAs (CONT.27 r993: PDE_UPDATE + * for pdeIndex 2306/2307 status=0x0, yet GPC2/GPC5 still fault + * FAULT_PDE @ +0x4000/+0xa000 - same class as the r929 v2 FE faults; + * the GPC engines only see the scratch-PT pool). + * The surviving route is the proven one: a CONTIGUOUS run of never-used + * scratch-PT fetch lines (one line = 16 PTEs = 64 KB VA; lines are + * monotonic and consecutive lines are VA-contiguous inside each pool + * block), one PTE per surface page, so the engine reaches all 900 pages + * through one contiguous alias VA and the raster walk never crosses into + * an unmapped page. The pair operands are rewritten to <alias>|<offset> + * by gsp_phys_rmcall_gr_3d_reloc. The table is per-boot: the alias PTEs + * are never rewritten (fetch-line rule) and the screen's HPAs are fixed + * by its single allocation, so every later launch is a pure rewrite. + */ +struct gsp_phys_gr_surf_keep { + u64 va; /* guest surface base VA (page aligned) */ + u64 alias; /* engine-visible contiguous shadow alias base */ + u64 npages; /* pages aliased (declared + slack tail) */ +}; +static struct gsp_phys_gr_surf_keep gsp_phys_gr_surf[4]; +static u32 gsp_phys_gr_surf_n; + +/* + * SESSION 153-CONTINUED-27: slack tail for the alias. r994 measured the + * replay's colour writes landing past the declared w*h*4 footprint + * (1280x720x4 = 0x384000) by up to +0xE000 (11.2 rows) - FAULT_PTE on + * the last fetch line's unwritten PTE slots (0x5d6000..0x5e2000 with the + * alias at 0x250000). The tail pages are aliased too: a page that still + * resolves (inside the guest allocation's own padding - exactly where + * native overdraw would land) maps the real HPA; one that does not + * resolves to a dedicated zeroed module page so the stray writes vanish + * instead of faulting. The chunk is allocated lazily and lives until + * module unload (4 KB). + */ +#define GSP_PHYS_GR_SURF_SLACK_PAGES 32u +static struct gsp_phys_boot_chunk gsp_phys_gr_surf_dummy; + +static u64 gsp_phys_gr_surf_dummy_dma_locked(void) +{ + struct gsp_phys_boot_chunk *d = &gsp_phys_gr_surf_dummy; + + if (!d->pages) { + if (gsp_phys_chunk_alloc(boot_stage.pdev, d, 1u)) + return 0; + gsp_phys_chunk_zero(d); + } + return (u64)gsp_phys_chunk_page_dma(d, 0); +} + +/* + * r929 v3 (Path A): relocate one 3D-window address operand to a shadow-pool + * VA. The guest VAs the 3D methods name (uploads 0x1_214f8f00.., vertex, + * program, tex pools) are installed in the ladder page tree by the Tier-1 + * mapper yet the engine's FE/PD walks still fault FAULT_PDE on them (r929 + * v2 evidence: "ENGINE GRAPHICS HUBCLIENT_FE faulted @ 0x1_214f8000 ... + * FAULT_PDE"); the shadow pool lines (map_shadow_locked) ARE the proven + * engine-visible region (every CUDA data/page relocation lives there). The + * page offset is preserved; a VA whose 4 KB page has no resolved hpa stays + * verbatim (it faults later and is named in the drain). Called with + * gsp_phys_rm_lock held. + */ +/* + * SESSION 153-CONTINUED-59: window/attr operand-miss state for the + * pre-doorbell abort (see gsp_phys_rmcall_gr_3d_reloc's level-2 fallback + * and the MISS-ABORT site in gsp_phys_backend_guest_gr_submit). Reset at + * the start of each 3D-window patch pass; a nonzero count at the submit + * point means the emitted block still carries a raw client VA the physical + * engine cannot resolve, which is the r1104/r1105 lane-killing class. + * sema_gl_miss_abort=0 restores the historical submit-anyway behaviour. + */ +static u32 gsp_phys_rmcall_gr_3d_miss_n; +static u64 gsp_phys_rmcall_gr_3d_junk_n; +static bool gsp_phys_rmcall_gr_miss_abort = true; +static u64 gsp_phys_gr_miss_abort_n; +module_param_named(sema_gl_miss_abort, gsp_phys_rmcall_gr_miss_abort, bool, + 0644); +MODULE_PARM_DESC(sema_gl_miss_abort, + "Abort a 3D-window launch before the doorbell while a window/attr operand relocation MISSes (page demanded via the pin path; retried on the next forward; fail-open after 3000 aborts). Default 1"); + +/* + * SESSION 153-CONTINUED-73: direct-write witness over the launch's OWN + * relocated operand destinations. The retirement scan only sees stores into + * the data-rec bounce copies; a pair rewritten to a shadow or surface-alias + * VA makes the engine store STRAIGHT into the guest page behind it (the + * 3d_reloc alias comment calls this class "direct guest writes, invisible + * to the witness"). Every such rewrite records its destination guest page + * here (with the source VA for the report); the submit path fingerprints + * the set right before the doorbell and the retirement wait re-fingerprints + * it, so one healthy run answers: do the draw's stores land in the bounce + * (counted today), in guest pages through a relocated operand (this + * witness), or in NEITHER (the draw rasterized nothing and the missing + * pipeline/vertex state is the wall). The MME-family payload rewrites + * (sema_gl_mme_reloc -> mme-reloc-rsv) feed the same set, so the MME + * shadow-scratch destinations are covered too. Static scratch: the only + * caller is the single-inflight ordered gsp-phys-work item. + */ +#define GSP_PHYS_GR_WIT_MAX 4096u +static u64 gsp_phys_gr_wit_pa[GSP_PHYS_GR_WIT_MAX]; +static u64 gsp_phys_gr_wit_va[GSP_PHYS_GR_WIT_MAX]; +static u64 gsp_phys_gr_wit_csum[GSP_PHYS_GR_WIT_MAX]; +static u64 gsp_phys_gr_wit_first8[GSP_PHYS_GR_WIT_MAX]; +static u32 gsp_phys_gr_wit_n; +static u32 gsp_phys_gr_wit_rep_log; + +/* + * CONT-111: the changed-page line budget was a hardcoded 4; a store-rich + * launch (r1378 reached 305/903 relocated-operand pages) needs the whole + * list to find the render target among the operand pages. 0 keeps the + * summary line only. + */ +static u32 gsp_phys_gr_wit_shown = 4; +module_param_named(sema_gl_wit_shown, gsp_phys_gr_wit_shown, uint, 0644); +MODULE_PARM_DESC(sema_gl_wit_shown, + "Max GUEST-WIT changed va/pa lines per report (default 4; CONT-111 RT hunt)"); + +static void gsp_phys_gr_wit_reset(void) +{ + gsp_phys_gr_wit_n = 0; +} + +static void gsp_phys_gr_wit_note(u64 va, u64 hpa) +{ + u32 k; + + if (!hpa) + return; + hpa &= ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + for (k = 0; k < gsp_phys_gr_wit_n; k++) + if (gsp_phys_gr_wit_pa[k] == hpa) + return; + if (gsp_phys_gr_wit_n >= GSP_PHYS_GR_WIT_MAX) + return; + gsp_phys_gr_wit_pa[gsp_phys_gr_wit_n] = hpa; + gsp_phys_gr_wit_va[gsp_phys_gr_wit_n] = + va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + gsp_phys_gr_wit_n++; +} + +static void gsp_phys_gr_wit_capture(const struct gsp_phys_gr_submit *req) +{ + u32 k; + + /* + * Also witness the guest pages behind the launch's tracked data + * records (and the late-resolved gap pages): a store through a + * surface ALIAS VA lands in the surface's own guest HPA - invisible + * to the bounce scan even when the same page is a tracked record - + * so the direct-write pass needs these HPAs in the set. The + * pre-doorbell timing (after every module-side writer) makes the + * baseline the guest content the engine starts from. + */ + for (k = 0; req && k < req->n_data; k++) + gsp_phys_gr_wit_note(req->data[k].va, req->data[k].hpa); + for (k = 0; k < gsp_phys_gr_gap_n; k++) + gsp_phys_gr_wit_note(gsp_phys_gr_gap[k].va, + gsp_phys_gr_gap[k].hpa); + for (k = 0; k < gsp_phys_gr_wit_n; k++) { + const u8 *p = (const u8 *)__va(gsp_phys_gr_wit_pa[k]); + + gsp_phys_gr_wit_first8[k] = p ? + get_unaligned_le64(p) : 0; + gsp_phys_gr_wit_csum[k] = p ? + gsp_phys_gr_fnv1a64(p, GSP_PHYS_PAGE_SIZE) : 0; + } +} + +static u32 gsp_phys_gr_wit_report(const char *where) +{ + u32 k, changed = 0, shown = 0; + u64 first_va = 0, first_pa = 0; + + if (!gsp_phys_gr_wit_n) + return 0; + for (k = 0; k < gsp_phys_gr_wit_n; k++) { + const u8 *p = (const u8 *)__va(gsp_phys_gr_wit_pa[k]); + u64 fp = p ? + gsp_phys_gr_fnv1a64(p, GSP_PHYS_PAGE_SIZE) : 0; + + if (fp != gsp_phys_gr_wit_csum[k]) { + changed++; + if (!first_pa) { + first_pa = gsp_phys_gr_wit_pa[k]; + first_va = gsp_phys_gr_wit_va[k]; + } + /* + * CONT-73 attribution: the pre/post first word of up + * to four changed pages. A QMD/descriptor word or an + * MME-scratch datum names the writer; freshly built + * vertex/float data means the guest CPU rewrote its + * buffer while the engine ran. + */ + if (shown < gsp_phys_gr_wit_shown) { + shown++; + pr_info("vgsp-gr: GUEST-WIT %s changed va=0x%llx pa=0x%llx pre=0x%016llx post=0x%016llx\n", + where, + gsp_phys_gr_wit_va[k], + gsp_phys_gr_wit_pa[k], + gsp_phys_gr_wit_first8[k], + p ? get_unaligned_le64(p) : 0); + } + } + } + /* + * One line per launch while the report budget lasts (a full boot can + * carry hundreds of X-channel launches; the beef rows are the ones + * being chased). + */ + if (gsp_phys_gr_wit_rep_log < 512u) { + gsp_phys_gr_wit_rep_log++; + pr_info("vgsp-gr: GUEST-WIT %s: %u/%u relocated-operand guest page(s) changed (direct shadow/alias stores%s)\n", + where, changed, gsp_phys_gr_wit_n, + changed ? "" : " - NONE (no direct guest writes)"); + if (changed) + pr_info("vgsp-gr: GUEST-WIT %s first changed va=0x%llx pa=0x%llx\n", + where, first_va, first_pa); + } + /* + * CONT-73 settle control: the engine is retired by now, so a second + * scan after a short quiet delay separates engine stores (one-shot, + * will not grow) from LIVE guest-CPU rewrites (still growing). A + * nonzero "late" count here means the changed rows above are guest + * noise, not engine writes. + */ + if (changed && gsp_phys_gr_wit_rep_log <= 128u) { + u32 late = 0; + + msleep(150); + for (k = 0; k < gsp_phys_gr_wit_n; k++) { + const u8 *p = + (const u8 *)__va(gsp_phys_gr_wit_pa[k]); + u64 fp = p ? + gsp_phys_gr_fnv1a64(p, + GSP_PHYS_PAGE_SIZE) : 0; + + if (fp != gsp_phys_gr_wit_csum[k]) + late++; + } + pr_info("vgsp-gr: GUEST-WIT %s settle: %u page(s) changed at +150 ms (was %u)%s\n", + where, late, changed, + late > changed ? + " - GUEST-ACTIVE (growing; noise)" : + " - quiescent (engine-final writes)"); + } + return changed; +} + +/* + * SESSION 153-CONTINUED-90: SPH-alias receipt. The DS I-fetch of a bound + * stage reaches the program-family VA through the shadow VA the relocation + * picked, and the PTE of that shadow VA names the physical page the engine + * reads. Log the mapping choice (launch page set vs trusted resolver), the + * shadow VA and the mapped HPA, then read the fetch words back through the + * host direct map of that very page (for a RAM-backed HPA the direct map is + * content-identical to the engine's PTE-based read), so "which physical page + * does the DS I-fetch actually read" is provable from the log alone. Bounded + * per boot; foreign (non-RAM) HPAs are named but not dereferenced. + */ +static void gsp_phys_gr_sph_alias_receipt(u64 page, u64 va, u64 sh, u64 hpa, + const char *via) +{ + static u32 sph_alias_n; + u32 w[8] = { 0 }; + u32 off = (u32)(va & (GSP_PHYS_PAGE_SIZE - 1u)); + bool rd = false; + + if (sph_alias_n >= 48u) + return; + sph_alias_n++; + if (pfn_valid((unsigned long)(hpa >> PAGE_SHIFT))) { + const u8 *p = (const u8 *)__va(hpa + off); + u32 i; + + for (i = 0; i < 8u; i++) + w[i] = get_unaligned_le32(p + (size_t)i * 4u); + rd = true; + } + pr_info("vgsp-gr: SPH-ALIAS-%s page=0x%llx va=0x%llx off=0x%x shadow=0x%llx hpa=0x%llx rd=%u w8=%08x %08x %08x %08x %08x %08x %08x %08x\n", + via, page, va, off, sh, hpa, rd ? 1u : 0u, + w[0], w[1], w[2], w[3], w[4], w[5], w[6], w[7]); +} + +/* + * SESSION 153-CONTINUED-90: A/B canary for the DS fetch path. When set, + * every program-family operand the window relocates maps a dedicated canary + * page (a deterministic 0xcc00cc00|idx pattern) instead of the resolved + * guest page. The DS error set either changes (the DS does fetch through + * this shadow VA) or stays byte-identical (something else supplies the + * stage start, e.g. the QMD program pointer path). Live-flippable. + */ +static bool gsp_phys_gr_sph_canary; +module_param_named(sema_gl_sph_canary, gsp_phys_gr_sph_canary, bool, 0644); +MODULE_PARM_DESC(sema_gl_sph_canary, + "Map program-family shadow VAs to a canary page (SPH fetch-path A/B)"); +static struct gsp_phys_boot_chunk sph_canary_chunk; +static bool sph_canary_ready; + +static u64 gsp_phys_gr_sph_canary_hpa(void) +{ + if (!sph_canary_ready) { + u8 *p; + u32 i; + + if (!boot_stage.pdev || + gsp_phys_chunk_alloc(boot_stage.pdev, &sph_canary_chunk, 1)) + return 0; + p = gsp_phys_chunk_page_va(&sph_canary_chunk, 0); + if (!p) + return 0; + for (i = 0; i < GSP_PHYS_PAGE_SIZE / 4u; i++) + ((u32 *)p)[i] = 0xcc00cc00u | (i & 0xffu); + sph_canary_ready = true; + } + return (u64)gsp_phys_chunk_page_dma(&sph_canary_chunk, 0); +} + +/* + * SESSION 153-CONTINUED-93: 8K program-pair shadow line (mission task 2 + * fallback). The DS validates a bound program through its fetched SPH; a + * per-page shadow line maps exactly one 4 KB page at PTE0 and leaves the + * line's PTE1 invalid, so any read past the first 4 KB of the program page + * (the SASS tail of a bind whose delivered length crosses 0x1000, or any + * DS full-program walk) leaves the authored bytes behind. Map the program + * family page (PTE0) and family+0x1000 (PTE1) into ONE fresh 128 B fetch + * line BEFORE the engine can fetch it (a line's contents are frozen once + * fetched), so a contiguous 8 KB is engine-visible from its first window + * on. The line is cached per hpa0 (pair cache) so every later window reuses + * the identical never-rewritten mapping; the alias cache is re-pointed at + * the pair line so every other consumer (program binds, code-fetch + * operands) sees the same 8 KB view. Live A/B via sema_gl_sph_8k. Called + * with gsp_phys_rm_lock held. + */ +static bool gsp_phys_rmcall_gr_sph_8k; +module_param_named(sema_gl_sph_8k, gsp_phys_rmcall_gr_sph_8k, bool, 0644); +MODULE_PARM_DESC(sema_gl_sph_8k, + "Map the program family page and family+0x1000 in one contiguous 8K shadow line (fresh fetch line; default 0)"); +#define GSP_PHYS_RMCALL_GR_PAIR_CACHE 64u +static u64 gsp_phys_gr_pair_hpa[GSP_PHYS_RMCALL_GR_PAIR_CACHE]; +static u64 gsp_phys_gr_pair_va[GSP_PHYS_RMCALL_GR_PAIR_CACHE]; +static u32 gsp_phys_gr_pair_n; + +/* + * Resolve the HPA of `page0 + 4K` through the same sources the relocation + * chain uses: the launch's own page set (the pages the copy-in authored), + * the tracked data recs, the sema service's trusted resolver, then the + * late-resolved gap pages. 0 = not resolvable. + */ +static u64 gsp_phys_rmcall_gr_prog_next_hpa_locked( + const struct gsp_phys_gr_submit *req, u64 page0) +{ + u64 page1 = page0 + GSP_PHYS_PAGE_SIZE; + u32 i; + + for (i = 0; i < req->n_pages; i++) + if ((req->pages[i].guest_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) == page1 && + req->pages[i].hpa) + return req->pages[i].hpa; + for (i = 0; i < req->n_data; i++) + if ((req->data[i].va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) == page1 && + req->data[i].hpa) + return req->data[i].hpa; + { + struct gsp_sema_gr_reloc r; + + if (!gsp_sema_gr_reloc_resolve(page1, &r) && r.hpa) + return r.hpa; + } + for (i = 0; i < gsp_phys_gr_gap_n; i++) + if ((gsp_phys_gr_gap[i].va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) == page1 && + gsp_phys_gr_gap[i].hpa) + return gsp_phys_gr_gap[i].hpa; + return 0; +} + +static void gsp_phys_rmcall_gr_alias_cache_set_locked(u64 hpa, u64 va) +{ + u32 i; + + for (i = 0; i < boot_stage.rmcall_gr_alias_count; i++) + if (boot_stage.rmcall_gr_alias_hpa[i] == hpa) { + boot_stage.rmcall_gr_alias_va[i] = va; + return; + } + if (boot_stage.rmcall_gr_alias_count < GSP_PHYS_RMCALL_GR_ALIAS_CACHE) { + u32 ci = boot_stage.rmcall_gr_alias_count; + + boot_stage.rmcall_gr_alias_hpa[ci] = hpa; + boot_stage.rmcall_gr_alias_va[ci] = va; + boot_stage.rmcall_gr_alias_count = ci + 1u; + } +} + +/* + * Allocate (or reuse) the pair line. Returns the line base VA on success, + * 0 when the second half cannot be resolved or the pool is exhausted (the + * caller then falls through to the single-page path). The fresh line is + * allocated on first use for an hpa0; the pair cache reuses it afterwards + * (the PTEs are identical on every window by construction). + */ +static u64 gsp_phys_rmcall_gr_sph_pair_locked( + const struct gsp_phys_gr_submit *req, u64 page0, u64 hpa0, u32 slot) +{ + struct gsp_phys_boot_chunk *sh = &boot_stage.rmcall_gr_shadow; + u32 i, line, page, line_in_page; + u64 hpa1, va0; + u8 *pt; + int ret; + + for (i = 0; i < gsp_phys_gr_pair_n; i++) + if (gsp_phys_gr_pair_hpa[i] == hpa0) + return gsp_phys_gr_pair_va[i]; + hpa1 = gsp_phys_rmcall_gr_prog_next_hpa_locked(req, page0); + if (!hpa1) + return 0; + if (!sh->pages) { + ret = gsp_phys_chunk_alloc(boot_stage.pdev, sh, + GSP_PHYS_RMCALL_GR_SHADOW_PAGES); + if (ret) + return 0; + gsp_phys_chunk_zero(sh); + } + if (slot >= sh->pages || !boot_stage.rmcall_va_guest_pt) + return 0; + line = boot_stage.rmcall_guest_pt_next_line; + if (line >= GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES) + return 0; + page = line / GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; + line_in_page = line % GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; + if (gsp_phys_rmcall_gr_scratch_win_install_locked(page)) + return 0; + pt = gsp_phys_rmcall_guest_pt_page_va(page); + if (!pt) + return 0; + put_unaligned_le64(gsp_phys_rmcall_gr_pte_leaf(hpa0), + pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * + line_in_page) * 8); + put_unaligned_le64(gsp_phys_rmcall_gr_pte_leaf(hpa1), + pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * + line_in_page + 1u) * 8); + dma_wmb(); + boot_stage.rmcall_guest_pt_next_line = line + 1u; + va0 = gsp_phys_rmcall_guest_pt_win_va(page) + + ((u64)line_in_page << 16); + gsp_phys_rmcall_gr_alias_cache_set_locked(hpa0, va0); + gsp_phys_rmcall_gr_alias_cache_set_locked(hpa1, + va0 + GSP_PHYS_PAGE_SIZE); + if (gsp_phys_gr_pair_n < GSP_PHYS_RMCALL_GR_PAIR_CACHE) { + gsp_phys_gr_pair_hpa[gsp_phys_gr_pair_n] = hpa0; + gsp_phys_gr_pair_va[gsp_phys_gr_pair_n] = va0; + gsp_phys_gr_pair_n++; + } + pr_info("vgsp-gr: SPH-PAIR line=0x%llx hpa0=0x%llx hpa1=0x%llx (8K contiguous program page) h=0x%x\n", + va0, hpa0, hpa1, req->h_object); + return va0; +} + +/* + * SESSION 153-CONTINUED-92: identity-fetch keep test (see the + * sema_gl_prog_guest_va knob above). Resolve the operand's page (the + * launch's own page set first - the pages the copy-in authored - then the + * sema service's trusted resolver), install the Tier-1 identity PTE at the + * guest VA itself and report whether the operand may stay verbatim. The + * receipt names the exact VA the DS will now fetch through (the guest VA, + * shadow column = the guest VA) and reads the fetch words back through the + * host direct map of the mapped HPA, so "the DS reads the authored page at + * its own heap VA" is provable from the log alone. Anything the identity + * mapper refuses (no window slot, stale fetch line, unresolved HPA) returns + * false and the caller falls back to the shadow rewrite. Called with + * gsp_phys_rm_lock held. + */ +static bool gsp_phys_rmcall_gr_prog_keep_locked(void __iomem *bar0, + u32 epoch, u32 up_w, u32 lo_w, + const struct gsp_phys_gr_submit *req) +{ + u64 va = ((u64)(up_w & 0xffu) << 32) | (u64)(lo_w & 0xfffffffcu); + u64 page = va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + u64 hpa = 0; + u32 i; + + if (!va || !page) + return false; + for (i = 0; i < req->n_pages; i++) { + if ((req->pages[i].guest_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) == page && + req->pages[i].hpa) { + hpa = req->pages[i].hpa; + break; + } + } + if (!hpa) { + struct gsp_sema_gr_reloc r; + + if (!gsp_sema_gr_reloc_resolve(page, &r) && r.hpa) + hpa = r.hpa; + } + if (!hpa) + return false; + if (gsp_phys_rmcall_gr_map_va_locked(bar0, page, hpa, epoch)) + return false; + gsp_phys_gr_sph_alias_receipt(page, va, va, hpa, "prog-guest"); + return true; +} + +static bool gsp_phys_rmcall_gr_3d_reloc(u32 *up_w, u32 *lo_w, + const struct gsp_phys_gr_submit *req, + u32 *n_shadow) +{ + u64 va = ((u64)(*up_w & 0xffu) << 32) | (u64)(*lo_w & 0xfffffffcu); + u64 page = va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + u64 sh = 0; + u32 line = 0, i; + + if (!va || !page) + return false; + /* + * SESSION 153-CONTINUED-90: must run before every resolution stage so + * the canary cannot be shadowed by an earlier (surd) alias hit. + */ + if (READ_ONCE(gsp_phys_gr_sph_canary) && req->prog_va && + page == (req->prog_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1))) { + u64 chpa = gsp_phys_gr_sph_canary_hpa(); + u32 line3 = 0; + + if (chpa && + *n_shadow < GSP_PHYS_RMCALL_GR_SHADOW_PAGES && + !gsp_phys_rmcall_gr_map_shadow_locked(*n_shadow, chpa, + &sh, &line3)) { + if (line3 != 0xffffffffu) + (*n_shadow)++; + sh |= va & (GSP_PHYS_PAGE_SIZE - 1); + gsp_phys_gr_sph_alias_receipt(page, va, sh, chpa, + "canary"); + *lo_w = (*lo_w & 0x3u) | ((u32)sh & 0xfffffffcu); + *up_w = (*up_w & ~0xffu) | + ((u32)(sh >> 32) & 0xffu); + return true; + } + } + /* + * SESSION 153-CONTINUED-63: rec-over-alias preference. When the fill + * tracked the render target through the bounce lane (every page + * resolved into contiguous data recs), rewrite the operand to the + * rec's bounce VA so the engine's stores land where the retire wait + * can count them and the copy-out can ship them back to the guest's + * own pages. The completeness gate (gsp_sema_gr_surf_recs) makes a + * partial run impossible to point at. Runs before the alias so the + * identity alias (direct guest writes, invisible to the witness) + * cannot win for a tracked target. + */ + { + u64 sva = 0; + u32 sn = 0; + + if (gsp_sema_gr_surf_recs(&sva, &sn) && + va >= sva && va < sva + ((u64)sn << 12)) { + for (i = 0; i < req->n_data; i++) { + if (req->data[i].va != page || + !req->data[i].reloc_va) + continue; + sh = req->data[i].reloc_va | + (va & (GSP_PHYS_PAGE_SIZE - 1)); + *lo_w = (*lo_w & 0x3u) | + ((u32)sh & 0xfffffffcu); + *up_w = (*up_w & ~0xffu) | + ((u32)(sh >> 32) & 0xffu); + return true; + } + } + } + /* + * SESSION 153-CONTINUED-27: a big surface (the X screen) is aliased + * as ONE contiguous run by the pre-pass; rewrite the operand to the + * alias base | offset. A one-page shadow relocation of it would + * break the raster walk at the first page crossing (r992), and the + * raw VA has no PTE chain the GPC engines can see (r993). + */ + for (i = 0; i < gsp_phys_gr_surf_n; i++) { + if (va >= gsp_phys_gr_surf[i].va && + va < gsp_phys_gr_surf[i].va + + (gsp_phys_gr_surf[i].npages << 12)) { + /* + * CONT-73: the alias VA maps the surface's guest + * HPAs 1:1 (the alias PTEs were built from + * req->pages[] first, the trusted resolver else), so + * a store through it lands straight in that guest + * page - record it for the direct-write witness. + */ + { + u32 pi; + + for (pi = 0; pi < req->n_pages; pi++) { + if ((req->pages[pi].guest_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) != + page || + !req->pages[pi].hpa) + continue; + gsp_phys_gr_wit_note(page, + req->pages[pi].hpa); + break; + } + if (pi == req->n_pages) { + struct gsp_sema_gr_reloc ra; + + if (!gsp_sema_gr_reloc_resolve(page, + &ra) && + ra.hpa) + gsp_phys_gr_wit_note(page, + ra.hpa); + } + } + sh = gsp_phys_gr_surf[i].alias + + (va - gsp_phys_gr_surf[i].va); + *lo_w = (*lo_w & 0x3u) | ((u32)sh & 0xfffffffcu); + *up_w = (*up_w & ~0xffu) | + ((u32)(sh >> 32) & 0xffu); + return true; + } + } + for (i = 0; i < req->n_pages; i++) { + if ((req->pages[i].guest_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) != page || + !req->pages[i].hpa) + continue; + if (*n_shadow >= GSP_PHYS_RMCALL_GR_SHADOW_PAGES) + return false; + /* + * SESSION 153-CONTINUED-93: 8K program-pair line (see the + * pair helper). Only the program family page rides this + * path; an unresolvable second half falls through to the + * single-page map below. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_sph_8k) && req->prog_va && + page == (req->prog_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1))) { + u64 pv = gsp_phys_rmcall_gr_sph_pair_locked( + req, page, req->pages[i].hpa, *n_shadow); + + if (pv) { + (*n_shadow)++; + sh = pv | (va & + (GSP_PHYS_PAGE_SIZE - 1)); + gsp_phys_gr_wit_note(page, req->pages[i].hpa); + gsp_phys_gr_sph_alias_receipt(page, va, pv, + req->pages[i].hpa, "pair"); + *lo_w = (*lo_w & 0x3u) | + ((u32)sh & 0xfffffffcu); + *up_w = (*up_w & ~0xffu) | + ((u32)(sh >> 32) & 0xffu); + return true; + } + } + if (gsp_phys_rmcall_gr_map_shadow_locked(*n_shadow, + req->pages[i].hpa, + &sh, &line)) + return false; + if (line != 0xffffffffu) + (*n_shadow)++; /* alias hit consumes no slot */ + /* CONT-73: shadow VA maps this guest HPA 1:1. */ + gsp_phys_gr_wit_note(page, req->pages[i].hpa); + sh |= va & (GSP_PHYS_PAGE_SIZE - 1); + if (req->prog_va && + page == (req->prog_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1))) + gsp_phys_gr_sph_alias_receipt(page, va, sh, + req->pages[i].hpa, + "pages"); + *lo_w = (*lo_w & 0x3u) | ((u32)sh & 0xfffffffcu); + *up_w = (*up_w & ~0xffu) | ((u32)(sh >> 32) & 0xffu); + return true; + } + /* + * SESSION 153-CONTINUED-59: level-2 late resolution, the same trusted + * chain the MME payload branch uses (ipin/pin/lat/map/walk then shadow + * map). The r1104 first beat window MISSed the program pair 0x2054 + * (va=0x4350900) and the r1105 windows MISSed the index-buffer pairs + * (0x17c8, va=0x44fedf0...) whose pages were not in the launch page + * set at build time; the raw guest VA then let the engine fault the + * MMU (HUBCLIENT_PD) and the guest driver reset the channel. Resolve + * the page here if any stage knows it, so a MISS only survives when + * every source fails; a surviving MISS is recorded for the pin demand + * set and counts toward the pre-doorbell abort. + */ + { + struct gsp_sema_gr_reloc r; + u32 line2 = 0; + + if (!gsp_sema_gr_reloc_resolve(va, &r) && r.hpa && + *n_shadow < GSP_PHYS_RMCALL_GR_SHADOW_PAGES && + !gsp_phys_rmcall_gr_map_shadow_locked(*n_shadow, r.hpa, + &sh, &line2)) { + if (line2 != 0xffffffffu) + (*n_shadow)++; + /* CONT-73: shadow VA maps this guest HPA 1:1. */ + gsp_phys_gr_wit_note(page, r.hpa); + sh |= va & (GSP_PHYS_PAGE_SIZE - 1); + if (req->prog_va && + page == + (req->prog_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1))) + gsp_phys_gr_sph_alias_receipt(page, va, sh, + r.hpa, + "resolve"); + *lo_w = (*lo_w & 0x3u) | ((u32)sh & 0xfffffffcu); + *up_w = (*up_w & ~0xffu) | ((u32)(sh >> 32) & 0xffu); + return true; + } + } + /* + * SESSION 153-CONTINUED-61: junk-operand filter. The window + * recognizer is byte-shape based, so float/immediate DATA words in + * the method stream also parse as address pairs. The observed junk + * class sits in the 256 MB..4 GB numeric range (0x3d000000..0xc2...... + * float bit patterns: 0x3f800000 = 1.0f, 0xbf800000 = -1.0f, ...) + * and can NEVER resolve - r1109 measured miss=20 constant across + * every attempt, all of it this class, which pinned the abort census + * nonzero forever and blocked every submit. Real client VAs of this + * driver live below 0x5000000 (the 64-80 MB GL/heap range, e.g. + * 0x4250000/0x4350000/0x462fc40) or at/above 0x1_00000000 (the 4G+ + * push-buffer/UVM ranges). A miss outside both is a data-word + * misidentification, not a raw client VA the engine would read as an + * address: it is neither demanded (the demand set was 60 % junk by + * r1109, diverting the pin cycle) nor counted toward the pre-doorbell + * abort. Witnessed per class, bounded. + */ + if (page >= 0x10000000ULL && page < 0x100000000ULL) { + static u32 gr_missjunk_log; + + gsp_phys_rmcall_gr_3d_junk_n++; + if (gr_missjunk_log < 24 || + !(gsp_phys_rmcall_gr_3d_junk_n % 512u)) { + gr_missjunk_log++; + pr_info("vgsp-gr: MISS-JUNK va=0x%llx (data-word class; not demanded, not counted toward the abort) n=%llu\n", + page, (u64)gsp_phys_rmcall_gr_3d_junk_n); + } + return false; + } + /* + * Unresolvable: demand the page for the pin path and count the miss + * (the submit path aborts pre-doorbell while the count is nonzero). + */ + gsp_phys_rmcall_gr_3d_miss_n++; + gsp_sema_demand_note(page); + return false; +} + +/* + * r929 v3: copy the 3D window and relocate every address operand a 3D-class + * method run carries (same recognizers as the sema service's svc_gl_field: + * LINE_LENGTH_IN 0x180 OFFSET_OUT pair, SET_VERTEX_STREAM_SUBSTITUTE 0xf84, + * SET_TEX_SAMPLER_POOL 0x155c, SET_TEX_HEADER_POOL 0x1574 and the + * SET_PIPELINE_PROGRAM_ADDRESS 0x2014/0x2018+j*0x40 pairs). Returns the + * number of relocated pair(s). Run widths follow the service's decoder: + * op 1/3/5 are 1+count dwords, everything else resyncs one dword. + */ +/* + * SESSION 153-CONTINUED-13 (draw-window strip experiment): comma-separated + * hex NVC597 method byte-addresses to REMOVE from a captured 3D window + * before the verbatim replay (whole methods - a dropped run takes its + * count data words with it). Answers whether a DRAW window's non-retire + * lives in the draw content (BEGIN/END/immediate vertices/DRAW_ARRAY) or + * in the state binds/method-stream shape. Empty = off (historical + * behaviour); live-tunable via sysfs so one boot can sweep the variants. + */ +/* + * SESSION 153-CONTINUED-47 (A/B knob): gates the newly recognised pair + * classes (SET_INDEX_BUFFER_A/B 0x17c8 and the vertex-stream banks + * 0x1c20..0x1cf0) in the verbatim-replay relocation pass. Default 0 keeps + * the r1035-r1041 baseline relocation set exactly; flipping it live + * (`echo Y > /sys/module/vgsp/parameters/sema_gl_idx_reloc`) rewrites the + * next launches' index/vertex operands to shadow VAs, so one boot can A/B + * the effect instead of comparing across boots. + */ +static bool gsp_phys_rmcall_gr_idx_reloc; +module_param_named(sema_gl_idx_reloc, gsp_phys_rmcall_gr_idx_reloc, bool, 0644); +MODULE_PARM_DESC(sema_gl_idx_reloc, + "Relocate SET_INDEX_BUFFER (0x17c8) and vertex-stream banks 0x1c20..0x1cf0 operands to shadow VAs in 3D replays (default 0 = baseline set only)"); + +/* + * SESSION 153-CONTINUED-48: MME-family payload relocation. The C597 class + * carries CALL_MME_MACRO(j) = 0x3800+j*8 and CALL_MME_DATA(j) = 0x3804+j*8 + * (class/clc597.h); the GLX client's macro(2) run (observed at byte + * address 0x3810, cnt=5) inlines raw 32-bit client VAs as macro arguments + * (`... 0450f800 ... 0450f000` in the first beef window of r1042/r1043). + * Those words are the last raw-operand class left in the faulting window + * after the index/vertex relocation: the engine walks the macro data + * through its own fetch path, and a raw client VA has no PTE chain the + * GPC engines can see (the r993/r1043 FAULT_PDE->PRI-fault class). The + * pass rewrites any payload word of a run whose byte address lies in + * [0x3800, 0x3a00) through the same vetted shadow mapping the pair + * classes use: only a word whose page the launch's own page set (or the + * big-surface alias) resolves is rewritten (MISS stays verbatim), the + * rewrite maps the SAME HPA at a shadow VA so the value is + * content-equivalent, and a shadow VA that would need an upper byte (a + * 64-bit alias in a 32-bit slot) is refused. Default 0 (baseline windows + * byte-identical); live-tunable so one boot can A/B. + */ +static bool gsp_phys_rmcall_gr_mme_reloc; +module_param_named(sema_gl_mme_reloc, gsp_phys_rmcall_gr_mme_reloc, bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_reloc, + "Relocate CALL_MME_MACRO/DATA(j) run payload words whose page resolves in the launch page set to shadow VAs in 3D replays (default 0 = off)"); + +/* + * SESSION 153-CONTINUED-58: torn-read zero guard for the vertex-attribute + * families. The r1098/r1099 first-launch class error + * (`Class Error: ChId 0001, Class 0000c597, Offset 00001168, + * Data 00000000, ErrorCode 00000005` = SET_VERTEX_ATTRIBUTE_A(2) written + * with NUMERICAL_TYPE=0, the header's UNUSED_ENUM_DO_NOT_USE value -> + * INVALID_ENUM) is written by a replayed window run + * `20030458 38400000 38200040 00000000`: a cnt=3 INC run at method 0x1160 + * whose THIRD data word is exactly zero and is immediately followed by a + * 52-word zero region - the read caught the client's push buffer after + * part of it had been reused/cleared. The same client writes valid A(2) + * values in the other windows of the same push (0x38200040/0x38200600/ + * 0x38400600), so the zero is a torn read, not client content; and + * NUMERICAL_TYPE=0 is never a legitimate client write. Guard: for INC + * runs in the SET_VERTEX_ATTRIBUTE_A/B families (0x1160..0x11dc) drop a + * data word that is exactly zero, splitting the run so the remaining + * words keep their methods (the attribute then keeps the value the + * ATTR-PRELUDE or the previous frame wrote). Default 1; live-tunable. + */ +static u32 gsp_phys_rmcall_gr_attr_zero_fix = 1; +module_param_named(sema_gl_attr_zero_fix, gsp_phys_rmcall_gr_attr_zero_fix, uint, 0644); +MODULE_PARM_DESC(sema_gl_attr_zero_fix, + "Drop torn-read zero data words in the SET_VERTEX_ATTRIBUTE_A/B runs (0x1160..0x11dc) of 3D replays, splitting the run (default 1)"); + +/* (SESSION 153-CONTINUED-53 MME prelude knob is declared before + * gsp_phys_rmcall_build_gr_launch, its reader.) */ + +/* + * SESSION 153-CONTINUED-48: ladder-wedge recovery. CONT-47 measured that + * the FIRST beef launch per boot whose window carries an unresolvable raw + * operand PRI-faults the GR: every SKED/FIFO register then reads the + * priv-error poison (cp. kernel_falcon_ga102.c `privErrVal = 0xbadf0000`) + * and the ladder channel stays wedged for the whole boot (engine GP_GET + * stuck; every later publish unconsumed, even an emptied window) - which + * makes content bisection a one-shot per boot. When enabled, the launch + * poll watches the GR FIFO status (0x400504, the sked recorder's + * `grfifo`) for the 0xbadf-tagged poison and performs ONE bounded + * recovery per launch: the r445 hard re-arm (TSG PREEMPT bWait=1 + the + * m155 lane-enable tail + GPFIFO/TSG schedules + doorbell) followed by + * the warm GR semaphore probe, exactly the mechanism behind + * gsp_phys_gr_wedge_rearm. Bounded: one detection per launch, poison + * gated, all RPCs keep their own timeouts. Default 1 (the poisoned state + * is already a guaranteed-dead boot; the recovery only adds the chance + * that later launches in the same boot still execute). + */ +static bool gsp_phys_gr_wedge_recover = true; +module_param_named(sema_gr_wedge_recover, gsp_phys_gr_wedge_recover, bool, + 0644); +MODULE_PARM_DESC(sema_gr_wedge_recover, + "On the 0xbadf priv-error poison in the GR FIFO status, run one bounded TSG-preempt+resched hard re-arm per launch (default 1)"); + +/* + * SESSION 153-CONTINUED-93: parked-engine recovery on the warm-sem ladder + * gate. The r1311-r1313 class: after the first few DS-validated draws the + * front-end stops fetching (grfifo=0x00000001, no poison, no trap) and the + * warm GR semaphore probe times out on every later launch - the ladder + * then fails fast and the whole boot is dead for bisection. When this knob + * is set, a wedged probe triggers ONE bounded hard re-arm (the proven + * r445 TSG-preempt bWait=1 + resched + probe) before giving up; an ALIVE + * probe after the re-arm lets the fresh-entry retries resume, and the MME + * prelude latch is cleared so the re-preload is owed (the preempt reset + * the context). Bounded: one per launch; all RPCs keep their timeouts. + * Default 1; A/B with 0 restores the fail-fast ladder. + */ +static bool gsp_phys_gr_park_recover = true; +module_param_named(sema_gr_park_recover, gsp_phys_gr_park_recover, bool, + 0644); +MODULE_PARM_DESC(sema_gr_park_recover, + "On a wedged warm-sem probe in the launch timeout ladder, run one bounded TSG-preempt+resched hard re-arm per launch and retry (default 1)"); + +static char *gsp_phys_rmcall_gr_win_drop; +module_param_named(sema_gl_win_drop, gsp_phys_rmcall_gr_win_drop, charp, 0644); +MODULE_PARM_DESC(sema_gl_win_drop, + "Hex method byte-addresses (comma-separated) to strip from a captured 3D window before replay (draw isolation; default empty)"); + +static bool gsp_phys_rmcall_gr_win_drop_match(u32 addr) +{ + const char *p = gsp_phys_rmcall_gr_win_drop; + u32 n = 0; + + if (!p || !*p) + return false; + while (*p && n < 64) { + u32 v = 0; + u32 digits = 0; + + while (*p == ' ' || *p == ',' || *p == ':' || *p == ';') + p++; + /* + * SESSION 153-CONTINUED-46: accept an optional 0x/0X prefix. + * The original parser scanned hex digits only, so the natural + * "0x3810,0x39c8" input parsed as 0 (the 'x' ended the digit + * run after the leading '0') and silently never matched - + * cost a full bisect round. + */ + if (p[0] == '0' && (p[1] == 'x' || p[1] == 'X')) + p += 2; + for (;;) { + char c = *p; + u32 d; + + if (c >= '0' && c <= '9') + d = c - '0'; + else if (c >= 'a' && c <= 'f') + d = c - 'a' + 10; + else if (c >= 'A' && c <= 'F') + d = c - 'A' + 10; + else + break; + if (v <= 0xfffffffu) + v = (v << 4) | d; + p++; + digits++; + } + if (!digits) + break; + /* + * SESSION 153-CONTINUED-47: `lo-hi` drops every method + * byte-address in the closed range (both ends accept the + * 0x prefix). The content bisect needs whole families + * (`0x3800-0x3a00` MME, `0x3400-0x3500` MME scratch) and + * the emptied-window control (`0-0x7ffc`): with the window + * stripped whole, a launch that still retires proves the + * SET_OBJECT + release tail path is healthy and that the + * wedge lives in the stripped content. + */ + if (*p == '-') { + const char *q = p + 1; + u32 hi = 0; + u32 hd = 0; + + if (q[0] == '0' && (q[1] == 'x' || q[1] == 'X')) + q += 2; + for (;;) { + char c = *q; + u32 h; + + if (c >= '0' && c <= '9') + h = c - '0'; + else if (c >= 'a' && c <= 'f') + h = c - 'a' + 10; + else if (c >= 'A' && c <= 'F') + h = c - 'A' + 10; + else + break; + if (hi <= 0xfffffffu) + hi = (hi << 4) | h; + q++; + hd++; + } + if (hd) { + p = q; + n++; + if (addr >= v && addr <= hi) + return true; + continue; + } + } + n++; + if (v == addr) + return true; + } + return false; +} + +/* + * SESSION 153-CONTINUED-27: auto-drop of the software-emulated lanes from + * every verbatim 3D replay. Three method triggers in a captured window + * cause the physical engine to repeat work the sema service already + * executes in software - and, on the replay channel, with the WRONG + * context state (no previous-window sticky registers): + * - 0x1b0 LAUNCH_DMA (C597 CE lane): triggers both the CE copy and the + * LOAD_INLINE_DATA upload. The service authors every LID payload into + * the guest pages itself (svc_gl_lid/svc_grce_inline_write) and + * emulates the CE strips (svc_op2d); physically re-running them with + * an unset source/context walks unbounded memory (r992-r996: the + * "march" - faults at whatever page the alias cap ended, e.g. + * 0x5d6000/0x5f4000, after having written every mapped page). + * - 0x5e0 (TWOD RECTS fill trigger): the second point pair of the + * client's screen clear. The service's FERMI_TWOD executor runs every + * fill (r986: 46x `sema-twod-fill ... done=3686400 rc=0`); the + * physical re-run uses the replay channel's unset dst PITCH and + * marches (same fault class). + * - 0x10c NOTIFY: the completion write; without the preceding + * SET_NOTIFY_A/B (which live in other pushes) the engine writes + * address 0 (r996: `faulted @ 0x0_00000000 FAULT_PTE VIRT_WRITE`). + * The kick-notify lane + the backend's release tail author every + * completion the client actually waits on. + * All three are pure triggers - their setup methods stay in the window, + * so dropping the run can never misalign display state. The emulated + * lanes are the live ones (the r986 pixel proof: the screen paints via + * svc_twod fills, and every CE strip retires rc=0 in software). Default + * on; set 0 to replay the raw window unmodified. + */ +static bool gsp_phys_gr_win_drop_auto = 1; +module_param_named(sema_gl_win_drop_auto, gsp_phys_gr_win_drop_auto, bool, + 0644); +MODULE_PARM_DESC(sema_gl_win_drop_auto, + "Drop the software-emulated trigger methods (0x10c NOTIFY, 0x1b0 LAUNCH_DMA, 0x5e0 TWOD fill) from verbatim 3D replays (default 1)"); + +static bool gsp_phys_rmcall_gr_win_drop_auto_match(u32 addr) +{ + if (!READ_ONCE(gsp_phys_gr_win_drop_auto)) + return false; + /* + * SESSION 153-CONTINUED-48: NVC597_SET_REPORT_SEMAPHORE_A=0x1b00 - + * the C597 twin of the 0x10c NOTIFY trigger. Every glcore draw + * window ends with 4-word runs [A=upper(7:0), B=lower(31:0), + * C=payload, D=flags] whose B word is a raw client VA (observed + * 0x41a5bb0/0x759e870/0x251800; payloads = frame counters 0xb4f3+). + * The replayed engine executing them writes a client VA the GPC + * engines cannot reach - the VIRT_WRITE fault class (r1061: the + * kept [0,0x2000) window whose only fetch/write triggers besides + * the index binds are these raw 0x1b00 writes is exactly the one + * that still faulted with grfifo=0xbadf1201; r1062: the full window + * stalls the front-end right after the PBDMA consumes it and + * fecs_cur never loads). Their page cannot join the launch page set + * (n_ptr is capped at 7 by the QMD pointer region and the draw's + * program/index/vertex/MME pages already fill it), so relocation is + * impossible; dropping is the same call the 0x10c NOTIFY drop + * already makes: the module appends its own SET_REPORT_SEMAPHORE + * tail and the host-side completion lanes author every completion + * the client actually waits on (the r1061 keep-prefix window + * retired with ch_rel=0xc0ffee02 and the client kept pushing + * frames without any 0x1b00 in the replay). + */ + return addr == 0x010cu || addr == 0x01b0u || addr == 0x05e0u || + addr == 0x1b00u; +} + +/* + * SESSION 153-CONTINUED-14 (FECS residency / draw-stall isolation): patch + * the DATA of one method inside a replayed 3D window before it reaches the + * engine. Format: comma-separated `addr:data` hex pairs, addr = method + * byte-address (e.g. 1618 = BEGIN, 1614 = END), data = the replacement + * value. Header-only methods (IMMD) get their 13-bit data field rewritten + * in place; count-carrying methods (INC/NON_INC/INC_ONCE) get their FIRST + * payload word replaced. Live-tunable via sysfs so one boot can sweep + * variants (BEGIN OP=7 QUADS -> TRIANGLES etc.) against the draw-stall. + */ +static char *gsp_phys_rmcall_gr_win_patch; +module_param_named(sema_gl_win_patch, gsp_phys_rmcall_gr_win_patch, charp, + 0644); +MODULE_PARM_DESC(sema_gl_win_patch, + "Hex addr:data pairs (comma-separated) patched into a captured 3D window method before replay (default empty)"); + +static bool gsp_phys_rmcall_gr_win_patch_find(u32 addr, u32 *out_data) +{ + const char *p = gsp_phys_rmcall_gr_win_patch; + u32 n = 0; + + if (!p || !*p) + return false; + while (*p && n < 64) { + u32 v = 0, d = 0; + u32 digits = 0; + bool have_data = false; + + while (*p == ' ' || *p == ',' || *p == ':' || *p == ';') + p++; + for (;;) { + char c = *p; + u32 h; + + if (c >= '0' && c <= '9') + h = c - '0'; + else if (c >= 'a' && c <= 'f') + h = c - 'a' + 10; + else if (c >= 'A' && c <= 'F') + h = c - 'A' + 10; + else + break; + if (v <= 0xfffffffu) + v = (v << 4) | h; + p++; + digits++; + } + if (!digits) + break; + if (*p == ':' || *p == '=') { + p++; + for (;;) { + char c = *p; + u32 h; + + if (c >= '0' && c <= '9') + h = c - '0'; + else if (c >= 'a' && c <= 'f') + h = c - 'a' + 10; + else if (c >= 'A' && c <= 'F') + h = c - 'A' + 10; + else + break; + if (d <= 0xfffffffu) + d = (d << 4) | h; + p++; + have_data = true; + } + } + n++; + if (have_data && v == addr) { + *out_data = d; + return true; + } + } + return false; +} + +static u32 gsp_phys_rmcall_gr_3d_last_subdrop; +static u32 gsp_phys_rmcall_gr_3d_last_sdmdrop; + +/* + * SESSION 153-CONTINUED-62: an MME-family payload word that neither the + * launch page set nor the level-2 resolver could translate has to stay + * verbatim in the emitted stream (never guessed) - but a raw client VA in + * an 0x3810 macro payload is exactly the operand the engine faults on + * (`MMU Fault HUBCLIENT_PD`). Until this helper the word was left silent: + * the abort census stayed 0 and the launch went out to fault seconds + * later (r1115: fault @0x0_04350000, the 0x4350900 program-page payload). + * Now the surviving miss counts toward the pre-doorbell MISS-ABORT and + * joins the pin demand set, so the launch is held (fail-closed, bounded) + * until the page resolves, exactly like a window pair miss. The float/ + * data-word class (0x10000000..0x100000000) is excluded - it can never + * resolve and must not pin the census nonzero (the r1109 lesson). + */ +static void gsp_phys_gr_3d_mme_miss(u32 wv) +{ + u64 page = (u64)wv & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + + if (page >= 0x10000000ULL && page < 0x100000000ULL) + return; + gsp_phys_rmcall_gr_3d_miss_n++; + gsp_sema_demand_note(page); +} + + +static u32 gsp_phys_rmcall_gr_3d_last_progkeep; +static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out, + u32 *out_n, u32 *n_dropped, + const struct gsp_phys_gr_submit *req, + u32 *n_shadow, + void __iomem *bar0, u32 epoch) +{ + u32 off = 0, wout = 0, reloc = 0, dropped = 0, subdrop = 0, sdm_drop = 0; + u32 wp_match = 0, wp_lowrun = 0, prog_keep = 0; + + *out_n = n; + *n_dropped = 0; + while (off < n) { + u32 hdr = in[off]; + u32 op = (hdr >> 29) & 0x7u; + u32 cnt = (hdr >> 16) & 0xfffu; + u32 addr = (hdr & 0x1fffu) << 2; + u32 sub = (hdr >> 13) & 0x7u; + u32 span = 1u; + + if ((op == 1 || op == 3 || op == 5) && cnt && + off + 1u + cnt <= n) + span = 1u + cnt; + /* + * SESSION 153-CONTINUED-52: drop METHOD WRITES aimed at a + * non-zero subchannel. The glcore draw window ends with + * `2001c00d 70000000/68000000` = INC m=0x34 cnt=1 SUBCHANNEL 6 + * (channel-class MEM_OP_D lane) two-word runs. The ladder + * channel has nothing bound on subchannel 6, so the physical + * PBDMA hands those methods to its host-software lane and + * raises NV_PPBDMA_INTR_0_METHOD (bit 21): the r1075 relay + * records `Channel ID 00000004 intr 00200000` at every launch, + * and the real host driver's software-method handler answers + * "Invalid host sofware method received" -> rcRequired -> + * robust-channel error recovery kills the channel mid-window + * (fifo_gk104.c). The methods are channel-level barriers whose + * effects the module's own release/fence lanes already author, + * so dropping them from the replay is the same call the 0x10c + * NOTIFY / 0x1b0 LAUNCH_DMA auto-drops make. Only count- + * carrying writes (op 1/3/5) are candidates: the single-word + * TERT encodings (op 0/2/4/7) reuse bits 15:13 as mask/type + * payload. + */ + if ((op == 1u || op == 3u || op == 5u) && sub != 0u) { + dropped++; + subdrop++; + off += span; + continue; + } + /* + * SESSION 153-CONTINUED-59: single-word SDM-type entries + * (SEC_OP 0/2/6, see gsp_phys_rmcall_gr_sdm_drop). The + * retiring X-family windows never yield one; this guards a + * window that carries the client's SDM markers (the + * 0x1fff0-class marker words or the prelude 0x046x6dXX + * family), which the physical PBDMA refuses with PBENTRY. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_sdm_drop) && + hdr && (op == 0u || op == 2u || op == 6u)) { + /* + * SESSION 153-CONTINUED-86: name every dropped SDM + * entry (bounded) so the A/B arm can be audited: the + * "keep the drop only for entries proven unrelated" + * decision needs the actual words, not the count. + */ + static u32 sdm_log; + + if (sdm_log < 24u) { + sdm_log++; + pr_info("vgsp-gr: SDM-DROP in[%u]=%08x op=%u addr=%#x sub=%u cnt/val=%#x (span=%u)\n", + off, hdr, op, addr, sub, + (cnt ? cnt : ((hdr >> 16) & 0x1fffu)), + span); + } + dropped++; + sdm_drop++; + off += span; + continue; + } + if (gsp_phys_rmcall_gr_win_drop_match(addr) || + gsp_phys_rmcall_gr_win_drop_auto_match(addr)) { + dropped++; + off += span; + continue; + } + memcpy(&out[wout], &in[off], (size_t)span * 4); + /* + * SESSION 153-CONTINUED-58: torn-read zero guard - see the + * sema_gl_attr_zero_fix comment. Only INC runs (op 1) whose + * data words advance one method each; the attribute runs are + * exactly this shape. Rebuild the run in place, skipping + * exactly-zero words and re-targeting the surviving words' + * methods (a zero word's method is simply never written, so + * the attribute keeps its prelude/previous-frame value). + */ + if (READ_ONCE(gsp_phys_rmcall_gr_attr_zero_fix) && + op == 1u && cnt && addr >= 0x1160u && addr <= 0x11dcu) { + u32 k, k0 = 0, L = 0, wr = wout; + u32 subch = (hdr >> 13) & 0x7u; + + for (k = 0; k <= cnt; k++) { + u32 wv = (k < cnt) ? out[wout + 1u + k] : 0u; + /* + * SESSION 153-CONTINUED-62: the bad predicate + * also covers a torn ACTIVE attribute whose + * COMPONENT_BIT_WIDTHS enum is 0 (no such + * encoding exists): the r1113 setup-push + * capture carried + * `20030458 38400000 38200040 000081a4` + * (A(2)=0x81a4, widths=0) and the replayed + * prelude write trapped the engine - + * `Class Error: Class 0000c597, Offset + * 00001168, Data 000081a4, ErrorCode 0xc` + * -> grfifo=0xbadf1201 poison the moment the + * surf prelude let the draw execute. An + * inactive attribute (SOURCE=1, bit 6) keeps + * its word; only the invalid active encoding + * is dropped, so the attribute keeps its + * prelude/previous value until the window's + * own (valid) write lands. + */ + bool bad = (k < cnt) && + (wv == 0u || + (((wv >> 21) & 0x3fu) == 0u && + !(wv & 0x40u))); + + if (!bad && k < cnt) { + if (!L) + k0 = k; + out[wr + 1u] = out[wout + 1u + k]; + wr++; + L++; + continue; + } + if (L) { + u32 naddr = addr + 4u * k0; + u32 nhdr = (1u << 29) | + (L << 16) | + ((naddr >> 2) & 0x1fffu) | + (subch << 13); + + out[wr - L] = nhdr; + L = 0; + } + if (bad) { + static u32 az_log; + + if (az_log < 64) { + az_log++; + pr_info("vgsp-gr: ATTR-ZERO addr=0x%x word=%u/%u off=%u dropped (torn zero/invalid-width; attribute keeps its prelude value)\n", + addr + 4u * k, k, + cnt, off); + } + } + } + /* +1: the rebuilt run still needs its (new) header word */ + span = (wr - wout) + 1u; + } + { + /* + * SESSION 153-CONTINUED-14: live data patch (BEGIN + * OP sweep etc.). Header-only forms rewrite the 13-bit + * data field; count-carrying forms rewrite the first + * payload word. + */ + u32 pv; + + if (gsp_phys_rmcall_gr_win_patch_find(addr, &pv)) { + static u32 patch_log; + u32 old; + + wp_match++; + if (span > 1u) { + old = out[wout + 1]; + out[wout + 1] = pv; + } else { + old = (out[wout] >> 16) & 0x1fffu; + out[wout] = (out[wout] & ~0x1fff0000u) | + ((pv & 0x1fffu) << 16); + } + if (patch_log < 32) { + patch_log++; + pr_info("vgsp-gr: 3D win-patch addr=0x%x off=%u data 0x%x -> 0x%x\n", + addr, off, old, pv); + } + } else if (addr >= 0x100u && addr < 0x200u) { + /* + * SESSION 153-CONTINUED-84: the win_patch + * knob (sema_gl_win_patch=124:1) reaches + * this pass only if the window carries a + * run at the patch address; log the low-page + * runs so a decode mismatch is visible + * instead of silent. + */ + wp_lowrun++; + if (wp_lowrun <= 4u && + gsp_phys_rmcall_gr_win_patch && + *gsp_phys_rmcall_gr_win_patch) + pr_info("vgsp-gr: WIN-PATCH lowrun addr=0x%x op=%u cnt=%u hdr=%08x\n", + addr, op, cnt, hdr); + } + } + if ((op == 1 || op == 3 || op == 5) && cnt && + off + 1u + cnt <= n) { + u32 *d = &out[wout + 1]; + u32 lo_i = 0; + bool pair = false; + + if (op == 1 && addr == 0x0180 && cnt >= 4) { + lo_i = 2; /* OFFSET_OUT is v2/v3 */ + pair = true; + } else if (op == 1 && addr == 0x0f84 && cnt >= 2) { + pair = true; + } else if (op == 1 && addr == 0x155c && cnt >= 3) { + pair = true; + } else if (op == 1 && addr == 0x1574 && cnt >= 3) { + pair = true; + } else if (op == 1 && addr >= 0x2014 && + addr < 0x4000 && (addr & 0x3fu) == 0x14 && + cnt >= 2) { + pair = true; + } else if (op == 1 && (addr == 0x1c00 || addr == 0x1c10) && + cnt >= 3) { + /* SESSION 153-CONTINUED-10: SET_VERTEX_STREAM_ + * A_FORMAT(j)+BASE(j) run [fmt, upper, lower] + * (j*0x10 stride) - the vertex data the draw + * reads. Observed [00001000 1 2150b790]. */ + lo_i = 1; + pair = true; + } else if (READ_ONCE(gsp_phys_rmcall_gr_idx_reloc) && + op == 1 && addr >= 0x1c20 && addr <= 0x1cf0 && + (addr & 0xfu) == 0u && cnt >= 3) { + /* SESSION 153-CONTINUED-47 (A/B knob): widen + * the vertex-stream banks from j=0/1 to every + * FORMAT slot (0x1c20..0x1cf0). The GLX + * client's 3D window binds streams j>=2 that + * svc_gl_field already collects into the page + * set but the window never had rewritten. */ + lo_i = 1; + pair = true; + } else if (READ_ONCE(gsp_phys_rmcall_gr_idx_reloc) && + op == 1 && addr == 0x17c8 && cnt >= 2) { + /* SESSION 153-CONTINUED-47 (A/B knob): SET_ + * INDEX_BUFFER_A/B [ADDRESS_UPPER(7:0), + * ADDRESS_LOWER(31:0)] - the indexed draw + * fetches its indices through this VA. The + * r1042 boot showed an X window carrying + * index binds PRI-stalling once relocated + * (relocation of exactly this pair is the + * only kernel delta vs the r1035 baseline + * where index-bearing windows did not exist + * and every X window retired), so the + * rewrite is gated off by default and A/B-ed + * live per boot. Observed [00000000 + * 0450fcc8] -> VA 0x450fcc8. */ + lo_i = 0; + pair = true; + } else if (READ_ONCE(gsp_phys_rmcall_gr_idx_reloc) && + op == 1 && addr == 0x1b00 && cnt >= 2) { + /* SESSION 153-CONTINUED-48: NVC597_SET_REPORT_ + * SEMAPHORE_A/B..D (0x1b00) - the engine's own + * completion write [upper(7:0), lower(31:0), + * payload, flags] (observed [0, 0x4181bb0, + * 0x4f9f6, 0x1000f010] at the end of every + * glcore window). With the VA raw the replayed + * engine writes a client VA the GPC engines + * cannot reach - the VIRT_WRITE fault class + * (r1061: the kept [0,0x2000) window that + * faults is exactly the one still carrying the + * raw 0x1b00 VAs). Rewrite the pair to the + * shadow VA (same HPA, page collected by + * svc_gl_field 0x1b00), so the write lands in + * the client's own page - its wait then sees a + * real hardware completion. MISS stays + * verbatim like every other pair. */ + lo_i = 0; + pair = true; + } else if (op == 1 && addr == 0x2380 && cnt >= 3) { + /* SET_CONSTANT_BUFFER_SELECTOR_A/B: + * [size, upper, lower]; observed + * [00010000 1 2150a500]. */ + lo_i = 1; + pair = true; + } else if (op == 1 && addr >= 0x0800 && addr < 0x0c00 && + (addr & 0x3fu) == 0u && cnt >= 2) { + /* SET_COLOR_TARGET_A/B(j) offset pair: + * [upper, lower, w, h, fmt]; the surface the + * draw rasterizes into (must live in guest + * RAM for the present lane to see pixels). + * Observed [1 222fa200 40 4 f7 0]. */ + lo_i = 0; + pair = true; + } else if (op == 1 && addr == 0x218 && cnt >= 4 && + (d[2] & ~0xffu) == 0u && + (d[3] & 0xfffu) == 0u) { + /* SESSION 153-CONTINUED-27: the TWOD twin + * of the colour target - INC 0x218 cnt>=4 + * covers 0x218..0x224 = [w, h, upper, + * lower] (the emulator's SET_DST sequence; + * the glxgears client window fills the X + * screen through it). Shape-checked so a + * non-surface run can never be rewritten. */ + lo_i = 2; + pair = true; + } + if (pair) { + u64 pva = ((u64)(d[lo_i] & 0xffu) << 32) | + (u64)(d[lo_i + 1] & 0xfffffffcu); + static u32 pair_log; + bool ok = false, kept = false; + + /* + * SESSION 153-CONTINUED-92: the program-family + * space experiment (see sema_gl_prog_guest_va). + * The DS rejects every bound stage whose SPH it + * fetches through a relocated shadow VA while + * the bytes at those VAs are provably correct; + * leave the operand at its ORIGINAL guest VA + * whenever the page resolves + identity-maps, + * so the DS fetches/validates in the same VA + * space the client's program heap describes. + * Kept pairs count as relocated so the SPH-BIND + * guard (orel == nrun) is unaffected; a page the + * identity mapper refuses falls through to the + * shadow rewrite exactly as before. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_prog_guest_va) && + bar0 && req->prog_va && + addr >= 0x2014u && addr < 0x4000u && + (addr & 0x3fu) == 0x14u && + gsp_phys_rmcall_gr_prog_keep_locked( + bar0, epoch, d[lo_i], d[lo_i + 1], + req)) { + static u32 progva_log; + + kept = true; + prog_keep++; + if (progva_log < 24u) { + progva_log++; + pr_info("vgsp-gr: PROG-GUEST-VA keep addr=0x%x va=0x%llx (Tier-1 identity; operand left verbatim) h=0x%x\n", + addr, pva, + req->h_object); + } + } else { + ok = gsp_phys_rmcall_gr_3d_reloc( + &d[lo_i], &d[lo_i + 1], req, + n_shadow); + } + if (ok || kept) + reloc++; + if (!kept && pair_log < 48) { + pair_log++; + pr_info("vgsp-gr: pair addr=0x%x va=0x%llx -> %s0x%llx\n", + addr, pva, ok ? "" : + "MISS ", + ((u64)(d[lo_i] & 0xffu) << 32) | + (u64)(d[lo_i + 1] & 0xfffffffcu)); + } + } + /* + * SESSION 153-CONTINUED-48: MME-family payload + * relocation (see the sema_gl_mme_reloc param). The + * C597 MME macro/data family spans 0x3800..0x3a00; + * its runs carry the macro arguments as raw payload + * words, which include 32-bit client VAs + * (0x450f800/0x450f000 observed in the beef macro(2) + * run at 0x3810). Rewrite only a word whose page the + * launch's own page set resolves (the same vetted + * shadow mapping the pairs use); a MISS, an + * unaligned/too-small word, or a shadow VA that + * cannot fit the 32-bit slot stays verbatim. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_mme_reloc) && + addr >= 0x3800u && addr < 0x3a00u) { + u32 wi; + + for (wi = 0; wi < cnt; wi++) { + u32 wv = d[wi]; + u32 up = 0, lo = wv; + static u32 mme_log; + + if (wv < 0x10000u || (wv & 3u)) + continue; + if (!gsp_phys_rmcall_gr_3d_reloc( + &up, &lo, req, n_shadow)) { + /* + * SESSION 153-CONTINUED-51: + * the payload page missed the + * launch's page set (the QMD + * pointer region caps the set + * far below a full window's + * operand count - r1071: the + * macro payload pages + * 0x44fd200..0x44fde00 were + * ~12 distinct pages while the + * set held 7; the words stayed + * RAW and the engine fetched an + * untranslatable guest VA on + * every full window - the + * 0xbadf1201 priv-error + * poison). Resolve the page + * through the SAME trusted + * ledger/pin/walk chain the + * launch pages use (the + * RELOC-FIXED resolver) and + * shadow-map it exactly like a + * matched page. A resolve/map + * miss or a shadow VA that + * cannot fit the 32-bit slot + * stays verbatim (never + * guessed) - and, SESSION + * 153-CONTINUED-62, it now + * COUNTS toward the abort + * census and joins the pin + * demand set: r1115's residual + * fault was exactly this class + * (`MMU Fault HUBCLIENT_PD @ + * 0x0_04350000` 2.5 s after a + * launch whose census read 0 + * because an 0x3810 payload + * word 0x4350900 missed here + * silently). + */ + struct gsp_sema_gr_reloc r; + u64 va2 = (u64)wv; + u64 sh2 = 0; + u32 line2 = 0; + + if (gsp_sema_gr_reloc_resolve( + va2, &r) || + !r.hpa) { + gsp_phys_gr_3d_mme_miss( + wv); + continue; + } + if (*n_shadow >= + GSP_PHYS_RMCALL_GR_SHADOW_PAGES) { + gsp_phys_gr_3d_mme_miss( + wv); + continue; + } + if (gsp_phys_rmcall_gr_map_shadow_locked( + *n_shadow, r.hpa, + &sh2, &line2)) { + gsp_phys_gr_3d_mme_miss( + wv); + continue; + } + if (line2 != 0xffffffffu) + (*n_shadow)++; + /* CONT-73: MME shadow-scratch + * destination maps this guest + * HPA 1:1; record it so a store + * through the MME path is + * witnessed too. */ + gsp_phys_gr_wit_note( + va2 & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1), + r.hpa); + sh2 |= va2 & + (GSP_PHYS_PAGE_SIZE - 1); + up = (u32)(sh2 >> 32) & 0xffu; + lo = (wv & 0x3u) | + ((u32)sh2 & 0xfffffffcu); + if (up != 0) { + gsp_phys_gr_3d_mme_miss( + wv); + continue; + } + d[wi] = lo; + reloc++; + if (mme_log < 32) { + mme_log++; + pr_info("vgsp-gr: mme-reloc-rsv addr=0x%x wi=%u w=0x%x -> 0x%x\n", + addr, wi, wv, + lo); + } + continue; + } + if (up != 0) { + gsp_phys_gr_3d_mme_miss(wv); + continue; + } + d[wi] = lo; + reloc++; + if (mme_log < 32) { + mme_log++; + pr_info("vgsp-gr: mme-reloc addr=0x%x wi=%u w=0x%x -> 0x%x\n", + addr, wi, wv, lo); + } + } + } + wout += span; + off += 1u + cnt; + continue; + } + off++; /* header-only / resync, like the service decoder */ + wout++; + } + /* + * SESSION 153-CONTINUED-84: prove the win_patch knob reached the + * pass (the r1236 "never logged" was a late write, not a decode + * gap - this line makes the distinction explicit on every boot). + */ + if (gsp_phys_rmcall_gr_win_patch && *gsp_phys_rmcall_gr_win_patch) { + static u32 wp_dbg; + + if (wp_dbg < 24u) { + wp_dbg++; + pr_info("vgsp-gr: WIN-PATCH sweep knob=%s runs=%u matched=%u lowruns=%u h=0x%x\n", + gsp_phys_rmcall_gr_win_patch, n, wp_match, + wp_lowrun, + req ? req->h_object : 0u); + } + } + *out_n = wout; + *n_dropped = dropped; + gsp_phys_rmcall_gr_3d_last_subdrop = subdrop; + gsp_phys_rmcall_gr_3d_last_sdmdrop = sdm_drop; + gsp_phys_rmcall_gr_3d_last_progkeep = prog_keep; + return reloc; +} + +/* + * SESSION 153-CONTINUED-27: alias-build for a big colour target. Consumes + * ceil(npages/16) never-used scratch-PT fetch lines and writes one PTE per + * surface page, so the whole surface is reachable through one CONTIGUOUS + * alias VA (lines are monotonic; consecutive lines are VA-contiguous + * inside a pool block - both pool blocks are checked). Every page's HPA + * comes from the launch's own page set when present, else from the + * service's trusted resolver - the identical vetted translation the + * shadow relocation uses. Lines are consumed even on failure (they are + * never reused, so a half-written line can never be shadowed later); + * the caller then falls back to the existing per-page path. Called with + * gsp_phys_rm_lock held. + */ +static bool gsp_phys_gr_surface_alias_locked(const struct gsp_phys_gr_submit *req, + u64 va, u64 npages, u64 *alias_out) +{ + u64 map_pages = npages + GSP_PHYS_GR_SURF_SLACK_PAGES; + u32 nlines; + u32 base_line; + u32 i, j; + u64 first_va, last_va; + + if (map_pages > 4096u) + map_pages = 4096u; + nlines = (u32)((map_pages + + GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES - 1) / + GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES); + base_line = boot_stage.rmcall_guest_pt_next_line; + if (!nlines || base_line + nlines > GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES) + return false; + first_va = gsp_phys_rmcall_guest_pt_win_va( + base_line / GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE) + + ((u64)(base_line % + GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE) << 16); + last_va = gsp_phys_rmcall_guest_pt_win_va( + (base_line + nlines - 1u) / + GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE) + + ((u64)((base_line + nlines - 1u) % + GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE) << 16); + /* one contiguous run: never straddle a pool block boundary */ + if (last_va != first_va + ((u64)(nlines - 1u) << 16)) + return false; + /* + * Consume the lines before writing: a failure mid-way leaves the + * partially written lines consumed (never reused), so the monotonic + * line invariant - a cached line can never shadow a later mapping - + * holds unconditionally. + */ + boot_stage.rmcall_guest_pt_next_line = base_line + nlines; + for (i = 0; i < nlines; i++) { + u32 line = base_line + i; + u32 page = line / GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; + u32 lip = line % GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; + u8 *pt; + + if (gsp_phys_rmcall_gr_scratch_win_install_locked(page)) + return false; + pt = gsp_phys_rmcall_guest_pt_page_va(page); + if (!pt) + return false; + for (j = 0; j < GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES; j++) { + u64 k = (u64)i * GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES + + j; + u64 pv, hpa = 0; + u32 pi; + + if (k >= map_pages) + break; + pv = va + (k << 12); + for (pi = 0; pi < req->n_pages; pi++) { + if ((req->pages[pi].guest_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) == pv && + req->pages[pi].hpa) { + hpa = req->pages[pi].hpa; + break; + } + } + if (!hpa) { + struct gsp_sema_gr_reloc r; + + if (gsp_sema_gr_reloc_resolve(pv, &r) || + !r.hpa) { + /* + * A declared page must resolve; a + * slack-tail page past the guest + * allocation gets the dummy so the + * engine's overdraw vanishes. + */ + if (k < npages) + return false; + hpa = gsp_phys_gr_surf_dummy_dma_locked(); + if (!hpa) + return false; + } else { + hpa = r.hpa; + } + } + put_unaligned_le64( + gsp_phys_rmcall_pte(hpa), + pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * + lip + + j) * + 8); + } + dma_wmb(); + } + *alias_out = first_va; + return true; +} + +/* + * SESSION 153-CONTINUED-27: pre-pass over a raw 3D window. Scans the + * colour-target runs (C597 INC 0x800+j*0x40 [upper, lower, w, h, fmt, + * blksz] and - same surface, shape seen on the TWOD subchannel - INC + * 0x218 cnt>=4 [w, h, upper, lower]) and aliases each big surface once + * per boot so the relocation pass can rewrite its operands to the alias + * (r992/r993 evidence: the X screen 0x120400000, 1280x720 fmt 0xe6, is + * the draw's render target AND the TWOD fill's destination). Small + * surfaces keep the historical one-pair shadow relocation. Called with + * gsp_phys_rm_lock held. + */ +static void gsp_phys_gr_surface_alias_scan_locked( + const u32 *win, u32 n, const struct gsp_phys_gr_submit *req) +{ + u32 off = 0; + + while (off < n) { + u32 hdr = win[off]; + u32 op = (hdr >> 29) & 0x7u; + u32 cnt = (hdr >> 16) & 0xfffu; + u32 addr = (hdr & 0x1fffu) << 2; + u32 span = 1u; + + if ((op == 1 || op == 3 || op == 5) && cnt && + off + 1u + cnt <= n) { + span = 1u + cnt; + } else { + off++; + continue; + } + if (op == 1 && ((addr >= 0x0800 && addr < 0x0c00 && + (addr & 0x3fu) == 0u && cnt >= 4) || + (addr == 0x218 && cnt >= 4))) { + const u32 *d = &win[off + 1u]; + u64 va; + u32 w, h, fmt; + u64 bytes, npages; + u32 sdi; + u64 alias = 0; + + if (addr == 0x218) { + va = ((u64)(d[2] & 0xffu) << 32) | + (u64)(d[3] & 0xfffff000u); + w = d[0]; + h = d[1]; + fmt = 0xe6u; /* TWOD run carries fmt on + * its own IMMD; the surface + * class here is 32bpp */ + } else { + va = ((u64)(d[0] & 0xffu) << 32) | + (u64)(d[1] & 0xfffff000u); + w = d[2]; + h = d[3]; + fmt = d[4]; + } + if (!va || (va & 0xfffu) || fmt != 0xe6u || + !w || !h || w > 8192u || h > 8192u) + goto next; + bytes = (u64)w * 4u * (u64)h; + npages = (bytes + GSP_PHYS_PAGE_SIZE - 1) >> 12; + if (npages < 2u || npages > 4096u) + goto next; + for (sdi = 0; sdi < gsp_phys_gr_surf_n; sdi++) + if (gsp_phys_gr_surf[sdi].va == va) + goto next; /* already aliased */ + if (gsp_phys_gr_surface_alias_locked(req, va, npages, + &alias)) { + if (gsp_phys_gr_surf_n < + ARRAY_SIZE(gsp_phys_gr_surf)) { + gsp_phys_gr_surf[gsp_phys_gr_surf_n].va = + va; + gsp_phys_gr_surf[gsp_phys_gr_surf_n] + .alias = alias; + gsp_phys_gr_surf[gsp_phys_gr_surf_n] + .npages = npages; + gsp_phys_gr_surf_n++; + } + pr_info("vgsp-gr: SURFACE alias va=0x%llx pages=%llu (%ux%u fmt=0x%x) -> 0x%llx (contiguous shadow run; operands rewritten)\n", + va, npages, w, h, fmt, alias); + } else { + pr_info("vgsp-gr: SURFACE alias FAILED va=0x%llx (%ux%u fmt=0x%x): operands fall to the per-page path\n", + va, w, h, fmt); + } + } +next: + off += span; + } +} + +/* + * SESSION 153-CONTINUED-66: write a built method block into the module + * pushbuffer. The block may exceed one 4 KB page (GSP_PHYS_RMCALL_GR_METHOD_ + * WORDS is 2048), so copy page-wise: page 0 is the registered BUF_PB chunk, + * page 1 is the PTE-planted rmcall_gr_pb2 page at VA_PB2. The two pages are + * contiguous in GPU VA (VA_PB..VA_PB+0x2000), so the GPFIFO entry's single + * fetch address + length walks both. A short/absent page-2 plant leaves the + * tail unwritten and the caller's bound check keeps such launches <= 4 KB. + */ +static void gsp_phys_gr_pb_copy(const u32 *methods, u32 words) +{ + struct gsp_phys_boot_chunk *pb = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB]; + u32 per = GSP_PHYS_PAGE_SIZE / 4u; + u32 done = 0; + + while (done < words) { + struct gsp_phys_boot_chunk *c; + u32 page = done / per; + u32 off = done % per; + u32 n = words - done; + u8 *dst; + + if (n > per - off) + n = per - off; + if (!page) + c = pb; + else if (page == 1u) + c = &boot_stage.rmcall_gr_pb2; + else + return; + dst = gsp_phys_chunk_page_va(c, 0); + if (page == 2u || page == 3u) { + /* + * SESSION 153-CONTINUED-77: pages 1 and 2 of the pb2 + * chunk are the third/fourth pushbuffer pages + * (VA_PB3 slot 0x103, VA_PB4 slot 0x104). + */ + if (boot_stage.rmcall_gr_pb2.pages < page) + return; + dst = gsp_phys_chunk_page_va(&boot_stage.rmcall_gr_pb2, + page - 1u); + } + if (!dst) + return; + memcpy(dst + (size_t)off * 4u, methods + done, (size_t)n * 4u); + done += n; + } +} + +/* + * SESSION 153-CONTINUED-95: CTX-BIND launch receipt - is the client's + * promoted GR context buffer engine-visible on the replay channel? + * + * The guest driver promotes its per-channel context buffers through + * NV2080_CTRL_CMD_GPU_PROMOTE_CTX entries (MAIN bufferId 0 and PATCH + * bufferId 2 carry bInitialize=1 on a fresh context). The emulated GSP-RM + * answers SUCCESS and records the statements (gsp_sema_service_ctx_ + * promote_note), but it does not install them anywhere: the replay + * channel's engine context is the buffer the kernel's own ladder built + * (server FBMEM MAIN, mapped at GSP_PHYS_RMCALL_VA_GR_CTX and committed by + * the mode-35 lane promote). This receipt logs, at the first launches of a + * boot, the exact promote statements next to the engine's binding so the + * "context state never replayed" hypothesis is decided by bytes: the + * client's MAIN/PATCH pa/va/size/bInitialize, and the first words of the + * client MAIN buffer read through the GPA->HPA map (a zeroed buffer is the + * expected state before a server-side ctx init that the emulation never + * performs). + */ +static void gsp_phys_ctx_bind_receipt(void) +{ + static u32 receipt_n; + u64 cpa = 0, cva = 0, csz = 0, ppa = 0, pva = 0, psz = 0; + u32 cattr = 0, cinit = 0, pattr = 0, pinit = 0; + bool have_main, have_patch; + + if (receipt_n >= 4u) + return; + receipt_n++; + have_main = gsp_sema_service_ctx_promote_get(GSP_PHYS_RMCALL_ENGINE_GR0, + 0u, &cpa, &cva, &csz, + &cattr, &cinit) == 0; + have_patch = gsp_sema_service_ctx_promote_get(GSP_PHYS_RMCALL_ENGINE_GR0, + 2u, &ppa, &pva, &psz, + &pattr, &pinit) == 0; + pr_info("vgsp-gr: CTX-BIND client-main=%s pa=0x%llx va=0x%llx size=0x%llx attr=0x%x binit=%u; client-patch=%s pa=0x%llx va=0x%llx size=0x%llx binit=%u; replay-ctx pa=0x%llx size=0x%x fbmem=%u va=0x%llx srv_ctxva=%u; installed=0 (engine ctx is the ladder's buffer, NOT the client promote)\n", + have_main ? "seen" : "none", cpa, cva, csz, cattr, cinit, + have_patch ? "seen" : "none", ppa, pva, psz, pinit, + boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_size, + boot_stage.rmcall_ctx_fbmem ? 1u : 0u, + boot_stage.rmcall_ctx_va, boot_stage.rmcall_srv_ctxva ? 1u : 0u); + if (have_main && cpa) { + u64 hpa = 0; + + if (gsp_phys_gpa_to_hpa_vgpu(NULL, cpa & ~(u64)(GSP_PHYS_PAGE_SIZE - 1), &hpa) == 0 && + pfn_valid(hpa >> PAGE_SHIFT)) { + const u32 *w = (const u32 *)__va(hpa); + + pr_info("vgsp-gr: CTX-BIND client-main content[0..7]=%08x %08x %08x %08x %08x %08x %08x %08x (gpa=0x%llx hpa=0x%llx)\n", + get_unaligned_le32((const u8 *)&w[0]), + get_unaligned_le32((const u8 *)&w[1]), + get_unaligned_le32((const u8 *)&w[2]), + get_unaligned_le32((const u8 *)&w[3]), + get_unaligned_le32((const u8 *)&w[4]), + get_unaligned_le32((const u8 *)&w[5]), + get_unaligned_le32((const u8 *)&w[6]), + get_unaligned_le32((const u8 *)&w[7]), + cpa, hpa); + } else { + pr_info("vgsp-gr: CTX-BIND client-main content: GPA 0x%llx -> HPA translation unavailable (no guest RAM registration yet)\n", + cpa); + } + } +} + +/* + * SESSION 153-CONTINUED-95: engine context readback (gsp-phys-ctx-dump). + * + * What the DS validates SPHs against is the *engine's* context buffer, not + * this host's view of anything the client promoted. The replay channel's + * context is the ladder's buffer (server FBMEM MAIN) mapped into the + * channel VAS at GSP_PHYS_RMCALL_VA_GR_CTX with VIDMEM PTEs. The host has + * no aperture to FBMEM (no BAR1 window), so the copy engine reads it: one + * 4 KB CE copy per page from the ctx VA to a host page placed on a fresh + * never-reused scratch fetch line, polled on the module semaphore exactly + * like the P6a guest copies. Runs once per boot at the first launch (the + * channel is idle: single-inflight work item), while the ladder's channel + * is not server-memory and the identity DMA domain holds. Bounded: stops + * at the first refused/timed-out page and records how many bytes landed. + */ +#define GSP_PHYS_CTX_RB_BYTES 0x140000u /* MAIN 0x133000 + PATCH 0x5000 + slack */ + +static u8 *gsp_phys_ctx_rb_buf; +static u32 gsp_phys_ctx_rb_len; +static bool gsp_phys_ctx_rb_tried; + +static const u8 *gsp_phys_ctx_readback_get(u32 *len) +{ + if (len) + *len = READ_ONCE(gsp_phys_ctx_rb_len); + return gsp_phys_ctx_rb_buf; +} + +/* + * SESSION 153-CONTINUED-96: re-arm the one-shot readback. A write to the + * gsp-phys-ctx-read debugfs file clears the tried flag (and the length), so + * the next forwarded launch re-runs the copy with the current server state. + * Safe from any context: the readback itself only runs in the single-inflight + * launch path and re-checks every gate. + */ +static void gsp_phys_ctx_readback_rearm(void) +{ + WRITE_ONCE(gsp_phys_ctx_rb_len, 0); + WRITE_ONCE(gsp_phys_ctx_rb_tried, false); +} + +static void gsp_phys_ctx_readback_try(void) +{ + struct gsp_phys_boot_chunk *sem = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM]; + void __iomem *bar0; + struct page *pg; + u64 hpa; + u32 i, bytes = 0; + u32 ctx_pages, patch_pages = 0; + + if (READ_ONCE(gsp_phys_ctx_rb_tried)) + return; + if (!boot_stage.rmcall_ctx_valid || !boot_stage.rmcall_ctx_pa) + return; + /* + * SESSION 153-CONTINUED-96: the copies run on the secondary COPY0 + * lane (gsp_phys_rmcall_ce_lane_bringup). CE class methods are never + * legal on the GR0 ladder channel, so the lane's readiness (alloc + + * BIND/schedules + its own token) is the hard requirement here; the + * GR channel pieces are not used by the copy. Gate on the pieces the + * lane submission needs: claimed, RM init-done, not unloading. + */ + if (!boot_stage.rmcall_ce_lane_valid || !boot_stage.rmcall_ce_token) + return; + if (gsp_phys_backend_state() != GSP_PHYS_BACKEND_READY) + return; + if (!gsp_phys_gpu_claimed() || !boot_stage.rm_init_done || + boot_stage.rm_unload_submitted) + return; + if (gsp_phys_dma_domain_identity() != 1) + return; + bar0 = gsp_phys_gpu_bar0(); + if (!bar0) + return; + if (!gsp_phys_ctx_rb_buf) { + gsp_phys_ctx_rb_buf = vmalloc(GSP_PHYS_CTX_RB_BYTES); + if (!gsp_phys_ctx_rb_buf) + return; + } + /* + * SESSION 153-CONTINUED-96: copy the WHOLE MAIN context (the golden + * register image the engine restores) and the PATCH buffer (the + * context patch area the DS's SPH region may live in), in that order, + * after the 64 KB r1323/r1324 first run only proved the head. The + * dump file serves MAIN first, then PATCH; lengths are logged. + */ + ctx_pages = DIV_ROUND_UP(boot_stage.rmcall_ctx_size, + GSP_PHYS_PAGE_SIZE); + if (ctx_pages > (GSP_PHYS_CTX_RB_BYTES / GSP_PHYS_PAGE_SIZE)) + ctx_pages = GSP_PHYS_CTX_RB_BYTES / GSP_PHYS_PAGE_SIZE; + if (boot_stage.rmcall_golden_fb_aux && boot_stage.rmcall_patch_size) { + patch_pages = DIV_ROUND_UP(boot_stage.rmcall_patch_size, + GSP_PHYS_PAGE_SIZE); + if (ctx_pages + patch_pages > + GSP_PHYS_CTX_RB_BYTES / GSP_PHYS_PAGE_SIZE) + patch_pages = GSP_PHYS_CTX_RB_BYTES / + GSP_PHYS_PAGE_SIZE - ctx_pages; + } + pg = alloc_page(GFP_KERNEL); + if (!pg) + return; + hpa = (u64)page_to_phys(pg); + for (i = 0; i < ctx_pages + patch_pages; i++) { + u64 src_base = (i < ctx_pages) ? + GSP_PHYS_RMCALL_VA_GR_CTX : + GSP_PHYS_RMCALL_VA_GR_PATCH; + u64 page_off = (i < ctx_pages) ? + (u64)i * GSP_PHYS_PAGE_SIZE : + (u64)(i - ctx_pages) * GSP_PHYS_PAGE_SIZE; + u64 copy_src_va = src_base + page_off; + u64 map_src_va = 0, dst_va = 0, sem_va = 0; + u32 poll = 0; + unsigned long deadline; + int ret; + + mutex_lock(&gsp_phys_rm_lock); + ret = gsp_phys_rmcall_guest_map_locked(hpa, hpa, 0, &map_src_va, + &dst_va, &sem_va); + if (ret) { + mutex_unlock(&gsp_phys_rm_lock); + break; + } + memset((u8 *)__va(hpa), 0xab, GSP_PHYS_PAGE_SIZE); + ret = gsp_phys_rmcall_ce_lane_submit_locked(bar0, + copy_src_va, dst_va, GSP_PHYS_RMCALL_VA_SEM, + GSP_PHYS_PAGE_SIZE, + GSP_PHYS_RMCALL_SEM_PAYLOAD, NULL); + mutex_unlock(&gsp_phys_rm_lock); + if (ret) { + pr_warn("vgsp-gr: CTX-READBACK page %u submit refused (%d) after %u bytes; stopping\n", + i, ret, bytes); + break; + } + deadline = jiffies + msecs_to_jiffies(250); + for (;;) { + poll = READ_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0)); + if (poll == GSP_PHYS_RMCALL_SEM_PAYLOAD) + break; + if (time_after_eq(jiffies, deadline)) { + poll = 0; + break; + } + gsp_phys_rmcall_wait_quantum(); + } + if (poll != GSP_PHYS_RMCALL_SEM_PAYLOAD) { + pr_warn("vgsp-gr: CTX-READBACK page %u (src VA 0x%llx) no CE release within 250 ms after %u bytes; stopping\n", + i, copy_src_va, bytes); + break; + } + memcpy(gsp_phys_ctx_rb_buf + (size_t)i * GSP_PHYS_PAGE_SIZE, + (const u8 *)__va(hpa), GSP_PHYS_PAGE_SIZE); + bytes += GSP_PHYS_PAGE_SIZE; + } + __free_page(pg); + WRITE_ONCE(gsp_phys_ctx_rb_len, bytes); + WRITE_ONCE(gsp_phys_ctx_rb_tried, true); + pr_info("vgsp-gr: CTX-READBACK engine ctx pa=0x%llx copied=%u bytes on the CE lane (MAIN %u pages then PATCH %u pages; server/own buffer, not the client promote; WPR2 untouched)\n", + boot_stage.rmcall_ctx_pa, bytes, ctx_pages, patch_pages); +} + int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, struct gsp_phys_backend_copy_result *out) { @@ -18592,6 +25711,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * under the warning bound. */ static u32 methods[GSP_PHYS_RMCALL_GR_METHOD_WORDS]; + static u32 stream_patch[GSP_PHYS_RMCALL_GR_METHOD_WORDS]; static u32 qmd[GSP_PHYS_GR_QMD_WORDS]; static struct gsp_phys_fault_info fault; unsigned long start = jiffies; @@ -18602,6 +25722,24 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, const u8 *guest_param_pg; u32 want_qmd = 0, want_ch = 0; bool qmd_rel = false, ch_rel = false; + /* + * r929 (Path A): this submission is a raw 3D method window. The QMD + * machinery (qmdwin relocation, QMD prewrite, release0 redirect, + * PCAS) does not apply: the block is SET_OBJECT C597 + the window + + * the channel release tail, and completion is the channel release + * alone (qmd_rel stays off so the poll does not wait for a QMD + * release word no engine will write). + */ + bool stream3d = gsp_phys_rmcall_gr_3d && req && req->stream_n; + /* + * CONT-73: fresh direct-write witness for this submission. The + * window/attr patch passes below record every relocated operand + * destination's guest page; the pre-doorbell capture and the retire + * report read exactly this launch's set (single-inflight, statics). + */ + gsp_phys_gr_wit_reset(); + const u32 *win3d = NULL; + u32 win3d_n = 0; bool qmd_rel_redirected = false; /* r420b: release0 -> module sem */ u64 qmd_rel_hpa = 0, ch_rel_hpa = 0; u64 ch_sem_va = GSP_PHYS_RMCALL_VA_SEM + 4; @@ -18609,9 +25747,10 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, u8 *qmd_dst_mem = NULL; u32 qmd_rel_off = 0, ch_rel_off = 0; u32 poll = 0, w0 = 0, w1 = 0, grpoll_n = 0; + static u32 post_spin_n; bool sem_gated = false; - static u32 sked_prev[11]; - static u32 sked_sample[11]; + static u32 sked_prev[28]; + static u32 sked_sample[28]; static u32 sked_tick; static bool sked_first; /* @@ -18624,7 +25763,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * stack). Safe as static: the only caller is the single-inflight * ordered gsp-phys-work item. */ - static u32 bounce_csum_in[GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES]; + static u64 bounce_csum_in[GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES]; /* * r378: which guest page's content each bounce slot holds this * launch (set by the copy-in fills, cleared per submission). The @@ -18638,6 +25777,14 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, sked_tick = 0; sked_first = true; guest_param_pg = NULL; + /* + * SESSION 153-CONTINUED-95: prove the client's promoted GR context + * buffer binding at the first launches, then read the engine's own + * context buffer back for byte-level inspection (both bounded, both + * diagnostic; no behavior change to the launch itself). + */ + gsp_phys_ctx_bind_receipt(); + gsp_phys_ctx_readback_try(); u32 qmd_win_pde = 0, qmd_win_installed = 0; int qmd_win_ret = 0; bool qmd_win_diag = false; @@ -18647,6 +25794,22 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, bool rearmed = false; u32 attempts = 0; /* + * SESSION 153-CONTINUED-48: one bounded priv-error recovery attempt + * per launch (sema_gr_wedge_recover), with a per-boot tally for the + * report. Static like the other submit scratch: the single-inflight + * ordered work item is the only caller. + */ + bool wedge_recover_done = false; + static u32 wedge_recoveries; + /* + * SESSION 153-CONTINUED-72: the MME-FAST pre-reset spin can observe + * the 0xbadf poison and then the FECS clears it before the poll + * loop's first recovery check reads grfifo (the r1149 race: recovery + * missed -> prelude latch stuck -> engine wedged for the boot). The + * observation is latched here and OR-ed into the WEDGE-RECOVER gate. + */ + bool fast_poison = false; + /* * F2/F4 state: the fast first-timeout re-arm runs once per launch; the * deposit readback is one-shot per launch (the first timeout is the * evidence window - the old r198 post-mortem only ran after attempt 3, @@ -18722,6 +25885,26 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, gsp_phys_gr_refused++; return -EINVAL; } + /* + * r564: refuse a degenerate data set BEFORE the engine ever sees it. + * r565 evidence: the fill dropped 4 argument bases (GRFILL-DROP), + * RDW showed nrecs=2 where healthy launches run 968-969, the launch + * was still submitted and the backend timed out after ~19 s + * ("GR forward failed rc=-110"), after which the tail authored the + * software release and the client kept spinning on a completion for a + * kernel that never ran - the worst of both (20 s of backend + * occupancy plus a false completion). Refusing up front fails fast + * into the same software-completion fallback without touching the + * engine, and cannot wedge it for the next client. + */ + if (req->n_data < gsp_phys_gr_min_data_value()) { + gsp_phys_gr_refused++; + pr_err_ratelimited("vgsp-gr: GR submit refused: degenerate data set n_data=%u (min %u) launch_va=0x%llx\n", + req->n_data, + gsp_phys_gr_min_data_value(), + req->push_va); + return -ENODATA; + } if (!gsp_phys_backend_gr_ready() || !gsp_phys_guest_mem_enabled() || gsp_phys_dma_domain_identity() != 1 || !gsp_phys_guest_ram_active()) { gsp_phys_gr_refused++; @@ -18741,6 +25924,41 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, ret = -EIO; goto out; } + /* + * SESSION 153-CONTINUED-60: all-zero 3D-window skip. The client's push + * page is cyclically zeroed/rewritten (CONT-45); a capture that lands + * on the zero phase still forwards an all-zero method window. Live + * r1107: such a window launched (3D raw dump words=252, every word 0), + * the engine trapped SET_VERTEX_ATTRIBUTE_A(2) (trap=00001168 + * data_lo=0x50) and escalated to a priv-error poison (grfifo= + * 0xbadf1201, mme_pc=513) that wedged the GR lane for the rest of the + * boot; r1031 had already seen the same class fault the PBDMA on the + * zero lead (PB-FAULT-DUMP P0). A zero window carries no method to + * execute and nothing to store: skip the hardware handoff entirely - + * no shadow relocation, no engine contact, no wedge - and return the + * distinct rc -EBADMSG so the service authors the software release and + * suppresses the synthetic fault (a frame with no content deserves no + * client-visible engine error). + */ + if (stream3d) { + u32 zwi; + bool zerowin = req->stream_n != 0; + + for (zwi = 0; zwi < req->stream_n; zwi++) { + if (req->stream[zwi]) { + zerowin = false; + break; + } + } + if (zerowin) { + gsp_phys_gr_zerowin_skips++; + pr_warn_ratelimited("vgsp-gr: ZEROWIN-SKIP h=0x%x push=0x%llx window=%u all-zero (no engine submit; software release) n=%llu\n", + req->h_object, req->push_va, req->stream_n, + gsp_phys_gr_zerowin_skips); + ret = -EBADMSG; + goto out; + } + } gsp_phys_gr_submitted++; gsp_phys_gr_override_apply_param(); @@ -18802,6 +26020,52 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, } } /* + * r449 SHADER-CONSTANT-CACHE invalidate (gsp_phys_gr_ccache_inv; default + * 0 = off): the same cache-control word's bit 31 (the V01_06-layout + * SHADER_CONSTANT_CACHE position) is never set by the captured guest + * QMDs. The forwarded param page is rewritten in place between launches + * (the two-bank win0 pool), so a stale constant-cache line can serve + * the PREVIOUS launch's parameters and the current launch's output + * pages stay untouched - the post-dispatch no-store signature. Mode 1 + * sets bit 31; mode 2 forces the whole invalidate set (bits 24..31). + * Only when the QMD carries the driver's standard cache-control + * signature (bits 26..29) so an unknown QMD revision is never touched. + */ + if (gsp_phys_gr_ccache_inv_value() && + (qmd[5] & 0x3c000000u) == 0x3c000000u) { + u32 mode = gsp_phys_gr_ccache_inv_value(); + + if (mode == 2) + qmd[5] |= 0xff000000u; + else + qmd[5] |= 1u << 31; + if (!gsp_phys_r449_ccache_log) { + gsp_phys_r449_ccache_log = true; + pr_info("vgsp-gr: r449 constant-cache invalidate mode=%u applied (w5=0x%08x)\n", + mode, qmd[5]); + } + } + /* + * r433 RELEASE_MEMBAR_TYPE A/B (gsp_phys_gr_rel_membar): force the + * forwarded QMD's RELEASE_MEMBAR_TYPE to FE_SYSMEMBAR (QMDV00_06 + * word 11 bit 14) so the front-end flushes its pending system-memory + * stores before writing the release semaphore. The captured guest + * QMDs carry FE_NONE, so a grid that completes with its data still + * in the GPU's L2 releases the client while no store is host-visible + * - the post-dispatch store-loss signature. The release ordering then + * guarantees the host's retire scan and copy-out can never observe a + * completion whose data has not landed. 0 (default) keeps the + * guest's captured value. + */ + if (gsp_phys_gr_rel_membar_value() && !(qmd[11] & (1u << 14))) { + qmd[11] |= 1u << 14; + if (!gsp_phys_gr_rel_membar_log) { + gsp_phys_gr_rel_membar_log = true; + pr_info("vgsp-gr: r433 RELEASE_MEMBAR_TYPE forced to FE_SYSMEMBAR in the forwarded QMD (w11=0x%08x)\n", + qmd[11]); + } + } + /* * Two-step PCAS (gsp_phys_rmcall_gr_pcas_two_step): set the QMD's * REQUIRE_SCHEDULING_PCAS (w4 bit12) so the copy PCAS does not * schedule and the trailing schedule-only PCAS does. @@ -18815,13 +26079,16 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * a fresh shadow line so the guest observes its own QMD release; else * point release0 at the module semaphore so the host observes engine * completion. A reduction release is never redirected (its arithmetic - * would change). + * would change). r929: skipped entirely for a 3D stream replay - + * there is no QMD, so there is no release0 to point anywhere. */ - if (((qmd[4] >> 10) & 1)) { + if (!stream3d && ((qmd[4] >> 10) & 1)) { u32 red_en = (qmd[24] >> 26) & 1; u32 one_word = (qmd[24] >> 31) & 1; bool rel_mod = gsp_phys_rmcall_gr_rel_mod_value() != 0; + int rel_map_ret = 0; + if (!rel_mod && req->rel_hpa && one_word && !red_en && n_shadow < GSP_PHYS_RMCALL_GR_SHADOW_PAGES) { u64 sh_va = 0; @@ -18841,6 +26108,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, 0xffu); qmd[25] = req->rel_payload; } else { + rel_map_ret = ret; ret = 0; } } @@ -18869,6 +26137,23 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * completed on the guest word (the r414 saxpy hang). */ qmd_rel_redirected = true; + /* + * r633 diag: name the exact redirect cause. rel_hpa==0 + * means the caller could not resolve the release page + * (capture resolution or PA->HPA); map_ret!=0 means the + * page was resolved but its shadow mapping failed. + */ + { + static u32 qmdrel_redir_log; + + if (qmdrel_redir_log < 32) { + qmdrel_redir_log++; + pr_info("vgsp-gr: qmd-rel redirect: rel_hpa=0x%llx map_ret=%d one_word=%u red_en=%u rel_mod=%d n_shadow=%u pay=%u\n", + req->rel_hpa, rel_map_ret, + one_word, red_en, rel_mod ? 1 : 0, + n_shadow, req->rel_payload); + } + } } } @@ -18966,7 +26251,18 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, pg_va = pg->guest_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); pr_info("vgsp-gr: page[%u] va=0x%llx hpa=0x%llx qmd_word=%u\n", i, pg->guest_va, pg->hpa, pg->qmd_word); - if (pg->hpa) { + /* + * SESSION 149 R15 (M2-S1): with the window-0 shadow path + * enabled, force the program page (QMD word 48) OFF the + * post-commit Tier-1 install. r801b proved that install is + * ineffective for a late GR page (the engine kept reading + * the empty guest page: CODEHASH h0=0, FAULT-CAP + * code_src=guest-hpa); skipping it lets the page fall to + * the win0 shadow copy + w48/49 rewrite line, which is the + * proven one. Every other page keeps its Tier-1 route. + */ + if (pg->hpa && !(pg->qmd_word == 48u && + gsp_phys_rmcall_gr_shadow_win0_value())) { ret = gsp_phys_rmcall_gr_map_va_locked(bar0, pg_va, pg->hpa, epoch); if (!ret) { @@ -19073,6 +26369,59 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, pg->guest_va, sh_va, slot); } } + /* + * SESSION 153-CONTINUED-89: inline records that land on a + * qmd_word-NONE page. The record-apply loop below belongs to + * the QMD-word shadow-*copy* path and never runs for these + * pages; they are resolved to the guest HPA directly (Tier-1 + * identity, contiguous shadow-line extension, or the alias + * above), so authoring the record into pg->hpa is exactly + * what the engine will read. The concrete case: the real + * captured upload at 0x4350f00 (1408 B) crosses its 4 KB + * page - the head (256 B) lands on the program page 0x4350000 + * and applies, but the tail page 0x4351000 is qmd_word=NONE + * and was silently dropped, leaving the draw's winning j=1 + * program (the later bind) truncated at 256 B at the engine + * (the DS rejects the SASS it cannot fetch: SPH 9/11). + */ + if (mapped && pg->hpa && + pg->qmd_word == GSP_PHYS_GR_QMD_WORD_NONE) { + u32 jx; + + for (jx = 0; jx < req->n_inl; jx++) { + const struct gsp_phys_gr_inline *inl = + &req->inl[jx]; + u32 len; + + if ((inl->dst_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) != pg_va) + continue; + len = inl->len; + if (len > GSP_PHYS_GR_INLINE_BYTES) + len = GSP_PHYS_GR_INLINE_BYTES; + if ((u64)(inl->dst_va & + (GSP_PHYS_PAGE_SIZE - 1)) + len > + GSP_PHYS_PAGE_SIZE) + continue; + memcpy((u8 *)__va(pg->hpa) + + (inl->dst_va & + (GSP_PHYS_PAGE_SIZE - 1)), + inl->data, len); + { + static u32 inl_alias_log; + + if (inl_alias_log < 12u) { + inl_alias_log++; + pr_info("vgsp-gr: INLINE-APPLY-ALIAS dst=0x%llx len=%u pa=0x%llx first=%08x (qmd_word=NONE page authored)\n", + inl->dst_va, len, + pg->hpa, + len >= 4 ? + get_unaligned_le32(inl->data) : + 0u); + } + } + } + } if (!mapped && pg->qmd_word != GSP_PHYS_GR_QMD_WORD_NONE && pg->qmd_word + 1 < GSP_PHYS_GR_QMD_WORDS) { struct gsp_phys_boot_chunk *sh = @@ -19254,6 +26603,74 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, memcpy(sh_page + (inl->dst_va & (GSP_PHYS_PAGE_SIZE - 1)), inl->data, len); + /* + * SESSION 153-CONTINUED-10: a stream3d (3D + * method-window) submission maps the engine's + * VAs straight at the GUEST page HPA + * (gsp_phys_rmcall_gr_map_shadow_locked + * installs the guest PTE - there is no + * shadow-chunk copy in that path), so the + * engine's I-fetch reads the guest page, not + * sh_page. The GLX program page is never + * written by the guest itself (the driver's + * uploads never ran), and the r945 evidence is + * exactly that: "Graphics Exception: Shader + * Program Header 11/18 Error" on the draw + * whose inline prog-hex blob had only reached + * this shadow copy. Author the inline bytes + * into the guest page too, emulating the + * upload that never executed. + */ + if (stream3d && pg->hpa) + memcpy((u8 *)__va(pg->hpa) + + (inl->dst_va & + (GSP_PHYS_PAGE_SIZE - 1)), + inl->data, len); + /* + * SESSION 153-CONTINUED-11: bounded witness of + * every inline record applied (which program + * page received which image), so the SPH + * delivery is auditable from the log. + */ + { + static u32 inl_apply_log; + + if (inl_apply_log < 24) { + inl_apply_log++; + pr_info("vgsp-gr: INLINE-APPLY dst=0x%llx len=%u pg_va=0x%llx guest=%u first=%08x %08x\n", + inl->dst_va, len, pg_va, + stream3d && pg->hpa ? + 1u : 0u, + len >= 4 ? get_unaligned_le32(inl->data) : 0u, + len >= 8 ? get_unaligned_le32(inl->data + 4) : 0u); + } + } + /* + * SESSION 153-CONTINUED-87: readback witness. + * For a stream3d window the engine's I-fetch + * walks the GUEST page (the inline bytes were + * just authored there above); read the exact + * engine-visible location back so the fetch + * content is provable from the log - if the DS + * still rejects the header, the bytes are known + * present at the VA it fetched. + */ + if (stream3d && pg->hpa) { + static u32 inl_rb_log; + const u32 *gw = (const u32 *) + ((u8 *)__va(pg->hpa) + + (inl->dst_va & + (GSP_PHYS_PAGE_SIZE - 1))); + + if (inl_rb_log < 12u) { + inl_rb_log++; + pr_info("vgsp-gr: INLINE-RB dst=0x%llx pa=0x%llx rb=%08x %08x %08x %08x %08x %08x %08x %08x\n", + inl->dst_va, pg->hpa, + gw[0], gw[1], gw[2], + gw[3], gw[4], gw[5], + gw[6], gw[7]); + } + } copied++; } /* @@ -19520,8 +26937,9 @@ gr_page_unmapped: * GPFIFO/PB/REPORT_SEM legs already use. The 2 MiB window is installed * by the ladder itself, so no window ensure is sent for it (a * pdeIndex-0 Tier-1 install would touch the ladder's own PDE). + * r929: a 3D stream replay has no inline QMD at all (skipped). */ - if (gsp_phys_rmcall_gr_qmdwin_value()) { + if (!stream3d && gsp_phys_rmcall_gr_qmdwin_value()) { u8 *qw = boot_stage.rmcall_gr_qmdwin.pages ? gsp_phys_chunk_page_va(&boot_stage.rmcall_gr_qmdwin, 0) : NULL; @@ -19548,9 +26966,9 @@ gr_page_unmapped: * use it as the destination when the caller resolved it; a Tier-1 * install keeps the guest VA (the guest sees its own QMD), a Tier-2 * alias keeps the launch runnable, and an unresolved page keeps the - * module scratch VA. + * module scratch VA. r929: skipped for a 3D stream replay. */ - if (!qmdwin_on && req->qmd_hpa) { + if (!stream3d && !qmdwin_on && req->qmd_hpa) { u64 qmd_page = req->qmd_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); u32 qmd_off = (u32)(req->qmd_va & (GSP_PHYS_PAGE_SIZE - 1)); bool dst_ok = false; @@ -19628,6 +27046,27 @@ gr_page_unmapped: /* A stale release from a previous submission must not be observed. */ WRITE_ONCE(*(u32 *)sem_va, 0); WRITE_ONCE(*(u32 *)(sem_va + 4), 0); + /* + * r441: zero the GUEST-VISIBLE release words for the first attempt + * too, not only for outcome retries. The r439 evidence shows a first + * attempt whose poll broke on a STALE value (qmd_rel=0x00000005 from + * the previous launch's engine write, want=6) while the engine never + * fetched this launch's deposit (sentinel intact, "engine fetch + * NONE"): the host then shipped a completion for a launch that did + * not run. Zeroing forces the poll to wait for THIS launch's own + * engine write; when the engine does not write it, the timeout ladder + * and the software-authored completion take over (the honest path), + * and the wedge classification stops being satisfiable by a stale + * value. The write guard re-validates the target like the retry path. + */ + if (qmd_rel_hpa && qmd_rel_off < GSP_PHYS_PAGE_SIZE && + gsp_sema_gr_write_guard(req->rel_va, qmd_rel_hpa, + "gr-rel-zero-first")) + WRITE_ONCE(*(u32 *)((u8 *)__va(qmd_rel_hpa) + qmd_rel_off), 0); + if (ch_rel_hpa && ch_rel_off < GSP_PHYS_PAGE_SIZE && + gsp_sema_gr_write_guard(req->rel_va, ch_rel_hpa, + "gr-chrel-zero-first")) + WRITE_ONCE(*(u32 *)((u8 *)__va(ch_rel_hpa) + ch_rel_off), 0); wmb(); /* @@ -19647,13 +27086,603 @@ gr_page_unmapped: GSP_PHYS_GR_QMD_SENTINEL); } + /* + * r929 v3 (Path A): relocate the 3D window's address operands to + * shadow-pool VAs the engine walkers can reach (see + * gsp_phys_rmcall_gr_3d_patch_window; the r929 v2 FE faulted + * FAULT_PDE on a Tier-1-mapped guest VA). Runs with the rm lock + * held, before the method block is built. + */ + if (stream3d) { + u32 win_n = req->stream_n; + u32 ndrop = 0; + u32 nrel; + + /* + * SESSION 153-CONTINUED-27: a big colour target (the X + * screen) is aliased as ONE contiguous shadow run BEFORE + * the relocation pass runs, so the pass rewrites its + * operands to the alias (see the table above). + */ + gsp_phys_gr_surface_alias_scan_locked(req->stream, + req->stream_n, req); + /* SESSION 153-CONTINUED-59: fresh miss census for this pass + * (window + attr patch below both feed it). */ + gsp_phys_rmcall_gr_3d_miss_n = 0; + nrel = gsp_phys_rmcall_gr_3d_patch_window( + req->stream, req->stream_n, stream_patch, &win_n, + &ndrop, req, &n_shadow, bar0, epoch); + + win3d = stream_patch; + win3d_n = win_n; + /* + * SESSION 153-CONTINUED-69: drop the CALL_MME_MACRO/DATA runs + * of stubbed slots (see gsp_phys_gr_mme_drop_calls). With the + * stub as a pure NOP terminator the parameter port no longer + * drains; the r1142 stall parked the PBDMA on the 16-arg + * CALL_MME_MACRO(57) run and the block never completed. The + * drop turns those calls into nothing instead of a call whose + * parameters have nowhere to go. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_call_drop) && + gsp_sema_gr_sa_stub_value()) { + u32 sa_ent[64]; + u64 sa_mask = gsp_sema_mme_sa_get(sa_ent, 64); + u64 dmask = 0; + u32 ndrop_c = gsp_phys_gr_mme_drop_calls( + stream_patch, win3d_n, sa_mask, sa_ent, + &dmask); + + if (ndrop_c) { + win3d_n -= ndrop_c; + pr_info("vgsp-gr: MME-CALL-DROP %u word(s) of stubbed-slot CALL runs (window=%u) dropped_mask=0x%llx h=0x%x push=0x%llx\n", + ndrop_c, win3d_n, + (unsigned long long)dmask, + req->h_object, req->push_va); + } + } + pr_info("vgsp-gr: 3D window relocated %u operand pair(s) to shadow VAs (window=%u shadow=%u dropped=%u subdrop=%u sdmdrop=%u progkeep=%u)\n", + nrel, win3d_n, n_shadow, ndrop, + gsp_phys_rmcall_gr_3d_last_subdrop, + gsp_phys_rmcall_gr_3d_last_sdmdrop, + gsp_phys_rmcall_gr_3d_last_progkeep); + /* + * SESSION 153-CONTINUED-87: emit the client's missing + * pipeline-stage program binds (see the sema_gl_sph_bind + * comment). The window carries only the j=1 (VTG) pair; the + * real client state also binds j=2 (TESS_CONTROL at + * family+0x200) and j=5 (PIXEL at family+0x0) - both present + * verbatim in the init-push image scraped from guest RAM + * (`20020825 [0,04400200]`, `20020855 [0,04400000]`). Build + * the two runs and push them through the same window + * relocation pass (miss -> the pair stays raw and the whole + * bind is skipped, never a raw client VA to the engine), + * then prepend them ahead of the window. Beef client + * channels only: the family layout is that client's. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_sph_bind) && + (req->h_object & 0xfffff000u) == 0xbeef0000u && + req->prog_va && + win3d_n + 64u <= GSP_PHYS_RMCALL_GR_METHOD_WORDS) { + static u32 sph_bind_log; + u64 fam = req->prog_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + u32 in[64], out[64]; + u32 nfin = 0, nrun = 0, on = 0, odrop = 0, orel; + u32 bi; + + /* + * SESSION 153-CONTINUED-89 (2nd half): the client's + * full channel-init pipeline block, harvested from the + * guest-RAM init-push image (guest PA 0x81efc5b4, this + * boot) and identical in the X channel's own captured + * window (methods 0x21c, 0x155c, 0x1574 in that exact + * order). The per-frame window carries none of it - + * on real hardware the client wrote it ONCE at + * channel init, and the replay channel's fresh context + * never receives it, so the DS validates the fetched + * SPHs against unset context state (9/11). Emitted + * verbatim in the client's own order, ahead of the + * stage binds: + * 0x21c <- 0x1001 INVALIDATE_SHADER_CACHES + * (INSTRUCTION|CONSTANT) + * 0x155c <- 1, 0x214fa200, 0 + * TEX_SAMPLER_POOL A/B/C + * 0x1574 <- 1, 0x214fa220, 0x10 + * TEX_HEADER_POOL A/B/C + * 0x310 <- 0x705 SET_SPA_VERSION + * 0x22c <- 0x800 SET_GO_IDLE_TIMEOUT + * 0x1128 <- ffffffff, ffffffff + * DA_OUTPUT_ATTRIBUTE_SKIP_MASK + * ZCULL_BOUNDS(0x196c) is already carried by the + * window itself. All constant runs (no relocation). + */ + in[nfin++] = 0x20000000u | (1u << 16) | + (0x21cu >> 2); + in[nfin++] = 0x1001u; + { + /* + * The pool runs are address-bearing (the patch + * pass's own rules: addr 0x155c/0x1574, cnt >= + * 3) and count toward the relocated-pair + * guard, so emit them only when their page is + * in this launch's page set (r1290: emitted + * unconditionally -> unaccounted pairs -> + * `SPH-BIND skipped (reloc=6/4)` and the whole + * prelude was lost). Both VAs share one page. + */ + bool pool_pg = false; + u32 pk; + + for (pk = 0; pk < req->n_pages; pk++) + if ((req->pages[pk].guest_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) == + (0x214fa200u & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1))) + pool_pg = true; + if (pool_pg) { + in[nfin++] = 0x20000000u | (3u << 16) | + (0x155cu >> 2); + in[nfin++] = 0x1u; + in[nfin++] = 0x214fa200u; + in[nfin++] = 0x0u; + in[nfin++] = 0x20000000u | (3u << 16) | + (0x1574u >> 2); + in[nfin++] = 0x1u; + in[nfin++] = 0x214fa220u; + in[nfin++] = 0x10u; + nrun += 2; + } + } + in[nfin++] = 0x20000000u | (1u << 16) | + (0x310u >> 2); + in[nfin++] = 0x705u; + in[nfin++] = 0x20000000u | (1u << 16) | + (0x22cu >> 2); + in[nfin++] = 0x800u; + in[nfin++] = 0x20000000u | (2u << 16) | + (0x1128u >> 2); + in[nfin++] = 0xffffffffu; + in[nfin++] = 0xffffffffu; + + /* + * SESSION 153-CONTINUED-89: the client's init-push + * pipeline state for the window's own stage 1. The + * guest-RAM init-push image (live at guest PA + * 0x81efc5b4, this boot) carries, beside the SPH + * version pair: + * 0x2040 <- 0x11 SET_PIPELINE_SHADER(1) + * ENABLE|TYPE=VERTEX + * 0x2054 <- 1, 0x21201b00 program address pair + * 0x204c <- 0x1a, 0x0 REGISTER_COUNT(1)=26, + * BINDING(1)=0 + * The per-frame window carries only the address pair, + * so the replay channel's TYPE/COUNT/BINDING stage + * registers stay unset - the state the DS validates + * the fetched SPH against (9/11). Emit the TYPE run + * before the binds and the COUNT/BINDING pair after + * them, in the client's own order. Both are constant + * runs (no relocation). + */ + in[nfin++] = 0x20000000u | (1u << 16) | + (0x2040u >> 2); + in[nfin++] = 0x11u; + + /* + * SESSION 153-CONTINUED-87 (second half): the window's + * OWN binds first. The captured per-frame window + * carries draws (BEGIN at word 93) BEFORE its program + * binds (j=1 pair at word 184): on real hardware the + * bind happened in an earlier push, but the replay + * channel's fresh context has nothing bound when the + * early draws validate their stages - the + * Shader-Program-Header poison (r1277+ 9/11). Re-emit + * every bind the window itself carries ahead of it. + */ + for (bi = 0; bi < req->n_prog_binds && bi < 8u; bi++) { + u64 va = req->prog_bind_va[bi]; + u32 sl = req->prog_bind_slot[bi] & 0x1fu; + + if (!va || nfin + 3u > 64u) + continue; + in[nfin++] = 0x20000000u | (2u << 16) | + ((0x2014u + sl * 0x40u) >> 2); + in[nfin++] = (u32)(va >> 32) & 0xffu; + in[nfin++] = (u32)va & 0xfffffffcu; + nrun++; + } + /* + * Then the client's missing stage binds (see the + * sema_gl_sph_bind comment): TESS_CONTROL (j=2 at + * family+0x200) and PIXEL (j=5 at family+0x0), both + * present verbatim in the init-push image scraped + * from guest RAM (`20020825 [0,04400200]`, + * `20020855 [0,04400000]`). + */ + if (nfin + 13u <= 64u) { + in[nfin++] = 0x20000000u | (2u << 16) | + (0x2094u >> 2); + in[nfin++] = (u32)((fam + 0x200u) >> 32) & 0xffu; + in[nfin++] = (u32)(fam + 0x200u) & 0xfffffffcu; + in[nfin++] = 0x20000000u | (2u << 16) | + (0x2154u >> 2); + in[nfin++] = (u32)(fam >> 32) & 0xffu; + in[nfin++] = (u32)fam & 0xfffffffcu; + /* + * SESSION 153-CONTINUED-88: the per-slot + * SET_PIPELINE_SHADER(j) TYPE state. The + * client's init push writes it once per + * channel (init-push image: `20010810 + * 00000011` = SET_PIPELINE_SHADER(1) <- + * ENABLE|TYPE=VTG); the captured per-frame + * windows carry only the program-address + * pairs, so the replay context's stage + * registers for the synthesized binds would + * stay unset. Emit them for the two + * synthesized slots with the known types + * (j=2 TESS_CONTROL -> 2, j=5 PIXEL -> 5). + */ + in[nfin++] = 0x20000000u | (1u << 16) | + (0x2080u >> 2); + in[nfin++] = 1u | (2u << 4); + in[nfin++] = 0x20000000u | (1u << 16) | + (0x2140u >> 2); + in[nfin++] = 1u | (5u << 4); + /* + * SESSION 153-CONTINUED-89: the client's + * init-push stage-1 REGISTER_COUNT/BINDING + * pair (`20020813 0000001a 00000000` in the + * guest-RAM init image): REGISTER_COUNT(1) = + * 26 (the window SPH's reg count), BINDING(1) + * = group 0. Emitted after the address pair, + * exactly as the client's own push orders it. + */ + in[nfin++] = 0x20000000u | (2u << 16) | + (0x204cu >> 2); + in[nfin++] = 0x1au; + in[nfin++] = 0x0u; + /* + * nrun counts only the two ADDRESS-bearing + * binds above: the guard below matches it + * against the relocation pass's + * relocated-pair count, and the constant + * type runs can never be relocated. + */ + nrun += 2; + } + orel = gsp_phys_rmcall_gr_3d_patch_window( + in, nfin, out, &on, &odrop, req, &n_shadow, + bar0, epoch); + if (orel == nrun && on == nfin) { + memmove(&stream_patch[nfin], stream_patch, + (size_t)win3d_n * 4u); + memcpy(stream_patch, out, (size_t)nfin * 4u); + win3d_n += nfin; + if (sph_bind_log < 16u) { + sph_bind_log++; + pr_info("vgsp-gr: SPH-BIND emitted fam=0x%llx win-binds=%u j2=0x%llx j5=0x%llx j1-type=0x11 rc=0x1a initblk=17 win=%u (pre=%u) h=0x%x push=0x%llx\n", + fam, req->n_prog_binds, + fam + 0x200u, fam, + win3d_n, nfin, + req->h_object, + req->push_va); + } + } else if (sph_bind_log < 16u) { + sph_bind_log++; + pr_info("vgsp-gr: SPH-BIND skipped (reloc=%u/%u kept=%u/%u fam=0x%llx h=0x%x push=0x%llx)\n", + orel, nrun, on, nfin, fam, + req->h_object, req->push_va); + } + } + /* + * SESSION 153-CONTINUED-62: surface (render-target) prelude - + * see the gsp_phys_gr_surf_pre comment. Injected only when the + * window carries no usable colour-target bind of its own; the + * alias scan + relocation run exactly as they would for a + * window-carried bind (the captured VA is the surface the + * machine's draws target, X screen observed). The miss census + * feeds the MISS-ABORT below, so an unresolvable surface pair + * holds the launch (fail-closed) instead of faulting. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_surf_prelude) && + !gsp_phys_gr_window_has_surface(req->stream, + req->stream_n)) { + const u32 *sp; + u32 spn = 0, spg = 0; + + sp = gsp_sema_surf_prelude_get(&spn, &spg); + if (sp && spn && spg) { + if (spg != READ_ONCE(gsp_phys_gr_surf_pre_gen)) { + u32 sdrop = 0, sn = spn; + u32 srel; + + if (sn > GSP_PHYS_GR_SURF_PRE_WORDS) + sn = GSP_PHYS_GR_SURF_PRE_WORDS; + memcpy(gsp_phys_gr_surf_pre_in, sp, + (size_t)sn * 4u); + gsp_phys_gr_surface_alias_scan_locked( + gsp_phys_gr_surf_pre_in, sn, + req); + srel = gsp_phys_rmcall_gr_3d_patch_window( + gsp_phys_gr_surf_pre_in, sn, + gsp_phys_gr_surf_pre, + &gsp_phys_gr_surf_pre_n, + &sdrop, req, &n_shadow, + bar0, epoch); + WRITE_ONCE(gsp_phys_gr_surf_pre_gen, + spg); + pr_info("vgsp-gr: SURF-PRELUDE patched gen=%u raw=%u kept=%u dropped=%u reloc=%u h=0x%x push=0x%llx\n", + spg, sn, + gsp_phys_gr_surf_pre_n, + sdrop, srel, + req->h_object, + req->push_va); + } + if (gsp_phys_gr_surf_pre_n && + win3d_n + gsp_phys_gr_surf_pre_n + 2u + + 8u <= + GSP_PHYS_RMCALL_GR_METHOD_WORDS) { + static u32 surf_emit_log; + + memmove(&stream_patch[gsp_phys_gr_surf_pre_n], + stream_patch, + (size_t)win3d_n * 4u); + memcpy(stream_patch, + gsp_phys_gr_surf_pre, + (size_t)gsp_phys_gr_surf_pre_n * + 4u); + win3d_n += gsp_phys_gr_surf_pre_n; + if (surf_emit_log < 16) { + surf_emit_log++; + pr_info("vgsp-gr: SURF-PRELUDE emitted words=%u window=%u h=0x%x push=0x%llx\n", + gsp_phys_gr_surf_pre_n, + win3d_n, + req->h_object, + req->push_va); + } + } + } + } + /* + * SESSION 153-CONTINUED-57: patch the attr/pipeline prelude for + * this launch (relocation exactly like the window: the + * SET_PIPELINE_PROGRAM_ADDRESS pairs and any other recognised + * operand land in engine-visible shadow VAs; sub!=0 and + * win_drop runs are removed by the same pass). Only done when + * the capture generation moved - the shadow VAs are stable + * per HPA for the boot, so an older patch stays valid while it + * drains across launches. See gsp_phys_gr_attr_pre*. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_attr_prelude)) { + const u32 *ap; + u32 apn = 0, apg = 0; + + ap = gsp_sema_attr_prelude_get(&apn, &apg); + if (ap && apn && apg && + apg != READ_ONCE(gsp_phys_gr_attr_pre_gen)) { + u32 adrop = 0, an = apn; + + if (an > GSP_PHYS_GR_ATTR_PRE_WORDS) + an = GSP_PHYS_GR_ATTR_PRE_WORDS; + gsp_phys_rmcall_gr_3d_patch_window(ap, an, + gsp_phys_gr_attr_pre, + &gsp_phys_gr_attr_pre_n, + &adrop, req, &n_shadow, + bar0, epoch); + if (gsp_phys_gr_attr_pre_n) { + WRITE_ONCE(gsp_phys_gr_attr_pre_gen, + apg); + pr_info("vgsp-gr: ATTR-PRELUDE patched gen=%u raw=%u kept=%u dropped=%u h=0x%x push=0x%llx\n", + apg, an, gsp_phys_gr_attr_pre_n, + adrop, req->h_object, + req->push_va); + } + } + } + if (ndrop) { + static u32 gr3d_drop_log; + + if (gr3d_drop_log < 32) { + gr3d_drop_log++; + pr_info("vgsp-gr: 3D win-drop h=0x%x push=0x%llx raw=%u kept=%u dropped=%u list=[%s]\n", + req->h_object, req->push_va, + req->stream_n, win3d_n, ndrop, + gsp_phys_rmcall_gr_win_drop ? + gsp_phys_rmcall_gr_win_drop : ""); + } + } + /* + * SESSION 153-CONTINUED-59: pre-doorbell MISS abort (see the + * gsp_phys_rmcall_gr_miss_abort comment). A window/attr + * operand whose relocation stayed a raw client VA after the + * level-2 resolver would make the physical engine read an + * unmapped page (r1104 MMU Fault HUBCLIENT_PD @0x4350000 -> + * guest channel reset; r1105 index-buffer misses). Ride the + * same failure path as RELOC-GAP-STRICT: the launch is not + * submitted, the caller falls back and authors the software + * release, the page is already in the pin demand set, and + * the next forward retries. Bounded: after 3000 aborts the + * check fails open so a page the resolver can never see + * cannot livelock the lane forever. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_miss_abort) && + gsp_phys_rmcall_gr_3d_miss_n && + gsp_phys_gr_miss_abort_n < 3000u) { + static u32 miss_abort_log; + + gsp_phys_gr_miss_abort_n++; + if (miss_abort_log < 16u || + !(gsp_phys_gr_miss_abort_n % 256u)) { + miss_abort_log++; + pr_info("vgsp-gr: MISS-ABORT h=0x%x push=0x%llx miss=%u n=%llu (operand page unresolvable; demanded via pin path, launch not submitted - retry next forward)\n", + req->h_object, req->push_va, + gsp_phys_rmcall_gr_3d_miss_n, + gsp_phys_gr_miss_abort_n); + } + mutex_unlock(&gsp_phys_rm_lock); + gsp_phys_gr_refused++; + /* + * SESSION 153-CONTINUED-61: the dedicated verdict lets + * the sema service keep the captured candidate armed + * and re-forward it (re-arm) instead of consuming it + * with a software completion: the demanded pages + * resolve through the pin path and the retry then runs + * the real launch. + */ + ret = GSP_PHYS_GR_RC_MISSABORT; + goto out; + } + /* + * SESSION 153-CONTINUED-56: bounded RAW (pre-drop, pre-reloc) + * window dump so the emitted window can be diffed against the + * client's authentic bytes. r1091 faulted at method 0x1168 + * (SET_VERTEX_ATTRIBUTE_A(2)) with Data 00000000 in the emitted + * stream; this copy tells whether that zero is the client's own + * value or an emission artifact (drop/reloc/align). + */ + { + static u32 gr3d_raw_log; + u32 wi; + + if (gr3d_raw_log < 8) { + gr3d_raw_log++; + pr_info("vgsp-gr: 3D raw dump h=0x%x push=0x%llx words=%u\n", + req->h_object, req->push_va, + req->stream_n); + for (wi = 0; + wi + 16 <= req->stream_n && wi < 448; + wi += 16) + pr_info("vgsp-gr: 3D raw[%3u]: %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n", + wi, + req->stream[wi + 0], + req->stream[wi + 1], + req->stream[wi + 2], + req->stream[wi + 3], + req->stream[wi + 4], + req->stream[wi + 5], + req->stream[wi + 6], + req->stream[wi + 7], + req->stream[wi + 8], + req->stream[wi + 9], + req->stream[wi + 10], + req->stream[wi + 11], + req->stream[wi + 12], + req->stream[wi + 13], + req->stream[wi + 14], + req->stream[wi + 15]); + } + } + /* + * SESSION 153-CONTINUED-11: bounded raw window dump - the + * exact C597 method stream the engine walks (program binds + * 0x2014/0x2018+j*0x40, SET_PIPELINE_SHADER 0x2000+j*0x40, + * draws 0x1618/0xd74). Two windows per boot. + */ + { + static u32 gr3d_win_log; + u32 wi; + + if (gr3d_win_log < 48) { + gr3d_win_log++; + pr_info("vgsp-gr: 3D window dump h=0x%x push=0x%llx words=%u\n", + req->h_object, req->push_va, win3d_n); + for (wi = 0; wi < win3d_n && wi < 384; + wi += 16) + pr_info("vgsp-gr: 3D win[%3u]: %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n", + wi, + win3d[wi + 0], win3d[wi + 1], + win3d[wi + 2], win3d[wi + 3], + win3d[wi + 4], win3d[wi + 5], + win3d[wi + 6], win3d[wi + 7], + win3d[wi + 8], win3d[wi + 9], + win3d[wi + 10], win3d[wi + 11], + win3d[wi + 12], win3d[wi + 13], + win3d[wi + 14], win3d[wi + 15]); + } + } + } + words = gsp_phys_rmcall_build_gr_launch(methods, req, qmd, qmd_dst, - ch_sem_va, want_ch); - memcpy(gsp_phys_chunk_page_va(pb, 0), methods, words * 4); + ch_sem_va, want_ch, + win3d, win3d_n, + gsp_phys_gr_attr_pre, + gsp_phys_gr_attr_pre_n); + gsp_phys_gr_pb_copy(methods, words); + /* SESSION 153-CONTINUED-12: identify this block to a later PBDMA dump. */ + WRITE_ONCE(gsp_phys_gr_last_words, words); + WRITE_ONCE(gsp_phys_gr_last_h, req->h_object); + WRITE_ONCE(gsp_phys_gr_last_push_va, req->push_va); + WRITE_ONCE(gsp_phys_gr_last_pb_dma, + (u64)gsp_phys_chunk_page_dma(pb, 0)); + /* + * SESSION 153-CONTINUED-52: stream validator for the block the engine + * will walk. The r1072 fault class ("Channel ID 4 intr0 00040000" = + * PBDMA PBENTRY, "invalid method encoding used in method", + * fifo_tu102.c) is raised by a method header the PBDMA refuses, or by + * a run whose data is not fully present so the walk desyncs into + * following words. Walk the exact published dwords with the FIFO DMA + * grammar (dev_ram.ref NV_FIFO_DMA_*: SEC_OP 31:29; 1/3/5 carry 1+cnt + * data words, 4/7 and TERT mask ops are single-word, 6 is RESERVED) + * and flag the first occurrences of a reserved encoding or a run + * overrunning the block. Read-only; bounded per boot. + */ + { + static u32 blkval_log; + const u32 *bv = methods; + u32 bo = 0, nbad = 0; + + if (blkval_log < 48) { + while (bo < words && nbad < 4) { + u32 h = bv[bo]; + u32 bop = (h >> 29) & 7u; + u32 bcnt = (h >> 16) & 0xfffu; + u32 bspan = 1u; + + if (bop == 1u || bop == 3u || bop == 5u) + bspan = 1u + bcnt; + if (bop == 6u) { + nbad++; + pr_warn("vgsp-gr: BLOCKVAL reserved-encoding word=%u/%u val=%08x (h=%x push=0x%llx prev=%08x %08x)\n", + bo, words, h, req->h_object, + req->push_va, + bo ? bv[bo - 1] : 0, + bo + 1 < words ? + bv[bo + 1] : 0); + } + if (bo + bspan > words) { + nbad++; + pr_warn("vgsp-gr: BLOCKVAL run-overrun word=%u/%u hdr=%08x span=%u (needs %u more) (h=%x push=0x%llx)\n", + bo, words, h, bspan, + bo + bspan - words, + req->h_object, req->push_va); + } + bo += bspan; + } + blkval_log++; + if (nbad) + pr_warn("vgsp-gr: BLOCKVAL done words=%u bad=%u (h=%x push=0x%llx)\n", + words, nbad, req->h_object, + req->push_va); + } + } pr_info("vgsp-gr: qmd w0..3=%08x %08x %08x %08x w4..7=%08x %08x %08x %08x w23..25=%08x %08x %08x w32/33=%08x %08x w48/49=%08x %08x\n", qmd[0], qmd[1], qmd[2], qmd[3], qmd[4], qmd[5], qmd[6], qmd[7], qmd[23], qmd[24], qmd[25], qmd[32], qmd[33], qmd[48], qmd[49]); + /* + * r430: full 64-word deposit dump for the wedge's healthy-vs-wedged + * QMD diff (hardware-only fields w50/w51 INNER_GET/PUT, w52 span-list, + * w53 next-QMD pointer are not in the partial line above). Gated by + * the skedrec param (default on); one long line per launch. + */ + if (gsp_phys_rmcall_gr_skedrec_value()) { + char qb[64 * 9 + 1]; + int qo = 0; + + for (i = 0; i < GSP_PHYS_GR_QMD_WORDS && qo < (int)sizeof(qb) - 10; i++) + qo += scnprintf(qb + qo, sizeof(qb) - qo, "%08x ", qmd[i]); + pr_info("vgsp-gr: QMD64 %s\n", qb); + } /* m120: dump the main PT page slots 0x100..0x11f so the PTE content * the walker would read (win0 + bounce window) is visible in the log. */ { @@ -19823,6 +27852,14 @@ gr_page_unmapped: (u64)gcopied << 12); } + /* + * CONT-73: fingerprint the direct-write witness set (relocated + * operand destinations that map guest pages) immediately before the + * doorbell - after every pre-launch writer (copy-in, mirrors) has + * run, so the retire re-scan measures the ENGINE's stores alone. + */ + gsp_phys_gr_wit_capture(req); + ret = gsp_phys_rmcall_publish_locked(bar0, boot_stage.rmcall_va_pb, words, &sub); mutex_unlock(&gsp_phys_rm_lock); @@ -19956,19 +27993,53 @@ gr_page_unmapped: break; } } - pr_info("vgsp-gr: GR LAUNCH submitted h=0x%x push=0x%llx len=%u pages=%u shadow=%u qmd_va=0x%llx slot=%u words=%u prelude=%d qmdvirt=%d token=0x%x; polling qmd_rel=%u ch_rel=0x%08x (<=%u ms)%s\n", + pr_info("vgsp-gr: GR LAUNCH submitted h=0x%x push=0x%llx len=%u pages=%u shadow=%u qmd_va=0x%llx slot=%u gpget=%u gpput=%u words=%u e0=0x%08x e1=0x%08x prelude=%d qmdvirt=%d token=0x%x; polling qmd_rel=%u ch_rel=0x%08x (<=%u ms)%s\n", req->h_object, req->push_va, req->push_len, req->n_pages, - n_shadow, req->qmd_va, sub.slot, words, + n_shadow, req->qmd_va, sub.slot, sub.gpget, sub.gpput, words, + sub.e0, sub.e1, gsp_phys_rmcall_gr_prelude_value(), gsp_phys_rmcall_gr_qmdvirt_value(), boot_stage.rmcall_token, qmd_rel, want_ch, poll_ms, qmdwin_note); + { + /* + * r596 diag: the raw-prelude geometry the verbatim gate tests. + * The verbatim replay needs + * raw && raw_words >= hdr + 67 && + * raw_qmd_end == hdr + 67 && + * INIT_WORDS + hdr + 67 + TAIL_WORDS <= METHOD_WORDS + * Bounded to the first eight launches so the geometry is on + * record without flooding a battery run. + */ + static u32 prelude_geom_log; + + if (gsp_phys_rmcall_gr_prelude_value() && prelude_geom_log < 8) { + prelude_geom_log++; + pr_info("vgsp-gr: prelude geom raw=%u raw_words=%u hdr=%u end=%u want_end=%u v=%d\n", + req->raw ? 1u : 0u, req->raw_words, + req->raw_qmd_hdr, req->raw_qmd_end, + req->raw_qmd_hdr + + GSP_PHYS_RMCALL_GR_QMD_WINDOW_WORDS, + gsp_phys_rmcall_gr_prelude_value()); + } + } if (qmd_win_diag) pr_info("vgsp-gr: qmd dest va=0x%llx pdeIndex=%u installed=%u win_mask=0x%llx ensure_ret=%d\n", qmd_dst, qmd_win_pde, qmd_win_installed, (unsigned long long)boot_stage.rmcall_guest_pt_win_mask, qmd_win_ret); + /* + * SESSION 153-CONTINUED-69: pre-reset MME fault sample. The engine + * faults within the first tens of ms of the launch; sample the MME + * ESR/INFO/INFO2 + DS ESR + GRFIFO/trap regs at MMIO rate from the + * moment the doorbell rang (in publish) so the faulting values are + * caught before the GSP-RM clears them. + */ + if (stream3d && bar0) + fast_poison = gsp_phys_mme_fast_sample(bar0, "launch", + req->h_object, 0); + poll_start_us = (u32)ktime_to_us(ktime_get()); deadline = jiffies + msecs_to_jiffies(poll_ms); for (;;) { @@ -19978,8 +28049,14 @@ gr_page_unmapped: * cancel or dispatch sequence of the launch window is on the * record even when the poll times out. */ - if (gsp_phys_rmcall_gr_skedrec_value() && bar0 && - (++sked_tick & 7u) == 0u) { + if (gsp_phys_rmcall_gr_skedrec_value() && bar0) { + /* + * r430: sample EVERY poll iteration (not every 8th): + * the wedge's release word lands ~7 ms after the + * doorbell, so the old 8-iteration gate saw at most one + * sample of the dispatch window. Changes only. + */ + sked_tick++; sked_sample[0] = ioread32(bar0 + 0x00407060u); sked_sample[1] = ioread32(bar0 + 0x00407064u); sked_sample[2] = ioread32(bar0 + 0x00407068u); @@ -19991,23 +28068,208 @@ gr_page_unmapped: sked_sample[8] = ioread32(bar0 + 0x00409b00u); sked_sample[9] = ioread32(bar0 + 0x00409b04u); sked_sample[10] = ioread32(bar0 + 0x00409400u); + /* + * SESSION 153-CONTINUED-14: FECS/FE ctxsw handshake + * lanes. fe0 = FECS_CTXSW_STATUS_FE_0 (0x409c00), + * fehost = FECS_HOST_INT_STATUS (0x409c18), fecsirq + * = FECS_IRQSTAT (0x409008). If the FECS dies in a + * switch, it should die visibly here first. + */ + sked_sample[11] = ioread32(bar0 + 0x00409c00u); + sked_sample[12] = ioread32(bar0 + 0x00409c18u); + sked_sample[13] = ioread32(bar0 + 0x00409008u); + /* + * SESSION 153-CONTINUED-52: in-flight fault registers. + * The DS shader-program-header error latch (0x405840), + * the MME exception latch (0x404490) with its PC/macro + * address (0x404494) and instruction register + * (0x40449c, fixed in SESSION 153-CONTINUED-106) are + * cleared by the real RM within ~ms of + * the fault, so the post-timeout reads are all clean. + * Sampling them every poll tick records the faulting + * values at the first change. + */ + sked_sample[14] = ioread32(bar0 + 0x00405840u); + sked_sample[15] = ioread32(bar0 + 0x00404490u); + sked_sample[16] = ioread32(bar0 + 0x00404494u); + /* SESSION 153-CONTINUED-106: INFO2 0x40449c (was CONFIG). */ + sked_sample[17] = ioread32(bar0 + 0x0040449cu); + /* + * SESSION 153-CONTINUED-52: the PBDMA host-software + * METHOD latch (NV_PPBDMA_METHOD0 0x4000c0: ADDR + * 13:2, SUBCH 18:16, INCR 0) - the method whose + * delivery raises NV_PPBDMA_INTR_0_METHOD (bit 21). + * The real driver clears the entry when it processes + * the interrupt, so only an in-flight sample can name + * it. + */ + sked_sample[18] = ioread32(bar0 + 0x004000c0u); + sked_sample[19] = ioread32(bar0 + 0x004000c4u); + /* + * SESSION 153-CONTINUED-57: the class-error trapped + * register set (gr_error_gk104.c grGetExceptionChannel): + * NV_PGRAPH_TRAPPED_ADDR 0x40704 (MTHD offset + SUBCH + + * INVALID status bit), TRAPPED_DATA_LOW 0x40708 (the + * class error's "Data" field), TRAPPED_DATA_HIGH + * 0x4070c, and TRAPPED_DATA_MME 0x40710 (PC 11:0 + + * MACRO_ADDRESS 19:12) - the last one names whether the + * offending write came from MME macro execution. The + * real RM clears these within ~ms of the fault; a + * per-tick sample catches the first change. The + * persistent 0x1168/Data=0 class error (r1091/r1098) + * needs exactly this decode. + */ + sked_sample[20] = ioread32(bar0 + 0x00400704u); + sked_sample[21] = ioread32(bar0 + 0x00400708u); + sked_sample[22] = ioread32(bar0 + 0x0040070cu); + sked_sample[23] = ioread32(bar0 + 0x00400710u); + /* + * SESSION 153-CONTINUED-58: the front-end DECODE + * STATE EX registers (NV_PGRAPH_PRI_FE_DECODE_STATE_EX + * 0x40a630..0x40a63c, 4 slots): MME_PC 11:0, + * MME_MACRO_ADDR 21:12, MME_GENERATED bit 31 - the + * front end's own record of the last MME-generated + * method write, which names the macro (address) and + * its pc for the class error even when the trapped + * registers are already back to the 0xbadf priv + * poison. Sampled per tick like the trap set. + */ + sked_sample[24] = ioread32(bar0 + 0x0040a630u); + sked_sample[25] = ioread32(bar0 + 0x0040a634u); + sked_sample[26] = ioread32(bar0 + 0x0040a638u); + sked_sample[27] = ioread32(bar0 + 0x0040a63cu); if (sked_first || memcmp(sked_sample, sked_prev, sizeof(sked_sample))) { - pr_info("vgsp-gr: sked rec +%ums cwd0=%08x cwd1=%08x cwd2=%08x info1=%08x info2=%08x act=%08x esr=%08x grfifo=%08x fecs_cur=%08x fecs_new=%08x ctxsw1=%08x\n", - (sked_tick / 8u) * + pr_info("vgsp-gr: sked rec +%ums cwd0=%08x cwd1=%08x cwd2=%08x info1=%08x info2=%08x act=%08x esr=%08x grfifo=%08x fecs_cur=%08x fecs_new=%08x ctxsw1=%08x fe0=%08x fehost=%08x fecsirq=%08x ds_esr=%08x mme_esr=%08x mme_pc=%u mme_mac=0x%x mme_ir=%08x mth0=%08x mth1=%08x(pbaddr=0x%x sub=%u incr=%u) trap=%08x(data_lo=%08x hi=%08x sub=%u mmegen=%u dhv=%u) mmepc=%u mmemac=0x%x\n", + sked_tick * GSP_PHYS_RMCALL_POLL_MS, sked_sample[0], sked_sample[1], sked_sample[2], sked_sample[3], sked_sample[4], sked_sample[5], sked_sample[6], sked_sample[7], sked_sample[8], sked_sample[9], - sked_sample[10]); + sked_sample[10], sked_sample[11], + sked_sample[12], sked_sample[13], + sked_sample[14], sked_sample[15], + sked_sample[16] & 0xfffu, + (sked_sample[16] >> 12) & 0xffu, + sked_sample[17], + sked_sample[18], sked_sample[19], + (sked_sample[18] >> 2) << 2, + (sked_sample[18] >> 16) & 7u, + sked_sample[18] & 1u, + sked_sample[20], sked_sample[21], + sked_sample[22], + (sked_sample[20] >> 16) & 7u, + (sked_sample[20] >> 20) & 1u, + (sked_sample[20] >> 24) & 1u, + sked_sample[23] & 0xfffu, + (sked_sample[23] >> 12) & 0xffu); + if (sked_sample[24] || sked_sample[25] || + sked_sample[26] || sked_sample[27]) + pr_info("vgsp-gr: sked fed0=%08x fed1=%08x fed2=%08x fed3=%08x (pc0=%u mac0=0x%x gen0=%u)\n", + sked_sample[24], + sked_sample[25], + sked_sample[26], + sked_sample[27], + sked_sample[24] & 0xfffu, + (sked_sample[24] >> 12) & 0x3ffu, + (sked_sample[24] >> 31) & 1u); memcpy(sked_prev, sked_sample, sizeof(sked_sample)); sked_first = false; } } + /* + * SESSION 153-CONTINUED-48: priv-error (0xbadf-tagged) wedge + * recovery - see sema_gr_wedge_recover. The poison latch is + * checked before the completion break and the ladder so a + * launch whose window faulted the GR hands the engine to the + * next launch recovered instead of wedged. One attempt per + * launch (local flag); the hard re-arm is the r445 mechanism, + * bounded by its own RPC and sem-probe timeouts. + */ + if (READ_ONCE(gsp_phys_gr_wedge_recover) && bar0 && + !wedge_recover_done) { + u32 g0 = ioread32(bar0 + 0x00400504u); + + /* + * SESSION 153-CONTINUED-72: either the poison is still + * asserted here, or the MME-FAST spin caught it before + * the FECS cleared it (fast_poison latch). Both mean + * the window priv-errored and the engine context is + * gone; the earlier live-register-only check missed + * the self-cleared case (r1149) and wedged the boot. + */ + if ((g0 & 0xffff0000u) == 0xbadf0000u || fast_poison) { + u32 g1; + + wedge_recover_done = true; + wedge_recoveries++; + pr_warn("vgsp-gr: WEDGE-RECOVER priv-error poison seen (grfifo=0x%08x%s) h=0x%x push=0x%llx - TSG preempt + resched\n", + g0, + fast_poison && + (g0 & 0xffff0000u) != 0xbadf0000u ? + " via MME-FAST latch (self-cleared)" : + "", + req->h_object, req->push_va); + gsp_phys_mme_regs_log(bar0, "wedge-poison"); + mutex_lock(&gsp_phys_rm_lock); + gsp_phys_gr_wedge_hard_rearm_locked(bar0); + mutex_unlock(&gsp_phys_rm_lock); + g1 = ioread32(bar0 + 0x00400504u); + /* + * SESSION 153-CONTINUED-61: the TSG preempt + + * resched above resets the engine context, so + * the MME program the emitted prelude loaded is + * gone. Clear the emission latch: the next + * launch that can carry the prelude re-preloads + * it (without this every later replay ran its + * macros against an unloaded MME and stalled + * pre-retire - the r1110 mme_pc=513 class). + */ + WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, 0); + /* + * SESSION 153-CONTINUED-79: also rewind the + * split-emission drain cursor, else a cleared + * latch with an exhausted cursor would skip + * the owed re-preload. + */ + WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0); + /* + * SESSION 153-CONTINUED-109: rewind the PACK + * cursor too (the r1377 deadlock: recovery reset + * pre_emit_off but left pack_off at its end, so + * the split emitter hit `poff >= maxa` -> can=0 + * -> packed=1 with zero words on every launch: + * `MME-PRELUDE part gen=3 off=0 words=0/3083` + * forever, latch never re-set, gates never + * release, MME never starts). The gate-fail + * path (gsp_phys_gr_mme_iram_gate_fail) already + * rewinds both cursor pairs. + */ + WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_sa, false); + /* + * SESSION 153-CONTINUED-109: the hard re-arm + * wiped the engine context, so the generation + * whose carrier executed is no longer resident. + * Invalidate it; the re-preload's own GP_GET + * advance (readback-free witness) re-promotes + * (fetch_miss/poll-break paths). Without this + * the stale live_gen would mark a wiped context + * live and the next window+calls launch would + * run against an unloaded MME. + */ + WRITE_ONCE(gsp_phys_gr_mme_live_gen, 0); + pr_warn("vgsp-gr: WEDGE-RECOVER after re-arm grfifo=0x%08x (%s); recovery #%u this boot; MME prelude latch cleared (context re-preload owed)\n", + g1, + (g1 & 0xffff0000u) == 0xbadf0000u ? + "still poisoned" : "cleared", + wedge_recoveries); + } + } if (qmd_rel) { if (qmd_rel_hpa) w0 = READ_ONCE(*(u32 *)((u8 *)__va(qmd_rel_hpa) + @@ -20015,15 +28277,147 @@ gr_page_unmapped: else w0 = READ_ONCE(*(u32 *)sem_va); } + /* + * SESSION 153-CONTINUED-59: PBENTRY first-sighting watch. + * The r1102 fault was only captured via the GSP's 0x1006 + * relay (seconds later, intr0 already cleared by the RM): + * here every 5 ms poll of a 3D-stream launch scans the 12 + * PBDMA instances for NV_PPBDMA_INTR_0 bit 18 (PBENTRY) and + * on the FIRST sighting dumps the live PB_HEADER / PB_COUNT + * / PB_DATA0 / HDR_SHADOW set plus the raw 0x100..0x120 + * block for the faulting instance, before any re-arm or RM + * clear can overwrite them. Bounded (8 lines per boot); + * read-only, no behavior change to the launch. + */ + if (stream3d && bar0) { + static u32 pbentry_log; + u32 pi2; + + for (pi2 = 0; pi2 < 12 && pbentry_log < 8; pi2++) { + u32 b2 = 0x40000u + pi2 * 0x2000u; + u32 i0 = ioread32(bar0 + b2 + 0x108u); + + if (!(i0 & 0x00040000u)) + continue; + pbentry_log++; + pr_warn("vgsp-gr: PBENTRY-FIRST-SIGHTING P%u intr0=%08x gpf=%u pbfetch=0x%llx hdr=%08x cnt=%08x data0=%08x pbhd=%08x h=%x push=0x%llx words=%u tick=%u\n", + pi2, i0, + ioread32(bar0 + b2 + 0x050u), + ((u64)(ioread32(bar0 + b2 + 0x058u) & + 0xffu) << 32) | + (ioread32(bar0 + b2 + 0x054u) & + ~3u), + ioread32(bar0 + b2 + 0x118u), + ioread32(bar0 + b2 + 0x088u), + ioread32(bar0 + b2 + 0x08cu), + ioread32(bar0 + b2 + 0x084u), + req->h_object, req->push_va, words, + sked_tick); + pr_info("vgsp-gr: PBENTRY-FIRST-SIGHTING P%u raw[0x100]=%08x [104]=%08x [10c]=%08x [110]=%08x [114]=%08x [11c]=%08x [120]=%08x\n", + pi2, + ioread32(bar0 + b2 + 0x100u), + ioread32(bar0 + b2 + 0x104u), + ioread32(bar0 + b2 + 0x10cu), + ioread32(bar0 + b2 + 0x110u), + ioread32(bar0 + b2 + 0x114u), + ioread32(bar0 + b2 + 0x11cu), + ioread32(bar0 + b2 + 0x120u)); + gsp_phys_pbdma_fault_dump(0xffffffffu); + break; + } + } + /* + * SESSION 153-CONTINUED-13: early PBDMA fetch probe for a 3D + * stream launch - where is the front end while a DRAW window + * does not retire? One shot per launch at ~150 ms and ~1.2 s + * (5 ms poll), bounded per boot. pbfetch inside the module + * pushbuffer page = the walk/exec is stuck in our block; + * pbfetch past it = the whole stream was decoded and the + * stall is in execution/completion. + */ + if (stream3d && bar0 && + (sked_tick == 30 || sked_tick == 240)) { + static u32 winprobe_log; + u32 pi; + + if (winprobe_log < 64) { + winprobe_log++; + pr_info("vgsp-gr: WINPROBE +%ums push=0x%llx words=%u block_dma=0x%llx grfifo=%08x trapped=%08x trapped_lo=%08x\n", + sked_tick * GSP_PHYS_RMCALL_POLL_MS, + req->push_va, words, sub.pb_dma, + ioread32(bar0 + 0x00400504u), + ioread32(bar0 + 0x00400704u), + ioread32(bar0 + 0x00400708u)); + for (pi = 0; pi < 12; pi++) { + u32 b = 0x40000u + pi * 0x2000u; + u32 intr0 = ioread32(bar0 + b + 0x108u); + u32 gpf = ioread32(bar0 + b + 0x050u); + u32 pbf = ioread32(bar0 + b + 0x054u); + u32 pbfhi = ioread32(bar0 + b + 0x058u); + u32 hdrsh = ioread32(bar0 + b + 0x118u); + u32 wbh = ioread32(bar0 + b + 0x084u); + u32 wbcnt = ioread32(bar0 + b + 0x088u); + u32 wbd0 = ioread32(bar0 + b + 0x08cu); + u64 fetch64 = + ((u64)(pbfhi & 0xffu) << 32) | + (u64)(pbf & ~3u); + s64 woff = -1; + + if (!intr0 && !gpf && !pbf && !hdrsh) + continue; + if (gsp_phys_gr_last_pb_dma && + fetch64 >= gsp_phys_gr_last_pb_dma && + fetch64 < + gsp_phys_gr_last_pb_dma + + (u64)gsp_phys_gr_last_words * + 4u) + woff = (s64)(fetch64 - + gsp_phys_gr_last_pb_dma) / + 4; + pr_info("vgsp-gr: WINPROBE P%u intr0=%08x gp_put=%u pbfetch=0x%llx off=%lld hdr=%08x(%s m=0x%x cnt=%u sub=%u) pbhd=%08x(type=%s) pbcnt=%08x pbd0=%08x\n", + pi, intr0, gpf, fetch64, woff, hdrsh, + gsp_phys_pbdma_type_name(hdrsh >> 29), + (hdrsh & 0x1fffu) << 2, + (hdrsh >> 16) & 0xfffu, + (hdrsh >> 13) & 7u, wbh, + gsp_phys_pbdma_type_name(wbh >> 29), + wbcnt, wbd0); + } + } + } if (ch_rel_hpa) w1 = READ_ONCE(*(u32 *)((u8 *)__va(ch_rel_hpa) + ch_rel_off)); else w1 = READ_ONCE(*(u32 *)(sem_va + 4)); if ((!qmd_rel || w0 == want_qmd) && w1 == want_ch) { + /* + * SESSION 153-CONTINUED-107: if THIS launch carried + * the final packed prelude part, gate its residency on + * the IRAM readback BEFORE the break log promotes the + * generation to live (see gsp_phys_gr_mme_iram_gate). + * A failed readback clears the pending gen, the + * prelude latch and both emission cursors: the calls + * stay held and the next launch re-emits from word 0. + */ + if (bar0 && + READ_ONCE(gsp_phys_gr_mme_pending_gen) && + READ_ONCE(gsp_phys_rmcall_gr_mme_prelude_pack) && + READ_ONCE(gsp_phys_rmcall_gr_mme_iram_gate) && + READ_ONCE(gsp_phys_rmcall_gr_mme_iram_gate_break) && + !gsp_phys_gr_mme_iram_gate(bar0, req->h_object, + READ_ONCE(gsp_phys_gr_mme_pending_gen), + "break")) + gsp_phys_gr_mme_iram_gate_fail(); /* F4-a: first poll break + USERD witnesses. */ gsp_phys_gr_poll_break_log(w0, want_qmd, w1, - poll_start_us, &sub); + poll_start_us, &sub, + req->h_object); + /* + * SESSION 153-CONTINUED-72: GR-STATE readback at the + * retire (fed MME-generated-write records + latches). + */ + gsp_phys_gr_state_rb_log(req->h_object); break; } /* @@ -20041,6 +28435,9 @@ gr_page_unmapped: pr_err("vgsp-gr: GR LAUNCH FAILED: wait budget crossed (qmd_rel=0x%08x want=0x%08x ch_rel=0x%08x want=0x%08x) h=0x%x push=0x%llx; ladder stopped\n", w0, want_qmd, w1, want_ch, req->h_object, req->push_va); + gsp_phys_mme_regs_log(bar0, "wait-max"); + gsp_phys_gr_fetch_miss_log(req->h_object, &sub, + "wait-max"); ret = -ETIMEDOUT; goto out; } @@ -20052,6 +28449,23 @@ gr_page_unmapped: pr_err("vgsp-gr: GR LAUNCH FAILED: timeout after %u ms (qmd_rel=0x%08x want=0x%08x ch_rel=0x%08x want=0x%08x) h=0x%x push=0x%llx\n", poll_ms, w0, want_qmd, w1, want_ch, req->h_object, req->push_va); + gsp_phys_mme_regs_log(bar0, "timeout"); + gsp_phys_gr_fetch_miss_log(req->h_object, &sub, + "timeout"); + /* + * SESSION 153-CONTINUED-106: the r1363 trap sat in the + * ~2.6 s window AFTER this timeout (launch failed + * 403088.34, ILLEGAL_OPCODE relayed 403090.96). Re-spin + * once here, bounded to the first few timeouts of the + * boot, so the latch is caught at first sight with its + * INFO/IR word instead of being missed between spins. + */ + if (gsp_phys_mme_post_ms && post_spin_n < 8u) { + post_spin_n++; + (void)gsp_phys_mme_fast_sample(bar0, "post", + req->h_object, + gsp_phys_mme_post_ms); + } /* * F4-b: one-shot deposit readback at the FIRST timeout * (the old post-mortem only ran after attempt 3, which @@ -20156,6 +28570,39 @@ gr_page_unmapped: mutex_lock(&gsp_phys_rm_lock); sp_ret = gsp_phys_rmcall_gr_sem_push_locked(bar0); mutex_unlock(&gsp_phys_rm_lock); + /* + * SESSION 153-CONTINUED-93: parked-engine + * recovery (see the sema_gr_park_recover + * knob). A wedged probe on a poison-free, + * trap-free front-end is the r1311-r1313 + * park class: the context is gone and only a + * context reload brings it back. One bounded + * hard re-arm per launch; ALIVE afterwards + * resumes the fresh-entry retries. + */ + if (sp_ret && bar0 && + READ_ONCE(gsp_phys_gr_park_recover)) { + static u32 park_rec_n; + + mutex_lock(&gsp_phys_rm_lock); + sp_ret = + gsp_phys_gr_wedge_hard_rearm_locked(bar0); + mutex_unlock(&gsp_phys_rm_lock); + park_rec_n++; + if (!sp_ret) { + WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, 0); + WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_sa, false); + WRITE_ONCE(gsp_phys_gr_mme_live_gen, 0); + pr_warn("vgsp-gr: PARK-RECOVER #%u warm sem ALIVE after TSG preempt + resched; fresh-entry retries resume push=0x%llx\n", + park_rec_n, req->push_va); + } else { + pr_warn("vgsp-gr: PARK-RECOVER #%u still wedged after hard re-arm (rc=%d) push=0x%llx\n", + park_rec_n, sp_ret, + req->push_va); + } + } if (sp_ret) { gsp_phys_gr_sem_wedged++; pr_info("vgsp-gr: timeout ladder: warm sem WEDGED (rc=%d) - failing fast instead of fresh-entry retries\n", @@ -20450,6 +28897,16 @@ gr_page_unmapped: ioread32(bar0 + 0x00405b00u), ioread32(bar0 + 0x00400100u), ioread32(bar0 + 0x0040013cu)); + if (bar0) + pr_info("vgsp-gr: FECS readback: CUR=0x%08x NEW=0x%08x ST1=0x%08x FE0=0x%08x HOSTINT=0x%08x IRQSTAT=0x%08x CPUCTL=0x%08x OS=0x%08x\n", + ioread32(bar0 + 0x00409b00u), + ioread32(bar0 + 0x00409b04u), + ioread32(bar0 + 0x00409400u), + ioread32(bar0 + 0x00409c00u), + ioread32(bar0 + 0x00409c18u), + ioread32(bar0 + 0x00409008u), + ioread32(bar0 + 0x00409100u), + ioread32(bar0 + 0x00409080u)); if (bar0) { u32 g; @@ -20487,7 +28944,7 @@ gr_page_unmapped: ret = -ETIMEDOUT; goto out; } - msleep(GSP_PHYS_RMCALL_POLL_MS); + gsp_phys_rmcall_wait_quantum(); } gsp_phys_gr_ok++; @@ -20531,9 +28988,23 @@ gr_page_unmapped: if (req->n_data && gsp_phys_gr_data_chunk()->pages) { u32 n_tracked = 0; u32 nw_attempts = 0; + bool idem = false; u32 wrote = gsp_phys_gr_output_retire_wait(req, bounce_csum_in, &n_tracked); + /* CONT-73: where did the draw write? bounce count above; + * direct guest stores (relocated alias/shadow operands and + * MME shadow-scratch destinations) here. */ + gsp_phys_gr_wit_report("retire"); + + /* + * r434: post-launch hardware-only QMD dump (w50/w51 INNER + * get/put = the SKED's CTA dispatch counters) so the + * healthy-vs-wedged diff has the fields the submission-time + * dump cannot show. + */ + gsp_phys_gr_qmd64_post(qmd_dst_mem, "retire"); + /* * m179: the zero-write test is specific to launches with a * non-empty TRACKED data set. A launch whose data entries do @@ -20546,11 +29017,35 @@ gr_page_unmapped: if (wrote == 0 && n_tracked) { bool ff = gsp_phys_gr_qmd_fetched(qmd_dst_mem); - pr_info("vgsp-gr: first-attempt witness: engine fetch %s (qmd sentinel %s) tracked=%u\n", - ff ? "OK" : "NONE", - ff ? "overwritten" : "intact", n_tracked); + /* CONT-103: no QMD deposit on the stream3d lane, so the + * sentinel is vacuous there; the GP_GET FETCH-WIT is the + * fetch witness for that lane. */ + if (qmd_dst_mem) + pr_info("vgsp-gr: first-attempt witness: engine fetch %s (qmd sentinel %s) tracked=%u\n", + ff ? "OK" : "NONE", + ff ? "overwritten" : "intact", n_tracked); + else + pr_info("vgsp-gr: first-attempt witness: engine fetch n/a (no QMD deposit on this lane - sentinel vacuous; see FETCH-WIT) tracked=%u\n", + n_tracked); } - if (wrote == 0 && n_tracked && + /* + * r427 idempotent fast-accept: a zero-write launch whose identity + * (QMD minus release0 + staged content checksums) matches an + * earlier WRITING launch of this boot is a legitimate idempotent + * re-launch - the result is already in place. Accept immediately: + * no retries, no quiet holds. (See gsp_phys_gr_launch_fp.) + */ + if (wrote == 0 && n_tracked) { + u64 fp = gsp_phys_gr_launch_fp(qmd, req, bounce_csum_in); + + if (gsp_phys_gr_idem_match(fp)) { + pr_warn("vgsp-gr: IDEMPOTENT-ACCEPT h=0x%x push=0x%llx fp=0x%llx tracked=%u (same QMD+content as an earlier writing launch; no retries, no hold)\n", + req->h_object, req->push_va, fp, + n_tracked); + idem = true; + } + } + if (wrote == 0 && n_tracked && !idem && gsp_phys_rmcall_gr_nowrite_retry_value()) { u32 nw_retries = (u32)gsp_phys_gr_nowrite_retries_value(); @@ -20606,6 +29101,26 @@ gr_page_unmapped: pr_info("vgsp-gr: outcome retry %u/%u (wrote=%u tracked=%u): refill + re-publish\n", nw_attempts, nw_retries, wrote, n_tracked); + /* + * r445 wedge hard re-arm (gsp_phys_gr_wedge_rearm, + * default off): the r445 evidence shows every + * post-dispatch no-store episode leaves the warm + * ladder sem probe ALIVE (the ladder block executes) + * while the client launch keeps writing nothing, so + * the plain refill+re-publish retries are futile. + * On the FINAL retry attempt, force one context + * save/unload (TSG PREEMPT bWait=1) and re-run the + * mode-35 lane-enable tail + schedules before the + * re-publish, so the retried launch meets a freshly + * restored context. Bounded: one preempt per launch + * (2 s RPC timeout), only on the last retry. + */ + if (nw_attempts == nw_retries && + gsp_phys_gr_wedge_rearm_value()) { + mutex_lock(&gsp_phys_rm_lock); + gsp_phys_gr_wedge_hard_rearm_locked(bar0); + mutex_unlock(&gsp_phys_rm_lock); + } if (quiet_ms) { gsp_phys_gr_wedge_quiet++; pr_warn("vgsp-gr: WEDGE-QUIET hold %u ms before outcome retry %u (push=0x%llx; %u consecutive zero-write launch(es); no submissions in between)\n", @@ -20777,9 +29292,12 @@ gr_page_unmapped: words = gsp_phys_rmcall_build_gr_launch( methods, req, qmd, qmd_dst, ch_sem_va, - want_ch); - memcpy(gsp_phys_chunk_page_va(pb, 0), - methods, words * 4); + want_ch, win3d, + win3d_n, + gsp_phys_gr_attr_pre, + gsp_phys_gr_attr_pre_n); + gsp_phys_gr_pb_copy(methods, + words); } if (qmdwin_swap) pr_info("vgsp-gr: outcome retry %u: release words zeroed + fresh qmdwin bank %u dest 0x%llx + win0 bank %u%s + PB rebuilt\n", @@ -20799,6 +29317,9 @@ gr_page_unmapped: bounce_staged_hpa); gsp_phys_gr_fill_gap_bounce(bounce_csum_in, bounce_staged_hpa); + /* CONT-73: refingerprint the direct-write + * witness set for this retry publish. */ + gsp_phys_gr_wit_capture(req); mutex_lock(&gsp_phys_rm_lock); ret = gsp_phys_rmcall_publish_locked( bar0, boot_stage.rmcall_va_pb, @@ -20846,7 +29367,7 @@ gr_page_unmapped: } if (time_after_eq(jiffies, dl)) break; - msleep(GSP_PHYS_RMCALL_POLL_MS); + gsp_phys_rmcall_wait_quantum(); } pr_info("vgsp-gr: outcome retry %u: fresh ack %s (qmd_rel=0x%08x want=0x%08x ch_rel=0x%08x after %u ms)\n", nw_attempts, @@ -20886,6 +29407,10 @@ gr_page_unmapped: } wrote = gsp_phys_gr_output_retire_wait( req, bounce_csum_in, &n_tracked); + /* CONT-73: direct-write witness per retry. */ + gsp_phys_gr_wit_report("retry-retire"); + gsp_phys_gr_qmd64_post(qmd_dst_mem, + "retry-retire"); pr_info("vgsp-gr: outcome retry %u done: engine wrote %u page(s)\n", nw_attempts, wrote); /* r410: the quiet-window verdict. */ @@ -20916,7 +29441,7 @@ gr_page_unmapped: * loop is disabled (gsp_phys_rmcall_gr_nowrite_retry=0) so a * zero-write outcome is never silent. */ - if (wrote == 0 && n_tracked) { + if (wrote == 0 && n_tracked && !idem) { bool restored = false; bool fetched = gsp_phys_gr_qmd_fetched(qmd_dst_mem); @@ -20938,7 +29463,8 @@ gr_page_unmapped: fetched ? "AMBIGUOUS" : "CONFIRMED", req->h_object, req->push_va, req->n_data, n_tracked, nw_attempts, want_qmd, - fetched ? "OK" : "NONE", + qmd_dst_mem ? (fetched ? "OK" : "NONE") : + "n/a(noQMD-stream3d)", READ_ONCE(gsp_phys_gr_nw_streak) + 1u, gsp_phys_gr_wedge_quiet, gsp_phys_gr_wedge_heals); @@ -21008,6 +29534,32 @@ gr_page_unmapped: readl(bar0 + 0x00409008u), readl(bar0 + 0x00409b00u), readl(bar0 + 0x00409b04u)); + /* r425: the wedged-launch SKED/FECS baseline. */ + gsp_phys_gr_sked_wit(bar0, "accept"); + /* + * r425 stale-store witness: re-scan the last completed + * launch's first engine-changed page. A change since its + * copy-out proves a late store landed after the launch + * was shipped (a stuck/queued grid or a stale-QMD + * execution), the mechanism behind the "1 page" heals. + */ + if (gsp_phys_gr_prev_out_valid && + bounce_staged_hpa[gsp_phys_gr_prev_out_bi] == 0 && + gsp_phys_gr_data_chunk()->pages) { + const u8 *pp = gsp_phys_chunk_page_va( + gsp_phys_gr_data_chunk(), + gsp_phys_gr_prev_out_bi); + u64 pc; + + if (pp) { + pc = gsp_phys_gr_fnv1a64(pp, GSP_PHYS_PAGE_SIZE); + if (pc != gsp_phys_gr_prev_out_csum) + pr_warn("vgsp-gr: PREV-OUT-CHANGED bi=%u fp %016llx -> %016llx (late store after its launch shipped)\n", + gsp_phys_gr_prev_out_bi, + gsp_phys_gr_prev_out_csum, + pc); + } + } /* * (c) Late-store check: a store ramp that retires after * the host's window is indistinguishable from a lost @@ -21028,6 +29580,18 @@ gr_page_unmapped: late, late_tracked, late ? "LATE-STORES - copy-out will ship them" : "NO-STORES - dispatch lost or read-only kernel"); + /* + * SESSION 153-CONTINUED-83: the no-writes moment + * is the store wall's fingerprint - snapshot the + * DS/SPH/MME/FE-decode evidence here so the + * stage that never received a program header (or + * the MME PC that never emitted the draw) is on + * the same line as the failing launch. + */ + if (!late) + gsp_phys_store_side_log(gsp_phys_gpu_bar0(), + "no-writes", + req->h_object); } if (qmd_rel_hpa) { @@ -21058,6 +29622,14 @@ gr_page_unmapped: want_qmd, want_ch); } /* + * r427: record the launch identity of every launch that WROTE, + * so a later idempotent re-launch (zero writes, same identity) + * can accept immediately (see gsp_phys_gr_launch_fp). + */ + if (wrote && n_tracked) + gsp_phys_gr_idem_record( + gsp_phys_gr_launch_fp(qmd, req, bounce_csum_in)); + /* * r410: streak bookkeeping for the quiet-window trigger. A * launch with tracked data that wrote nothing extends the * zero-write streak; any launch that wrote resets it. Runs @@ -21087,11 +29659,12 @@ gr_page_unmapped: u32 aliased = 0, unchanged = 0, skipped = 0; u32 chg[8] = { }; u32 outwit = 0; /* COPYWIT: bounded per-launch witness lines */ + u32 outwit_chg = 0; /* CONT-112: own budget, changed pages */ for (wbi = 0; wbi < req->n_data; wbi++) { u32 bi = gsp_phys_gr_data_bi(req->data[wbi].reloc_va); const u8 *src; - u32 ci, csum = 0; + u64 fp; if (bi == GSP_PHYS_GR_DATA_BI_NONE || !req->data[wbi].hpa) @@ -21112,9 +29685,8 @@ gr_page_unmapped: aliased++; continue; } - for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) - csum += get_unaligned_le32(src + ci); - if (csum == bounce_csum_in[bi]) { + fp = gsp_phys_gr_fnv1a64(src, GSP_PHYS_PAGE_SIZE); + if (fp == bounce_csum_in[bi]) { /* * r378: the engine did not touch this page, * so the guest page still holds exactly this @@ -21123,6 +29695,37 @@ gr_page_unmapped: * stale-content corruption at worst - skip. */ unchanged++; + /* + * r452: witness the UNCHANGED page's chunk + * content (first 4 words) and the guest + * page's, bounded per launch. This is the + * only place the two can be compared for a + * page the csum rule classified as "engine + * did not touch" - it separates "the store + * never landed in the chunk" from "the + * store landed but the csum rule missed + * it" (or "the reader resolves a different + * page than the writer wrote"). + */ + if (gsp_phys_gr_copywit_enabled() && + outwit < GSP_PHYS_GR_COPYWIT_MAX) { + const u8 *gp = req->data[wbi].hpa ? + (const u8 *)__va(req->data[wbi].hpa) : + NULL; + + outwit++; + pr_info("vgsp-gr: OUT-UNCH bi=%u va=0x%llx fp=%016llx pre=%016llx chunk=%08x %08x %08x %08x guest=%08x %08x %08x %08x\n", + bi, req->data[wbi].va, fp, + bounce_csum_in[bi], + get_unaligned_le32(src), + get_unaligned_le32(src + 4), + get_unaligned_le32(src + 8), + get_unaligned_le32(src + 12), + gp ? get_unaligned_le32(gp) : 0, + gp ? get_unaligned_le32(gp + 4) : 0, + gp ? get_unaligned_le32(gp + 8) : 0, + gp ? get_unaligned_le32(gp + 12) : 0); + } continue; } { @@ -21157,7 +29760,7 @@ gr_page_unmapped: * sema-bigcopy resolution lines. */ if (gsp_phys_gr_copywit_enabled() && - outwit < GSP_PHYS_GR_COPYWIT_MAX) { + outwit_chg < GSP_PHYS_GR_COPYWIT_CHG_MAX) { /* * r391 diagnostic (sum=0 class): the * landing-table resolution of this VA @@ -21172,14 +29775,14 @@ gr_page_unmapped: req->data[wbi].va, &rpa, &rap, &rval); - outwit++; - pr_info("vgsp-gr: OUT-PAGE-WIT va=0x%llx hpa=0x%llx bi=%u first8=%016llx last8=%016llx pre=%08x post=%08x res=%s pa=0x%llx ap=%u val8=%016llx vbase=%px\n", + outwit_chg++; + pr_info("vgsp-gr: OUT-PAGE-WIT va=0x%llx hpa=0x%llx bi=%u first8=%016llx last8=%016llx pre=%016llx post=%016llx res=%s pa=0x%llx ap=%u val8=%016llx vbase=%px\n", req->data[wbi].va, req->data[wbi].hpa, bi, f8, (u64)get_unaligned_le64( src + GSP_PHYS_PAGE_SIZE - 8), - bounce_csum_in[bi], csum, + bounce_csum_in[bi], fp, rhave ? "ledger" : "none", rpa, rap, rval, (void *)gsp_sema_gr_shadow_base()); @@ -21194,7 +29797,8 @@ gr_page_unmapped: * evidence). */ if (gsp_phys_gr_copywit_enabled() && - !gsp_phys_gr_page_all_zero(csum, src)) { + !gsp_phys_gr_page_all_zero( + gsp_phys_gr_page_sum32(src), src)) { u32 wi; for (wi = 0; wi < gsp_phys_gr_outwit_n; @@ -21223,6 +29827,16 @@ gr_page_unmapped: } if (changed < 8) chg[changed] = bi; + /* + * r425: remember the first engine-changed page + * (post-copy-out content) for the next zero-write + * accept's late-store witness. + */ + if (changed == 0) { + gsp_phys_gr_prev_out_bi = bi; + gsp_phys_gr_prev_out_csum = fp; + gsp_phys_gr_prev_out_valid = true; + } changed++; } /* @@ -21251,13 +29865,13 @@ gr_page_unmapped: * write target and the reader's page are different * (aperture/staleness disagreement), not a lost write. */ - if (outwit < GSP_PHYS_GR_COPYWIT_MAX) { + if (outwit_chg < GSP_PHYS_GR_COPYWIT_CHG_MAX) { u64 rpa2 = 0, rval2 = 0; u32 rap2 = 0; if (gsp_sema_gr_va_snap(req->data[wbi].va, &rpa2, &rap2, &rval2)) { - outwit++; + outwit_chg++; pr_info("vgsp-gr: OUT-PAGE-POST va=0x%llx hpa=0x%llx res_pa=0x%llx ap=%u val8=%016llx vbase=%px\n", req->data[wbi].va, req->data[wbi].hpa, rpa2, @@ -21301,8 +29915,7 @@ gr_page_unmapped: if (bi == GSP_PHYS_GR_DATA_BI_NONE) continue; wp = gsp_phys_chunk_page_va(dchunk, bi); - for (k = 0; k < GSP_PHYS_PAGE_SIZE; k += 4) - wc += get_unaligned_le32(wp + k); + (void)k; (void)wc; pr_info("vgsp-gr: window[%u] reloc=0x%llx first=%08x %08x %08x %08x changed=%u\n", bi, req->data[wi].reloc_va, @@ -21310,7 +29923,9 @@ gr_page_unmapped: get_unaligned_le32(wp + 4), get_unaligned_le32(wp + 8), get_unaligned_le32(wp + 12), - wc != bounce_csum_in[bi]); + gsp_phys_gr_fnv1a64( + wp, GSP_PHYS_PAGE_SIZE) != + bounce_csum_in[bi]); } } pr_info("vgsp-gr: bounce copy-out: %u pages written back, %u changed by engine, %u unchanged-skipped, %u alias-skipped, %u stale-skipped (first bi: %u %u %u %u %u %u %u %u)\n", @@ -21333,7 +29948,7 @@ gr_page_unmapped: for (gi = 0; gi < gsp_phys_gr_gap_n; gi++) { u32 bi = gsp_phys_gr_gap[gi].bi; const u8 *src = gsp_phys_chunk_page_va(dchunk, bi); - u32 ci, csum = 0; + u64 fp; if (!src) continue; @@ -21346,9 +29961,8 @@ gr_page_unmapped: gskipped++; continue; } - for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) - csum += get_unaligned_le32(src + ci); - if (csum == bounce_csum_in[bi]) { + fp = gsp_phys_gr_fnv1a64(src, GSP_PHYS_PAGE_SIZE); + if (fp == bounce_csum_in[bi]) { /* Engine did not touch it: nothing to ship. */ gunchanged++; continue; @@ -21417,8 +30031,20 @@ gr_page_unmapped: jiffies_to_msecs(jiffies - start), gsp_phys_gr_ok); /* m128: post-success qmdwin words so a probe that stores a value into * the qmdwin page (e.g. p6's constant-read dump at w48) is readable - * from the success log (the post-mortem only prints on timeout). */ - if (boot_stage.rmcall_gr_qmdwin.pages) { + * from the success log (the post-mortem only prints on timeout). + * r434: read the launch's OWN bank (qmd_dst_mem) when available - + * the old bank-0-only read showed a stale page for odd epochs. */ + if (qmd_dst_mem) { + const u32 *qw = (const u32 *)qmd_dst_mem; + + pr_info("vgsp-gr: post-OK qmdwin w48..59=%08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n", + get_unaligned_le32(qw + 48), get_unaligned_le32(qw + 49), + get_unaligned_le32(qw + 50), get_unaligned_le32(qw + 51), + get_unaligned_le32(qw + 52), get_unaligned_le32(qw + 53), + get_unaligned_le32(qw + 54), get_unaligned_le32(qw + 55), + get_unaligned_le32(qw + 56), get_unaligned_le32(qw + 57), + get_unaligned_le32(qw + 58), get_unaligned_le32(qw + 59)); + } else if (boot_stage.rmcall_gr_qmdwin.pages) { const u32 *qw = (const u32 *)gsp_phys_chunk_page_va( &boot_stage.rmcall_gr_qmdwin, 0); @@ -21560,6 +30186,181 @@ static void gsp_phys_rmcall_ctx_diag_decode(const char *tag) ramfc_valid); } +/* + * SESSION 153-CONTINUED-96: secondary COPY0 lane enable. On a GR0 ladder + * the ladder channel speaks GR and CE class methods are never legal on it, + * so the ladder's own FBMEM reads (gsp_phys_ctx_readback_try) need a second + * engine. Default 1 builds the lane (best-effort: a failed build only + * disables the readback, the GR ladder is unchanged); 0 skips it entirely. + */ +static bool gsp_phys_rmcall_ce_lane = 1; +module_param_named(gsp_phys_rmcall_ce_lane, gsp_phys_rmcall_ce_lane, bool, + 0644); +MODULE_PARM_DESC(gsp_phys_rmcall_ce_lane, + "Session 153-CONTINUED-96: build and schedule the secondary COPY0 lane (COPY0 TSG + channel on the ladder VAS, engineType 9) on a GR0 ladder so the ladder's own CE copy blocks (engine FBMEM readback) have a legal engine (default 1; 0 disables the lane, the GR ladder is unchanged)"); + +static bool gsp_phys_rmcall_ce_lane_value(void) +{ + return gsp_phys_rmcall_ce_lane; +} + +/* + * SESSION 153-CONTINUED-96: secondary COPY0 lane bringup (GR0 ladders only). + * + * The ladder's own FBMEM readback (and any later diagnostic copy of engine + * state the host cannot aperture) needs a CE engine, but CE class methods + * are never legal on the GR0 ladder channel. This builds the smallest + * second lane on the SAME VAS: its own COPY0 TSG (parent device, same + * ladder VAS) and its own V5-shape channel (engineType 9, TSG parent, + * lane memdescs, GPFIFO at VA_CE_GPFIFO), then runs the historical + * COPY0 stage-4 sequence - BIND(COPY0) -> GPFIFO_SCHEDULE -> + * TSG_GPFIFO_SCHEDULE -> notif index (advisory) -> GET_WORK_SUBMIT_TOKEN - + * and stores the lane's own work-submit token. Best-effort by design: a + * failure is logged and the GR ladder continues unchanged (no stage-4 + * abort, no retry); rmcall_ce_lane_valid stays false and the readback is + * skipped. The PDB slots for the lane's GPFIFO/PB pages were planted in + * pdb_build (stage 3, before any engine fetch). + */ +static int gsp_phys_rmcall_ce_lane_bringup(void __iomem *bar0) +{ + struct gsp_phys_rmcall_tsg_params tsg; + struct gsp_phys_rmcall_channel_params ch; + struct gsp_phys_rmcall_channel_variant v; + u32 token = 0, rpc_result = 0, status = 0; + u8 sched_params[2] = { 1, 0 }; + u32 bind_params = GSP_PHYS_RMCALL_ENGINE_COPY0; + u32 notif_index = GSP_PHYS_RMCALL_NOTIF_INDEX; + int ret; + + if (!gsp_phys_rmcall_engine_is_gr()) + return 0; /* COPY0 ladder: the lane already is it */ + if (!gsp_phys_rmcall_ce_lane_value()) + return 0; + if (boot_stage.rmcall_ce_lane_valid) + return 0; + + /* (1) TSG on the ladder VAS bound to the COPY0 engine. */ + memset(&tsg, 0, sizeof(tsg)); + tsg.h_va_space = cpu_to_le32(boot_stage.rmcall_vas); + tsg.engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_COPY0); + ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client, + boot_stage.rmcall_device, + GSP_PHYS_RMCALL_HANDLE_CE_TSG, + GSP_PHYS_RMCALL_CLASS_TSG, &tsg, sizeof(tsg), + false, NULL, "KEPLER_CHANNEL_GROUP_A (CE lane)", + true, NULL, NULL); + if (ret) { + pr_warn("vgsp-boot: rm-call: CE lane: COPY0 TSG alloc failed (%d); the lane is not built and the GR ladder continues unchanged\n", + ret); + return ret; + } + boot_stage.rmcall_ce_tsg = GSP_PHYS_RMCALL_HANDLE_CE_TSG; + + /* (2) The lane channel: V5 shape, COPY0 engine, lane memdescs. */ + memset(&v, 0, sizeof(v)); + v.name = "ce-lane-tsg-vas-pdb-skip-scrubber"; + v.id = 5; + v.tsg_parent = true; + v.flags = GSP_PHYS_RMCALL_CHANNEL_FLAG_SKIP_SCRUBBER; + v.internal_flags = GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL; + v.server_mem = false; + v.handle = GSP_PHYS_RMCALL_HANDLE_CE_CHANNEL; + gsp_phys_rmcall_channel_params_init(&ch, &v); + ch.engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_COPY0); + ch.h_context_share = 0; /* a CE channel never joins a GR share */ + ch.gp_fifo_offset = cpu_to_le64(GSP_PHYS_RMCALL_VA_CE_GPFIFO); + ch.instance_mem = gsp_phys_rmcall_mem_desc( + GSP_PHYS_RMCALL_BUF_CE_INSTANCE); + ch.userd_mem = gsp_phys_rmcall_mem_desc( + GSP_PHYS_RMCALL_BUF_CE_USERD); + ch.ramfc_mem = gsp_phys_rmcall_mem_desc( + GSP_PHYS_RMCALL_BUF_CE_RAMFC); + ch.mthdbuf_mem = gsp_phys_rmcall_mem_desc( + GSP_PHYS_RMCALL_BUF_CE_MTHDBUF); + ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client, + boot_stage.rmcall_ce_tsg, + GSP_PHYS_RMCALL_HANDLE_CE_CHANNEL, + GSP_PHYS_RMCALL_CLASS_CHANNEL, &ch, sizeof(ch), + false, NULL, "CE lane channel", true, NULL, + &status); + if (ret) { + pr_warn("vgsp-boot: rm-call: CE lane: channel alloc failed (%d, status=0x%x); the lane is not built and the GR ladder continues unchanged\n", + ret, status); + return ret; + } + boot_stage.rmcall_ce_channel = GSP_PHYS_RMCALL_HANDLE_CE_CHANNEL; + + /* (3) BIND to the COPY0 engine (the historical stage-4 sequence). */ + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_ce_channel, + GSP_PHYS_RMCALL_CTRL_BIND_CHANNEL, + &bind_params, sizeof(bind_params), 16, NULL, 0, + &rpc_result, &status, + "CE lane BIND_CHANNEL COPY0"); + if (ret) { + pr_warn("vgsp-boot: rm-call: CE lane: BIND_CHANNEL (engineType=0x%x) failed (rpc_result=0x%x status=0x%x); lane unusable, GR ladder continues\n", + bind_params, rpc_result, status); + return ret; + } + + /* (4) GPFIFO_SCHEDULE + TSG_GPFIFO_SCHEDULE. */ + rpc_result = 0; + status = 0; + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_ce_channel, + GSP_PHYS_RMCALL_CTRL_GPFIFO_SCHEDULE, + sched_params, sizeof(sched_params), 16, NULL, 0, + &rpc_result, &status, "CE lane GPFIFO_SCHEDULE"); + if (ret) { + pr_warn("vgsp-boot: rm-call: CE lane: GPFIFO_SCHEDULE failed (rpc_result=0x%x status=0x%x); lane unusable, GR ladder continues\n", + rpc_result, status); + return ret; + } + rpc_result = 0; + status = 0; + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_ce_tsg, + GSP_PHYS_RMCALL_CTRL_TSG_GPFIFO_SCHEDULE, + sched_params, sizeof(sched_params), 16, NULL, 0, + &rpc_result, &status, "CE lane TSG_GPFIFO_SCHEDULE"); + if (ret) + pr_warn("vgsp-boot: rm-call: CE lane: TSG_GPFIFO_SCHEDULE advisory failure (rpc_result=0x%x status=0x%x); continuing\n", + rpc_result, status); + + /* (5) notif index (advisory) + the lane's own work-submit token. */ + rpc_result = 0; + status = 0; + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_ce_channel, + GSP_PHYS_RMCALL_CTRL_SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX, + ¬if_index, sizeof(notif_index), 16, NULL, 0, + &rpc_result, &status, + "CE lane SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX"); + if (ret) + pr_info("vgsp-boot: rm-call: CE lane: SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX advisory failure (rpc_result=0x%x status=0x%x); continuing to the token\n", + rpc_result, status); + token = 0; + rpc_result = 0; + status = 0; + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_ce_channel, + GSP_PHYS_RMCALL_CTRL_GET_WORK_SUBMIT_TOKEN, + &token, sizeof(token), 28, &token, sizeof(token), + &rpc_result, &status, "CE lane GET_WORK_SUBMIT_TOKEN"); + if (ret && token != 0) { + pr_warn("vgsp-boot: rm-call: CE lane: GET_WORK_SUBMIT_TOKEN failed (rpc_result=0x%x status=0x%x) but the echoed params carry token=0x%x; proceeding\n", + rpc_result, status, token); + ret = 0; + } + if (ret || !token) { + pr_warn("vgsp-boot: rm-call: CE lane: no work-submit token (ret=%d rpc_result=0x%x status=0x%x token=0x%x); lane unusable, GR ladder continues\n", + ret, rpc_result, status, token); + return ret ? ret : -EIO; + } + boot_stage.rmcall_ce_token = token; + boot_stage.rmcall_ce_lane_valid = true; + pr_info("vgsp-boot: rm-call: CE lane READY (secondary COPY0 on the ladder VAS): hTsg=0x%x hChannel=0x%x token=0x%x (runlistId=%u chId=0x%x) gpFifoVa=0x%llx pbVa=0x%llx; CE copies on it read FBMEM the host cannot aperture\n", + boot_stage.rmcall_ce_tsg, boot_stage.rmcall_ce_channel, token, + (token >> 16) & 0x7fu, token & 0xfffu, + GSP_PHYS_RMCALL_VA_CE_GPFIFO, GSP_PHYS_RMCALL_VA_CE_PB); + return 0; +} + static int gsp_phys_rmcall_stage4(void __iomem *bar0) { struct gsp_phys_boot_chunk *userd_c = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD]; @@ -21741,6 +30542,22 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) boot_stage.rmcall_token = token; /* + * SESSION 153-CONTINUED-96: on a GR0 ladder, build the secondary + * COPY0 lane (own TSG/channel on the same VAS, own token) so the + * ladder's own CE copy blocks - the engine-context FBMEM readback - + * have a legal engine. Best-effort: a failure disables the readback + * only; the GR ladder below is byte-identical. On a COPY0 ladder the + * bringup is a no-op (the ladder channel already is the CE lane). + */ + { + int ce_ret = gsp_phys_rmcall_ce_lane_bringup(bar0); + + if (ce_ret) + pr_warn("vgsp-boot: rm-call: CE lane bringup failed (%d); engine FBMEM readback unavailable this boot (GR ladder unchanged)\n", + ce_ret); + } + + /* * H1: a GR0 ladder releases a semaphore on the compute engine instead * of running a CE copy. The dedicated helper performs the same guarded * submission (gr=true), doorbell and module-sem poll, logs REAL GR SEM @@ -21810,7 +30627,7 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) gsp_phys_status_drain(NULL); return -ETIMEDOUT; } - msleep(GSP_PHYS_RMCALL_POLL_MS); + gsp_phys_rmcall_wait_quantum(); } if (memcmp(gsp_phys_chunk_page_va(src, 0), @@ -22287,6 +31104,8 @@ static void gsp_phys_rmcall_fini(struct pci_dev *pdev) gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_pt[i]); gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_shadow); gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_qmdwin); + gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_pb2); + gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_probe); /* * B-2: the old 208-page bounce window (previously leaked here) and the * grown data-bounce region chunks. The region's PT chunk and data chunk @@ -22431,6 +31250,17 @@ static int gsp_phys_state_show(struct seq_file *m, void *v) boot_stage.rmcall_gr_promote_step, boot_stage.rmcall_gr_promote_size, boot_stage.rmcall_gr_hvirt); + /* SESSION 153-CONTINUED-96: secondary COPY0 lane (GR0 ladders only). */ + if (boot_stage.rmcall_ce_tsg || boot_stage.rmcall_ce_lane_valid) + seq_printf(m, + "rm_call_ce_lane: valid=%u hTsg=0x%x hChannel=0x%x token=0x%x gpFifoVa=0x%llx pbVa=0x%llx ceLaneParam=%u\n", + boot_stage.rmcall_ce_lane_valid, + boot_stage.rmcall_ce_tsg, + boot_stage.rmcall_ce_channel, + boot_stage.rmcall_ce_token, + GSP_PHYS_RMCALL_VA_CE_GPFIFO, + GSP_PHYS_RMCALL_VA_CE_PB, + gsp_phys_rmcall_ce_lane_value() ? 1u : 0u); seq_printf(m, "rm_call_guest: gpa=0x%llx hpa=0x%llx g2h_ok=%u w2g_ok=%u selftest_ok=%u gpa_valid=%u phys_domain_identity=%d\n", boot_stage.rmcall_guest_gpa, boot_stage.rmcall_guest_hpa, @@ -23320,6 +32150,91 @@ static const struct file_operations gsp_phys_guest_gpa_fops = { }; /* + * gsp-phys-shared-bit0 (write-only, mode 0200): development-harness assist + * for the NVIDIA X driver's per-client shared-data slot. + * + * Every direct GLX client receives a shared memory page from the X driver + * (nvidia_drv.so creates a memfd and passes it over a unix socket) whose + * first words are {u32 zero, u32 cookie, u32 state, u32 zero, u32 hClient, + * u32 hObject}; glcore's device init reads the state word and skips its own + * class-c46f channel creation while bit 0 of it is set. In this emulated + * environment the X driver publishes state=3 (bits 0+1) and direct clients + * therefore never allocate a channel and never render. The value is written + * by the X driver before the fd reaches the client, so it cannot be + * corrected from the client side. + * + * Input: "0x<hex GPA>" (4 KB aligned) naming the guest page that carries the + * slot. The GPA must fall inside a range registered through + * gsp-phys-guest-ram; it is translated with gsp_phys_gpa_to_hpa_vgpu() + * (first open mdev) and the state word is cleared of bit 0 through the + * kernel direct map (the translator validates pfn_valid, so __va() is a + * valid kernel mapping for the shmem-backed guest RAM). As a sanity guard + * the write is only performed when the page's +0x10 word looks like an RM + * client handle (0xc1d0xxxx / 0xc1e0xxxx) or the token carries a "force" + * suffix. Old and new values are always logged. + * + * This is a diagnostic aid for the development VM, matching the spirit of + * the existing guest-GPA pin; it is not part of the product behaviour. + */ +static ssize_t gsp_phys_shared_bit0_write(struct file *file, + const char __user *ubuf, size_t len, + loff_t *ppos) +{ + char buf[48]; + u64 gpa, hpa = 0; + u32 *word; + u32 h1032; + bool force = false; + int ret; + + if (len == 0 || len >= sizeof(buf)) + return -EINVAL; + if (copy_from_user(buf, ubuf, len)) + return -EFAULT; + buf[len] = '\0'; + + { + char *p = strim(buf); + char *tok = strsep(&p, " \t"); + + if (p && strstr(p, "force")) + force = true; + if (kstrtoull(tok, 0, &gpa) || !PAGE_ALIGNED(gpa)) { + pr_info("vgsp-boot: gsp-phys-shared-bit0: write ignored: expected \"0x<hex GPA>\" (4 KB aligned, optional 'force')\n"); + return -EINVAL; + } + } + + ret = gsp_phys_gpa_to_hpa_vgpu(NULL, gpa, &hpa); + if (ret) { + pr_err("vgsp-boot: gsp-phys-shared-bit0: GPA 0x%llx translation failed (%d: no open mdev / GPA not inside a gsp-phys-guest-ram range / not RAM)\n", + gpa, ret); + return -ENOENT; + } + + word = (u32 *)__va(hpa + 8); + h1032 = *(u32 *)__va(hpa + 0x10); + if (!force && (h1032 >> 16) != 0xc1d0 && (h1032 >> 16) != 0xc1e0) { + pr_info("vgsp-boot: gsp-phys-shared-bit0: gpa=0x%llx word=0x%x but +0x10=0x%x does not look like an RM client handle; refusing (append 'force' to override)\n", + gpa, *word, h1032); + return -EINVAL; + } + + pr_info("vgsp-boot: gsp-phys-shared-bit0: gpa=0x%llx hpa=0x%llx hClient=0x%x word=0x%x -> 0x%x\n", + gpa, hpa, h1032, *word, *word & ~1u); + *word &= ~1u; + return len; +} + +static const struct file_operations gsp_phys_shared_bit0_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = gsp_phys_shared_bit0_write, + .llseek = no_llseek, +}; + + +/* * gsp-phys-guest-ram (write-only, mode 0200): configure a guest RAM backing * file that gsp_phys_gpa_to_hpa_vgpu() translates through (QEMU * memory-backend-file,...,share=yes). One write is @@ -23522,12 +32437,54 @@ static int gsp_phys_reg_probe_show(struct seq_file *m, void *unused) 0x0040070cu); fe_hww_esr = gsp_phys_reg_probe_line(m, bar0, "FE_HWW_ESR", 0x00404000u); + /* + * SESSION 153-CONTINUED-52: MME exception latches (see + * gsp_phys_mme_regs_log). + */ + (void)gsp_phys_reg_probe_line(m, bar0, "PGRAPH_PRI_MME_HWW_ESR", + 0x00404490u); + (void)gsp_phys_reg_probe_line(m, bar0, + "PGRAPH_PRI_MME_HWW_ESR_INFO", + 0x00404494u); + (void)gsp_phys_reg_probe_line(m, bar0, + "PGRAPH_PRI_MME_HWW_ESR_INFO2", + 0x0040449cu); fe_chip_def = gsp_phys_reg_probe_line(m, bar0, "FE_CHIP_DEF_INFO", 0x00404030u); fe_go_idle = gsp_phys_reg_probe_line(m, bar0, "FE_GO_IDLE_INFO", 0x00404194u); sked_hww_esr = gsp_phys_reg_probe_line(m, bar0, "SKED_HWW_ESR", 0x00407020u); + /* + * SESSION 153-CONTINUED-11: the shader-program-header fault window. + * DS_HWW_ESR bit n = "the SPH with index n failed validation" (hwref: + * NV_PGRAPH_PRI_DS_HWW_ESR_SPH0_ERR..SPH23_ERR, ESR_2 for 24..30); + * the SPH_INFO registers snapshot the START address of each stage's + * faulting SPH, so a post-fault read names the exact VA the engine + * fetched. + */ + (void)gsp_phys_reg_probe_line(m, bar0, "DS_HWW_ESR", + 0x00405840u); + (void)gsp_phys_reg_probe_line(m, bar0, "DS_HWW_ESR_2", + 0x00405848u); + (void)gsp_phys_reg_probe_line(m, bar0, + "DS_HWW_ESR_SPH_INFO_PROGRAM_REGION_A", + 0x00405c1cu); + (void)gsp_phys_reg_probe_line(m, bar0, + "DS_HWW_ESR_SPH_INFO_PROGRAM_REGION_B", + 0x00405c20u); + { + u32 si; + + for (si = 0; si < 6u; si++) { + gsp_phys_reg_probe_line(m, bar0, + "DS_HWW_ESR_SPH_INFO_STAGE_START", + 0x00405c00u + si * 4u); + gsp_phys_reg_probe_line(m, bar0, + "DS_HWW_ESR_SPH_INFO_STAGE_START_UPPER", + 0x00405c24u + si * 4u); + } + } sked_activity = gsp_phys_reg_probe_line(m, bar0, "SKED_ACTIVITY", 0x00407054u); sked_cwd0 = gsp_phys_reg_probe_line(m, bar0, @@ -23536,6 +32493,30 @@ static int gsp_phys_reg_probe_show(struct seq_file *m, void *unused) sked_cwd1 = gsp_phys_reg_probe_line(m, bar0, "SKED_DEBUG_CWD_TABLE_1", 0x00407064u); + /* r425: the wedge diagnostics - the rest of the SKED CWD table, the + * per-8-group GR TPC masks (a zero compute mask would dispatch no + * TPCs), and the FE QMD-virtualization control for VEID 0. */ + (void)gsp_phys_reg_probe_line(m, bar0, "SKED_DEBUG_CWD_TABLE_2", + 0x00407068u); + (void)gsp_phys_reg_probe_line(m, bar0, "SKED_DEBUG_CWD_TABLE_3", + 0x0040706cu); + (void)gsp_phys_reg_probe_line(m, bar0, "SKED_DEBUG_INFO_1", + 0x00407028u); + { + u32 gi; + + for (gi = 0; gi < 8u; gi++) { + gsp_phys_reg_probe_line(m, bar0, + "FE_TPCM_GR_TPC_MASK", + 0x0040a680u + gi * 4u); + gsp_phys_reg_probe_line(m, bar0, + "FE_LAST_GR_TPC_MASK", + 0x0040a6a0u + gi * 4u); + gsp_phys_reg_probe_line(m, bar0, + "FE_CWD_OWN_TPC_MASK", + 0x0040a6c0u + gi * 4u); + } + } fecs_irq = gsp_phys_reg_probe_line(m, bar0, "FECS_IRQSTAT", 0x00409008u); fecs_cpuctl = gsp_phys_reg_probe_line(m, bar0, "FECS_CPUCTL", @@ -23658,6 +32639,306 @@ static const struct file_operations gsp_phys_reg_probe_fops = { .release = single_release, }; +/* + * SESSION 153-CONTINUED-91: DS SPH fault spy. + * + * The DS_HWW_ESR SPH<n>_ERR bits (bit n = "the SPH with index n failed + * validation", hwref NV_PGRAPH_PRI_DS_HWW_ESR_SPH0_ERR..SPH23_ERR) and the + * per-stage SPH_INFO snapshot (STAGE0..STAGE5 START/UPPER at 0x405c00+si*4 / + * 0x405c24+si*4, PROGRAM_REGION_A/B at 0x405c1c/c20) are latched by the DS + * at fault time and consumed+cleared by the real GSP-RM exception handler + * before our relayed fault packet arrives (the fault-time register read in + * gsp_phys_fault_regs_log sees ESR=0x80000000 and all-zero stage starts + * one millisecond later). The RM's own diagnostic prints those STAGE + * registers (grErrorPrintDsSphInfo_GM200), so they name the exact SPH VAs + * the DS rejected - the missing datum behind the r1277+ Shader Program + * Header wall (CONT-90/91 could only move the fault SET by content, never + * identify the SPH entries). + * + * Reading this file runs a bounded tight poll of DS_HWW_ESR and snapshots + * the ESR pair, the six stage starts and the region pair on every sample + * that has any SPH bit set. Development-only, read-only, no state touched; + * run `cat /sys/kernel/debug/vgsp/gsp-phys-ds-spy` while the client draws. + * Duration via sema_ds_spy_ms (default 20 s). + */ +static u32 gsp_phys_ds_spy_ms = 20000; +module_param_named(sema_ds_spy_ms, gsp_phys_ds_spy_ms, uint, 0644); +MODULE_PARM_DESC(sema_ds_spy_ms, + "gsp-phys-ds-spy sampling duration in milliseconds (default 20000)"); + +#define GSP_PHYS_DS_ESR 0x00405840u +#define GSP_PHYS_DS_ESR_2 0x00405848u +#define GSP_PHYS_DS_REPORT_MASK 0x00405844u +#define GSP_PHYS_DS_REPORT_MASK_2 0x0040584cu +#define GSP_PHYS_DS_SPH_STAGE_LO 0x00405c00u +#define GSP_PHYS_DS_SPH_STAGE_UP 0x00405c24u +#define GSP_PHYS_DS_SPH_REGION_A 0x00405c1cu +#define GSP_PHYS_DS_SPH_REGION_B 0x00405c20u + +static int gsp_phys_ds_spy_show(struct seq_file *m, void *unused) +{ + void __iomem *bar0 = gsp_phys_gpu_bar0(); + u64 t0; + u32 hits = 0, samples = 0, maskholds = 0, mask0_last = 0, mask2_last = 0; + u32 ms = READ_ONCE(gsp_phys_ds_spy_ms); + + if (!bar0) + return -ENODEV; + if (!ms) + ms = 20000; + seq_printf(m, "# DS-SPY begin duration_ms=%u (poll DS_HWW_ESR 0x%08x)\n", + ms, GSP_PHYS_DS_ESR); + seq_printf(m, "# DS-SPY masks esr=%08x esr2=%08x mask=%08x mask2=%08x\n", + readl(bar0 + GSP_PHYS_DS_ESR), + readl(bar0 + GSP_PHYS_DS_ESR_2), + readl(bar0 + GSP_PHYS_DS_REPORT_MASK), + readl(bar0 + GSP_PHYS_DS_REPORT_MASK_2)); + t0 = ktime_get_ns(); + while (ktime_get_ns() - t0 < (u64)ms * 1000000ull) { + u32 esr = readl(bar0 + GSP_PHYS_DS_ESR); + u32 m0 = readl(bar0 + GSP_PHYS_DS_REPORT_MASK); + u32 m2 = readl(bar0 + GSP_PHYS_DS_REPORT_MASK_2); + + /* + * SESSION 153-CONTINUED-97: hold the HWW report masks + * cleared while sampling. A pending report POISONS the + * whole PRI block (every read returns 0xbadf1201), which + * hid the SPH stage snapshot at the validation instant + * (r1325 spy2: all 16+ hits badf1201, masks back at + * 00ffffff). The mask is (re)armed by the context load / + * the RM setup, not only by our replay path, so re-clear + * it every sample; the re-clear count is logged so the + * capture's provenance is visible. NOTE (r1326): poison + * still appeared on one pass despite 29 re-clears, so a + * clean latch is not guaranteed and the store-side read + * remains the authoritative fault-time datum. + * Development-only diagnostic tap. + */ + if (m0 || m2) { + iowrite32(0u, bar0 + GSP_PHYS_DS_REPORT_MASK); + iowrite32(0u, bar0 + GSP_PHYS_DS_REPORT_MASK_2); + maskholds++; + mask0_last = m0; + mask2_last = m2; + } + samples++; + if (esr & 0x7fffffffu) { + u32 esr2 = readl(bar0 + GSP_PHYS_DS_ESR_2); + u32 esr_a, s[6], u[6], ra, rb; + u32 si; + + for (si = 0; si < 6u; si++) { + s[si] = readl(bar0 + + GSP_PHYS_DS_SPH_STAGE_LO + + si * 4u); + u[si] = readl(bar0 + + GSP_PHYS_DS_SPH_STAGE_UP + + si * 4u); + } + ra = readl(bar0 + GSP_PHYS_DS_SPH_REGION_A); + rb = readl(bar0 + GSP_PHYS_DS_SPH_REGION_B); + esr_a = readl(bar0 + GSP_PHYS_DS_ESR); + hits++; + seq_printf(m, + "DS-SPY hit %u t=%lldus esr=%08x esr2=%08x esr_after=%08x region=%08x/%08x stage=%08x,%08x,%08x,%08x,%08x,%08x upper=%02x,%02x,%02x,%02x,%02x,%02x\n", + hits, div_u64(ktime_get_ns() - t0, 1000), + esr, esr2, esr_a, ra, rb, + s[0], s[1], s[2], s[3], s[4], s[5], + u[0] & 0xffu, u[1] & 0xffu, u[2] & 0xffu, + u[3] & 0xffu, u[4] & 0xffu, u[5] & 0xffu); + udelay(20); + } + } + seq_printf(m, "# DS-SPY end samples=%u hits=%u maskholds=%u last_mask=%08x/%08x\n", + samples, hits, maskholds, mask0_last, mask2_last); + return 0; +} + +static int gsp_phys_ds_spy_open(struct inode *inode, struct file *file) +{ + return single_open(file, gsp_phys_ds_spy_show, NULL); +} + +static const struct file_operations gsp_phys_ds_spy_fops = { + .owner = THIS_MODULE, + .open = gsp_phys_ds_spy_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +/* + * SESSION 153-CONTINUED-93: live PGRAPH register lab (debugfs + * gsp-phys-pgraph-rw). Read: a fixed snapshot of the DS/PD/FIFO registers + * the SPH investigation reads. Write: "0xADDR [0xVALUE]" (hex) inside the + * PGRAPH PRI window [0x400000,0x410000) - reads the old value, writes the + * new one (when given) and reads back, logging both. Diagnostic only; + * development bring-up tool. Writes outside the window are refused. + */ +static int gsp_phys_pgraph_show(struct seq_file *m, void *unused) +{ + void __iomem *bar0 = gsp_phys_gpu_bar0(); + + if (!bar0) + return -ENODEV; + seq_printf(m, "# PGRAPH register snapshot\n"); + seq_printf(m, "PD_CYA 0x00406020 %08x\n", readl(bar0 + 0x00406020u)); + seq_printf(m, "PD_HWW_ESR 0x00406018 %08x\n", readl(bar0 + 0x00406018u)); + seq_printf(m, "DS_HWW_ESR 0x00405840 %08x\n", readl(bar0 + 0x00405840u)); + seq_printf(m, "DS_HWW_ESR_2 0x00405848 %08x\n", readl(bar0 + 0x00405848u)); + seq_printf(m, "DS_REPORT_MASK 0x00405844 %08x\n", readl(bar0 + 0x00405844u)); + seq_printf(m, "DS_REPORT_MASK2 0x0040584c %08x\n", readl(bar0 + 0x0040584cu)); + seq_printf(m, "DS_SPH_REGION_A 0x00405c1c %08x\n", readl(bar0 + 0x00405c1cu)); + seq_printf(m, "DS_SPH_REGION_B 0x00405c20 %08x\n", readl(bar0 + 0x00405c20u)); + seq_printf(m, "DS_SPH_STAGE1 0x00405c04 %08x\n", readl(bar0 + 0x00405c04u)); + seq_printf(m, "DS_SPH_STAGE2 0x00405c08 %08x\n", readl(bar0 + 0x00405c08u)); + seq_printf(m, "DS_SPH_STAGE5 0x00405c14 %08x\n", readl(bar0 + 0x00405c14u)); + seq_printf(m, "PGRAPH_STATUS 0x00400700 %08x\n", readl(bar0 + 0x00400700u)); + seq_printf(m, "GR_FIFO_STATUS 0x00400504 %08x\n", readl(bar0 + 0x00400504u)); + return 0; +} + +static int gsp_phys_pgraph_open(struct inode *inode, struct file *file) +{ + return single_open(file, gsp_phys_pgraph_show, NULL); +} + +static ssize_t gsp_phys_pgraph_write(struct file *file, + const char __user *ubuf, size_t len, + loff_t *off) +{ + char buf[64]; + unsigned long long addr = 0, val = 0; + void __iomem *bar0 = gsp_phys_gpu_bar0(); + int n, has_val; + u32 old, now; + + if (!bar0) + return -ENODEV; + if (len == 0 || len >= sizeof(buf)) + return -EINVAL; + if (copy_from_user(buf, ubuf, len)) + return -EFAULT; + buf[len] = '\0'; + n = sscanf(buf, "%llx %llx", &addr, &val); + if (n < 1) + return -EINVAL; + has_val = (n >= 2); + if (addr < 0x00400000ull || addr >= 0x00410000ull) + return -EINVAL; + old = readl(bar0 + addr); + if (has_val) { + writel((u32)val, bar0 + addr); + now = readl(bar0 + addr); + pr_warn("vgsp-boot: pgraph-rw 0x%08llx old=%08x new=%08x (wrote %08llx)\n", + addr, old, now, val); + } else { + pr_info("vgsp-boot: pgraph-rw 0x%08llx = %08x (read)\n", + addr, old); + } + return (ssize_t)len; +} + +static const struct file_operations gsp_phys_pgraph_fops = { + .owner = THIS_MODULE, + .open = gsp_phys_pgraph_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, + .write = gsp_phys_pgraph_write, +}; + +/* + * SESSION 153-CONTINUED-94: gsp-phys-init-push - the first large beef push + * captured at decode time (see svc_init_push_scan). Prints a header with + * the push address/length, the local-memory-run count (SET_SHADER_LOCAL_ + * MEMORY_A..D seen in ANY beef push so far; 0 = the client never emitted + * the download-state local-memory block) and the bytes as 16-byte rows. + */ +static int gsp_phys_init_push_show(struct seq_file *m, void *unused) +{ + const u8 *buf; + u32 len, h, lmem, off; + u64 va; + + buf = gsp_sema_init_cap_get(&len, &h, &va, &lmem); + if (!len) { + seq_puts(m, "vgsp-init-push: empty (no >=4 KiB beef push with non-zero head captured yet)\n"); + return 0; + } + seq_printf(m, "vgsp-init-push: h=0x%x va=0x%llx len=%u lmem_runs=%u\n", + h, va, len, lmem); + for (off = 0; off + 4u <= len; off += 16u) { + u32 k, n = min_t(u32, 4u, (len - off) / 4u); + + seq_printf(m, "%06x:", off); + for (k = 0; k < n; k++) { + u32 w; + + memcpy(&w, buf + off + k * 4u, 4); + seq_printf(m, " %08x", w); + } + seq_putc(m, '\n'); + } + return 0; +} + +static int gsp_phys_init_push_open(struct inode *inode, struct file *file) +{ + return single_open(file, gsp_phys_init_push_show, NULL); +} + +static const struct file_operations gsp_phys_init_push_fops = { + .owner = THIS_MODULE, + .open = gsp_phys_init_push_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +/* + * SESSION 153-CONTINUED-95: gsp-phys-ctx-dump - raw binary readback of the + * engine context buffer (see gsp_phys_ctx_readback_try). Empty until the + * first launch of the boot has run the CE readback. + */ +static ssize_t gsp_phys_ctx_dump_read(struct file *filp, char __user *ubuf, + size_t count, loff_t *ppos) +{ + const u8 *buf; + u32 len = 0; + + buf = gsp_phys_ctx_readback_get(&len); + if (!buf || !len) + return 0; + return simple_read_from_buffer(ubuf, count, ppos, buf, len); +} + +static const struct file_operations gsp_phys_ctx_dump_fops = { + .owner = THIS_MODULE, + .read = gsp_phys_ctx_dump_read, + .llseek = default_llseek, +}; + +/* + * SESSION 153-CONTINUED-96: gsp-phys-ctx-read - write-only re-arm for the + * one-shot engine-context readback. `echo 1 > .../gsp-phys-ctx-read` clears + * the tried flag; the next forwarded launch re-copies MAIN+PATCH with the + * current server state. The copy itself still runs only in the launch path. + */ +static ssize_t gsp_phys_ctx_read_write(struct file *filp, + const char __user *ubuf, + size_t count, loff_t *ppos) +{ + gsp_phys_ctx_readback_rearm(); + pr_info("vgsp-gr: CTX-READBACK re-armed; the next forwarded launch re-copies the engine ctx (MAIN+PATCH)\n"); + return count; +} + +static const struct file_operations gsp_phys_ctx_read_fops = { + .owner = THIS_MODULE, + .write = gsp_phys_ctx_read_write, +}; + int gsp_phys_boot_debugfs_init(void) { struct dentry *file; @@ -23750,6 +33031,20 @@ int gsp_phys_boot_debugfs_init(void) err); return (int)err; } + file = debugfs_create_file("gsp-phys-shared-bit0", 0200, + gsp_phys_debugfs_root, NULL, + &gsp_phys_shared_bit0_fops); + if (IS_ERR_OR_NULL(file)) { + long err = IS_ERR(file) ? PTR_ERR(file) : -ENODEV; + + /* + * The shared-slot bit0 assist is optional; the other + * triggers and the state snapshot stay registered. + */ + pr_warn("vgsp-boot: debugfs file gsp-phys-shared-bit0 creation failed: %ld (other triggers and state snapshot stay available)\n", + err); + return (int)err; + } file = debugfs_create_file("gsp-phys-guest-ram", 0200, gsp_phys_debugfs_root, NULL, &gsp_phys_guest_ram_fops); @@ -23806,7 +33101,57 @@ int gsp_phys_boot_debugfs_init(void) err); return (int)err; } - pr_info("vgsp-boot: debugfs files /sys/kernel/debug/vgsp/{gsp-phys-state read-only, gsp-phys-log-dump read-only, gsp-phys-status-drain read-only, gsp-phys-reg-probe read-only, gsp-phys-unload-rpc write-only, gsp-phys-rm-probe write-only, gsp-phys-rm-call write-only, gsp-phys-guest-gpa write-only, gsp-phys-guest-ram write-only} ready\n"); + /* + * SESSION 153-CONTINUED-91: the DS SPH fault spy (see the show fn). + * Diagnostic only; failure to create it must not fail the init. + */ + file = debugfs_create_file("gsp-phys-ds-spy", 0444, + gsp_phys_debugfs_root, NULL, + &gsp_phys_ds_spy_fops); + if (IS_ERR_OR_NULL(file)) + pr_warn("vgsp-boot: debugfs file gsp-phys-ds-spy creation failed (DS SPH fault spy unavailable)\n"); + /* + * SESSION 153-CONTINUED-93: live PGRAPH register lab (read snapshot + + * bounded writes inside the PGRAPH PRI window). Diagnostic only. + */ + file = debugfs_create_file("gsp-phys-pgraph-rw", 0644, + gsp_phys_debugfs_root, NULL, + &gsp_phys_pgraph_fops); + if (IS_ERR_OR_NULL(file)) + pr_warn("vgsp-boot: debugfs file gsp-phys-pgraph-rw creation failed (PGRAPH register lab unavailable)\n"); + /* + * SESSION 153-CONTINUED-94: the captured first large beef push (the + * GL client's channel-init stream) verbatim, so the local-memory + * census and any other method in it can be verified offline after + * the guest's transient pushbuffer is gone. Diagnostic only. + */ + file = debugfs_create_file("gsp-phys-init-push", 0444, + gsp_phys_debugfs_root, NULL, + &gsp_phys_init_push_fops); + if (IS_ERR_OR_NULL(file)) + pr_warn("vgsp-boot: debugfs file gsp-phys-init-push creation failed (init-push capture unavailable)\n"); + /* + * SESSION 153-CONTINUED-95: raw binary dump of the engine context + * buffer readback (64 KB from GSP_PHYS_RMCALL_VA_GR_CTX, taken once + * per boot at the first launch). 0444; empty until the first launch + * happens. Read with `dd if=...gsp-phys-ctx-dump of=/tmp/ctx.bin`. + */ + file = debugfs_create_file("gsp-phys-ctx-dump", 0444, + gsp_phys_debugfs_root, NULL, + &gsp_phys_ctx_dump_fops); + if (IS_ERR_OR_NULL(file)) + pr_warn("vgsp-boot: debugfs file gsp-phys-ctx-dump creation failed (engine ctx readback unavailable)\n"); + /* + * SESSION 153-CONTINUED-96: write-only re-arm for the readback (see + * gsp_phys_ctx_read_write). `echo 1 >` it to re-copy MAIN+PATCH at + * the next forwarded launch without a module reload. + */ + file = debugfs_create_file("gsp-phys-ctx-read", 0200, + gsp_phys_debugfs_root, NULL, + &gsp_phys_ctx_read_fops); + if (IS_ERR_OR_NULL(file)) + pr_warn("vgsp-boot: debugfs file gsp-phys-ctx-read creation failed (readback re-arm unavailable)\n"); + pr_info("vgsp-boot: debugfs files /sys/kernel/debug/vgsp/{gsp-phys-state read-only, gsp-phys-log-dump read-only, gsp-phys-status-drain read-only, gsp-phys-reg-probe read-only, gsp-phys-ds-spy read-only, gsp-phys-ctx-dump read-only, gsp-phys-ctx-read write-only, gsp-phys-unload-rpc write-only, gsp-phys-rm-probe write-only, gsp-phys-rm-call write-only, gsp-phys-guest-gpa write-only, gsp-phys-guest-ram write-only} ready\n"); return 0; } @@ -24160,8 +33505,8 @@ static_assert(offsetof(struct gsp_phys_rmcall_channel_params, process_id) == 296 "NV_CHANNEL_ALLOC_PARAMS.ProcessID must be at offset 296"); static_assert(offsetof(struct gsp_phys_rmcall_channel_params, tpc_config_id) == 360, "NV_CHANNEL_ALLOC_PARAMS.tpcConfigID must be at offset 360"); -static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 81, - "the rm-call ladder registers exactly eighty-one DMA buffers (14 ladder + 16 single-page guest scratch PTs + 48 single-page extension scratch PTs + 3 golden9g VAS levels)"); +static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 87, + "the rm-call ladder registers exactly eighty-seven DMA buffers (14 ladder + 16 single-page guest scratch PTs + 48 single-page extension scratch PTs + 3 golden9g VAS levels + 6 secondary CE lane buffers)"); static_assert((GSP_PHYS_RMCALL_HANDLE_MEM_BASE + (GSP_PHYS_RMCALL_BUF_COUNT - 1) * GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE) < 0xcaf80000u, @@ -24175,11 +33520,26 @@ static_assert(((GSP_PHYS_RMCALL_VA_BASE >> 47) & 0x3) == 0 && ((GSP_PHYS_RMCALL_VA_BASE >> 21) & 0xff) == 0 && ((GSP_PHYS_RMCALL_VA_SEM >> 21) & 0xff) == 0, "the channel PDB VA window must sit in root/PD2/PD1/PD0 index 0 (GP10X 48:47/46:38/37:29/28:21 splits)"); -static_assert(((GSP_PHYS_RMCALL_VA_GPFIFO >> 12) & 0x1ff) != - ((GSP_PHYS_RMCALL_VA_PB >> 12) & 0x1ff) && - (((GSP_PHYS_RMCALL_VA_GPFIFO >> 12) & 0x1ff) + 4) == - ((GSP_PHYS_RMCALL_VA_SEM >> 12) & 0x1ff), - "the GPFIFO, pushbuffer and CE src/dst/semaphore VAs must take the five consecutive small-PT slots 0x100..0x104"); +static_assert(((GSP_PHYS_RMCALL_VA_GPFIFO >> 12) & 0x1ff) == 0x100 && + ((GSP_PHYS_RMCALL_VA_PB >> 12) & 0x1ff) == 0x101 && + ((GSP_PHYS_RMCALL_VA_PB2 >> 12) & 0x1ff) == 0x102 && + ((GSP_PHYS_RMCALL_VA_PB3 >> 12) & 0x1ff) == 0x103 && + ((GSP_PHYS_RMCALL_VA_PB4 >> 12) & 0x1ff) == 0x104, + "the GPFIFO + four pushbuffer pages must occupy the consecutive small-PT slots 0x100..0x104 for the engine's linear fetch walk (SESSION 153-CONTINUED-77: CE src/dst/sem moved off the walk to 0x109/0x10b/0x10c)"); +/* + * SESSION 153-CONTINUED-66: the two-page pushbuffer and the moved CE SRC / + * probe pages must stay inside the same 2 MB PD0 slot and the same 128 B PTE + * fetch line as the ladder, and must not collide with each other or with the + * QMD-window pair (0x107/0x108). + */ +static_assert(((GSP_PHYS_RMCALL_VA_PB2 >> 12) & 0x1ff) == + ((GSP_PHYS_RMCALL_VA_PB >> 12) & 0x1ff) + 1, + "the pushbuffer's second page must sit directly above the first (one contiguous GPU VA fetch)"); +static_assert(((GSP_PHYS_RMCALL_VA_SRC >> 21) & 0xff) == 0 && + ((GSP_PHYS_RMCALL_VA_GR_PROBE >> 21) & 0xff) == 0 && + ((GSP_PHYS_RMCALL_VA_SRC >> 12) & 0x1ff) == 0x109 && + ((GSP_PHYS_RMCALL_VA_GR_PROBE >> 12) & 0x1ff) == 0x10a, + "the moved CE SRC page and the warm-semaphore probe page must take the free slots 0x109/0x10a in the ladder's PTE line"); static_assert(((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 47) & 0x3) == 0 && ((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 38) & 0x1ff) == 0 && ((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 29) & 0x1ff) == 0 && @@ -24216,18 +33576,20 @@ static_assert(GSP_PHYS_RMCALL_VA_GR_CTX + static_assert(sizeof(struct gsp_phys_gr_page) == 24, "the GR forwarding page descriptor must stay {u64,u64,u32,u32} = 24 bytes"); static_assert(offsetof(struct gsp_phys_gr_inline, data) == 12 && - sizeof(struct gsp_phys_gr_inline) == 80, - "the GR forwarding inline record must stay {u64,u32,u8[64]} padded to 80 bytes"); + sizeof(struct gsp_phys_gr_inline) >= + 12 + GSP_PHYS_GR_INLINE_BYTES, + "the GR forwarding inline record must stay {u64,u32,u8[N]}"); static_assert((2u + 1u + GSP_PHYS_RMCALL_GR_QMD_INLINE_WORDS + 8u) <= GSP_PHYS_RMCALL_GR_METHOD_WORDS, "the GR launch method block (compute init + SET_OBJECT + inline-QMD run + PCAS + REPORT_SEMAPHORE) must fit the bound"); -static_assert(GSP_PHYS_RMCALL_GR_INIT_WORDS + GSP_PHYS_GR_RAW_WORDS + +static_assert(GSP_PHYS_RMCALL_GR_INIT_WORDS + + GSP_PHYS_RMCALL_GR_QMD_WINDOW_WORDS + GSP_PHYS_RMCALL_GR_TAIL_WORDS <= GSP_PHYS_RMCALL_GR_METHOD_WORDS, - "the verbatim GR launch block (compute init + raw tail + PCAS + REPORT_SEMAPHORE) must fit the bound"); + "the verbatim GR launch block minimum (compute init + window + PCAS + REPORT_SEMAPHORE) must fit the bound; the per-launch runtime check bounds raw_qmd_hdr for larger preludes"); static_assert(GSP_PHYS_RMCALL_GR_METHOD_WORDS * 4u <= - GSP_PHYS_RMCALL_COPY_SIZE, - "the GR launch method block must fit one module pushbuffer page"); + 4u * GSP_PHYS_RMCALL_COPY_SIZE, + "the GR launch method block may span the four module pushbuffer pages (SESSION 153-CONTINUED-77: prelude cap 3584 + largest chunked window)"); /* * GspSystemInfo wire ABI, pinned against the reference header compile |