diff options
Diffstat (limited to 'kernel/gsp_phys_booter.c')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 12630 |
1 files changed, 12486 insertions, 144 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 967dc25..189d8b7 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -46,6 +46,7 @@ #include <linux/vgaarb.h> #include <linux/acpi.h> #include <linux/mm.h> +#include <linux/vmalloc.h> #include <asm/barrier.h> #include <asm/unaligned.h> @@ -57,6 +58,15 @@ #include "gsp_phys_booter_load_img.h" #include "gsp_phys_gsp_rm_boot_img.h" #include "gsp_phys_registry_img.h" +/* + * SESSION 153-CONTINUED-114: golden MME slot library (untracked build + * input, .gitignore'd - see the file header for provenance). Used only + * when sema_gl_mme_golden=1: called-but-uncaptured slots are served + * from the driver's own generated macro code instead of being stubbed, + * so the DRAW_IAB call (slot 34) runs deterministically on every boot + * (the full-capture profile becomes a per-boot lottery otherwise). + */ +#include "golden-mme-tu104.h" #include "gsp_phys_registry_img_srvctx.h" static bool gsp_phys_boot_param; @@ -129,6 +139,16 @@ MODULE_PARM_DESC(gsp_phys_gr_reloc_resolve, * per-launch RELOC-CENSUS line. */ #define GSP_PHYS_GR_COPYWIT_MAX 12u +/* + * CONT-112: the changed-page witness gets its OWN budget. A store-rich + * launch walks the tracked pages in bi order and the unchanged pages come + * first, so the shared 12-line budget was always consumed by the OUT-UNCH + * flood before the loop reached the handful of engine-changed pages - the + * r1385 evidence has "2/900 pages changed" + OUT-ALIAS-WIT for bi=8960/8965 + * but no OUT-PAGE-WIT va->hpa pair for either. With the split counter both + * classes are witnessed on every launch (<=12 unchanged + <=16 changed). + */ +#define GSP_PHYS_GR_COPYWIT_CHG_MAX 16u static bool gsp_phys_gr_copyout_dump_param = true; module_param_named(gsp_phys_gr_copyout_dump, gsp_phys_gr_copyout_dump_param, bool, 0644); @@ -140,6 +160,7 @@ static inline bool gsp_phys_gr_copywit_enabled(void) return READ_ONCE(gsp_phys_gr_copyout_dump_param); } + /* * r405 late-write probes (the r405 "partial write" class: the scale launch * of streams2 wrote 1 of its 32 expected in-place pages and every other @@ -151,9 +172,19 @@ static inline bool gsp_phys_gr_copywit_enabled(void) * "ENGINE STILL WRITING" proves the copy-out shipped before the stores * landed (the fix is a longer settle window); "stable" proves the engine * genuinely stopped, which needs a different treatment. - * - gsp_phys_gr_settle_ms (default 300): the content-stable window the + * - gsp_phys_gr_settle_ms (default 100): the content-stable window the * partial-changed rule requires before it declares the stores settled. * Raise it to give a still-writing engine its time (A/B in one cycle). + * + * r552 (2026-10-23): measured A/B on real boots. Every launch with a + * partial changed-page count (read-mostly kernels: 1..64 of 1000+ pages + * written, so the changed>=n_all shortcut can never fire) paid the full + * window plus scan overhead: 683-901 ms per launch at 300 ms. At 100 ms + * the same boots measured 231-247 ms of retire wait and the launch + * latency p50 dropped 1119 ms -> 651 ms (n=22 launches; baseline 684 ms) + * with the battery unchanged (12/15, 0 hangs) and a green 2-round soak + * on the next boot. The content-stability hash still gates the verdict; + * only the minimum stable duration shrank. */ static bool gsp_phys_gr_late_probe_param; module_param_named(gsp_phys_gr_late_probe, gsp_phys_gr_late_probe_param, @@ -161,11 +192,11 @@ module_param_named(gsp_phys_gr_late_probe, gsp_phys_gr_late_probe_param, MODULE_PARM_DESC(gsp_phys_gr_late_probe, "Diagnostic: after the GR retire wait, re-scan the tracked pages +500 ms later and log whether the engine is still writing (default 0)"); -static int gsp_phys_gr_settle_ms_param = 300; +static int gsp_phys_gr_settle_ms_param = 100; module_param_named(gsp_phys_gr_settle_ms, gsp_phys_gr_settle_ms_param, int, 0644); MODULE_PARM_DESC(gsp_phys_gr_settle_ms, - "GR retire wait: content-stable window (ms) required before a partial changed-page count counts as settled (default 300)"); + "GR retire wait: content-stable window (ms) required before a partial changed-page count counts as settled (default 100; 300 was the pre-r552 value)"); static inline u32 gsp_phys_gr_settle_iters(void) { @@ -1120,18 +1151,61 @@ bool gsp_phys_boot_requested(void) * from gpFifoOffset, the pushbuffer from the GPFIFO ring entry GET, the CE * operands from the method block): SET_SEMAPHORE_A/B are always GPU VAs with * no mode bit (clc5b5.h:39-44; uvm_pascal_ce.c:42-63) and the user channel's - * copy is virtual-mode on both sides (uvm_maxwell_ce.c:179-205). All five - * pages live in one 2 MB PD0 slot below the VAS vaSize limit: with + * copy is virtual-mode on both sides (uvm_maxwell_ce.c:179-205). All the + * ladder pages live in one 2 MB PD0 slot below the VAS vaSize limit: with * VA_BASE = 0x00100000 every parent index (48:47, 46:38, 37:29, 28:21) is 0 - * and the five 4 KB pages take small-PT slots 0x100..0x104 - * (kern_gmmu_fmt_gp10x.c:16-83; uvm_pascal_mmu.c:25-32,172-204). + * and the 4 KB pages take small-PT slots in the single 128 B PTE line + * 0x100..0x10f (kern_gmmu_fmt_gp10x.c:16-83; uvm_pascal_mmu.c:25-32,172-204). + * + * SESSION 153-CONTINUED-66: the module pushbuffer is TWO pages. The MME + * context-setup prelude matured to 901 words while the beef window runs 390 + * words and the SA-RAM stubs add 29..47, so the single-launch block needs + * ~1400 words = 5.6 KB: the 1-page pushbuffer made every prelude-bearing + * launch print "MME-PRELUDE deferred (window=390 prelude=901 sa=47 exceeds + * block; replaying window alone)" and go out prelude=0. The block now spans + * VA_PB..VA_PB+0x2000 with the second page at small-PT slot 0x102. The + * slot 0x102 page used to be CE SRC; SRC moved to 0x109 (it is only read by + * the CE copy path and the GR inline-QMD destination default). Slot 0x10a + * is the dedicated warm-semaphore probe page: the probe used to share the + * pushbuffer's upper half, which a >0x800-byte forwarded block now occupies. */ #define GSP_PHYS_RMCALL_VA_BASE 0x00100000ULL #define GSP_PHYS_RMCALL_VA_GPFIFO GSP_PHYS_RMCALL_VA_BASE #define GSP_PHYS_RMCALL_VA_PB (GSP_PHYS_RMCALL_VA_BASE + GSP_PHYS_PAGE_SIZE) -#define GSP_PHYS_RMCALL_VA_SRC (GSP_PHYS_RMCALL_VA_BASE + 2 * GSP_PHYS_PAGE_SIZE) -#define GSP_PHYS_RMCALL_VA_DST (GSP_PHYS_RMCALL_VA_BASE + 3 * GSP_PHYS_PAGE_SIZE) -#define GSP_PHYS_RMCALL_VA_SEM (GSP_PHYS_RMCALL_VA_BASE + 4 * GSP_PHYS_PAGE_SIZE) +/* SESSION 153-CONTINUED-66: page 2 of the module pushbuffer (same PT line). */ +#define GSP_PHYS_RMCALL_VA_PB2 (GSP_PHYS_RMCALL_VA_BASE + 2 * GSP_PHYS_PAGE_SIZE) +/* + * SESSION 153-CONTINUED-77: page 3 of the module pushbuffer (slot 0x103) + * so a matured MME prelude (cap 2048 words) plus the largest chunked + * window fit one fetch block. The block walk is linear in VA, so the + * three pushbuffer VAs must stay contiguous: VA_DST (previously at + * base+0x3000, slot 0x103) moves to base+0xB000 (slot 0x10b, free). + */ +#define GSP_PHYS_RMCALL_VA_PB3 (GSP_PHYS_RMCALL_VA_BASE + 3 * GSP_PHYS_PAGE_SIZE) +/* + * SESSION 153-CONTINUED-77: page 4 (slot 0x104) after the r1163 cycle + * showed the capture cap-limited at 2048 words (ir_end=0x265, 34 slots + * covered). VA_SEM moves 0x104 -> 0x10c (free) so the four pushbuffer + * pages stay contiguous for the linear fetch. + */ +#define GSP_PHYS_RMCALL_VA_PB4 (GSP_PHYS_RMCALL_VA_BASE + 4 * GSP_PHYS_PAGE_SIZE) +#define GSP_PHYS_RMCALL_VA_DST (GSP_PHYS_RMCALL_VA_BASE + 0xB000ULL) +#define GSP_PHYS_RMCALL_VA_SEM (GSP_PHYS_RMCALL_VA_BASE + 0xC000ULL) +/* SESSION 153-CONTINUED-66: CE SRC moved off the pushbuffer's second page. */ +#define GSP_PHYS_RMCALL_VA_SRC (GSP_PHYS_RMCALL_VA_BASE + 0x9000ULL) +/* SESSION 153-CONTINUED-66: warm-semaphore probe page (own PTE, no sharing). */ +#define GSP_PHYS_RMCALL_VA_GR_PROBE (GSP_PHYS_RMCALL_VA_BASE + 0xA000ULL) +/* + * SESSION 153-CONTINUED-96: secondary CE (COPY0) lane. A COPY0 channel on + * the same VAS translates through the same 2 MiB window PDB, but CE class + * methods are never legal on the GR0 ladder channel, so the ladder's own + * FBMEM readback (gsp_phys_ctx_readback_try) needs its own engine. The + * lane's GPFIFO ring and one-page pushbuffer get their own small-PT slots + * 0x10d/0x10e in the same 128 B PTE line the ladder's pages occupy, planted + * in pdb_build (stage 3, before any engine fetch; there is no invalidate). + */ +#define GSP_PHYS_RMCALL_VA_CE_GPFIFO (GSP_PHYS_RMCALL_VA_BASE + 0xD000ULL) +#define GSP_PHYS_RMCALL_VA_CE_PB (GSP_PHYS_RMCALL_VA_BASE + 0xE000ULL) /* * m69: window-0 shadow pool - 16 module pages holding the Tier-2 copies of * the QMD's program/param pages, so the GR engine never has to read through @@ -1417,6 +1491,24 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RMCALL_HANDLE_USER_LANE_PM_FB 0xcaf000b8u #define GSP_PHYS_RMCALL_HANDLE_USER_LANE_FAULT_MEM0 0xcaf000c0u #define GSP_PHYS_RMCALL_HANDLE_USER_LANE_FAULT_MEM1 0xcaf000c4u +/* + * SESSION 153-CONTINUED-96: the secondary COPY0 lane's TSG/channel handles + * (free in the ladder's client range: 0xcaf000c8/0xcc sit above the mode-35 + * fault-mem handles and below the registered-buffer base 0xcaf01000). + */ +#define GSP_PHYS_RMCALL_HANDLE_CE_TSG 0xcaf000c8u +#define GSP_PHYS_RMCALL_HANDLE_CE_CHANNEL 0xcaf000ccu +/* + * Mode 35, graphics-execution plan §F first experiment: the TURING_A + * (0xc597, GR_OBJECT_TYPE_3D) class object constructed on the V7 + * user-lane channel AFTER its promote - the missing half the mode-35 + * capture never provided ("mode 35: ... the capture carries no 0xc597 on + * the user lane"). Best-effort like the TURING_COMPUTE_A: a rejection is + * logged and the lane keeps running; a construction on V7 must not hit + * the wired-golden deadlock class (m3a-m3g, that channel). 0xcaf00094 is + * free between 0xcaf00090 and 0xcaf000a0. + */ +#define GSP_PHYS_RMCALL_HANDLE_USER_LANE_GR3D 0xcaf00094u /* * Mode-35 V7 channel flags/privilege and fault/PM geometry, decoded from the @@ -1642,6 +1734,19 @@ enum gsp_phys_rmcall_buf { GSP_PHYS_RMCALL_BUF_GOLDEN_PT0, GSP_PHYS_RMCALL_BUF_GOLDEN_PT1, GSP_PHYS_RMCALL_BUF_GOLDEN_PT2, + /* + * SESSION 153-CONTINUED-96: secondary COPY0 lane buffers (one page + * each). The instance/USERD/RAMFC/mthdbuf are the lane's own memdescs + * in the channel alloc (a second channel must never share the GR + * lane's hardware USERD); the GPFIFO ring and pushbuffer page are + * engine-reached through their planted PTE slots. + */ + GSP_PHYS_RMCALL_BUF_CE_GPFIFO, + GSP_PHYS_RMCALL_BUF_CE_USERD, + GSP_PHYS_RMCALL_BUF_CE_INSTANCE, + GSP_PHYS_RMCALL_BUF_CE_RAMFC, + GSP_PHYS_RMCALL_BUF_CE_MTHDBUF, + GSP_PHYS_RMCALL_BUF_CE_PB, GSP_PHYS_RMCALL_BUF_COUNT }; @@ -1657,6 +1762,20 @@ enum gsp_phys_rmcall_buf { #define GSP_PHYS_RMCALL_TIMEOUT_MS 2000u #define GSP_PHYS_RMCALL_POLL_MS 5u /* + * SESSION 148 perf (termbin review, verified in-tree): every RPC reply wait + * polled with msleep(5). With several quanta per round trip and ~8 + * serialized PDE_UPDATE forwards per forwarded GR launch, the 5 ms quantum + * dominated per-launch latency (~95% of the ~189 ms host service time per + * launch). Wait with sub-millisecond quanta instead: the reply is observed + * within one quantum, cutting each round trip to ~1 ms. POLL_MS remains for + * the poll accounting prints. + */ +#define GSP_PHYS_RMCALL_POLL_US 150u +static inline void gsp_phys_rmcall_wait_quantum(void) +{ + usleep_range(GSP_PHYS_RMCALL_POLL_US, GSP_PHYS_RMCALL_POLL_US + 200u); +} +/* * GR launch release poll window. A CE copy completes in single-digit ms, but * a GR launch can force the engine's first context switch for the channel; * the FECS ctxsw path is measured in fractions of a second, so the window is @@ -1708,6 +1827,19 @@ enum gsp_phys_rmcall_buf { #define GSP_PHYS_RMCALL_CE_SUBCHANNEL 4u /* + * SESSION 153-CONTINUED-68: SEC_OP_ONE_INC = 5 (class/cl906f.h:181 + * NV906F_DMA_SEC_OP_ONE_INC, NVC06F_DMA_ONEINCR_* field layout). The + * client emits its MME IR uploads (method 0x114) as op=5 runs whose first + * data word is the IR instruction address and whose remaining words feed + * the IR data port (the r1099/r1116 captures: hdr 0xa0100045 = op 5, + * cnt 16, method 0x114, data [pc, code...]). op=1 (INC_METHOD) walks the + * METHOD address per data word instead (0x114 -> 0x118 -> 0x11c -> + * 0x120), so an IR upload emitted with op=1 writes one stray IR word and + * then clobbers the SA pointer/data registers. + */ +#define GSP_PHYS_RMCALL_METHOD_ONEINC 5u + +/* * CE (TURING_DMA_COPY_A 0xc5b5) method offsets and field values, from * class/clc5b5.h: SET_OBJECT 0x0, SET_SEMAPHORE_A/B/PAYLOAD 0x240/0x244/0x248, * LAUNCH_DMA 0x300, OFFSET_IN_UPPER/LOWER 0x400/0x404, OFFSET_OUT_UPPER/ @@ -2355,6 +2487,16 @@ static struct gsp_phys_boot_stage { u32 rmcall_gr_promote_size; u32 rmcall_gr_hvirt; /* mode-6 NV01_MEMORY_VIRTUAL handle */ u32 rmcall_token; /* GET_WORK_SUBMIT_TOKEN result */ + /* + * SESSION 153-CONTINUED-96: secondary COPY0 lane (same VAS). Alloc'd + * and scheduled only on a GR0 ladder (CE class is illegal on the GR + * channel); rmcall_ce_lane_valid means the token was obtained and the + * lane can accept CE copy blocks. Zero on a COPY0 ladder. + */ + u32 rmcall_ce_tsg; /* KEPLER_CHANNEL_GROUP_A handle */ + u32 rmcall_ce_channel; /* TURING_CHANNEL_GPFIFO_A handle */ + u32 rmcall_ce_token; /* its GET_WORK_SUBMIT_TOKEN result */ + bool rmcall_ce_lane_valid; /* alloc + BIND/schedules + token done */ u32 rmcall_notifier_mem; /* FBMEM error-notifier object handle */ u64 rmcall_notifier_base; /* its returned frame-buffer address */ bool rmcall_notifier_valid; /* the FBMEM notifier alloc succeeded */ @@ -2447,6 +2589,18 @@ static struct gsp_phys_boot_stage { */ struct gsp_phys_boot_chunk rmcall_gr_qmdwin; u64 rmcall_gr_qmdwin_va; /* planted VA; 0 = guest destination */ + /* + * SESSION 153-CONTINUED-66: page 2 of the module pushbuffer plus the + * dedicated warm-semaphore probe page. Allocated lazily by + * gsp_phys_rmcall_pdb_build() and planted in the ladder's 128 B PTE + * line at slots 0x102 (VA_PB2) and 0x10a (VA_GR_PROBE), before the + * engine's first fetch. Page 0 continues the registered BUF_PB chunk + * so stage 2's registration and every existing pushbuffer user are + * unchanged; only the >4 KB GR launch block reaches into these pages. + */ + struct gsp_phys_boot_chunk rmcall_gr_pb2; + struct gsp_phys_boot_chunk rmcall_gr_probe; + bool rmcall_gr_pb2_planted; /* m104: module page holding the injected mail_kernel message. */ struct gsp_phys_boot_chunk rmcall_gr_mail; /* @@ -2623,6 +2777,18 @@ EXPORT_SYMBOL_GPL(gsp_phys_boot_rm_flags); static void *gsp_phys_chunk_page_va(const struct gsp_phys_boot_chunk *c, u32 page) { + /* + * SESSION 141: an unallocated chunk (no per-page array AND no base + * va) must answer NULL. It previously returned + * (u8 *)NULL + page*PAGE_SIZE - a small non-NULL garbage pointer - + * so the caller's NULL guard passed and the empty-set post-mortem + * print dereferenced it: Oops #2 RIP + * gsp_phys_backend_guest_gr_submit+0x4394, CR2=0x1e93c with slot 30 + * (0x1e000 = 30*0x1000 + 0x900 off). min_data=0's no-external-data + * forward hit this on its first pass. + */ + if (!c->page_va && !c->va) + return NULL; if (c->page_va) return c->page_va[page]; return (u8 *)c->va + (size_t)page * GSP_PHYS_PAGE_SIZE; @@ -3169,6 +3335,10 @@ static void gsp_phys_rmcall_reset(void) boot_stage.rmcall_gr_promote_size = 0; boot_stage.rmcall_gr_hvirt = 0; boot_stage.rmcall_token = 0; + boot_stage.rmcall_ce_tsg = 0; + boot_stage.rmcall_ce_channel = 0; + boot_stage.rmcall_ce_token = 0; + boot_stage.rmcall_ce_lane_valid = false; boot_stage.rmcall_notifier_mem = 0; boot_stage.rmcall_notifier_base = 0; boot_stage.rmcall_notifier_valid = false; @@ -3204,6 +3374,11 @@ static void gsp_phys_rmcall_reset(void) memset(&boot_stage.rmcall_gr_qmdwin, 0, sizeof(boot_stage.rmcall_gr_qmdwin)); boot_stage.rmcall_gr_qmdwin_va = 0; + memset(&boot_stage.rmcall_gr_pb2, 0, + sizeof(boot_stage.rmcall_gr_pb2)); + memset(&boot_stage.rmcall_gr_probe, 0, + sizeof(boot_stage.rmcall_gr_probe)); + boot_stage.rmcall_gr_pb2_planted = false; memset(&boot_stage.rmcall_gr_shadow_win0, 0, sizeof(boot_stage.rmcall_gr_shadow_win0)); boot_stage.rmcall_gr_shadow_win0_used = 0; @@ -4911,6 +5086,773 @@ static u32 gsp_phys_engine_type_from_errstr(const char *err) } /* + * SESSION 153-CONTINUED-12: last published GR launch identity, set by the + * single-inflight submit path so a PBDMA fault dump can name the entry the + * PBDMA is stuck on (block length in dwords, guest push VA, object handle). + */ +static u32 gsp_phys_gr_last_words; +static u32 gsp_phys_gr_last_h; +static u64 gsp_phys_gr_last_push_va; + +/* + * SESSION 153-CONTINUED-12: on a robust-channel PBDMA error (exceptType + * 0x20) dump every PBDMA's latched instruction state and, for the PBDMA + * whose PB_FETCH sits inside the module pushbuffer page, the raw block words + * around the fetch point plus the whole block with indices. The latched + * raw instruction (HDR_SHADOW) can be matched against the block dump to + * name the exact word the PBDMA rejected ("invalid method encoding"); the + * fetch position shows how far the walk got. Read-only MMIO + a racy read + * of the pb page (best effort, diagnostic only); bounded per boot. + */ +#define GSP_PHYS_PBDMA_DUMP_MAX 64u + +/* + * SESSION 153-CONTINUED-52: the published method block's DMA address, so the + * PBDMA fault dump can name the exact word the PBDMA stopped on + * (offset = PB_FETCH - last block DMA). Set beside gsp_phys_gr_last_words. + */ +static u64 gsp_phys_gr_last_pb_dma; + +static const char *gsp_phys_pbdma_type_name(u32 t) +{ + static const char *names[8] = { "SSDM", "INC", "STORE_SDM", "NON_INC", + "IMMD", "INC_ONCE", "USE_SDM", "END_SEG" }; + + return names[t & 7u]; +} + +/* + * SESSION 153-CONTINUED-52: MME fault forensics. The C597 replay's dominant + * failure on draw-bearing windows is a real MME exception + * (NV_PGRAPH_PRI_MME_HWW_ESR 0x404490 latched ILLEGAL_OPCODE / + * MISSING_MACRO_DATA / ... - "unknown instruction fetched from the MME + * Instruction RAM", see gr_error_gk104.c grErrorLogMmeException). INFO + * carries the faulting MME PC and the macro address; INFO2 the instruction + * register. Read them post-failure (the latch survives) so the exact macro + * fetch point is on record. Read-only; bounded by its call sites. + */ +static void gsp_phys_mme_regs_log(void __iomem *bar0, const char *tag) +{ + u32 esr, info, info2, info3, info4, trapped; + + if (!bar0) + return; + esr = ioread32(bar0 + 0x00404490u); + info = ioread32(bar0 + 0x00404494u); + /* SESSION 153-CONTINUED-106: INFO2 is 0x40449c (0x404498 is CONFIG). */ + info2 = ioread32(bar0 + 0x0040449cu); + info3 = ioread32(bar0 + 0x004044a8u); + info4 = ioread32(bar0 + 0x004044acu); + trapped = ioread32(bar0 + 0x00400704u); + pr_warn("vgsp-gr: MME-REGS %s esr=%08x(pc_valid=%u illegal=%u miss=%u extra=%u) pc=%u macro_addr=0x%x ir=%08x,%08x,%08x trapped=%08x\n", + tag, esr, (info >> 28) & 1u, (esr >> 2) & 1u, + (esr >> 0) & 1u, (esr >> 1) & 1u, info & 0xfffu, + (info >> 12) & 0xffu, info2, info3, info4, trapped); +} + +/* + * SESSION 153-CONTINUED-107: MME IRAM debug port readback. + * + * CONT-106 found the debug port at 0x4044d0 host-side; the definitive field + * spec (tu104 dev_graphics_nobundle.h:2166 NV_PGRAPH_PRI_MME_DEBUG) is + * bit 0 STATUS (R; 0=RUNNING, 1=STOPPED) + * bit 1 ENABLE (RW; init DISABLED) + * bit 2 IRAM_RD (write 1 = TRIGGER, reads 0 = DONE) + * bits 26:15 IRAM_PC (RW, 12 bits = the IRAM word address) + * and the fetched group lands in NV_PGRAPH_PRI_MME_HWW_ESR_INFO4/3/2 + * (0x4044ac/a8/9c, field IR 31:0 each). CONT-106's r1364 read at PC 0 + * returned the reconstructed image's first three words + * (0x00003463,0x190d82f0,0x39804300), so the data path is the INFO + * triplet. The read is a two-step hardware handshake: set ENABLE, write + * the PC with IRAM_RD=1 (trigger), WAIT for IRAM_RD to read back 0 + * (DONE), then read the group. `a` is an IRAM WORD address (callers pass + * instruction * 3, matching gsp_phys_gr_mme_ir_pack[] indexing). + * Read-only diagnostic; bounded by its call sites. + */ +static void gsp_phys_mme_iram_read3(void __iomem *bar0, u32 a, u32 out[3]) +{ + u32 i; + + iowrite32(0x2u, bar0 + 0x004044d0u); /* ENABLE */ + iowrite32(((a & 0xfffu) << 15) | 6u, bar0 + 0x004044d0u); + for (i = 0; i < 1000u; i++) { + if (!(ioread32(bar0 + 0x004044d0u) & 0x4u)) + break; + cpu_relax(); + } + out[0] = ioread32(bar0 + 0x004044acu); + out[1] = ioread32(bar0 + 0x004044a8u); + out[2] = ioread32(bar0 + 0x0040449cu); +} + +/* + * Defined next to the packed-image array (gsp_phys_gr_mme_ir_pack); forward + * declaration so the post-doorbell forensics spin can call it. + */ +static void gsp_phys_mme_iram_probe(void __iomem *bar0, const char *tag, + u32 h_object); + +/* + * SESSION 153-CONTINUED-107: probe enable (see the sema_gl_mme_iram_probe + * knob beside sema_gl_mme_iram_gate). Declared here for the forensics spin + * above; the matching module_param sits with the gate knob. + */ +static bool gsp_phys_rmcall_gr_mme_iram_probe; + +/* + * SESSION 153-CONTINUED-69: pre-reset MME fault capture. Everything sampled + * so far was post-mortem: by the time the 5 ms launch poll ticks, the + * GSP-RM's FECS exception handler has already read NV_PGRAPH_PRI_MME_HWW_ESR + * (0x404490) - the relayed errString "ESR 0x404490=0x80000004" is the only + * surviving trace of the raw ILLEGAL_OPCODE value - and either cleared the + * latch back to 0x80000000 or left the whole PRI block in the 0xbadfxxxx + * priv-error poison. This bounded spin runs in the submit thread immediately + * after the publish/doorbell (execution starts within the first + * milliseconds) and records EVERY transition of the fault registers at + * MMIO read rate, so the faulting ESR (+INFO pc/macro when the valid bit is + * set, and the instruction register INFO2 that names the rejected + * instruction word) is on the record before any clear. Read-only; bounded + * per boot and per launch (the spin costs at most sema_mme_fast_ms of the + * launch thread's time and stops early once the poison is seen). + */ +static u32 gsp_phys_mme_fast_ms = 400u; +/* r1497: defined with the emission knob below; used by the post-launch + * DS ESR clear. */ +static bool gsp_phys_rmcall_gr_ds_mask_clear; +module_param_named(sema_mme_fast_ms, gsp_phys_mme_fast_ms, uint, 0644); +MODULE_PARM_DESC(sema_mme_fast_ms, + "Spin-sample the MME fault registers right after each GR launch doorbell for this many ms, logging every transition before the GSP-RM clears them (0 = off; default 400)"); + +/* + * SESSION 153-CONTINUED-106: post-timeout re-spin. The r1363 chronology + * shows the MME ILLEGAL_OPCODE relay arriving up to ~2.6 s AFTER the sem + * wait failed (launch failed 403088.34, ILLEGAL_OPCODE relayed 403090.96); + * every launch-window spin had already ended, so the latch was never seen + * with its INFO/IR. This bounded second spin runs on the first timeout + * exits of a boot, exactly where the r1363 trap sat. + */ +static u32 gsp_phys_mme_post_ms = 2000u; +module_param_named(sema_mme_post_ms, gsp_phys_mme_post_ms, uint, 0644); +MODULE_PARM_DESC(sema_mme_post_ms, + "Post-timeout MME fault-register re-spin length in ms on the first GR launch-timeout exits (0 = off; default 2000)"); + +/* + * SESSION 153-CONTINUED-83: store-side emission forensics. + * + * The store wall: draws retire but `output retirement wait: 0/N pages + * changed` and gen0=0 (no MME-generated method witnessed). The client's + * Turing draw is EMITTED BY AN MME MACRO (slot 34 = the draw macro: + * BEGIN(0x1618) -> SET_VERTEX_ARRAY_START(0xd74) -> END(0x1614), per the + * offline replay of sa_full_stream2.bin and the nvk reference builder), so + * "no stores" == "the draw macro never emitted". This probe answers three + * questions with one line, at the launch boundary: + * + * 1. DS_HWW_ESR (0x405840) + the per-stage SPH_INFO snapshot + * (0x405c00+si*4 START / 0x405c24+si*4 UPPER / 0x405c1c,c20 REGION): + * which pipeline stages failed SPH validation / what VA they fetched. + * 2. MME_HWW_ESR_INFO (0x404494) PC + MACRO_ADDRESS + PC_VALID, + * INFO_EX (0x4044a4) + IR words INFO2/3/4 (0x40449c/a8/ac): did the + * MME actually execute a macro, and where did it stop. + * 3. FE_DECODE_STATE_EX(i) (0x40a630+i*4): MME_PC 11:0, MACRO 21:12, + * GENERATED 31 - the generated-method witness, all four lanes, at + * the launch end (the retire-time read races the release writes). + * + * Bounded: 24 calls per boot, beef family only, one line per call. + */ +/* + * SESSION 153-CONTINUED-83: MME shadow-RAM read port (bar0 0x404488 index, + * 0x40448c data). The MME tracks class methods in its shadow RAM when the + * class's SET_MME_SHADOW_RAM_CONTROL mode selects tracking; reading the + * port tells us (a) whether the tracked window's methods ever reached the + * MME and (b) what values the MME holds for the draw-relevant methods. + * A read is: index = nvclass(15:0) | method_addr(27:16) | READ(30), then + * read data. Bounded by the caller; host-side diagnostic only. + */ +static u32 gsp_phys_mme_shadow_read(void __iomem *bar0, u32 nvclass, u32 mthd) +{ + iowrite32(nvclass | ((mthd & 0xfffu) << 16) | (1u << 30), + bar0 + 0x00404488u); + return ioread32(bar0 + 0x0040448cu); +} + +static void gsp_phys_store_side_log(void __iomem *bar0, const char *tag, + u32 h_object) +{ + static u32 ss_n; + u32 si, sph[6], sphu[6], rr_a, rr_b, ds, ds2, mminfo, mmex; + u32 mm2, mm3, mm4, fex[4], fed[4], fds[4]; + + if (!bar0 || ss_n >= 40u) + return; + ss_n++; + for (si = 0; si < 6u; si++) { + sph[si] = ioread32(bar0 + 0x00405c00u + si * 4u); + sphu[si] = ioread32(bar0 + 0x00405c24u + si * 4u); + } + for (si = 0; si < 4u; si++) { + fex[si] = ioread32(bar0 + 0x0040a630u + si * 4u); + fed[si] = ioread32(bar0 + 0x0040a620u + si * 4u); + fds[si] = ioread32(bar0 + 0x0040a640u + si * 4u); + } + rr_a = ioread32(bar0 + 0x00405c1cu); + rr_b = ioread32(bar0 + 0x00405c20u); + ds = ioread32(bar0 + 0x00405840u); + ds2 = ioread32(bar0 + 0x00405848u); + mminfo = ioread32(bar0 + 0x00404494u); + mmex = ioread32(bar0 + 0x004044a4u); + mm2 = ioread32(bar0 + 0x0040449cu); + mm3 = ioread32(bar0 + 0x004044a8u); + mm4 = ioread32(bar0 + 0x004044acu); + pr_info("vgsp-store: %s h=0x%x ds=%08x/%08x sph=%08x,%08x,%08x,%08x,%08x,%08x sphU=%08x,%08x,%08x,%08x,%08x,%08x rr=%08x,%08x\n", + tag, h_object, ds, ds2, + sph[0], sph[1], sph[2], sph[3], sph[4], sph[5], + sphu[0], sphu[1], sphu[2], sphu[3], sphu[4], sphu[5], + rr_a, rr_b); + pr_info("vgsp-store: %s h=0x%x mme_esr_info=%08x(pcv=%u pc=%u mac=0x%x) ex=%08x ir=%08x,%08x,%08x fed=%08x,%08x,%08x,%08x fex=%08x,%08x,%08x,%08x\n", + tag, h_object, mminfo, (mminfo >> 28) & 1u, mminfo & 0xfffu, + (mminfo >> 12) & 0xffu, mmex, mm2, mm3, mm4, + fed[0], fed[1], fed[2], fed[3], + fex[0], fex[1], fex[2], fex[3]); + /* + * MME shadow-RAM readback for the draw-relevant C597 methods (tracked + * methods show the last value the MME saw; zeros = never tracked). + * Also probe the MME setup pair and the draw trigger/vertex-start. + */ + pr_info("vgsp-store: %s h=0x%x shadow[c597] 0x114=%08x 0x11c=%08x 0x124=%08x 0x260=%08x 0x840=%08x 0xd74=%08x 0xd78=%08x 0x1618=%08x\n", + tag, h_object, + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x114u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x11cu), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x124u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x260u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x840u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0xd74u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0xd78u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x1618u)); + /* + * r1453 capture (LOG-ONLY, zero behaviour change): the engine's own + * macro-state shadow values for the slow-path STATE 0xD24/0xD25 + * inputs. d24 = SET_MME_SHADOW_SCRATCH(0x24) = method 0x3490, d25 = + * scratch 0x25 = method 0x3494 (the client/X-lane never writes + * either, so the MME shadow is the only non-blind source of these + * values). 0xd24/0xd25 are read as literal method indices through + * the same shadow port (same helper; masked to 12 bits) so the + * tracker interpretation is captured alongside the write methods. + * Rate-limited to 8 lines per boot, no new module parameter; the + * reads use the existing shadow-read path only (no emission change). + */ + { + static u32 d24_d25_cap_n; + + if (d24_d25_cap_n < 8u) { + d24_d25_cap_n++; + pr_info("vgsp-store: %s h=0x%x shadow[c597] d24d25 0x3490=%08x 0x3494=%08x 0xd24=%08x 0xd25=%08x\n", + tag, h_object, + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x3490u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x3494u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0xd24u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0xd25u)); + } + } + /* + * SESSION 153-CONTINUED-83 probe 2: the CALL trigger methods the + * window uses (are the calls even reaching the MME?), a repeat read + * (does a read clear?), and the dword-address interpretation of the + * draw/RT methods (0xd74>>2, 0x840>>2) to pin the index units. + */ + pr_info("vgsp-store: %s h=0x%x shadow2[c597] 0x3800=%08x 0x3810=%08x 0x3910=%08x 0x3938=%08x 0x3940=%08x 0x840r=%08x 0xd74dw=%08x 0x840dw=%08x 0x1618r=%08x\n", + tag, h_object, + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x3800u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x3810u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x3910u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x3938u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x3940u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x840u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0xd74u >> 2), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x840u >> 2), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x1618u >> 2)); +} + +static bool gsp_phys_mme_fast_sample(void __iomem *bar0, const char *tag, + u32 h_object, u32 ms_override) +{ + static u32 fast_boot_n; + u32 ms = ms_override ? ms_override : READ_ONCE(gsp_phys_mme_fast_ms); + u32 esr_p, ds_p, gf_p, tr_p, n = 0; + u32 fed0_p, fed1_p; + u32 ds_log_n = 0; + ktime_t t_end; + bool poisoned = false; + + /* + * SESSION 153-CONTINUED-106: the budget was 48, which a beef-lane boot + * with the wedged-engine relaunch train exhausts long before the + * first MME trap is relayed (r1363: 27 launch spins used by +300 s, + * trap traffic still arriving). Raised so every beef-family launch + * of a diagnostic boot keeps its spin. + */ + if (!bar0 || !ms || fast_boot_n >= 256u) + return false; + /* + * Beef-family channels get the full budget (the draw-class windows + * are the ones with the MME fault); other channels are sampled only + * a few times so a fast X-channel launch train cannot consume it. + */ + if ((h_object & 0xfffff000u) != 0xbeef0000u && fast_boot_n >= 2u) + return false; + fast_boot_n++; + esr_p = ioread32(bar0 + 0x00404490u); + ds_p = ioread32(bar0 + 0x00405840u); + gf_p = ioread32(bar0 + 0x00400504u); + tr_p = ioread32(bar0 + 0x00400704u); + /* + * SESSION 153-CONTINUED-109: the poison episode can already be + * latched when the sample's FIRST read happens. r1378's first + * window+live launch: `MME-FAST begin launch ... esr=badf1201 + * ds=badf1201 grfifo=badf1201 trap=badf1201` with the very next + * read clean (`esr badf1201->80000000 ... grfifo badf1201-> + * 00000001`). The historical detector only marks `poisoned` when a + * POST-entry read shows poison, so the episode was never handed to + * the WEDGE-RECOVER decision - no recovery ran, the prelude latch + * and live_gen stayed set, and every later calls-bearing window + * executed macro 57 against the wiped MME context: sked + * `mme_esr=80000004 (ILLEGAL_OPCODE) mme_mac=0x39`, then the FE + * parked at pc=513/mac=0xf1 and the stores stopped (0/905 after the + * 305-page burst). Treat an entry read of poison as the same + * witness; the FECS has already reset the context, so the recovery + * (latch clear + cursor rewind + live_gen invalidate) is exactly + * what is owed. + */ + if (((gf_p & 0xffff0000u) == 0xbadf0000u) || + ((esr_p & 0xffff0000u) == 0xbadf0000u) || + ((ds_p & 0xffff0000u) == 0xbadf0000u)) + poisoned = true; + /* + * SESSION 153-CONTINUED-84: FE_DECODE_STATE_EX(0)/(1) also drive a + * transition log - the MME's macro/generated witness can flip + * there without touching ESR/DS/grfifo, which is exactly the + * "did the MME start a macro after this doorbell" question the + * reloaded windows keep failing. + */ + fed0_p = ioread32(bar0 + 0x0040a630u); + fed1_p = ioread32(bar0 + 0x0040a634u); + pr_info("vgsp-gr: MME-FAST begin %s h=0x%x esr=%08x ds=%08x grfifo=%08x trap=%08x fe0=%08x fe1=%08x (spin %u ms)\n", + tag, h_object, esr_p, ds_p, gf_p, tr_p, fed0_p, fed1_p, ms); + /* + * SESSION 153-CONTINUED-107: IRAM chronology probe (see the + * gsp_phys_mme_iram_probe comment). Bounded to the first 8 fast + * samples of the boot so the launch train cannot flood; knob-gated + * because the debug-port writes disturb a live launch (see the + * sema_gl_mme_iram_probe comment). + */ + if (fast_boot_n <= 8u && + READ_ONCE(gsp_phys_rmcall_gr_mme_iram_probe)) + gsp_phys_mme_iram_probe(bar0, "begin", h_object); + t_end = ktime_get() + (s64)ms * 1000000ll; + for (;;) { + u32 esr = ioread32(bar0 + 0x00404490u); + u32 ds = ioread32(bar0 + 0x00405840u); + u32 gf = ioread32(bar0 + 0x00400504u); + u32 tr = ioread32(bar0 + 0x00400704u); + u32 fed0 = ioread32(bar0 + 0x0040a630u); + u32 fed1 = ioread32(bar0 + 0x0040a634u); + + if (esr != esr_p || ds != ds_p || gf != gf_p || tr != tr_p || + fed0 != fed0_p || fed1 != fed1_p) { + if (n < 12u) { + u32 info = ioread32(bar0 + 0x00404494u); + /* + * SESSION 153-CONTINUED-106: the IR word + * registers are 0x40449c (INFO2), 0x4044a8 + * (INFO3), 0x4044ac (INFO4). The old read at + * 0x404498 was NV_PGRAPH_PRI_MME_CONFIG + * (RW), which is why every "info2_ir" line so + * far showed the shadow-filter enable, not the + * rejected instruction. + */ + u32 info2 = ioread32(bar0 + 0x0040449cu); + u32 info3 = ioread32(bar0 + 0x004044a8u); + u32 info4 = ioread32(bar0 + 0x004044acu); + u32 mmeir = ioread32(bar0 + 0x0040a630u); + u32 mmeir1 = ioread32(bar0 + 0x0040a634u); + u32 mmedata = ioread32(bar0 + 0x00400708u); + /* + * SESSION 153-CONTINUED-106: the MME's own + * macro PC/state at the transition instant + * (0x4044b4: PC 11:0, MACRO 19:12, RUNNING + * 20) - the "is the MME mid-macro at the + * trap" witness. + */ + u32 mmemac = ioread32(bar0 + 0x004044b4u); + + /* + * r1151 (SESSION 153-CONTINUED-72): + * GR-STATE: the FE decode-state EX slot 0 + * records the last MME-GENERATED method + * write (bit31 generated, 21:12 macro + * address, 11:0 pc). gen0=1 with the + * kept slot's macro address is the + * "kept macro executes" witness during + * the launch; slot 1 is the second + * lane of the same record set. + */ + pr_warn("vgsp-gr: MME-FAST %s h=0x%x esr %08x->%08x info=%08x(pc_valid=%u pc=%u mac=0x%x) ir=%08x,%08x,%08x ds %08x->%08x grfifo %08x->%08x trap=%08x data=%08x fed=%08x,%08x (pc0=%u mac0=0x%x gen0=%u) fe1=%08x (pc1=%u mac1=0x%x gen1=%u)\n", + tag, h_object, esr_p, esr, + info, (info >> 28) & 1u, + info & 0xfffu, + (info >> 12) & 0xffu, + info2, info3, info4, + ds_p, ds, gf_p, gf, tr, + mmedata, mmeir, mmeir1, + mmeir & 0xfffu, + (mmeir >> 12) & 0x3ffu, + (mmeir >> 31) & 1u, + fed1, + fed1 & 0xfffu, + (fed1 >> 12) & 0x3ffu, + (fed1 >> 31) & 1u); + pr_warn("vgsp-gr: MME-FAST %s h=0x%x macro-status=%08x(mpc=%u mac=0x%x run=%u)\n", + tag, h_object, mmemac, + mmemac & 0xfffu, + (mmemac >> 12) & 0xffu, + (mmemac >> 20) & 1u); + /* + * SESSION 153-CONTINUED-107: sample the live + * IRAM at every fault-register transition, so + * the chronology (words loaded early vs wiped + * by the poison/DROP machinery) is on record + * for the same instant the ESR changes. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_mme_iram_probe)) + gsp_phys_mme_iram_probe(bar0, "trans", + h_object); + } + /* + * SESSION 153-CONTINUED-106: when the DS latch CHANGES + * into a fault (the SPH validation miss at draw + * dispatch), read the per-stage SPH_INFO START/UPPER + * and PROGRAM_REGION pair IMMEDIATELY - the FECS + * clears the whole block within ~1 ms (r1364: + * 404004.338 poison seen, 404004.339 clean), so this + * is the only in-flight window that names the VA each + * failing stage fetched. + */ + if (ds != ds_p && (ds & 0x7fffffffu) && ds_log_n < 8u) { + ds_log_n++; + u32 s0 = ioread32(bar0 + 0x00405c00u); + u32 s1 = ioread32(bar0 + 0x00405c04u); + u32 s2 = ioread32(bar0 + 0x00405c08u); + u32 s3 = ioread32(bar0 + 0x00405c0cu); + u32 s4 = ioread32(bar0 + 0x00405c10u); + u32 s5 = ioread32(bar0 + 0x00405c14u); + u32 u0 = ioread32(bar0 + 0x00405c24u); + u32 u1 = ioread32(bar0 + 0x00405c28u); + u32 u2 = ioread32(bar0 + 0x00405c2cu); + u32 u3 = ioread32(bar0 + 0x00405c30u); + u32 u4 = ioread32(bar0 + 0x00405c34u); + u32 u5 = ioread32(bar0 + 0x00405c38u); + u32 rr_a = ioread32(bar0 + 0x00405c1cu); + u32 rr_b = ioread32(bar0 + 0x00405c20u); + + pr_warn("vgsp-gr: DS-FAST %s h=0x%x ds %08x->%08x stage=%08x,%08x,%08x,%08x,%08x,%08x upper=%02x,%02x,%02x,%02x,%02x,%02x region=%08x,%08x esr=%08x\n", + tag, h_object, ds_p, ds, + s0, s1, s2, s3, s4, s5, + u0 & 0xffu, u1 & 0xffu, u2 & 0xffu, + u3 & 0xffu, u4 & 0xffu, u5 & 0xffu, + rr_a, rr_b, + ioread32(bar0 + 0x00404490u)); + /* + * SESSION 153-CONTINUED-106: the pending DS + * report poisons the whole PRI block + * (every register reads 0xbadf1201 - the + * first-capture above shows exactly that). + * Clear the report masks and re-read: the + * CONT-91 rule says a DS miss with the masks + * cleared latches without the poison, so the + * per-stage SPH snapshot should become + * readable. + */ + { + u32 m0 = ioread32(bar0 + 0x00405844u); + u32 m2 = ioread32(bar0 + 0x0040584cu); + + iowrite32(0u, bar0 + 0x00405844u); + iowrite32(0u, bar0 + 0x0040584cu); + udelay(80); + pr_warn("vgsp-gr: DS-FAST2 %s h=0x%x masks=%08x/%08x ds_after=%08x stage=%08x,%08x,%08x,%08x,%08x,%08x upper=%02x,%02x,%02x,%02x,%02x,%02x region=%08x,%08x\n", + tag, h_object, m0, m2, + ioread32(bar0 + 0x00405840u), + ioread32(bar0 + 0x00405c00u), + ioread32(bar0 + 0x00405c04u), + ioread32(bar0 + 0x00405c08u), + ioread32(bar0 + 0x00405c0cu), + ioread32(bar0 + 0x00405c10u), + ioread32(bar0 + 0x00405c14u), + ioread32(bar0 + 0x00405c24u) & 0xffu, + ioread32(bar0 + 0x00405c28u) & 0xffu, + ioread32(bar0 + 0x00405c2cu) & 0xffu, + ioread32(bar0 + 0x00405c30u) & 0xffu, + ioread32(bar0 + 0x00405c34u) & 0xffu, + ioread32(bar0 + 0x00405c38u) & 0xffu, + ioread32(bar0 + 0x00405c1cu), + ioread32(bar0 + 0x00405c20u)); + } + } + n++; + esr_p = esr; + ds_p = ds; + gf_p = gf; + tr_p = tr; + fed0_p = fed0; + fed1_p = fed1; + /* + * The priv-error poison is the terminal state: the + * FECS has already logged and the rest of the + * registers are 0xbadfxxxx. Stop early so the launch + * thread resumes its normal poll. + */ + if ((gf & 0xffff0000u) == 0xbadf0000u) { + poisoned = true; + break; + } + } + if (ktime_after(ktime_get(), t_end)) + break; + } + pr_info("vgsp-gr: MME-FAST end %s h=0x%x events=%u%s\n", + tag, h_object, n, poisoned ? " (poison)" : ""); + /* + * r1497: clear the DS HWW ESR latch (0x405840, W1C) after the + * launch as well as before it. The pre-clear makes each window's + * validation set readable; a draw that latches a6040e24 otherwise + * leaves the pending report to poison the next window's entry state + * (r1496: ds a6040e24 read at the following begin while poison was + * otherwise low). The clear is W1C and host-only, like the mask + * write above, so the A/B knob covers both sites. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_ds_mask_clear) && bar0) { + static u32 ds_post_log; + u32 esr = ioread32(bar0 + 0x00405840u); + + if (esr) { + /* + * SESSION 155 CONT 14: RESET (bit30, RESET_TASK) + * must be set with the W1C clears so the HWW + * latch is actually reset (reference-RM idiom); + * without it the per-draw attribution is unsound. + */ + iowrite32(esr | 0x40000000u, + bar0 + 0x00405840u); + if (ds_post_log < 24u) { + ds_post_log++; + pr_info("vgsp-gr: DS-ESR post-clear=%08x h=0x%x tag=%s\n", + esr, h_object, tag); + } + } + } + /* + * SESSION 153-CONTINUED-107: post-spin IRAM sample - the state left + * when the poison (if any) has just cleared, before the poll/retire + * work touches anything. + */ + if (fast_boot_n <= 8u && + READ_ONCE(gsp_phys_rmcall_gr_mme_iram_probe)) + gsp_phys_mme_iram_probe(bar0, "end", h_object); + /* + * SESSION 153-CONTINUED-83 store-side emission forensics: right after + * the launch's retire spin, snapshot the DS shader-program-header + * latches (which pipeline stages failed SPH validation), the MME + * execution state (did the MME run a macro, and where is its PC), and + * the FE decode-state EX slots (did any method decode carry the MME + * GENERATED bit). Bounded per boot (beef-family only). + */ + if ((h_object & 0xfffff000u) == 0xbeef0000u) + gsp_phys_store_side_log(bar0, tag, h_object); + /* + * SESSION 153-CONTINUED-72: return the poison observation to the + * launch site. The r1149 race: the poison flooded during this spin + * and the FECS cleared it before the poll loop's WEDGE-RECOVER check + * read grfifo, so recovery missed, the prelude latch stayed set and + * the engine wedged for the boot. The caller now latches this flag + * into the recovery gate instead of relying on a live register read. + */ + return poisoned; +} + +static void gsp_phys_pbdma_fault_dump(u32 chid) +{ + static u32 dump_n; + void __iomem *bar0 = gsp_phys_gpu_bar0(); + u64 pb_va = boot_stage.rmcall_va_pb; + u32 i; + + if (!bar0 || !gsp_phys_gr_last_words || dump_n >= GSP_PHYS_PBDMA_DUMP_MAX) + return; + dump_n++; + pr_info("vgsp-gr: PB-FAULT-DUMP chid=%u last_block words=%u h=0x%x push=0x%llx pb_va=0x%llx\n", + chid, gsp_phys_gr_last_words, gsp_phys_gr_last_h, + gsp_phys_gr_last_push_va, pb_va); + for (i = 0; i < 12; i++) { + u32 b = 0x40000u + i * 0x2000u; + u32 intr0 = readl(bar0 + b + 0x108u); + u32 gpf = readl(bar0 + b + 0x050u); + u32 pbf = readl(bar0 + b + 0x054u); + u32 pbfhi = readl(bar0 + b + 0x058u); + u32 pbhd = readl(bar0 + b + 0x084u); + u32 pbcnt = readl(bar0 + b + 0x088u); + u32 pbd0 = readl(bar0 + b + 0x08cu); + u32 hdrsh = readl(bar0 + b + 0x118u); + u32 fetch64 = ((u64)(pbfhi & 0xffu) << 32) | (pbf & ~3u); + u32 trig; + const char *trig_src; + s64 off_words = -1; + s64 voff = -1; + + if (!intr0 && !pbf && !hdrsh && !pbhd) + continue; + /* + * SESSION 153-CONTINUED-52: the real driver's rule for the + * instruction that triggered the PBDMA interrupt: if the + * DECODED header (PB_HEADER) type is IMMD or PB_COUNT's value + * field is zero, the raw instruction is PB_DATA0; otherwise it + * is HDR_SHADOW (fifo_tu102.c: "If the TYPE field of + * NV_PPBDMA_PB_HEADER is IMMD or the VALUE field of + * NV_PPBDMA_PB_COUNT is zero, then the raw PB instruction + * stored in NV_PPBDMA_PB_DATA0 is the one that triggered the + * interrupt. Otherwise ... HDR_SHADOW"). + */ + if (((pbhd >> 29) & 7u) == 4u || (pbcnt & 0x1fffu) == 0u) { + trig = pbd0; + trig_src = "DATA0"; + } else { + trig = hdrsh; + trig_src = "HDRSHADOW"; + } + if (gsp_phys_gr_last_pb_dma && + fetch64 >= gsp_phys_gr_last_pb_dma && + fetch64 < gsp_phys_gr_last_pb_dma + + (u64)gsp_phys_gr_last_words * 4u) + off_words = (s64)(fetch64 - gsp_phys_gr_last_pb_dma) / 4; + /* + * SESSION 153-CONTINUED-59: the SAME offset computed against + * the GPU VA the engine fetches (the r1102 WINPROBE/PB-FAULT + * agreement: pbfetch 0x101ebc with pb_va 0x101000 and 943 + * words = exactly the block end). voff >= words is the + * past-the-block-end prefetch position. + */ + voff = (s64)-1; + if (pb_va && fetch64 >= pb_va && + fetch64 < pb_va + (u64)GSP_PHYS_PAGE_SIZE) + voff = (s64)(fetch64 - pb_va) / 4; + pr_info("vgsp-gr: PB-FAULT-DUMP P%u intr0=%08x gpf=%u pbfetch=0x%llx len=%u off=%lld voff=%lld/%u hdr=%08x(%s m=0x%x cnt=%u sub=%u) cnt=%08x data0=%08x shadow=%08x pbhd=%08x(type=%s) TRIG=%08x(%s)\n", + i, intr0, gpf, fetch64, pbfhi >> 11, off_words, voff, + gsp_phys_gr_last_words, + hdrsh, gsp_phys_pbdma_type_name(hdrsh >> 29), + (hdrsh & 0x1fffu) << 2, + (hdrsh >> 16) & 0xfffu, (hdrsh >> 13) & 7u, + pbcnt, pbd0, hdrsh, pbhd, + gsp_phys_pbdma_type_name(pbhd >> 29), + trig, trig_src); + } + { + struct gsp_phys_boot_chunk *pbch = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB]; + u32 *pw = pbch->pages ? gsp_phys_chunk_page_va(pbch, 0) : NULL; + + if (pw && pb_va) { + u32 n = gsp_phys_gr_last_words < 384u ? + gsp_phys_gr_last_words : 384u; + u32 w; + + for (w = 0; w < n; w += 16) { + u32 m; + + pr_info("vgsp-gr: PB-FAULT-DUMP pb[%3u]= %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n", + w, pw[w], w + 1 < n ? pw[w + 1] : 0, + w + 2 < n ? pw[w + 2] : 0, w + 3 < n ? pw[w + 3] : 0, + w + 4 < n ? pw[w + 4] : 0, w + 5 < n ? pw[w + 5] : 0, + w + 6 < n ? pw[w + 6] : 0, w + 7 < n ? pw[w + 7] : 0, + w + 8 < n ? pw[w + 8] : 0, w + 9 < n ? pw[w + 9] : 0, + w + 10 < n ? pw[w + 10] : 0, w + 11 < n ? pw[w + 11] : 0, + w + 12 < n ? pw[w + 12] : 0, w + 13 < n ? pw[w + 13] : 0, + w + 14 < n ? pw[w + 14] : 0, w + 15 < n ? pw[w + 15] : 0); + } + /* + * SESSION 153-CONTINUED-59: the block TAIL and the + * words PAST the block end (the r1102 fault fetch + * sits exactly at the end - voff == words). The past + * words are what a prefetching PBDMA decodes next; + * the tail shows whether the block itself ends at an + * entry boundary. + */ + { + u32 words = gsp_phys_gr_last_words; + u32 lo = words > 48u ? words - 48u : 0u; + u32 hi = words + 32u; + + if (hi > 1024u) + hi = 1024u; + for (w = lo; w < hi; w += 16) { + pr_info("vgsp-gr: PB-FAULT-DUMP %s[%3u]= %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n", + w < words ? "tail" : "past", w, + pw[w], w + 1 < 1024u ? pw[w + 1] : 0, + w + 2 < 1024u ? pw[w + 2] : 0, w + 3 < 1024u ? pw[w + 3] : 0, + w + 4 < 1024u ? pw[w + 4] : 0, w + 5 < 1024u ? pw[w + 5] : 0, + w + 6 < 1024u ? pw[w + 6] : 0, w + 7 < 1024u ? pw[w + 7] : 0, + w + 8 < 1024u ? pw[w + 8] : 0, w + 9 < 1024u ? pw[w + 9] : 0, + w + 10 < 1024u ? pw[w + 10] : 0, w + 11 < 1024u ? pw[w + 11] : 0, + w + 12 < 1024u ? pw[w + 12] : 0, w + 13 < 1024u ? pw[w + 13] : 0, + w + 14 < 1024u ? pw[w + 14] : 0, w + 15 < 1024u ? pw[w + 15] : 0); + } + } + } + } +} + +/* + * SESSION 153-CONTINUED-86: the fault-time register witness. The relayed + * errString names the fault; these reads add the machine state behind it at + * the earliest host observation point (the 0x1006 consume), before the + * FECS/GSP-RM clear the latches: MME_HWW_ESR (0x404490) + INFO (0x404494: + * PC_VALID/PC/MACRO_ADDRESS) + INFO2 (IR word), DS_HWW_ESR (0x405840) + + * ESR_2 (0x405848) + the SPH_INFO stage starts (0x405c00+si*4) that name the + * exact VA each failing pipeline stage fetched its shader program header + * from, the SPH program-region pair (0x405c1c/c20), and the trapped method + * address/data (0x400704/8). GR-class faults only (0xd graphics exception, + * 0x45 class error; PBDMA 0x20 has its own dump), bounded per boot so a fault + * storm cannot flood dmesg. + */ +static void gsp_phys_fault_regs_log(u32 except_type) +{ + static u32 frl_n; + void __iomem *bar0 = gsp_phys_gpu_bar0(); + u32 esr, info, info2, ds, ds2, trap_a, trap_d, rr_a, rr_b, sph[6], si; + + if (!bar0 || frl_n >= 48u) + return; + if (except_type != 0xdu && except_type != 0x45u) + return; + frl_n++; + esr = ioread32(bar0 + 0x00404490u); + info = ioread32(bar0 + 0x00404494u); + /* SESSION 153-CONTINUED-106: INFO2 is 0x40449c (0x404498 is CONFIG). */ + info2 = ioread32(bar0 + 0x0040449cu); + ds = ioread32(bar0 + 0x00405840u); + ds2 = ioread32(bar0 + 0x00405848u); + for (si = 0; si < 6u; si++) + sph[si] = ioread32(bar0 + 0x00405c00u + si * 4u); + rr_a = ioread32(bar0 + 0x00405c1cu); + rr_b = ioread32(bar0 + 0x00405c20u); + trap_a = ioread32(bar0 + 0x00400704u); + trap_d = ioread32(bar0 + 0x00400708u); + pr_info("vgsp-fault: regs except=0x%x mme_esr=%08x(pcv=%u ill=%u miss=%u extra=%u) mme_info=%08x(pc=%u mac=0x%x) mme_ir=%08x ds=%08x/%08x sph=%08x,%08x,%08x,%08x,%08x,%08x rr=%08x,%08x trap=%08x data=%08x\n", + except_type, esr, (info >> 28) & 1u, (esr >> 2) & 1u, + esr & 1u, (esr >> 1) & 1u, info, info & 0xfffu, + (info >> 12) & 0xffu, info2, ds, ds2, + sph[0], sph[1], sph[2], sph[3], sph[4], sph[5], + rr_a, rr_b, trap_a, trap_d); +} + +/* * Note every consumed 0x1006 in the bounded last-4 ring, advance the fault * epoch (the relay submit snapshot compares against it) and log the designed * observation line when the fault relay is armed. Decoding and counting are @@ -4926,10 +5868,14 @@ static void gsp_phys_fault_observe(const struct gsp_phys_rx_record *rec, GSP_PHYS_FAULT_NOTE_MAX; gsp_phys_fault_seen++; gsp_phys_fault_epoch++; - if (gsp_phys_fault_relay_enabled()) + if (gsp_phys_fault_relay_enabled()) { pr_info_ratelimited("vgsp-fault: func=0x1006 seq=%u exceptType=0x%x runlistId=%u chid=%u prevXid=0x%x errString=\"%s\"\n", rec->seq_num, out->except_type, out->runlist_id, out->chid, out->prev_xid, out->err); + gsp_phys_fault_regs_log(out->except_type); + } + if (out->except_type == 0x20u) + gsp_phys_pbdma_fault_dump(out->chid); } /* Note every consumed 0x1020 for the C3 corroboration ring. */ @@ -6827,6 +7773,35 @@ int gsp_phys_backend_static_wait(struct gsp_phys_backend_static_config *out, EXPORT_SYMBOL_GPL(gsp_phys_backend_static_wait); /* + * Mode-35 KGR static-data snapshots. The user-lane query set + * (gsp_phys_rmcall_user_lane_kgr_queries) is answered by the LIVE GSP with + * the device's true GR static data; the replies are kept here so the + * guest-side emulation can serve the identical bytes instead of fabricating + * GR values. The concrete consumer is the X driver's 3D-engine selection: + * nv3dAllocDevice maps GR_GET_INFO index SM_VERSION (0x0c) through a table + * and fails with "Failed to select a 3D engine" when it reads back as + * SM_VERSION_NONE (0) - the fabricated fill never set it, while the live + * GSP's KGR_GET_INFO reply carries the real value (0x75 for TU104) along + * with every other GR static field (warps, caps, topology, ctx buffers). + * + * One entry per captured cmd; the generation key ties the cache to the + * backend session exactly like gsp_phys_backend_static above, and the + * getter refuses entries once the session has moved on. + */ +#define GSP_PHYS_KGR_SNAP_ENTRIES 10 +#define GSP_PHYS_KGR_SNAP_MAX_SIZE 3712u /* KGR_GET_INFO: 8 engines x 58 x 8 */ + +struct gsp_phys_kgr_snap { + u32 cmd; + u32 size; + bool valid; + u32 generation; + u8 data[GSP_PHYS_KGR_SNAP_MAX_SIZE]; +}; + +static struct gsp_phys_kgr_snap gsp_phys_kgr_snaps[GSP_PHYS_KGR_SNAP_ENTRIES]; + +/* * Drop the cached static-config snapshot. Called when the backend session * leaves READY (or a new claim starts): the cache generation key is bumped * past the current backend generation so a concurrent getter cannot serve the @@ -6835,9 +7810,13 @@ EXPORT_SYMBOL_GPL(gsp_phys_backend_static_wait); */ void gsp_phys_backend_static_invalidate(void) { + unsigned int i; + mutex_lock(&gsp_phys_rm_lock); gsp_phys_backend_static.valid = false; gsp_phys_backend_static.generation = gsp_phys_backend_generation() + 1; + for (i = 0; i < ARRAY_SIZE(gsp_phys_kgr_snaps); i++) + gsp_phys_kgr_snaps[i].valid = false; mutex_unlock(&gsp_phys_rm_lock); } @@ -6891,6 +7870,71 @@ int gsp_phys_backend_static_get(struct gsp_phys_backend_static_config *out) } EXPORT_SYMBOL_GPL(gsp_phys_backend_static_get); +/* + * Copy the captured reply for cmd into dst. Returns 0 on success (with + * *out_len set), -ENODATA when nothing was captured for cmd (or the entry's + * generation no longer matches the backend session), -ENOSPC when the + * snapshot does not fit dst_len, -EAGAIN while the backend is not READY. + */ +int gsp_phys_kgr_snap_get(u32 cmd, void *dst, u32 dst_len, u32 *out_len) +{ + unsigned int i; + + if (!dst || !out_len) + return -EINVAL; + + if (gsp_phys_backend_state() != GSP_PHYS_BACKEND_READY) + return -EAGAIN; + + for (i = 0; i < ARRAY_SIZE(gsp_phys_kgr_snaps); i++) { + struct gsp_phys_kgr_snap *s = &gsp_phys_kgr_snaps[i]; + + if (s->cmd != cmd) + continue; + if (!s->valid || + s->generation != gsp_phys_backend_generation()) + return -ENODATA; + if (s->size > dst_len) + return -ENOSPC; + memcpy(dst, s->data, s->size); + *out_len = s->size; + return 0; + } + return -ENODATA; +} +EXPORT_SYMBOL_GPL(gsp_phys_kgr_snap_get); + +/* + * Record one mode-35 KGR query reply. Called under the boot thread only + * (same thread as the query itself), so no lock is needed beyond the + * generation key. + */ +static void gsp_phys_kgr_snap_store(u32 cmd, const void *data, u32 size) +{ + unsigned int i; + struct gsp_phys_kgr_snap *slot = NULL; + + if (size > GSP_PHYS_KGR_SNAP_MAX_SIZE) + return; + + for (i = 0; i < ARRAY_SIZE(gsp_phys_kgr_snaps); i++) { + if (gsp_phys_kgr_snaps[i].cmd == cmd) { + slot = &gsp_phys_kgr_snaps[i]; + break; + } + if (!slot && !gsp_phys_kgr_snaps[i].valid) + slot = &gsp_phys_kgr_snaps[i]; + } + if (!slot) + return; + + slot->cmd = cmd; + slot->size = size; + slot->generation = gsp_phys_backend_generation(); + memcpy(slot->data, data, size); + slot->valid = true; +} + /* ============================================================ * Host RM client ladder (R1): gsp-phys-rm-call * ============================================================ */ @@ -9691,6 +10735,87 @@ static u64 gsp_phys_rmcall_pde(u64 pa) } /* + * SESSION 153-CONTINUED-92: leaf PTE for the engine-visible guest/scratch + * mappings with the configurable v2 KIND field (bits 63:56). + * + * m96 documented the symptom that motivates this: on Turing a kind=0 PTE is + * what the SM I-fetch refuses with MMU_NACK while the FE tolerates it, and + * `gsp_phys_rmcall_gr_pte_kind` was introduced to plant the win0/identity + * pool PTEs with a kind (UVM turing uses GENERIC_MEMORY 0x06). The knob was + * never applied to the scratch-PT pool mappings - every shadow VA the 3D + * window's program operands are relocated to (the VAs the DS fetches the + * SPH through) carried kind=0. The DS validates a program header, i.e. runs + * a program fetch; if that path is the SM-side one, kind=0 is a candidate + * cause for the SPH rejection that is independent of content (and explains + * why no content A/B ever moved the wall). With the knob 0 this is + * byte-identical to the previous PTE encoding; nonzero = experiment. + */ +static u64 gsp_phys_rmcall_gr_pte_leaf(u64 hpa) +{ + u64 pte = gsp_phys_rmcall_pte(hpa); + int kind = gsp_phys_rmcall_gr_pte_kind_value(); + + if (kind) + pte |= (u64)(u32)kind << 56; + return pte; +} + +/* + * r1497: a VID-aperture leaf pointing at the page's real frame-buffer + * offset. The guest's own page tables encode exactly this + * (VALID=1, APERTURE=VID(0), KIND=6 GENERIC_MEMORY, PA>>12 in [56:8]), + * so the engine reads the real FBMEM page the client uploaded instead of + * an emulated BAR1 shadow that no upload ever populated. + */ +static u64 gsp_phys_rmcall_gr_pte_leaf_vid(u64 fb_off) +{ + return 1ULL | (((fb_off >> 12) & 0x1ffffffULL) << 8) | + ((u64)6u << 56); +} + +/* + * r1497: is the engine-visible shadow page at `hpa` entirely empty? + * Only then may the real-FB VID leaf replace it; pages the module seeded + * (fixed CB), the engine wrote, or the client wrote through the emulated + * aperture keep their shadow identity. + */ +static bool gsp_phys_gr_shadow_page_empty(u64 hpa) +{ + const u8 *p; + u32 i; + + if (!hpa || !pfn_valid((unsigned long)(hpa >> PAGE_SHIFT))) + return false; + p = (const u8 *)__va(hpa); + for (i = 0; i < GSP_PHYS_PAGE_SIZE; i++) + if (p[i]) + return false; + return true; +} + +/* + * r1497: resolve a fetch VA to its real frame-buffer offset when the page + * is FB-aperture AND the emulated shadow copy of it is all-zero. This is + * the active indexed draw's vertex/index class (r1496): the client's page + * tables map the page to vidmem, the upload never reached an emulated + * path, and the engine fetched zeros. Returns true with *fb_off set. + */ +static bool gsp_phys_gr_fetch_vid_off_locked(u64 va, u64 *fb_off) +{ + struct gsp_sema_gr_reloc r; + + if (gsp_sema_gr_reloc_resolve(va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1), + &r)) + return false; + if (r.ap != 2 /* SVC_APERTURE_FB */ || !r.hpa) + return false; + if (!gsp_phys_gr_shadow_page_empty(r.hpa)) + return false; + *fb_off = r.pa; + return true; +} + +/* * GP10X leaf PTE for a frame-buffer (VIDEO_MEMORY aperture 0) address: * VALID=1 (bit 0), APERTURE=0 (bits 2:1), VOL=1 (bit 3), frame-buffer byte * offset >> 12 in the VIDEO address field (bits 32:8; dev_mmu.h:592-595, @@ -9820,6 +10945,26 @@ static void gsp_phys_rmcall_pdb_build(void) (u8 *)gsp_phys_chunk_page_va(pt, 0) + (size_t)sem_idx * 8); /* + * SESSION 153-CONTINUED-96: secondary CE lane GPFIFO ring (slot 0x10d) + * and its one-page pushbuffer (slot 0x10e), planted here in the same + * 128 B PTE line as the ladder's own pages - stage 3, before any + * engine fetch (no invalidate exists). Always planted (the lane's + * page/PTE cost is two pages; a disabled lane simply never rings its + * doorbell), so pdb_build stays deterministic. + */ + put_unaligned_le64(gsp_phys_rmcall_pte( + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_chunks[ + GSP_PHYS_RMCALL_BUF_CE_GPFIFO], 0)), + (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)((GSP_PHYS_RMCALL_VA_CE_GPFIFO >> 12) & 0x1ffu) * 8); + put_unaligned_le64(gsp_phys_rmcall_pte( + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_chunks[ + GSP_PHYS_RMCALL_BUF_CE_PB], 0)), + (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)((GSP_PHYS_RMCALL_VA_CE_PB >> 12) & 0x1ffu) * 8); + /* * Rank-2 qmdwin: plant the dedicated module page for the forwarded * inline-QMD destination in this same small-PT line (slot 0x107, VA * 0x107000; slots 0x105/0x106 stay the P6a guest slots below). This @@ -9866,6 +11011,66 @@ static void gsp_phys_rmcall_pdb_build(void) } } /* + * SESSION 153-CONTINUED-66: plant page 2 of the module pushbuffer + * (VA_PB2, small-PT slot 0x102) and the dedicated warm-semaphore + * probe page (VA_GR_PROBE, slot 0x10a) in the same 128 B PTE line, + * at stage 3 before the engine's first fetch (no invalidate exists). + * Page 0 stays the registered BUF_PB chunk (stage 2 unchanged); this + * chunk is engine-reached purely through these PTEs, exactly like + * the qmdwin pair above. Without it the GR launch block keeps the + * single-page bound (<= 4 KB). + */ + if (!boot_stage.rmcall_gr_pb2.pages && boot_stage.pdev) { + if (!gsp_phys_chunk_alloc(boot_stage.pdev, + &boot_stage.rmcall_gr_pb2, 3u)) + gsp_phys_chunk_zero(&boot_stage.rmcall_gr_pb2); + else + pr_warn("vgsp-boot: rm-call: pushbuffer page2/3/4 allocation failed; >4 KB launches keep the single-page block\n"); + } + if (!boot_stage.rmcall_gr_probe.pages && boot_stage.pdev) { + if (!gsp_phys_chunk_alloc(boot_stage.pdev, + &boot_stage.rmcall_gr_probe, 1u)) + gsp_phys_chunk_zero(&boot_stage.rmcall_gr_probe); + else + pr_warn("vgsp-boot: rm-call: warm-sem probe page allocation failed; the probe keeps the pushbuffer upper half\n"); + } + if (boot_stage.rmcall_gr_pb2.pages && boot_stage.rmcall_gr_probe.pages) { + u32 pb2_idx = (u32)((GSP_PHYS_RMCALL_VA_PB2 >> 12) & 0x1ffu); + u32 pb3_idx = (u32)((GSP_PHYS_RMCALL_VA_PB3 >> 12) & 0x1ffu); + u32 pb4_idx = (u32)((GSP_PHYS_RMCALL_VA_PB4 >> 12) & 0x1ffu); + u32 pr_idx = (u32)((GSP_PHYS_RMCALL_VA_GR_PROBE >> 12) & + 0x1ffu); + + put_unaligned_le64(gsp_phys_rmcall_pte( + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_gr_pb2, 0)), + (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)pb2_idx * 8); + if (boot_stage.rmcall_gr_pb2.pages >= 2u) + put_unaligned_le64(gsp_phys_rmcall_pte( + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_gr_pb2, 1)), + (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)pb3_idx * 8); + if (boot_stage.rmcall_gr_pb2.pages >= 3u) + put_unaligned_le64(gsp_phys_rmcall_pte( + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_gr_pb2, 2)), + (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)pb4_idx * 8); + put_unaligned_le64(gsp_phys_rmcall_pte( + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_gr_probe, 0)), + (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)pr_idx * 8); + dma_wmb(); + boot_stage.rmcall_gr_pb2_planted = true; + pr_info("vgsp-boot: rm-call: GR launch pb page2/3/4 + warm-sem probe planted VA=0x%llx/0x%llx/0x%llx/0x%llx ptSlots=0x%x/0x%x/0x%x/0x%x (up to four-page block; before first fetch)\n", + GSP_PHYS_RMCALL_VA_PB2, GSP_PHYS_RMCALL_VA_PB3, + GSP_PHYS_RMCALL_VA_PB4, GSP_PHYS_RMCALL_VA_GR_PROBE, + pb2_idx, pb3_idx, pb4_idx, pr_idx); + } + /* * m69: plant the window-0 shadow pool PTEs (current win0 slot range) in the * same proven small-PT line, before the engine's first fetch. The * launch copies the QMD's program/param page content into these pages @@ -13605,6 +14810,12 @@ static int gsp_phys_rmcall_user_lane_userd(void __iomem *bar0) * (CONTEXT_BUFFERS_INFO in particular) may be prepared by the query. All are * zero-in best-effort getters at the capture's payload sizes; a failure * warns and the lane continues. + * + * Every successful reply is retained via gsp_phys_kgr_snap_store(): the + * guest-side emulation serves these exact bytes for the same commands, so + * the guest's GL/3D stack reads the live GSP's true GR static values + * (SM_VERSION in particular - see the snapshot comment near + * gsp_phys_backend_static_get). */ static void gsp_phys_rmcall_user_lane_kgr_queries(void __iomem *bar0) { @@ -13643,6 +14854,8 @@ static void gsp_phys_rmcall_user_lane_kgr_queries(void __iomem *bar0) q[i].name); pr_info("vgsp-boot: rm-call: mode 35 KGR query %s: size=%u status=0x%x rpc_result=0x%x ret=%d\n", q[i].name, q[i].size, status, rpc_result, ret); + if (ret == 0 && rpc_result == 0 && status == 0) + gsp_phys_kgr_snap_store(q[i].cmd, buf, q[i].size); kvfree(buf); } } @@ -13915,6 +15128,16 @@ static int gsp_phys_rmcall_user_lane_main_promote(void __iomem *bar0) status, rpc_result, GSP_PHYS_RMCALL_HANDLE_USER_LANE, boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_va); kvfree(p); + /* + * SESSION 153-CONTINUED-96: the F2 mapped-MAIN promote is the server's + * acceptance of the ladder's own context buffer (PA+VA, bInitialize=1), + * i.e. exactly the condition rmcall_ctx_valid records. Modes 0-34 set it + * at the end of their promote step (line ~13675); mode 35 promotes here, + * so without this the state file read valid=0 and the engine-ctx + * readback gate silently skipped. + */ + if (!ret) + boot_stage.rmcall_ctx_valid = true; return ret; } @@ -14635,6 +15858,35 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) boot_stage.rmcall_gr_object = GSP_PHYS_RMCALL_HANDLE_USER_LANE_GR_OBJ; } + /* + * Graphics-execution plan §F first 3D-lane experiment: + * construct the TURING_A (0xc597) class object on V7 + * after the accepted promote, through the same generic + * alloc helper the golden lanes use. The plan's open + * question is whether a 0xc597 construct on the + * execution lane is accepted without the wired-golden + * deadlock class; the result line below is the + * experiment's evidence either way, and the construct + * never gates the lane. + */ + { + u32 g3_rpc = 0, g3_status = 0; + int g3_ret = gsp_phys_rmcall_alloc_gr_object_class(bar0, + GSP_PHYS_RMCALL_CLASS_GR_3D, + GSP_PHYS_RMCALL_HANDLE_USER_LANE_GR3D, + GSP_PHYS_RMCALL_HANDLE_USER_LANE, + "mode 35 TURING_A on V7 (plan-F first 3D-lane experiment, after promote)", + &g3_rpc, &g3_status); + + pr_info("vgsp-boot: rm-call: mode 35 plan-F experiment: TURING_A (0xc597) on V7 hChannel=0x%x -> ret=%d hObject=0x%x status=0x%x rpc_result=0x%x (%s)\n", + GSP_PHYS_RMCALL_HANDLE_USER_LANE, + g3_ret, + g3_ret ? 0 : + GSP_PHYS_RMCALL_HANDLE_USER_LANE_GR3D, + g3_status, g3_rpc, + g3_ret ? "rejected/absent; lane continues" : + "accepted; V7 carries a 3D class object for the method-stream replay"); + } pr_info("vgsp-boot: rm-call: mode 35: execution channel switched from V6 0x%x to V7 0x%x (captured flags=0x%x internalFlags=0x%x hContextShare=0x%x hVASpace=0, gpFifoVa=0x%llx entries=%u, sysmem USERD, sysmem instance/ramfc (m62 readback), TURING_COMPUTE_A 0x%x); V6 stays allocated and unscheduled, stage 4 + forwarding run on V7\n", boot_stage.rmcall_channel, GSP_PHYS_RMCALL_HANDLE_USER_LANE, @@ -14952,23 +16204,31 @@ static u32 gsp_phys_rmcall_userd_ctl_off(void) * launch path publish byte-identically; the CE caller's semaphore clear and * method-block build happen before this call. The caller must hold * gsp_phys_rm_lock. - */ -static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, - u32 words, - struct gsp_phys_rmcall_ce_submit *out) + * + * SESSION 153-CONTINUED-96: the lane carries the channel's GPFIFO ring, + * hardware USERD chunk, out-fill chunks and work-submit token, so the + * secondary COPY0 lane publishes through the identical body on its own + * ring/USERD/doorbell token (a second channel must never advance the GR + * lane's GP_PUT or ring the GR channel's doorbell). + */ +struct gsp_phys_rmcall_lane { + struct gsp_phys_boot_chunk *ring; + struct gsp_phys_boot_chunk *userd; + struct gsp_phys_boot_chunk *pb; + struct gsp_phys_boot_chunk *sem; + struct gsp_phys_boot_chunk *src; + struct gsp_phys_boot_chunk *dst; + u32 token; + const char *tag; +}; + +static int gsp_phys_rmcall_publish_lane_locked(void __iomem *bar0, + u64 pb_gpu_va, u32 words, + const struct gsp_phys_rmcall_lane *lane, + struct gsp_phys_rmcall_ce_submit *out) { - struct gsp_phys_boot_chunk *ring = - &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO]; - struct gsp_phys_boot_chunk *userd_c = - &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD]; - struct gsp_phys_boot_chunk *pb = - &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB]; - struct gsp_phys_boot_chunk *sem = - &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM]; - struct gsp_phys_boot_chunk *src = - &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC]; - struct gsp_phys_boot_chunk *dst = - &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST]; + struct gsp_phys_boot_chunk *ring = lane->ring; + struct gsp_phys_boot_chunk *userd_c = lane->userd; u32 *ring_va, *userd; u32 gpput, gpget, slot, e0, e1; u32 runlist_id, ch_id, doorbell; @@ -14981,11 +16241,39 @@ static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, gpput = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); gpget = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPGET_OFF); if (gpput >= GSP_PHYS_RMCALL_GPFIFO_ENTRIES) { - pr_err("vgsp-boot: rm-call: USERD GP_PUT=%u is outside the %u-entry ring; refusing to submit\n", - gpput, GSP_PHYS_RMCALL_GPFIFO_ENTRIES); + pr_err("vgsp-boot: rm-call: %s USERD GP_PUT=%u is outside the %u-entry ring; refusing to submit\n", + lane->tag, gpput, GSP_PHYS_RMCALL_GPFIFO_ENTRIES); return -EIO; } - slot = gpput; + /* + * SESSION 153-CONTINUED-46: publish at the ENGINE's GP_GET, not at + * the raw GP_PUT. Observed on every stalled beef boot: the USERD + * drifts (GPGet=5 while GPPut cycles 1/2/4/7 - the engine's GET ran + * past the publisher's PUT, e.g. after a wrapped or otherwise + * deranged ring), and a ring is MODULO: PUT < GET numerically means + * (PUT-GET) mod ENTRIES pending entries - hundreds - so the PBDMA + * walks stale slots, executes garbage method blocks and the launch + * never retires (the trapped=0x8000022c / no-dispatch class). The + * forwarded channel is strictly single-inflight, so the engine's + * GET names exactly the next free slot: write our entry there and + * advance PUT to GET+1 so the PBDMA sees exactly one pending entry - + * ours. When GP_GET is out of range (uninitialised USERD), fall back + * to the old PUT-derived slot. + */ + slot = (gpget < GSP_PHYS_RMCALL_GPFIFO_ENTRIES) ? gpget : gpput; + { + static u32 slotfix_log; + + if (gpget != gpput && slotfix_log < 64) { + slotfix_log++; + pr_info("vgsp-boot: rm-call: %s publish at engine GP_GET=%u (raw GP_PUT=%u was %s; pending-would-be=%u)\n", + lane->tag, gpget, gpput, + gpput == gpget ? "equal" : + (gpput > gpget ? "ahead" : "behind"), + (gpput + GSP_PHYS_RMCALL_GPFIFO_ENTRIES - gpget) % + GSP_PHYS_RMCALL_GPFIFO_ENTRIES); + } + } /* * e0: bits 31:2 hold the GET field, which itself is (fetch address @@ -15002,8 +16290,8 @@ static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, */ e0 = (u32)pb_gpu_va & ~3u; if (unlikely(e0 != (u32)((pb_gpu_va >> 2) << 2))) - pr_err("vgsp-boot: rm-call: GPFIFO GET encode mismatch: e0=0x%08x decodes to fetch 0x%llx, expected pb VA 0x%llx\n", - e0, (u64)(e0 & ~3u), pb_gpu_va); + pr_err("vgsp-boot: rm-call: %s GPFIFO GET encode mismatch: e0=0x%08x decodes to fetch 0x%llx, expected pb VA 0x%llx\n", + lane->tag, e0, (u64)(e0 & ~3u), pb_gpu_va); e1 = (u32)((pb_gpu_va >> 32) & 0xffu) | (words << 10); put_unaligned_le32(e0, (u8 *)ring_va + slot * 8); put_unaligned_le32(e1, (u8 *)ring_va + slot * 8 + 4); @@ -15012,8 +16300,10 @@ static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, * GP_PUT advances on the client sysmem USERD chunk, which is the * channel's hardware USERD because the params carry that same chunk; * the host write is therefore what the engine polls (cycle 21). + * CONT-46: PUT = slot+1 (slot was chosen from GP_GET above), so the + * single new entry is the entire pending set. */ - put_unaligned_le32((gpput + 1) % GSP_PHYS_RMCALL_GPFIFO_ENTRIES, + put_unaligned_le32((slot + 1) % GSP_PHYS_RMCALL_GPFIFO_ENTRIES, (u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); /* * Doorbell value: workSubmitToken = ((runlistId << 16) & 0x7f00000) | @@ -15022,12 +16312,12 @@ static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, * write the formula's result (identical unless the token carries stray * high bits, which is logged). */ - runlist_id = (boot_stage.rmcall_token >> 16) & 0x7fu; - ch_id = boot_stage.rmcall_token & 0xfffu; + runlist_id = (lane->token >> 16) & 0x7fu; + ch_id = lane->token & 0xfffu; doorbell = ((runlist_id << 16) & 0x7f00000u) | (ch_id & 0xfffu); - if (doorbell != boot_stage.rmcall_token) - pr_warn("vgsp-boot: rm-call: token 0x%x carries bits outside runlistId[22:16]|chId[11:0]; writing the recomposed doorbell 0x%x\n", - boot_stage.rmcall_token, doorbell); + if (doorbell != lane->token) + pr_warn("vgsp-boot: rm-call: %s token 0x%x carries bits outside runlistId[22:16]|chId[11:0]; writing the recomposed doorbell 0x%x\n", + lane->tag, lane->token, doorbell); /* * C3: snapshot the fault epoch immediately before the doorbell, so a * 0x1006 consumed by this submission's timeout drain is accepted only @@ -15047,14 +16337,33 @@ static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, out->doorbell = doorbell; out->runlist_id = runlist_id; out->ch_id = ch_id; - out->pb_dma = (u64)gsp_phys_chunk_page_dma(pb, 0); - out->sem_dma = (u64)gsp_phys_chunk_page_dma(sem, 0); - out->src_dma = (u64)gsp_phys_chunk_page_dma(src, 0); - out->dst_dma = (u64)gsp_phys_chunk_page_dma(dst, 0); + out->pb_dma = (u64)gsp_phys_chunk_page_dma(lane->pb, 0); + out->sem_dma = (u64)gsp_phys_chunk_page_dma(lane->sem, 0); + out->src_dma = (u64)gsp_phys_chunk_page_dma(lane->src, 0); + out->dst_dma = (u64)gsp_phys_chunk_page_dma(lane->dst, 0); } return 0; } +static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, + u32 words, + struct gsp_phys_rmcall_ce_submit *out) +{ + const struct gsp_phys_rmcall_lane lane = { + .ring = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO], + .userd = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD], + .pb = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB], + .sem = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM], + .src = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC], + .dst = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST], + .token = boot_stage.rmcall_token, + .tag = "ladder", + }; + + return gsp_phys_rmcall_publish_lane_locked(bar0, pb_gpu_va, words, + &lane, out); +} + /* * One CE copy block on the ladder channel: clear the module semaphore, build * the methods for bytes into the module pushbuffer, publish the GPFIFO entry @@ -15076,10 +16385,30 @@ static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, * site passes false, so the default COPY0 ladder emits the same bytes as * before. */ -static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, - u64 dst_va, u64 sem_va, u32 bytes, - u32 sem_payload, bool gr, - struct gsp_phys_rmcall_ce_submit *out) +/* + * SESSION 153-CONTINUED-12: pb_off selects where in the module pushbuffer + * page the method block is written and fetched from. The warm-semaphore + * probe (gsp_phys_rmcall_gr_sem_push_locked) used the upper half so it + * could never overwrite a forwarded launch's pending method block: + * measured with the PBDMA dump, the probe's 10-dword block landing on + * words 0..9 of a pending 139-word 3D replay made the PBDMA walk the + * probe's SET_OBJECT/REPORT_SEM tail and then decode the replay window's + * operand 0x00230200 as an instruction -> PBENTRY on chid 4 and no + * retire. Every forwarded GR/CE block stays at offset 0. + * + * SESSION 153-CONTINUED-66: pb_alt/pb_alt_va point the block at a + * dedicated one-page module buffer (the probe's VA_GR_PROBE page) instead + * of the registered pushbuffer; a forwarded GR block may now span both + * pushbuffer pages, so no fixed offset of that page is safe for the probe. + */ +static int gsp_phys_rmcall_ce_submit_off_locked(void __iomem *bar0, + u64 src_va, u64 dst_va, + u64 sem_va, u32 bytes, + u32 sem_payload, bool gr, + u32 pb_off, + struct gsp_phys_boot_chunk *pb_alt, + u64 pb_alt_va, + struct gsp_phys_rmcall_ce_submit *out) { struct gsp_phys_boot_chunk *pb = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB]; @@ -15108,11 +16437,28 @@ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, if (!bar0 || !boot_stage.rmcall_token || !bytes || bytes > GSP_PHYS_RMCALL_COPY_SIZE) return -EINVAL; + if (pb_off & 3u) + return -EINVAL; pb_gpu_va = boot_stage.rmcall_va_pb; if (!pb_gpu_va || (pb_gpu_va & 3)) return -EIO; + /* + * SESSION 153-CONTINUED-66: pb_alt selects a dedicated one-page module + * buffer instead of the registered pushbuffer page (the warm-semaphore + * probe uses its own PTE-planted page now that a forwarded GR block may + * span BOTH pushbuffer pages). pb_off stays relative to that page. + */ + if (pb_alt && pb_alt->pages) { + pb = pb_alt; + pb_gpu_va = pb_alt_va; + if (!pb_gpu_va || (pb_gpu_va & 3)) + return -EIO; + } - pb_va = gsp_phys_chunk_page_va(pb, 0); + if (!gsp_phys_chunk_page_va(pb, 0) || pb_off > GSP_PHYS_PAGE_SIZE) + return -EIO; + pb_va = (u32 *)((u8 *)gsp_phys_chunk_page_va(pb, 0) + pb_off); + pb_gpu_va += pb_off; ring_va = gsp_phys_chunk_page_va(ring, 0); userd = gsp_phys_chunk_page_va(userd_c, 0); if (!pb_va || !ring_va || !userd || @@ -15133,9 +16479,82 @@ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, dst_va, bytes, sem_payload); + if (pb_off + words * 4 > GSP_PHYS_PAGE_SIZE) + return -EIO; + return gsp_phys_rmcall_publish_locked(bar0, pb_gpu_va, words, out); } +static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, + u64 dst_va, u64 sem_va, u32 bytes, + u32 sem_payload, bool gr, + struct gsp_phys_rmcall_ce_submit *out) +{ + return gsp_phys_rmcall_ce_submit_off_locked(bar0, src_va, dst_va, + sem_va, bytes, sem_payload, + gr, 0u, NULL, 0, out); +} + +/* + * SESSION 153-CONTINUED-96: one CE copy block on the secondary COPY0 lane. + * Byte-for-byte the CE submission body (same method block, same guards, + * same module-semaphore clear) but on the lane's own GPFIFO ring, USERD + * chunk, one-page pushbuffer and work-submit token. Only legal when the + * lane's bringup obtained a token (rmcall_ce_lane_valid). The caller must + * hold gsp_phys_rm_lock and polls the module semaphore separately, exactly + * like the readback's bounded page loop. + */ +static int gsp_phys_rmcall_ce_lane_submit_locked( + void __iomem *bar0, u64 src_va, u64 dst_va, u64 sem_va, + u32 bytes, u32 sem_payload, + struct gsp_phys_rmcall_ce_submit *out) +{ + const struct gsp_phys_rmcall_lane lane = { + .ring = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_CE_GPFIFO], + .userd = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_CE_USERD], + .pb = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_CE_PB], + .sem = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM], + .src = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC], + .dst = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST], + .token = boot_stage.rmcall_ce_token, + .tag = "CE lane", + }; + struct gsp_phys_boot_chunk *sem = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM]; + resource_size_t bar0_len; + u32 *pb_va; + u32 words; + + if (!boot_stage.rmcall_ce_lane_valid) + return -EOPNOTSUPP; + if (!boot_stage.pdev) + return -EINVAL; + bar0_len = pci_resource_len(boot_stage.pdev, 0); + if (bar0_len < GSP_PHYS_RMCALL_DOORBELL_OFF + 4) + return -EIO; + if (!bar0 || !boot_stage.rmcall_ce_token || !bytes || + bytes > GSP_PHYS_RMCALL_COPY_SIZE) + return -EINVAL; + pb_va = gsp_phys_chunk_page_va(lane.pb, 0); + if (!pb_va || !gsp_phys_chunk_page_va(lane.ring, 0) || + !gsp_phys_chunk_page_va(lane.userd, 0) || + !gsp_phys_chunk_page_va(sem, 0)) + return -EIO; + + /* A stale release from a previous submission must not be observed. */ + WRITE_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0), 0); + wmb(); + + words = gsp_phys_rmcall_build_methods(pb_va, sem_va, src_va, dst_va, + bytes, sem_payload); + if (words * 4 > GSP_PHYS_PAGE_SIZE) + return -EIO; + + return gsp_phys_rmcall_publish_lane_locked(bar0, + GSP_PHYS_RMCALL_VA_CE_PB, + words, &lane, out); +} + /* * H1 stage-4 GR verification: one GR semaphore release on the ladder channel, * shared by stage 4's GR branch and the debugfs trigger 9 re-run. The caller @@ -15193,11 +16612,33 @@ static int gsp_phys_rmcall_gr_sem_push_locked(void __iomem *bar0) unsigned long deadline; int ret; - ret = gsp_phys_rmcall_ce_submit_locked(bar0, GSP_PHYS_RMCALL_VA_SRC, + /* + * SESSION 153-CONTINUED-12: this probe runs while a forwarded launch + * may still be pending in the same pushbuffer page. It must not land + * on words 0.. of that page: measured PB-FAULT-DUMP shows the probe's + * 10-dword block overwriting a pending 139-word 3D replay's head, so + * the PBDMA walks the probe then decodes a replay operand as an + * instruction (PBENTRY, no retire). + * + * SESSION 153-CONTINUED-66: the forwarded GR block now spans BOTH + * pushbuffer pages (up to 2048 dwords = 0x2000 bytes), so the old + * upper-half placement (0x800) would clobber it. The probe has its + * own PTE-planted page (VA_GR_PROBE, small-PT slot 0x10a); when that + * plant is unavailable the probe falls back to 0x800 as before. + */ + ret = gsp_phys_rmcall_ce_submit_off_locked(bar0, + GSP_PHYS_RMCALL_VA_SRC, GSP_PHYS_RMCALL_VA_DST, GSP_PHYS_RMCALL_VA_SEM, GSP_PHYS_RMCALL_COPY_SIZE, GSP_PHYS_RMCALL_SEM_PAYLOAD, true, + boot_stage.rmcall_gr_probe.pages ? + 0u : 0x800u, + boot_stage.rmcall_gr_probe.pages ? + &boot_stage.rmcall_gr_probe : + NULL, + boot_stage.rmcall_gr_probe.pages ? + GSP_PHYS_RMCALL_VA_GR_PROBE : 0, &sub); if (ret) { pr_err("vgsp-boot: rm-call: GR submission refused before the doorbell (%d)\n", @@ -15242,7 +16683,7 @@ static int gsp_phys_rmcall_gr_sem_push_locked(void __iomem *bar0) gsp_phys_status_drain(NULL); return -ETIMEDOUT; } - msleep(GSP_PHYS_RMCALL_POLL_MS); + gsp_phys_rmcall_wait_quantum(); } gpget = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPGET_OFF); @@ -15342,7 +16783,7 @@ static void gsp_phys_rmcall_ctx_image_probe(void __iomem *bar0) offs[k], poll); break; } - msleep(GSP_PHYS_RMCALL_POLL_MS); + gsp_phys_rmcall_wait_quantum(); } if (poll != GSP_PHYS_RMCALL_SEM_PAYLOAD) break; @@ -15443,7 +16884,7 @@ static int gsp_phys_rmcall_ce_copy_va(void __iomem *bar0, u64 src_va, *sem_out = poll; return -ETIMEDOUT; } - msleep(GSP_PHYS_RMCALL_POLL_MS); + gsp_phys_rmcall_wait_quantum(); } } @@ -15626,7 +17067,7 @@ int gsp_phys_backend_ce_copy_submit(u64 src_va, u64 dst_va, u64 sem_va, gsp_phys_work_timed_out); return -ETIMEDOUT; } - msleep(GSP_PHYS_RMCALL_POLL_MS); + gsp_phys_rmcall_wait_quantum(); } /* @@ -16145,7 +17586,7 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa, ret = -ETIMEDOUT; goto out; } - msleep(GSP_PHYS_RMCALL_POLL_MS); + gsp_phys_rmcall_wait_quantum(); } /* @@ -16381,15 +17822,33 @@ EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_ce_copy); #define GSP_PHYS_RMCALL_PRI_SKED_DEBUG_1 0x00407040u #define GSP_PHYS_RMCALL_PRI_SKED_DEBUG_1_EN_CWD_SLOT_COUNT 0x800u /* Method block bound: init(152) + inline run(67) + PCAS(4) + REPORT_SEM(8) + - * the optional QMD-virt pair(2). */ -#define GSP_PHYS_RMCALL_GR_METHOD_WORDS 512u + * the optional QMD-virt pair(2). Raised 512 -> 1024 (SESSION 153- + * CONTINUED-53) so a 3D stream replay can carry the captured MME setup + * prelude (<=512 words) ahead of the largest chunked window (<=448) in one + * block. SESSION 153-CONTINUED-66: 1024 -> 2048 because the matured prelude + * reached 901 words and the deferral test (window=390 + prelude=901 + sa=47 + * vs the bound) dropped it on every launch; the block now spans the module + * pushbuffer's TWO 4 KB pages (VA_PB..VA_PB2) and is copied page-wise by + * gsp_phys_gr_pb_copy(). */ +#define GSP_PHYS_RMCALL_GR_METHOD_WORDS 4096u static u64 gsp_phys_gr_submitted; static u64 gsp_phys_gr_ok; static u64 gsp_phys_gr_refused; static u64 gsp_phys_gr_timed_out; +/* + * SESSION 153-CONTINUED-60: 3D-stream launches whose window was all-zero + * (the client's push page caught mid-rewrite). They are never handed to the + * engine: the zero method stream trips the PBDMA (trap 0x1168 -> priv-error + * poison, see the ZEROWIN-SKIP site). Counted so a boot's zero-window rate + * is visible without dmesg archaeology. + */ +static u64 gsp_phys_gr_zerowin_skips; static u64 gsp_phys_gr_retries; /* fresh-entry launch retries published */ static u64 gsp_phys_gr_pool_exhausted; +/* r1497: launch fetch pages mapped through a real-FB VID leaf (empty shadow). */ +static u64 gsp_phys_gr_vid_map_pages; +static u32 gsp_phys_gr_vid_log; /* m156 (E5): zero-write retries where the warm sem push proved the engine * wedged (the persistent-lottery class); further retries are skipped. */ static u64 gsp_phys_gr_sem_wedged; @@ -16472,7 +17931,7 @@ static bool gsp_phys_rmcall_gr_init_emitted; * stale-QMD execution) landed after its launch was shipped. */ static u32 gsp_phys_gr_prev_out_bi; -static u32 gsp_phys_gr_prev_out_csum; +static u64 gsp_phys_gr_prev_out_csum; static bool gsp_phys_gr_prev_out_valid; /* r425: SKED/FECS/CWD register witness (one line; the healthy-vs-wedged @@ -16627,16 +18086,2799 @@ static u32 *gsp_phys_rmcall_build_gr_init(u32 *p) } /* + * r597: extract the SEM_EXECUTE ACQUIRE groups from the captured guest + * prelude and copy them, verbatim, ahead of the rebuilt inline-QMD window. + * Two encodings are recognised: + * - C46F host-semaphore group: INC_METHOD addr 0x5c count>=5, words + * {LO, HI, PAYLOAD_LO, PAYLOAD_HI, EXECUTE}; EXECUTE & 7 != 1 is an + * acquire (the pump's own op decode). + * - C5C0 SET_REPORT_SEMAPHORE A..D: addr 0x1b00 count>=4, word D's low + * two bits select release (0) / acquire (non-zero). + * Only the wait methods are taken, so the proven window (init + inline QMD + * + PCAS + release) is untouched. Returns the number of words emitted. + */ +static u32 gsp_phys_rmcall_gr_emit_acquires(u32 *p, + const struct gsp_phys_gr_submit *req, + u32 budget) +{ + u32 off = 0, words = 0; + + if (!gsp_phys_rmcall_gr_acq_value() || !req || !req->raw) + return 0; + while (off < req->raw_qmd_hdr && words + 8 < budget) { + u32 hdr = req->raw[off]; + u32 op = (hdr >> 29) & 0x7; + u32 count = (hdr >> 16) & 0xfff; + u32 addr = (hdr & 0x1fff) << 2; + u32 n = 1 + count; + + if ((op != 1 && op != 3) || !count || + off + n > req->raw_qmd_hdr) { + off++; + continue; + } + if (addr == 0x5c && count >= 5) { + u32 v4 = req->raw[off + 5]; + u64 va = ((u64)(req->raw[off + 2] & 0xff) << 32) | + (u64)(req->raw[off + 1] & 0xfffffffc); + + if ((v4 & 0x7) != 1 && va) { + memcpy(p + words, &req->raw[off], n * 4); + words += n; + } + } else if (addr >= 0x1b00 && addr <= 0x1b0c && + count >= 4) { + u32 d = req->raw[off + 4] & 0x3; + + if (d != 0) { + memcpy(p + words, &req->raw[off], n * 4); + words += n; + } + } + off += n; + } + return words; +} + +/* + * r929 (GL 3D execution, Path A): replay a captured raw 3D method window. + * The sema service marks stream captures (the X server's C597 sections that + * carried their own operands) with a raw window (req->stream); when this is + * on, gsp_phys_rmcall_build_gr_launch emits SET_OBJECT TURING_A + the window + * verbatim + the channel release tail instead of the compute prelude/QMD, so + * the engine executes the REAL 3D methods (state binds, LINE_LENGTH_IN + + * LAUNCH_DMA uploads, program/tex binds) and, once draws are in the window, + * rasterizes. No effect on any submit without a stream window (all CUDA + * launches). Default on; A/B with gsp_phys_rmcall_gr_3d=0. + */ +static bool gsp_phys_rmcall_gr_3d = true; +module_param_named(gsp_phys_rmcall_gr_3d, gsp_phys_rmcall_gr_3d, bool, 0644); +MODULE_PARM_DESC(gsp_phys_rmcall_gr_3d, + "Replay a captured raw C597 method window verbatim instead of the compute prelude (default 1)"); + +/* + * SESSION 153-CONTINUED-59: drop single-word SDM-type entries (SEC_OP 0 + * SSDM / 2 STORE_SDM / 6 USE_SDM, tu102 dev_esched_pbdma.h type map) from + * every replayed stream: the 3D window patch, the MME prelude and the attr + * prelude emission. The r1102 PBENTRY first-sighting forensics localized + * the fault decode point to the MME prelude head (WINPROBE/PB-FAULT-DUMP + * P0 HDR_SHADOW=04666d14 = prelude[2], PB_DATA0=04646d00 = prelude[3], + * pbfetch=0x101ebc = block end after the whole 943-word block was + * prefetched, intr0 bit 18 in the guest relay): the client's setup push + * carries a 21-word run of `04666d00 04666d10 04666d14 04646d00 ...` + * (SDM-family single-word encodings, method fields 0x3400/0x3440/0x3450) + * that svc_mme_prelude_capture stored verbatim; the physical PBDMA refuses + * them ("invalid method encoding used in method", NV_PPBDMA_INTR_0 + * PBENTRY), which also explains why the whole block was fetched but the + * REPORT_SEMAPHORE never retired. The guest-vGPU parts (subdevice masks) + * are meaningless on the single-subdevice ladder channel - the same call + * the sub!=0 multi-word drop makes. Zero words are kept (benign padding). + * Default 1; A/B with 0. + */ +static bool gsp_phys_rmcall_gr_sdm_drop = true; +module_param_named(sema_gl_sdm_drop, gsp_phys_rmcall_gr_sdm_drop, bool, + 0644); +MODULE_PARM_DESC(sema_gl_sdm_drop, + "Drop single-word SDM-type entries (SEC_OP 0/2/6) from 3D window replays, the MME prelude and the attr prelude (default 1)"); + +/* + * Whole-vidmem page-hash bracket for the forwarded GR launch path. When set, + * hash every 4 KB page of the first open device's BAR1 shadow immediately + * before the publish, then re-hash once the output retirement wait returns + * and log the pages whose content changed. This proves whether the engine + * wrote any vidmem page, not just the output pages the copy-out samples. + * Default 0: no allocation, no hashing and no change to the submit path. + */ +static bool gsp_phys_gr_vidmem_bracket; +module_param_named(sema_gl_vidmem_bracket, gsp_phys_gr_vidmem_bracket, bool, + 0644); +MODULE_PARM_DESC(sema_gl_vidmem_bracket, + "Hash every vidmem page before and after a forwarded GR launch to report writes outside the sampled output pages (default 0)"); + +/* + * SESSION 153-CONTINUED-53: MME context-setup prelude. The r1073-r1077 + * in-flight capture proved the draw-window replay raises a real MME + * ILLEGAL_OPCODE on CALL_MME_MACRO(2)=0x3810 (mme_pc=0, mme_ir=1, mme_mac + * =2) because the ladder context never received the client's macro + * program. The client does upload it (0x11c/0x114 pointer pairs, 0x118 + * code, 0x120 entry addresses, 0x124 shadow control; see + * svc_mme_prelude_capture); with this knob the launch builder preplays + * those captured runs verbatim - once per capture generation - right + * after the SET_OBJECT C597 bind and ahead of the window, so the macros + * the window calls are resident. Declared before build_gr_launch (its + * reader); the matching capture-side knob is sema_mme_prelude_cap. + * Default 1; A/B with 0. + */ +static bool gsp_phys_rmcall_gr_mme_prelude = true; +module_param_named(sema_gl_mme_prelude, gsp_phys_rmcall_gr_mme_prelude, bool, + 0644); +MODULE_PARM_DESC(sema_gl_mme_prelude, + "Preplay the captured client MME context-setup runs once per capture generation ahead of the first 3D window replay (default 1)"); + +/* + * SESSION 153-CONTINUED-63: MME START-ADDRESS RAM completion + stub knob. + * The captured SA table (svc_mme_sa_*) may be incomplete (the prelude + * buffer is capped at 512 words); every CALLED slot the table does not + * cover is loaded with a harmless end-noop stub at a fixed IR address so + * the call becomes a no-op instead of a jump to IR 0 with garbage state + * (the r1116 0x238c poison). Default 1; A/B with 0. + */ +#define GSP_PHYS_RMCALL_GR_MME_STUB_PC 0xC0u +/* + * SESSION 153-CONTINUED-69: the stub body, take 3. The CONT-68 body + * (20208c00 38c40046 038c4301) was copied from macro(0) instruction 3 - + * but that is a MID-MACRO instruction: the offline MME64 simulator shows + * it carries two output writes with mthd=NONE, so in isolation (method + * register still 0) it EMITS SET_OBJECT <- 0 twice - exactly the relayed + * trap "ChID 0001, Class 0000c597, Offset 00000000, Data 00000000" - and + * its end_next=1 requires the following instruction, so the macro also + * runs past the stub into whatever the IR holds there. + * + * The real minimal terminator is what Mesa's mme_tu104_builder_finish() + * emits: two default NOPs with end_next set on the FIRST (end_next - + * "end after the next instruction" - so the second NOP executes and then + * the macro returns). Both NOPs encode as w2=0x318c0300 (pred=ZERO, + * ALU0/1 dst=ZERO src=ZERO,ZERO, outs NONE) and emit NOTHING. The + * end_next bit is w2 bit 0. + */ +static const u32 gsp_phys_gr_mme_stub_code[6] = { + 0x00000000u, 0x318c0000u, 0x318c0301u, /* NOP, end_next=1 */ + 0x00000000u, 0x318c0000u, 0x318c0300u, /* NOP tail (executed) */ +}; +static bool gsp_phys_rmcall_gr_sa_stub = true; +module_param_named(sema_gl_mme_sa_stub, gsp_phys_rmcall_gr_sa_stub, bool, + 0644); +MODULE_PARM_DESC(sema_gl_mme_sa_stub, + "Load a harmless end-noop stub into every MME macro slot the window calls but the captured START-ADDRESS RAM table does not cover (default 1; the r1116 0x238c lane poison class)"); + +/* + * SESSION 153-CONTINUED-68: stub EVERY called slot, covered or not + * (attribution: with all calls no-op'd the window either still faults + * (the fault is in non-call content) or faults differently (the fault is + * macro execution). Default 0; A/B knob. + */ +static bool gsp_phys_rmcall_gr_sa_stub_all; +module_param_named(sema_gl_mme_sa_stub_all, gsp_phys_rmcall_gr_sa_stub_all, + bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_sa_stub_all, + "Stub every macro slot the window calls, including slots the captured SA table covers (attribution knob; default 0)"); + +static inline bool gsp_sema_gr_sa_stub_value(void) +{ + return READ_ONCE(gsp_phys_rmcall_gr_sa_stub); +} + +/* + * SESSION 153-CONTINUED-69: optional explicit stub IR address override. + * 0 = derive from the capture (ir_end, floor 0xC0). Nonzero pins the stub + * body/call target to this instruction address - the stub-landing A/B for + * the IR-boundary/ONEINC pointer-unit question. + */ +static u32 gsp_phys_rmcall_gr_mme_stub_pc; +module_param_named(sema_gl_mme_stub_pc, gsp_phys_rmcall_gr_mme_stub_pc, uint, + 0644); +MODULE_PARM_DESC(sema_gl_mme_stub_pc, + "Explicit MME stub IR instruction address (0 = derive from capture ir_end; try 0xf0/0x100 for the IR-boundary A/B)"); + +/* + * SESSION 153-CONTINUED-69: no-stall CALL drop. With the stub terminating + * macros as pure NOPs the macro consumes none of the CALL_MME_DATA + * parameter words the window pushes; the MME parameter port then + * backpressures on the window's larger calls - the r1142 stall parked the + * PBDMA mid-block with HDR_SHADOW = the 16-arg CALL_MME_MACRO(57) + * ONEINC run and the block never completed (while every <=4-arg window + * retired in ~100 us). Drop every CALL_MME_MACRO/CALL_MME_DATA run of a + * STUBBED slot from the emitted window instead: no call, no parameter + * words, no backpressure. Covered slots keep their calls (their real code + * consumes the parameters). Default 1; A/B with 0. + */ +static bool gsp_phys_rmcall_gr_call_drop = true; +module_param_named(sema_gl_mme_call_drop, gsp_phys_rmcall_gr_call_drop, bool, + 0644); +MODULE_PARM_DESC(sema_gl_mme_call_drop, + "Drop CALL_MME_MACRO/DATA runs of stubbed slots from emitted 3D windows (prevents the noop-stub parameter-port stall; default 1)"); + +/* + * SESSION 153-CONTINUED-70: per-slot macro re-enable mask, one slot at a + * time. The r1142-r1144 state drops EVERY stubbed-slot CALL run: the window + * retires on real HW (~94 us) but the macros that expand the driver's + * per-draw state (vertex stream binds, constant binds, ...) never execute, + * so no state the draw needs exists and every launch still stores 0 pages. + * A slot named in this mask is never treated as stubbed: its + * CALL_MME_MACRO/DATA words stay in the emitted window and the engine + * executes the macro against the replayed context (SA RAM + IR program as + * the client's own uploads left them; a slot the client never uploaded + * reads SA=0 exactly like silicon and lands in macro 0). A/B one bit per + * cycle; keep the proven-good set empty until the offline MME64 sim has + * shown the slot's macro emits sane methods for its caller args. + */ +static u64 gsp_phys_rmcall_gr_keep_mask; +module_param_named(sema_gl_mme_keep_mask, gsp_phys_rmcall_gr_keep_mask, + ullong, 0644); +MODULE_PARM_DESC(sema_gl_mme_keep_mask, + "Slots whose CALL_MME_MACRO/DATA runs are never dropped/stubbed (u64 bitmask; one slot at a time for macro re-enable)"); + +/* + * SESSION 153-CONTINUED-113: clamp the keep mask to capture-covered slots. + * The r1405 poison anatomy (traced live): the draw window calls + * CALL_MME_MACRO(34) and CALL_MME_MACRO(57); the capture holds 13 SA slots + * (0..12, IR <= 0xe9) while keep_mask=~0 exempted every slot from stubbing, + * so the calls ran, the MME fetched a byte the packed program does not hold + * (`info=badf1201 pc_valid=1 pc=513 mac=0x1f1`, `miss=1`, ir=badf1201) and + * the priv-error wedge wiped the context; the second recovery's hard re-arm + * left the engine unresponsive (probe rc=-110) and GP_GET frozen for the + * boot. A kept slot the capture cannot serve can only ever fetch + * non-resident code: the CONT-80 r1186 note already pairs "drop the runs + + * stub SA[slot]" as the safe combination for a nop_drop slot - apply the + * same combination to any kept slot outside the capture's coverage. The + * decision reads the live SA table per window, so a later capture + * generation that covers the slot releases it automatically. Default 1 + * (keep_clamp); 0 = the r1405 all-real behavior (A/B). + */ +static bool gsp_phys_rmcall_gr_keep_clamp = true; +module_param_named(sema_gl_mme_keep_clamp, gsp_phys_rmcall_gr_keep_clamp, + bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_keep_clamp, + "Clamp kept slots to capture-covered slots: a keep_mask slot with no captured SA entry (or uncovered IR) is stubbed + its CALL runs dropped instead of fetching non-resident code (default 1)"); + +static u32 gsp_phys_gr_keep_clamp_log; + +/* + * SESSION 153-CONTINUED-114: golden MME slot overlay (see the include + * comment). The capture is a per-boot lottery (r1407/r1385 got the full + * 55-slot program; r1405/6/10/11/12 only 13 slots), and a called slot + * outside the capture is stubbed with its CALL runs dropped - which is + * why the draw never executes on partial boots. The golden library is + * the same driver-emitted microcode word-for-word (validated against a + * live full capture), so a golden-filled slot is byte-shape-identical + * to a captured one downstream (pack image, 0x11c pairs, IRAM gate). + * + * gsp_phys_gr_golden_slot: returns 1 and the slot's instruction entry + + * word pointer when the library can serve slot j (and the knob is on). + * gsp_phys_gr_golden_active: the knob as a bool, for the cheap sites. + */ +static unsigned int gsp_phys_rmcall_gr_mme_golden; +module_param_named(sema_gl_mme_golden, gsp_phys_rmcall_gr_mme_golden, + uint, 0644); +MODULE_PARM_DESC(sema_gl_mme_golden, + "Golden MME slot overlay: serve called-but-uncaptured SA slots from the driver's own generated macro code (untracked kernel/golden-mme-tu104.h) instead of stubbing them, so CALL_MME_MACRO(DRAW_IAB etc.) runs deterministically (default 0; 1 = fill uncovered slots; 2 = force/overwrite all)"); +static unsigned int gsp_phys_rmcall_gr_mme_golden_force; +module_param_named(sema_gl_mme_golden_force, gsp_phys_rmcall_gr_mme_golden_force, + uint, 0644); +MODULE_PARM_DESC(sema_gl_mme_golden_force, + "Force the golden overlay over captured slots too (A/B for a stale capture; default 0 = fill uncovered only)"); + +/* + * r1448 STRATEGY 3.3 (golden-completion): with golden_force on, a called + * slot the golden library cannot serve is stubbed (SA pair -> harmless + * end-noop) and its CALL runs are dropped, and its captured body is + * zeroed in the packed IR - only golden bodies can execute. This closes + * the r1447 residual (slot 0, entry 0, was the one executed non-golden + * body). 0 = pre-r1448 behavior (capture bodies of non-golden slots may + * execute). + */ +static bool gsp_phys_rmcall_gr_mme_golden_only = true; +module_param_named(sema_gl_mme_golden_only, gsp_phys_rmcall_gr_mme_golden_only, + bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_golden_only, + "Golden-only completion: with golden_force on, blank the packed body and stub/drop CALL runs of every slot the golden library cannot serve (default 1; 0 = current behavior)"); + +static u32 gsp_phys_gr_golden_log; +static u32 gsp_phys_gr_golden0_log; + +static int gsp_phys_gr_golden_slot(u32 j, u32 *entry, const u32 **words, + u32 *nwords) +{ + const struct golden_mme_tu104_slot *gs; + + if (!READ_ONCE(gsp_phys_rmcall_gr_mme_golden)) + return 0; + if (j >= GOLDEN_MME_TU104_SLOTS || j >= 64u) + return 0; + gs = &golden_mme_tu104_sa[j]; + if (!(gs->flags & GMME_F_PRESENT) || !gs->words) + return 0; + if ((u64)gs->off + gs->words > GOLDEN_MME_TU104_WORDS) + return 0; + /* + * r1448 STRATEGY 3.3: entry 0 is a valid IR address - slot 0's + * SETUP_FIXED_CONST body begins at IR word 0, and the window calls + * slot 0 (CALL at win-off 56). The pre-r1448 `!gs->entry` reject + * left slot 0 as the only executed non-golden body (r1447 54/55 + * overlay). Accept it and log the acceptance once so the coverage + * change is visible in dmesg. + */ + if (!gs->entry && !READ_ONCE(gsp_phys_gr_golden0_log)) { + WRITE_ONCE(gsp_phys_gr_golden0_log, 1u); + pr_info("vgsp-gr: MME-GOLDEN slot 0 accepted (entry=0 valid; golden body %u words)\n", + gs->words); + } + *entry = gs->entry; + *words = &golden_mme_tu104_words[gs->off]; + *nwords = gs->words; + return 1; +} + +/* + * r1448 STRATEGY 3.3 golden-only completion: with golden_force on and + * the golden_only knob on, a slot the golden library cannot serve must + * never execute. Returns true when the slot is outside the golden corpus + * and has to be blanked/stubbed (fail-closed). + */ +static bool gsp_phys_gr_slot_golden_blank(u32 j) +{ + u32 gent, gn; + const u32 *gw; + + if (!READ_ONCE(gsp_phys_rmcall_gr_mme_golden_only) || + !READ_ONCE(gsp_phys_rmcall_gr_mme_golden_force)) + return false; + return !gsp_phys_gr_golden_slot(j, &gent, &gw, &gn); +} + +/* + * SESSION 153-CONTINUED-79: per-launch method-block word cap + prelude drain. + * The r1172-r1175 A/B: every beef launch whose emitted block exceeded ~2000 + * words (3083-word full-SA prelude + window, 3315-3484 words) NEVER STARTED + * on the physical engine (MME-FAST events=0, grfifo frozen, ch_rel never + * written) while every block <= ~1300 words (1002-word partial capture + + * window) executed (retires in 90-170 us when the executed macro stayed + * inside the loaded IR). The failure class is fetch-side (a stuck GPFIFO + * entry), not content: once parked, every later entry also failed until a + * poison-latch TSG re-arm reset the PBDMA. The full-SA capture (55 slots, + * 3083 words, macros that chain past the first 15 slots e.g. macro 0 -> + * IR 0x201 = slot-25 entry) therefore cannot be replayed in ONE block. + * Emit at most gsp_phys_rmcall_gr_block_cap words per launch and drain the + * (SDM-sanitized, run-aligned) prelude across consecutive launches with a + * generation-keyed cursor; the emission latch is set only when the final + * part has been emitted, so a completed program is re-requested after the + * WEDGE-RECOVER latch clear. Default 2048; 0 = legacy 4096 module buffer. + */ +static u32 gsp_phys_rmcall_gr_block_cap = 2048u; +module_param_named(sema_gl_mme_block_cap, gsp_phys_rmcall_gr_block_cap, uint, + 0644); +MODULE_PARM_DESC(sema_gl_mme_block_cap, + "Max words emitted per GR launch method block (0 = 4096 module buffer; the >2048-word blocks never start on the physical engine)"); + +/* Generation-keyed drain cursor for the split MME prelude emission. */ +static u32 gsp_phys_gr_mme_pre_emit_gen; +static u32 gsp_phys_gr_mme_pre_emit_h; +static u32 gsp_phys_gr_mme_pre_emit_off; +/* + * CONT-105: generation-keyed cursor for the SPLIT packed MME prelude + * (see the sema_gl_mme_pack_split comment). pack_off counts IR words + * (always a multiple of 3 = whole instructions); pack_sa records that the + * SA table has gone out for this pass. The pair is reset together with the + * drain cursor above whenever the emission restarts. + */ +static u32 gsp_phys_gr_mme_pack_off; +static bool gsp_phys_gr_mme_pack_sa; + +/* + * CONT-103: prelude residency witness + calls-kept block shrink. + * The CONT-79 split-emission re-carries the whole prelude prefix on EVERY + * launch while keep_mask is set (idempotent re-emission, CONT-72), so the + * first calls-kept launch is 1613 prelude words + the 410-word window = + * 2126-2142 words - over the ~2050-word fetch wall (CONT-79: oversized + * blocks never start, PBDMA parks). The deadlock: the calls can only run + * after the program is resident, and the launch that would run them is too + * big to start. The re-emission is only needed while the program is NOT + * known resident; the launch that carries the final part is the witness + * (its poll break with GP_GET advanced proves the engine consumed it). + * live_gen records the generation whose carrier executed; while it equals + * the capture generation the drain restart is skipped and the calls-kept + * launch ships window-only (~530 words, well under the wall). + * sema_gl_mme_prelude_live_skip=0 restores the unconditional re-emission + * for a one-flip A/B. + */ +static u32 gsp_phys_gr_mme_pending_gen; /* final part carried by the last + * launch; consumed at its poll */ +static u32 gsp_phys_gr_mme_live_gen; /* generation whose carrier executed */ +static bool gsp_phys_rmcall_gr_mme_live_skip = true; +module_param_named(sema_gl_mme_prelude_live_skip, + gsp_phys_rmcall_gr_mme_live_skip, bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_prelude_live_skip, + "Skip the forced per-launch prelude re-emission once the generation's final part has been carried by a launch the engine executed (default 1; 0 = always re-emit, pre-CONT-103 behavior)"); + +/* + * CONT-103: elide the program-address pair runs (method space + * [0x2054, 0x2158]) from a calls-kept window. With the prelude-live skip the + * calls-kept launch starts, and the engine then traps 0x80002058 on the + * relocated (shadow-pool) pair operand and parks - the CONT-101 trap class. + * The binds persist on the channel from the earlier calls-held launches, so + * dropping the re-binds removes the trapping write; the calls and the MME + * program stay. Off by default: the A/B against the trapping state. + */ +static bool gsp_phys_rmcall_gr_mme_nop_pair; +module_param_named(sema_gl_mme_nop_pair, gsp_phys_rmcall_gr_mme_nop_pair, + bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_nop_pair, + "Elide program-address pair runs (0x2054/0x2094/0x2154 families) from a calls-kept 3D window (default 0; the engine traps 0x2058 on the relocated shadow operand)"); + +/* + * r1453 PAIR-CHAIN (sema_gl_pair_chain): the offline EC 0x0d decode + * (BEGIN_END_ACTIVE, nv50.c:173) and the r1452b window dumps show the + * guest's draws arriving as back-to-back normal/normal BEGIN (0x1618) + * ... END (0x1614) pairs with no WAIT_FOR_IDLE (0x110) / SEND_GO_IDLE + * (0x13c) drain between them, and the FE then refuses the closing END + * (trap 0x80001614, Data 0, EC 0x0d) - and every later state write + * (0x2058 PROGRAM_ADDRESS_B) while a pair stays active. + * + * The guest's BEGIN data word is carried VERBATIM end to end (the + * window is a raw-push slice: svc_gl_flush -> req->stream -> + * gsp_phys_rmcall_gr_3d_patch_window memcpy). Nothing in the kernel + * tree constructs or normalises a 0x586/0x585 word: the only 32-bit + * BEGIN data form is a count-carrying run at method 0x1618 whose first + * payload word is the full data (SPLIT_MODE is bits 30:29 of THAT + * word); the 13-bit IMMD header form (op 4, data 8/7) cannot express + * the split bits and is what the observed raw pairs carry. This pass + * therefore never rebuilds a run: it (a) receipts the full BEGIN data + * word it carries, (b) inserts ONE WAIT_FOR_IDLE<-0 (INC 0x110, 1 data + * word) ahead of a following normal-begin pair so each END closes a + * drained pair, and (c) leaves split-mode chains (BEGIN 1/2/3) alone. + * 0 = legacy bytes (pass not run; held-call elision unchanged). + */ +static bool gsp_phys_rmcall_gr_pair_chain = true; +module_param_named(sema_gl_pair_chain, gsp_phys_rmcall_gr_pair_chain, + bool, 0644); +MODULE_PARM_DESC(sema_gl_pair_chain, + "One logical BEGIN..END chain per launch: carry the full 32-bit BEGIN data (SPLIT_MODE 30:29), insert WAIT_FOR_IDLE(0x110) drains between back-to-back normal/normal pairs, and extend pair elision to held-call windows (default 1; 0 = legacy bytes)"); +/* + * r1465 PAIR-ORPHAN-DROP (sema_gl_pair_orphan_drop): the pair_open/split + * tracker is a projection of the FE's pair state, so it must only advance + * through windows the FE actually consumed. A dropped/held/LOAD-ONLY + * window is a pair-state no-op (restore the launch-entry snapshot), and an + * END observed while the model is closed is provably orphaned (its opener + * window was dropped) - the FE has no pair to close and faults EC 0x0d, so + * the END run is elided. Default 1; 0 = legacy bytes (leak/0x1614). + */ +static bool gsp_phys_rmcall_gr_pair_orphan_drop = true; +module_param_named(sema_gl_pair_orphan_drop, + gsp_phys_rmcall_gr_pair_orphan_drop, bool, 0644); +MODULE_PARM_DESC(sema_gl_pair_orphan_drop, + "Pair-state sync: roll pair_open/open_split back on dropped/held/LOAD-ONLY windows and elide orphan END runs when no pair is open (default 1; 0 = legacy bytes)"); +/* + * SESSION 155-2: r1526's first calls-kept draw window faulted with a GR + * class error at SET_FRONT_FACE (0x191c <- 0x901) EC 0x0d + * ILLEGAL_METHOD_IN_BEGIN_END: the window entered with a BEGIN..END pair + * still open from the previous emitted window, so the first client state + * method ran inside it. Close the carried pair with the driver's own END + * form (0x1614 + event word, event 1 for split chains 1/2) before the + * window, then let the normal walk rebase the model. Gated: default 0 + * (legacy bytes). + */ +static bool gsp_phys_rmcall_gr_pair_close_entry = true; /* PAIR-SAFE: default ON (CONT.399) */ +module_param_named(sema_gl_pair_close_entry, + gsp_phys_rmcall_gr_pair_close_entry, bool, 0644); +MODULE_PARM_DESC(sema_gl_pair_close_entry, + "Emit an END (0x1614) for a BEGIN..END pair carried open into the window before its first run (default 0; A/B)"); +/* + * SESSION 155 CONT 9: the synthesized SPH-BIND prelude (j0/j2/j4/j5) is + * PREPENDED ahead of the window, so when the previous window left a + * BEGIN..END pair carried open the prelude's method runs land inside it - + * r1558 (mask=0x35) took EC 0x0d ILLEGAL_METHOD_IN_BEGIN_END at 0x2018. + * sema_gl_pair_close_entry closes the pair but is applied to the WHOLE + * window (prepends included) at the end of the pass; this knob makes the + * prelude itself self-contained: when the tracker says a pair is open, the + * END (0x1614 + event word, event 1 for split chains 1/2) is emitted as the + * FIRST word of the prelude, and each synthesized stage's + * SET_PIPELINE_SHADER TYPE and REGISTER_COUNT are emitted BEFORE its SPH + * address pair (the client's own stage-1 order). Default 0 = legacy bytes. + */ +static bool gsp_phys_rmcall_gr_sph_bind_pair_safe = true; /* PAIR-SAFE: default ON (CONT.399) */ +module_param_named(sema_gl_sph_bind_pair_safe, + gsp_phys_rmcall_gr_sph_bind_pair_safe, bool, 0644); +MODULE_PARM_DESC(sema_gl_sph_bind_pair_safe, + "Make the synthesized SPH-BIND prelude BEGIN/END-safe: close a carried-open pair with END(0x1614) before the prelude and emit TYPE/REGISTER_COUNT before each stage address pair (default 0; legacy bytes)"); +/* + * SESSION 155 CONT 9 (TASK B): the window replayed for a non-tessellated, + * non-geometry draw carries only the VSB (j=1) record; the client's + * channel state leaves VSA/TCS/TES/GS explicitly DISABLED. The replay + * channel instead inherits the setup-push program records (TCS enabled, + * 0x2080 <- 0x20 is a disable the ATTR-PRELUDE drops), so the DS validates + * VSB in tessellation mode and GS against a stale program - the r1560 + * 0x900400a0 SPH5/7/18 bits. Emit the client's own disable records, each + * value quoted from the driver's unchanged-program path: + * 0x2000 <- 0x00 SET_PIPELINE_SHADER(0) ENABLE=FALSE|TYPE=VERTEX_CULL_BEFORE_FETCH + * 0x2080 <- 0x20 SET_PIPELINE_SHADER(2) ENABLE=FALSE|TYPE=TESSELLATION_INIT + * 0x20c0 <- 0x30 SET_PIPELINE_SHADER(3) ENABLE=FALSE|TYPE=TESSELLATION + * 0x2100 <- 0x40 SET_PIPELINE_SHADER(4) ENABLE=FALSE|TYPE=GEOMETRY + * and suppress the synthesized j0/j2/j4 binds (a disabled stage has no + * program to bind). Default 0 = legacy bytes. + * + * SESSION 155 CONT 10: the all-four bool became a per-stage mask, bit j = + * synth slot j (0=VSA, 1=VSB, 2=TCS, 3=TES, 4=GS, 5=PS). A set bit emits + * that stage's client disable record (for the stages whose record the + * driver's unchanged-program path writes: 0/2/3/4) and skips that slot's + * synthesized bind (j0/j2/j4/j5; j1 is the window's own VTG record). + */ +static u32 gsp_phys_rmcall_gr_sph_stage_disable; +module_param_named(sema_gl_sph_stage_disable, + gsp_phys_rmcall_gr_sph_stage_disable, uint, 0644); +MODULE_PARM_DESC(sema_gl_sph_stage_disable, + "Per-stage synth-bind disable mask (bit j = synth slot j: 0=VSA,1=VSB,2=TCS,3=TES,4=GS,5=PS): emit the client's SET_PIPELINE_SHADER disable record for that stage and skip its synthesized bind (default 0; legacy bytes)"); + +/* + * SESSION 155 CONT 11: the last DS latch bit in the r1562 replay is SPH18 + * (GS check2, attribute map). r1562 proved the DS validates the GS stage + * even when the driver's own disable record (0x2100 <- 0x40) leaves it + * unbound: SPH18 latched with stage4 START=0. The client has no geometry + * program, so no GS SPH exists anywhere in its push. This knob binds a + * synthesized header-only GEOMETRY SPH (128 B, pass-through maps) to + * stage 4 while keeping SET_PIPELINE_SHADER(4) ENABLE=0, so the DS has a + * valid operand at SPH_INFO_STAGE4_START but the engine never executes + * it. The image is authored by the sema service at family+0x780 (same 4K + * page as the bound VTG/TCS/PS images) and the address pair below is + * relocated by the same window pass. Default 0 = legacy bytes. + */ +static bool gsp_phys_rmcall_gr_sph_gs_img; +module_param_named(sema_gl_sph_gs_img, gsp_phys_rmcall_gr_sph_gs_img, + bool, 0644); +MODULE_PARM_DESC(sema_gl_sph_gs_img, + "Bind a synthesized header-only GEOMETRY SPH to stage 4 with ENABLE=0 (family+0x780; satisfies the SPH18 GS attribute-map check; default 0)"); + +bool gsp_phys_rmcall_gr_sph_gs_img_value(void) +{ + return READ_ONCE(gsp_phys_rmcall_gr_sph_gs_img); +} + +/* + * SESSION 155 CONT 12: the synthesized GS SPH's attribute-map variant. + * The r1564 run proved the DS fetches the image (stage4 START nonzero) + * yet SPH18 persists and the VTG-class check2 bits return, so the map + * CONTENT is the discriminator: + * 0 = pass-through (IMAP = previous OMAP = POSITION+COLOR; + * OMAP = PS IMAP superset = POSITION+COLOR) + * 1 = all IMAP/OMAP zero (the mesh-dummy contract) + * 2 = PS-exact (IMAP = POSITION.W; OMAP = POSITION.XYZW) + * 3 = IMAP POSITION+COLOR, OMAP POSITION.XYZW only + * A/B live: the sema service reads this per capture. + */ +static u32 gsp_phys_rmcall_gr_sph_gs_maps; +module_param_named(sema_gl_sph_gs_maps, gsp_phys_rmcall_gr_sph_gs_maps, + uint, 0644); +MODULE_PARM_DESC(sema_gl_sph_gs_maps, + "Synth GS SPH attribute-map variant: 0=pass-through(POS+COLOR both), 1=zero maps, 2=PS-exact(POS.W/POS.XYZW), 3=IMAP POS+COLOR / OMAP POS.XYZW (default 0)"); + +u32 gsp_phys_rmcall_gr_sph_gs_maps_value(void) +{ + return READ_ONCE(gsp_phys_rmcall_gr_sph_gs_maps); +} + +/* + * SESSION 155 CONT 13: GS-mode context variant for the synthesized GS SPH. + * The mesh per-primitive dummy-GS contract (the only driver precedent for + * a fake GS SPH) requires ENABLE_FAST_G_S in the header and writes + * SET_GS_MODE(FAST_GS) + POST_VTG_SHADER_ATTRIBUTE_SKIP_MASK. Our replay + * channel never receives those (the client has no GS program). Bit j: + * 1 = set FASTGS in the SPH header and emit + * 0x0208 <- 1 (SET_GS_MODE FAST_GS) + * 0x1240..0x125c <- 0 (POST_VTG_SHADER_ATTRIBUTE_SKIP_MASK) + * 2 = enable the stage (final SET_PIPELINE_SHADER(4) = 0x41) + * Default 0 = the CONT 11 disabled/no-fastgs form. + */ +static u32 gsp_phys_rmcall_gr_sph_gs_mode; +module_param_named(sema_gl_sph_gs_mode, gsp_phys_rmcall_gr_sph_gs_mode, + uint, 0644); +MODULE_PARM_DESC(sema_gl_sph_gs_mode, + "Synth GS SPH mode bits: 1=FASTGS header + SET_GS_MODE(FAST_GS)+skip-mask, 2=enable stage (0x41); 0=disabled/no-fastgs (default)"); + +u32 gsp_phys_rmcall_gr_sph_gs_mode_value(void) +{ + return READ_ONCE(gsp_phys_rmcall_gr_sph_gs_mode); +} + +/* + * SESSION 155 CONT 14: raster-enable prelude. The client's per-frame 3D + * window carries no SET_CT_WRITE / SET_DEPTH_TEST / SET_DEPTH_WRITE / + * SET_STENCIL_TEST / SET_SCISSOR_ENABLE - those are channel state on the + * client's own channel - so the replay channel keeps the hardware + * defaults. If CT_WRITE defaults to 0 every rasterized fragment is + * discarded and the surface stays black even though the draw executes. + * Emit the safe render-enable state ahead of each replayed 3D window: + * SET_CT_WRITE(0..7) 0x1a00+i*4 <- 0x1111 (R|G|B|A write enable) + * SET_DEPTH_TEST 0x12cc <- 0 + * SET_DEPTH_WRITE 0x12e8 <- 0 + * SET_STENCIL_TEST 0x1380 <- 0 + * SET_SCISSOR_ENABLE(0) 0x0e00 <- 0 + * Default 0 = legacy bytes (A/B). + */ +static bool gsp_phys_rmcall_gr_raster_pre; +module_param_named(sema_gl_raster_prelude, gsp_phys_rmcall_gr_raster_pre, + bool, 0644); +MODULE_PARM_DESC(sema_gl_raster_prelude, + "Emit CT_WRITE(0..7)=0x1111 + depth/stencil/scissor disable ahead of replayed 3D windows (default 0; A/B)"); + +bool gsp_phys_rmcall_gr_raster_pre_value(void) +{ + return READ_ONCE(gsp_phys_rmcall_gr_raster_pre); +} + +/* + * SESSION 155 CONT 18: follow a migrated screen page instead of skipping + * its write-back. The guest's video-memory manager migrates the X screen + * BO (the WGUARD-SKIP "used != now" case); the drawn content then sits in + * the bounce copy of the old page and is currently discarded. When this + * knob is on and the stale VA belongs to the captured screen surface, the + * page is written to the VA's CURRENT mapping (receipt + * WGUARD-STALE-WRITE) so the guest sees the frame. Off = the historical + * safe skip. + */ +static bool gsp_phys_gr_stale_write; +static u32 gsp_phys_gr_stale_log; /* CONT 18 */ +module_param_named(sema_gl_stale_write, gsp_phys_gr_stale_write, bool, + 0644); +MODULE_PARM_DESC(sema_gl_stale_write, + "Write a copy-out page to the screen VA's current mapping when the write guard reports a migration (default 0; safe skip)"); + +bool gsp_phys_rmcall_gr_stale_write_value(void) +{ + return READ_ONCE(gsp_phys_gr_stale_write); +} + +/* + * SESSION 155 CONT 22: extra raster/store-disable A/B. The client's window + * carries none of the blend/alpha/cull/point/ZCULL channel state; if the + * replay's defaults kill fragments (blend discard, cull, alpha, ZCULL), no + * color reaches the ROP. Emit the neutral disabling form ahead of the + * window (constant runs): + * 0x12e4 SET_BLEND_STATE_PER_TARGET <- 0 + * 0x1e00 SET_BLEND_PER_TARGET_SEPARATE<- 0 + * 0x12ec SET_ALPHA_TEST <- 0 + * 0x1308 D3D_SET_CULL_MODE <- 0 (no cull) + * 0x1520 SET_POINT_SPRITE <- 0 + * 0x02e4 SET_ZCULL_ROP_BYPASS <- 0 + * 0x02e8 SET_ZCULL_SUBREGION <- 0 + * 0x02fc ASSIGN_ZCULL_SUBREGIONS <- 0 + * Default 0 = legacy bytes. + */ +static bool gsp_phys_rmcall_gr_raster_extra; +module_param_named(sema_gl_raster_extra, gsp_phys_rmcall_gr_raster_extra, + bool, 0644); +MODULE_PARM_DESC(sema_gl_raster_extra, + "Emit blend/alpha/cull/point/ZCULL neutral-disabled state ahead of replayed 3D windows (default 0; A/B)"); + +bool gsp_phys_rmcall_gr_raster_extra_value(void) +{ + return READ_ONCE(gsp_phys_rmcall_gr_raster_extra); +} + +/* + * SESSION 155 CONT 15: where the synthesized GS SPH lives inside the + * program-family page. 0 = family+0x780 (below the window's own VTG bind + * at family+0x900), 1 = family+0xe80 (above it; the DS's PROGRAM_REGION + * snapshot reads the bind VA, so a region starting there does not cover + * 0x780). Default 0 = legacy. + */ +static u32 gsp_phys_rmcall_gr_sph_gs_addr; +module_param_named(sema_gl_sph_gs_addr, gsp_phys_rmcall_gr_sph_gs_addr, + uint, 0644); +MODULE_PARM_DESC(sema_gl_sph_gs_addr, + "Synth GS SPH offset in the family page: 0=+0x780 (legacy), 1=+0xe80 (above the VTG bind)"); + +u32 gsp_phys_rmcall_gr_sph_gs_addr_value(void) +{ + return READ_ONCE(gsp_phys_rmcall_gr_sph_gs_addr); +} + +/* + * SESSION 155 CONT 15: GS context A/B words emitted with the synth GS bind. + * bit0: SET_MESH_CONTROL (0x114c) <- 0 + * bit1: SET_GS_MODE (0x0208) <- 0 (TYPE_ANY) + * The client's per-frame push carries neither; if the init push left them + * set, the GS macro/mesh path may treat the stage specially. Default 0. + */ +static u32 gsp_phys_rmcall_gr_sph_gs_ctx; +module_param_named(sema_gl_sph_gs_ctx, gsp_phys_rmcall_gr_sph_gs_ctx, + uint, 0644); +MODULE_PARM_DESC(sema_gl_sph_gs_ctx, + "Extra GS context with the synth bind: bit0=SET_MESH_CONTROL(0x114c)<-0, bit1=SET_GS_MODE(0x0208)<-0 (default 0)"); + +u32 gsp_phys_rmcall_gr_sph_gs_ctx_value(void) +{ + return READ_ONCE(gsp_phys_rmcall_gr_sph_gs_ctx); +} +static bool gsp_phys_gr_mme_pairs_emitted; /* launch emitted a pair run */ +static u32 gsp_phys_gr_mme_pairs_gen; /* gen whose pairs retired OK */ +static u32 gsp_phys_gr_mme_launch_gen; /* last built launch's gen */ +static bool gsp_phys_gr_mme_pair_open; /* BEGIN..END open across windows */ +static u32 gsp_phys_gr_mme_pair_open_split; /* split mode of that BEGIN */ +static bool gsp_phys_gr_mme_pair_open_entry; /* open seen when THIS window was fixed */ +static u32 gsp_phys_gr_mme_pair_open_entry_split; /* split of the entry-open BEGIN */ +static u32 gsp_phys_gr_mme_pair_entry_elide_n; /* entry-open pair-run elisions */ + +/* + * True when a method run in [w, w+n) writes NVC597_BEGIN (0x1618) or + * NVC597_END (0x1614) - the per-launch proof the pair-elision extension + * gates on (pairs must have reached the FE once for the generation). + */ +static bool gsp_phys_gr_pair_present(const u32 *w, u32 n) +{ + u32 r = 0; + + while (r < n) { + u32 h = w[r]; + u32 op = (h >> 29) & 7u; + u32 cnt = (h >> 16) & 0xfffu; + u32 span = 1u; + u32 m = (h & 0xfffu) << 2; + + if ((op == 1u || op == 3u || op == 5u) && cnt && + r + 1u + cnt <= n) + span = 1u + cnt; + if ((op == 1u || op == 3u || op == 4u || op == 5u) && + (m == 0x1618u || m == 0x1614u)) + return true; + r += span; + } + return false; +} + +/* + * SESSION 154-1: true when the method run in [w, w+n) already writes + * `method` (byte offset). Used to keep the channel-init re-emission from + * duplicating a write the captured window itself carries: the X channel's + * per-frame window contains the client's own SET_SHADER_EXCEPTIONS + * (0x1528) run (sema-pushraw `[79] off=0x5a4 op=4 addr=0x1528`), so the + * booter's extra INC-form write is the first method the FE sees and the + * one its trapped-method register reports (r1483 `trap=80001528`). + * + * The run grammar is the same as every window walk in this file: op 1/3/5 + * carry 1+count data words, op 4 is the client's one-word immd13 header + * (the 13-bit method field is in the header, no data word), everything + * else is a single header word. The low 13 bits are the method >> 2 for + * both the 0x2000 and 0x8000 header forms. + */ +static bool gsp_phys_gr_method_present(const u32 *w, u32 n, u32 method) +{ + u32 r = 0; + + while (r < n) { + u32 h = w[r]; + u32 op = (h >> 29) & 7u; + u32 cnt = (h >> 16) & 0x1fffu; + u32 span = 1u; + + if ((op == 1u || op == 3u || op == 5u) && cnt && + r + 1u + cnt <= n) + span = 1u + cnt; + if (((h & 0x1fffu) << 2) == method) + return true; + r += span; + } + return false; +} + +/* + * SESSION 154-1: first SET_PIPELINE_PROGRAM_ADDRESS pair in an already + * relocated window. Returns the pair's engine-visible (shadow) VA. The + * DS validates each fetched SPH against the channel's PROGRAM REGION; + * the reference driver programs that region with the program's own GPU + * base (nv3dGetProgramGpuAddress -> NV9097_SET_PROGRAM_REGION_A/B), not + * with the whole shader heap. The replay's engine fetch goes through the + * relocated shadow VA, so the matching region base is the bound VA's 4K + * page (e.g. beef bind 0x19a0f00 -> region 0x19a0000), not the shadow + * pool's base (the r1483/r1484 forced 0x1500000, which does not contain + * the beef program's 0x19a0000 page and makes the DS reject the fetched + * SPH: r1484 `GR-STATE sph=0,019a0f00,019a0200,...,019a0000` together + * with `ds=86040646`). + * + * Only op 1 (INC) can carry the two-word A/B pair; the run grammar is the + * same as the window walkers. + */ +static bool gsp_phys_gr_first_prog_bind(const u32 *w, u32 n, u64 *va_out) +{ + u32 r = 0; + + while (r < n) { + u32 h = w[r]; + u32 op = (h >> 29) & 7u; + u32 cnt = (h >> 16) & 0x1fffu; + u32 addr = (h & 0x1fffu) << 2; + u32 span = 1u; + + if ((op == 1u || op == 3u || op == 5u) && cnt && + r + 1u + cnt <= n) + span = 1u + cnt; + if (op == 1u && cnt >= 2u && addr >= 0x2014u && + addr < 0x4000u && (addr & 0x3fu) == 0x14u) { + u64 pva = ((u64)(w[r + 1u] & 0xffu) << 32) | + (u64)(w[r + 2u] & 0xfffffffcu); + + if (pva) { + *va_out = pva; + return true; + } + } + r += span; + } + return false; +} + +/* + * Apply the pair-chain rule to the final emitted window (prepends + * included). Returns the number of inserted drains; *np is updated. + * The scan is method-run aligned and insertion copies whole runs, so a + * run is never split and the BEGIN data word is never rewritten. + */ +static u32 gsp_phys_gr_pair_chain_fix(u32 *w, u32 *np, u32 cap) +{ + u32 n = *np, r = 0, begins = 0, ends = 0, drains = 0; + u32 chains = 0, orphans = 0, orphan_dropped = 0, nested = 0, badv = 0; + u32 entry_closes = 0; + bool open = READ_ONCE(gsp_phys_gr_mme_pair_open); + bool seen = false, begin_logged = false; + u32 open_split = READ_ONCE(gsp_phys_gr_mme_pair_open_split); + + WRITE_ONCE(gsp_phys_gr_mme_pair_open_entry, open); + WRITE_ONCE(gsp_phys_gr_mme_pair_open_entry_split, open_split); + + if (READ_ONCE(gsp_phys_rmcall_gr_pair_close_entry) && open) { + u32 expect = ((open_split == 1u || open_split == 2u) ? 1u : 0u); + + if (n + 2u <= cap) { + memmove(&w[2], &w[0], (size_t)n * 4u); + w[0] = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + (1u << 16) | (0x1614u >> 2); + w[1] = expect; + n += 2u; + r = 2u; + open = false; + seen = true; + entry_closes++; + pr_info_ratelimited("vgsp-gr: PAIR-CLOSE-ENTRY split=%u ev=%u window=%u (carried pair closed before the window)\n", + open_split, expect, n); + } else { + pr_warn_ratelimited("vgsp-gr: PAIR-CLOSE-ENTRY skipped (window=%u cap=%u)\n", + n, cap); + } + } + + while (r < n) { + u32 h = w[r]; + u32 op = (h >> 29) & 7u; + u32 cnt = (h >> 16) & 0xfffu; + u32 span = 1u; + u32 m = (h & 0xfffu) << 2; + bool is_begin = false, is_end = false; + u32 data = 0, split = 0, ev = 0; + + if ((op == 1u || op == 3u || op == 5u) && cnt && + r + 1u + cnt <= n) + span = 1u + cnt; + if (op == 4u) { + if (m == 0x1618u) { + is_begin = true; + data = (h >> 16) & 0x1fffu; + } else if (m == 0x1614u) { + is_end = true; + ev = (h >> 16) & 1u; + } + } else if ((op == 1u || op == 3u || op == 5u) && cnt && + m == 0x1618u) { + is_begin = true; + data = w[r + 1u]; + split = (data >> 29) & 3u; + } else if ((op == 1u || op == 3u || op == 5u) && cnt && + m == 0x1614u) { + is_end = true; + ev = w[r + 1u] & 1u; + } + if (is_begin) { + if (!begin_logged) { + begin_logged = true; + pr_info_ratelimited("vgsp-gr: PAIR-BEGIN-DATA off=%u data=%08x split=%u form=%s (full 32-bit data carried verbatim)\n", + r, data, split, + (op == 4u) ? "immd13(no-split)" : + "run32"); + } + if (!open && seen && split == 0u) { + /* + * EC 0x0d: a normal/normal pair begun while + * the previous pair is not drained is refused + * at its END. Insert the class's own stall + * (WAIT_FOR_IDLE <- 0) ahead of it. + */ + if (n + 2u <= cap) { + memmove(&w[r + 2u], &w[r], + (size_t)(n - r) * 4u); + w[r] = ((u32)GSP_PHYS_RMCALL_METHOD_IMM + << 29) | (1u << 16) | + (0x110u >> 2); + w[r + 1u] = 0u; + n += 2u; + r += 2u; + drains++; + pr_info_ratelimited("vgsp-gr: PAIR-DRAIN inserted WAIT_FOR_IDLE(0x110)<-0 before pair #%u at off=%u window=%u\n", + begins + 1u, r, n); + } else { + pr_warn_ratelimited("vgsp-gr: PAIR-DRAIN skipped (window=%u cap=%u): back-to-back pairs can still raise EC0x0d\n", + n, cap); + } + } + if (open) { + /* + * Legal only as a split chain (the previous + * BEGIN left its end open). + */ + if (open_split == 1u || open_split == 2u) + chains++; + else + nested++; + } else { + open = true; + seen = true; + } + open_split = split; + begins++; + } else if (is_end) { + if (!open) { + /* + * r1465 PAIR-ORPHAN-DROP: the opener's + * window was dropped, so the FE has no + * pair to close and would raise EC 0x0d + * on this END. Remove the whole END run + * (header 0x80000585/0x80000584 immd13 + * form or a 1+cnt run32) in place; never + * rewrite it. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_pair_orphan_drop) && + span <= n - r) { + memmove(&w[r], &w[r + span], + (size_t)(n - r - span) * 4u); + n -= span; + orphan_dropped++; + pr_info_ratelimited("vgsp-gr: PAIR-ORPHAN-DROP off=%u span=%u window=%u\n", + r, span, n); + continue; + } + orphans++; + } else { + u32 expect = ((open_split == 1u || + open_split == 2u) ? 1u : 0u); + + if (ev != expect) + badv++; + open = false; + ends++; + } + } + r += span; + } + if (begins > 1u || drains || chains || orphans || orphan_dropped || + nested || badv || open || entry_closes) + pr_info_ratelimited("vgsp-gr: PAIR-CHAIN begin=%u end=%u drains=%u chain=%u orphan_end=%u orphan_drop=%u nested=%u endv_mismatch=%u open_at_end=%u entry=%u close=%u window=%u\n", + begins, ends, drains, chains, orphans, orphan_dropped, + nested, badv, open ? 1u : 0u, + READ_ONCE(gsp_phys_gr_mme_pair_open_entry) ? 1u : 0u, + entry_closes, n); + *np = n; + WRITE_ONCE(gsp_phys_gr_mme_pair_open, open); + WRITE_ONCE(gsp_phys_gr_mme_pair_open_split, open_split); + return drains; +} + +/* + * SESSION 153-CONTINUED-79: parameter-conserving NOP for kept parking slots. + * The r1177 bisect: keep={0,2,57} (full 3089-word program resident, split + * emission) parks the PBDMA on the emitted block (MME-FAST events=0, grfifo + * frozen, warm sem WEDGED) while keep={0,2} retires every launch - slot 57's + * CALL_MME_MACRO(57) run is the r1142 park class. Rather than drop the run + * (which also removes its parameter words from the stream), rewrite the call + * run's header to target the CALL_MME_DATA port (method+4) so the parameter + * words are still written (conserved) but no macro executes: the trigger + * word becomes a plain port write and the PBDMA never opens a macro context + * whose parameter drain can back up. A slot named in this mask must also be + * in sema_gl_mme_keep_mask (kept = retained in the window). Default 0. + */ +static u64 gsp_phys_rmcall_gr_nop_param_mask; +module_param_named(sema_gl_mme_nop_param_mask, gsp_phys_rmcall_gr_nop_param_mask, + ullong, 0644); +MODULE_PARM_DESC(sema_gl_mme_nop_param_mask, + "Kept slots whose CALL_MME_MACRO trigger is rewritten to a plain CALL_MME_DATA port write (params conserved, macro not executed; u64 bitmask)"); + +/* + * SESSION 153-CONTINUED-80: the r1181 live A/B validated the retarget above + * and it PARKS the PBDMA exactly like the raw kept call - with + * keep=5|1<<57 + nop=1<<57 the first post-change launch froze + * (MME-FAST begin grfifo=0x00000001, end events=0, no poison, warm sem + * WEDGED) while keep=5 (slot 57 call+params elided) retired in 94-106 us + * on the same boot (r1176/r1181 baseline). The bare DATA-port run without an + * open macro context is not a safe landing for the trigger. With this knob + * set, a nop-mask slot joins the drop path instead (the r1144-r1177 + * proven-safe behavior: whole run elided, retires clean, params NOT + * conserved). Use for the all-real-macro A/B: sa_stub_all=0 leaves + * 34/39/40/48/51 real while this drops only 57. Default 0 preserves the + * retarget behavior so the r1179 transform evidence stays reproducible. + */ +static u64 gsp_phys_rmcall_gr_nop_drop; +module_param_named(sema_gl_mme_nop_drop, gsp_phys_rmcall_gr_nop_drop, ullong, + 0644); +MODULE_PARM_DESC(sema_gl_mme_nop_drop, + "Slots whose call runs are dropped entirely (proven-safe retire, SA[slot] stubbed) instead of retargeted to the CALL_MME_DATA port (u64 bitmask; default 0)"); + +/* + * SESSION 153-CONTINUED-82: hold real CALL runs until the prelude latch. + * The r1220 (all-real, nop_drop=1<<57, sa_stub_all=N) wedge: the window's + * real CALL runs executed in the SAME launch that emitted only the first + * PART of a fresh capture generation (`MME-PRELUDE part gen=3 off=1776 + * words=1776/3083 ... (continuation next launch)`), so the called macros + * fetched IR code that was not resident yet - `MME-FAST launch ... + * pc_valid=1 pc=513`, priv-error poison, `Shader Program Header 18 Error`, + * `Class Error ... Offset 0000238c, Data 00020020` (the r1116 lane-poison + * class) and the boot wedged. The prelude latch (gsp_phys_gr_mme_prelude_* + * gen/h) is set only when the FINAL part has gone out; with this knob the + * launch builder holds (elides, the proven-safe drop shape) every remaining + * CALL_MME_MACRO/DATA run of the emitted window while EITHER the latch for + * this generation+channel is not set after this launch, OR this launch + * uploaded generation code for the first time (latch invalid before the + * emission). The r1222 all-real boot proved the second clause necessary: + * the drain-completing launch (part-2 IR uploads + window real calls in the + * SAME block, latch set by its own emission) still hit ILLEGAL_OPCODE - + * real calls resume on the next launch, when only the idempotent keep-forced + * re-emission of the already-resident prefix is in flight. Default 1 (the + * fix); A/B with 0 to reproduce the r1209/r1220/r1222 wedge. + */ +static bool gsp_phys_rmcall_gr_mme_hold_calls = true; +module_param_named(sema_gl_mme_hold_calls, gsp_phys_rmcall_gr_mme_hold_calls, + bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_hold_calls, + "Hold (elide) window CALL_MME_MACRO/DATA runs until the MME prelude latches for the current generation+channel (default 1; 0 = legacy, real calls run against a partial prelude)"); + +/* + * SESSION 153-CONTINUED-107: separately control the CONT-104 FE-idle clause + * of the hold (see the call site). With the IRAM gate proving residency the + * clause's only value is the r1359 race, and r1373 showed the FE wedged at + * 0x07071602 for minutes while every calls-kept window was held. Default 1 + * (pre-CONT-107 behavior); 0 releases the calls as soon as the prelude + * latch + IRAM gate allow. + */ +static bool gsp_phys_rmcall_gr_mme_hold_fe_idle = true; +module_param_named(sema_gl_mme_hold_fe_idle, + gsp_phys_rmcall_gr_mme_hold_fe_idle, bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_hold_fe_idle, + "Also hold the real CALL runs while the front-end is not settled (0x400504 != 1; default 1)"); + +/* + * SESSION 153-CONTINUED-84: emit NVC597_SET_MME_VERSION=1 (0x234) with the + * prelude step. The channel-init defaults the client pushes once per channel + * (Turing+) are never replayed, so the replayed context's MME stays in its + * reset/disabled state and every CALL_MME_MACRO run parks the front-end + * waiting for a macro that cannot start (r1245: call-bearing launches hang + * with all MME registers idle). Default 1; A/B with 0. + */ +static bool gsp_phys_rmcall_gr_mme_version = true; +module_param_named(sema_gl_mme_version, gsp_phys_rmcall_gr_mme_version, + bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_version, + "Emit SET_MME_VERSION=1 ahead of the replayed MME prelude (default 1; the Turing channel-init method the guest pushes once and the capture never carries)"); + +/* + * SESSION 153-CONTINUED-99: IR-only prelude drain. The call-bearing launch + * must carry the macros' SA pairs + IR words IN THE SAME BLOCK as the + * CALL_MME_MACRO runs: the split drain ships the full 3083-word capture + * across two launches (r1337 W1/W2), so the calls only ever ran on a launch + * whose block held part of the program - and every such launch faults MME + * ILLEGAL_OPCODE (relayed ESR 0x404490=0x80000004) the moment the first + * CALL executes (r1336/r1337/r1339/r1340, four boots; with {48,51,57} + * dropped in r1340 the remaining {0,2,34,39,40} calls fault identically). + * Shipping the FULL capture in one block does not work either: the 3441- + * and 3623-word blocks of r1341 never retired at all (MME-FAST begin + * grfifo=0d0d0001/00000001, no ch_rel, timeout ladder). The SA pairs + + * IR-load runs of the whole capture are only ~1277 of the 3083 words; the + * remainder is the setup family the macros do not fetch. With this knob the + * drain emits ONLY the runs targeting the MME SA/IR ports (0x114/0x118/ + * 0x11c/0x120), whole runs, so a single <=2048-word block can carry SA+IR + * (1277) + SA re-arm + window and the calls execute against the program + * the same launch uploaded. Default 0 (A/B; the r1337 drain behavior). + */ +static bool gsp_phys_rmcall_gr_mme_prelude_ir_only; +module_param_named(sema_gl_mme_prelude_ir_only, + gsp_phys_rmcall_gr_mme_prelude_ir_only, bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_prelude_ir_only, + "Emit only the MME SA/IR port runs (0x114/0x118/0x11c/0x120) from the prelude drain so SA+IR fit one working-size block (default 0)"); + +/* + * SESSION 153-CONTINUED-99b: packed (partitioned) prelude. The captured + * prelude buffer is the sum of the client's overlapping MME upload runs: + * each run writes its macro's body at its SA entry, and the runs overlap + * (runs for later slots rewrite shared tail regions). Replaying them + * verbatim in capture order therefore leaves the IR in a state where a + * macro's own body tail has been overwritten with the NEXT macro's bytes - + * executing it fetches non-code and the MME raises ILLEGAL_OPCODE + * (relayed ESR 0x404490=0x80000004; r1336/r1337/r1339/r1340/r1342, five + * boots). Verified against the driver's own generated macro source: the + * slot-34 run equals mmeCodeOGL_C597_DRAW_IAB word-for-word (75/75) but + * the capture-order final image differs at 50 of them. + * + * This knob rebuilds the program the way the engine must see it: each IR + * address takes the value from the run of the slot whose SA region owns it + * (run j clipped at the next SA entry), then emits the whole IR as ONE + * contiguous 0x114 ONE_INC run plus the full SA table as [slot, entry] + * pairs - about 1270 words, so a single <=2048-word block carries SA+IR + + * SA re-arm + window and the window's CALL_MME_MACRO runs execute against + * the program the same launch uploaded. Default 0 (capture-order drain). + */ +static bool gsp_phys_rmcall_gr_mme_prelude_pack; +module_param_named(sema_gl_mme_prelude_pack, + gsp_phys_rmcall_gr_mme_prelude_pack, bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_prelude_pack, + "Rebuild the MME IR as the partitioned program (each SA region from its own run) and emit it compactly with the SA table (default 0)"); + +/* Packed IR image; indexed by IR word address (<= 0x420 in captures). */ +static u32 gsp_phys_gr_mme_ir_pack[4096]; + +/* + * CONT-112: A/B IR-word override for the packed MME program. The offline + * replay of the captured SA stream (mmeref/, sim_run.py) shows slot 34 (the + * draw macro) emitting `0x1618<-0x61000008, 0xd74<-0, 0xd78<-count(0x3c), + * 0x1614<-0` from the CLIENT'S OWN call params - i.e. the emitted sequence + * is parameter-correct offline. This knob overrides ONE word of the packed + * image (the exact words the engine loads into its IRAM) so a live A/B can + * mutate the emitted method words one at a time (BEGIN data / SVS / DRAW + * count / END) and observe the engine's writes: the packed image is rebuilt + * per launch, so the value is live-tunable without a reload. Off by + * default (~0u); both params runtime-settable. + */ +static unsigned int gsp_phys_gr_mme_irpatch_off = ~0u; +module_param_named(sema_gl_mme_irpatch_off, gsp_phys_gr_mme_irpatch_off, + uint, 0644); +MODULE_PARM_DESC(sema_gl_mme_irpatch_off, + "IR word index in the packed MME program to override for A/B (default 4294967295 = off)"); +static unsigned int gsp_phys_gr_mme_irpatch_val; +module_param_named(sema_gl_mme_irpatch_val, gsp_phys_gr_mme_irpatch_val, + uint, 0644); +MODULE_PARM_DESC(sema_gl_mme_irpatch_val, + "Value written at sema_gl_mme_irpatch_off in the packed MME program (default 0)"); + +/* + * SESSION 153-CONTINUED-107: IRAM chronology + load-gate helpers. + * + * CONT-106 proved the trap-time fetch at PC 0x364 (macro 57 entry) reads + * ZERO IR words while the host-side pack is verbatim (CONT-105), and the + * IRAM debug port at 0x4044d0 also reads zero at 0x364 on that boot while + * the same port read the golden first words at pc=0 on r1364. The one + * unproven link is the LOAD: did the emitted 0x114 pack writes land in the + * MME's IRAM before the macro executed? These helpers read the live IRAM + * back and (a) log the timeline around each launch's fault-register + * transitions, (b) gate the real CALL runs on the readback matching the + * packed image at every covered SA entry. + * + * The probe reads the macro-57 entry group (0x364) under BOTH port address + * conventions - instruction base (write 0x364) and word base (write + * 3*0x364 = 0xA2C) - plus word 0, and prints the live words next to the + * pack-image expectation, so one line answers "is the program resident + * right now and which address unit does the port use". + */ +static void gsp_phys_mme_iram_probe(void __iomem *bar0, const char *tag, + u32 h_object) +{ + u32 a[3], bi[3], bw[3]; + + if (!bar0) + return; + gsp_phys_mme_iram_read3(bar0, 0u, a); + gsp_phys_mme_iram_read3(bar0, 0x364u, bi); + gsp_phys_mme_iram_read3(bar0, 3u * 0x364u, bw); + pr_info("vgsp-gr: MME-IRAM %s h=0x%x w0=%08x,%08x,%08x pack0=%08x,%08x,%08x i364=%08x,%08x,%08x w364=%08x,%08x,%08x pack364=%08x,%08x,%08x\n", + tag, h_object, a[0], a[1], a[2], + gsp_phys_gr_mme_ir_pack[0], + gsp_phys_gr_mme_ir_pack[1], + gsp_phys_gr_mme_ir_pack[2], + bi[0], bi[1], bi[2], + bw[0], bw[1], bw[2], + gsp_phys_gr_mme_ir_pack[3u * 0x364u], + gsp_phys_gr_mme_ir_pack[3u * 0x364u + 1u], + gsp_phys_gr_mme_ir_pack[3u * 0x364u + 2u]); +} + +/* + * SESSION 153-CONTINUED-107 gate knob (see the helper comment above). + * Default 0: A/B flip - with the gate on, a launch whose readback shows the + * IRAM not holding the packed image holds its real CALL runs and owes a + * full re-emission (prelude latch cleared; pack/drain cursors rewound), so + * the calls only ever execute against a verified-resident program. + */ +static bool gsp_phys_rmcall_gr_mme_iram_gate; +module_param_named(sema_gl_mme_iram_gate, gsp_phys_rmcall_gr_mme_iram_gate, + bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_iram_gate, + "Read the MME IRAM back (0x4044d0 port) and hold/re-emit the packed prelude until it reads back the packed image at the SA entries (default 0; the CONT-106 all-zero fetch)"); + +/* + * SESSION 153-CONTINUED-108: site control for the IRAM-gate readbacks. + * + * r1374 chronology (first gate boot that released calls): the carrier + * launch (window dropped, no calls) read grfifo=01010001 (empty) at +5 ms; + * the poll-break gate then wrote the MME debug port (0x4044d0) DURING that + * live launch; the first sample of the next launch 2.9 s later read + * grfifo=07071602 (not-empty + FULL, COUNT=22) with NO launch active in + * between - the front end never drained again for the rest of the boot + * (26 launches, all beginning FULL). CONT-107 already recorded the sibling + * evidence: every probe-bearing cycle (r1369..r1371) parked the engine on + * the first X-channel launches while the readback-free cycle (r1368) ran + * the full lane. Both gate sites write the same port, so each gets its own + * knob for the A/B: + * - break (default 0): the poll-break readback after the carrier + * release. Off by default - it is the r1374 suspect. Skipping it does + * not skip the live-gen promotion (that keys on the engine GP_GET + * fetch witness); the build site still gates the calls. + * - build (default 1): the pre-doorbell readback that authorizes the + * calls-bearing launch. Kept on; flip to 0 at runtime to isolate it. + */ +static bool gsp_phys_rmcall_gr_mme_iram_gate_break; +module_param_named(sema_gl_mme_iram_gate_break, + gsp_phys_rmcall_gr_mme_iram_gate_break, bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_iram_gate_break, + "Do the IRAM-gate readback at the launch poll break (writes the debug port while the carrier is finishing; default 0 - the r1374 FIFO-fill suspect)"); + +static bool gsp_phys_rmcall_gr_mme_iram_gate_build = true; +module_param_named(sema_gl_mme_iram_gate_build, + gsp_phys_rmcall_gr_mme_iram_gate_build, bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_iram_gate_build, + "Do the IRAM-gate readback in the launch builder before the doorbell (default 1; the calls-authorizing check)"); + +/* + * SESSION 153-CONTINUED-107: separate knob for the per-launch chronology + * probe lines. They write the MME debug port DURING a live launch, and the + * first three probe-bearing cycles (r1369..r1371) all parked the engine on + * the first X-channel launches while the last probe-free cycle (r1368) ran + * the full lane; until the port's side effects are cleared, keep the probe + * off by default and enable it only on a known-good boot (A/B). + */ +module_param_named(sema_gl_mme_iram_probe, gsp_phys_rmcall_gr_mme_iram_probe, + bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_iram_probe, + "Per-launch MME-IRAM chronology probe lines in the post-doorbell spin (writes the debug port during a live launch; default 0)"); +MODULE_PARM_DESC(sema_gl_mme_iram_probe, + "Per-launch MME-IRAM chronology probe lines in the post-doorbell spin (writes the debug port during a live launch; default 0)"); + +/* + * Read every covered SA entry from the live IRAM and compare with the packed + * image. Auto-calibrates the port's address unit once (word base 3*entry vs + * instruction base entry - the r1364 validation could only settle address 0). + * Returns true when every checked entry matches (or there is nothing to + * check); false on any mismatch (all-zero included). Bounded logging. + */ +static bool gsp_phys_gr_mme_iram_gate(void __iomem *bar0, u32 h_object, + u32 gen, const char *tag) +{ + static u32 gate_run_n, gate_fail_n, gate_pass_n; + static int base_sel = -1; /* -1 unresolved; 0 word; 1 instruction */ + u32 sa_ent[64], i; + u32 checked = 0, match = 0, zero = 0; + u32 bad_slot = 0xffffffffu, bad_entry = 0, checked_to_bad = 0; + u32 got0 = 0, got1 = 0, got2 = 0, exp0 = 0, exp1 = 0, exp2 = 0; + u64 sa_mask; + + if (!bar0) + return true; + sa_mask = gsp_sema_mme_sa_get(sa_ent, 64); + for (i = 0; i < 64u; i++) { + u32 entry, base, g0, g1, g2, got[3]; + + if (!(sa_mask & (1ull << i)) || !sa_ent[i]) + continue; + entry = sa_ent[i]; + if (entry * 3u + 2u >= 4096u) + continue; + g0 = gsp_phys_gr_mme_ir_pack[entry * 3u]; + g1 = gsp_phys_gr_mme_ir_pack[entry * 3u + 1u]; + g2 = gsp_phys_gr_mme_ir_pack[entry * 3u + 2u]; + /* + * Zero-gold entries are stub-covered (no captured code) or + * not part of the packed image; they carry nothing to gate. + */ + if (!g0 && !g1 && !g2) + continue; + if (base_sel < 0 && entry) { + /* + * One-time unit calibration on the first non-zero + * entry: try the word base, then the instruction + * base. Leaving it unresolved simply keeps the word + * base (the documented form). + */ + gsp_phys_mme_iram_read3(bar0, 3u * entry, got); + if (got[0] == g0 && got[1] == g1 && got[2] == g2) { + base_sel = 0; + pr_info("vgsp-gr: MME-IRAM-BASE resolved=word (3*entry) at entry=0x%x\n", + entry); + } else { + gsp_phys_mme_iram_read3(bar0, entry, got); + if (got[0] == g0 && got[1] == g1 && + got[2] == g2) { + base_sel = 1; + pr_info("vgsp-gr: MME-IRAM-BASE resolved=instruction (entry) at entry=0x%x\n", + entry); + } + } + } + base = (base_sel == 1) ? entry : 3u * entry; + gsp_phys_mme_iram_read3(bar0, base, got); + checked++; + if (got[0] == g0 && got[1] == g1 && got[2] == g2) { + match++; + continue; + } + if (!got[0] && !got[1] && !got[2]) + zero++; + if (bad_slot == 0xffffffffu) { + bad_slot = i; + bad_entry = entry; + checked_to_bad = checked; + got0 = got[0]; got1 = got[1]; got2 = got[2]; + exp0 = g0; exp1 = g1; exp2 = g2; + } + } + if (!checked || match == checked) { + if (gate_pass_n < 8u) { + gate_pass_n++; + pr_info("vgsp-gr: MME-IRAM-GATE pass %s h=0x%x gen=%u base=%d checked=%u run=%u\n", + tag, h_object, gen, base_sel, checked, + gate_run_n); + } + gate_run_n++; + return true; + } + gate_run_n++; + if (gate_fail_n < 24u) { + gate_fail_n++; + pr_warn("vgsp-gr: MME-IRAM-GATE FAIL %s h=0x%x gen=%u base=%d checked=%u match=%u zero=%u bad slot=%u(entry=0x%x @%u) got=%08x,%08x,%08x want=%08x,%08x,%08x run=%u\n", + tag, h_object, gen, base_sel, checked, match, zero, + bad_slot, bad_entry, checked_to_bad, + got0, got1, got2, exp0, exp1, exp2, gate_run_n); + } + return false; +} + +/* + * SESSION 153-CONTINUED-86: the missing SPH (shader-program-header) state. + * + * The executed beef launches fault `Graphics Exception: Shader Program + * Header 9/11/18 Error` + `DS_HWW_ESR 0x405840=0xa2040a00` (r1272): the DS + * unit validates the header of every pipeline program bound when a draw + * issues, including the SPH VERSION against the channel's programmed + * window. The Turing channel-init (the 3D driver's own + * _nv3dInitChannelTuring) writes the SPH version pair ONCE per channel: + * + * INC run @0x16a4, count 2: + * METHOD 0x16a4 (SET_SPH_VERSION) <- 4 | (4 << 16) = 0x00040004 + * METHOD 0x16a8 (CHECK_SPH_VERSION) <- 0x00040004 + * + * (fields: CURRENT 15:0, OLDEST_SUPPORTED 31:16; the value 4 is the SPH V4 + * the Turing shader images carry). The replayed context never receives it: + * the capture families (MME prelude 0x114..0x124/0x234/0x550..0x568/ + * 0x3400..0x34fc, attr prelude 0x1100..0x11fc/0x2000..0x205c) do not cover + * 0x16a4, and the captured 3D windows contain no such write (verified in the + * emitted-window decode and the raw guest-RAM stream). With the register + * unset (0/0) every fetched SPH fails its version window -> per-SPH error + * bits -> 0 GR-STATE retires and no stores. The client's own emitted init + * pair was located verbatim in the guest pushbuffer image (guest PA + * 0x8672d5b4: `200205a9 00040004 00040004`), so the launch builder emits + * exactly that run ahead of the window (right after SET_OBJECT, beside the + * MME version write). 3 words, idempotent; emitted on every replayed 3D + * window. Default 1; A/B with 0 reproduces the SPH fault. + */ +static bool gsp_phys_rmcall_gr_sph_version = true; +module_param_named(sema_gl_sph_version, gsp_phys_rmcall_gr_sph_version, + bool, 0644); +MODULE_PARM_DESC(sema_gl_sph_version, + "Emit the Turing channel-init SET_SPH_VERSION/CHECK_SPH_VERSION pair (0x16a4/0x16a8 <- 0x00040004) ahead of each replayed 3D window (default 1; A/B with 0 reproduces the Shader Program Header 9/11/18 fault)"); + +/* + * SESSION 153-CONTINUED-93: forced SPH version-window value. 0 = auto (the + * blob-derived window, see the emission site); nonzero = emit exactly this + * 32-bit value for both 0x16a4/0x16a8. Live A/B for the DS version-window + * hypothesis: the emitted 4/4 window may reject the fetched SPHs while a + * wider/older window (e.g. 0x00040003) accepts them. + */ +static u64 gsp_phys_rmcall_gr_sph_version_val; +module_param_named(sema_sph_version_val, gsp_phys_rmcall_gr_sph_version_val, + ullong, 0644); +MODULE_PARM_DESC(sema_sph_version_val, + "Forced SET_SPH_VERSION/CHECK_SPH_VERSION value (0 = auto from the loaded blobs)"); + +/* + * SESSION 153-CONTINUED-90: SET_SHADER_EXCEPTIONS (0x1528 <- 0). + * + * NVIDIA's own Turing 3D channel init (_nv3dInitChannelTuring) pushes this + * FIRST, with the comment "Disable shader exceptions. This matches OpenGL + * driver behavior." The client's captured init stream carries the write too + * (sema-pushraw `off=0x5a4 op=4 addr=0x1528 v0=0`). The replayed context + * never receives it: the emitted push (PB-FAULT-DUMP) goes SET_OBJECT -> + * SPH version pair -> MME version with no 0x1528 run, so the DS keeps + * raising "Graphics Exception: Shader Program Header 5/9/10/11 Error" on + * every draw whose fetched SPH it would tolerate with exceptions disabled - + * and the exception poisons the channel before any store retires. An + * executing r1294 launch names the consequence: the first post-exception + * method write reports `ChID 0001, Class 0000c597, Offset 0000022c, Data + * 00000800` (SET_GO_IDLE_TIMEOUT poisoned by the exception state). + * Emitted right after SET_OBJECT, exactly like the reference init order. + * Default 1; A/B with 0 restores the old behavior. + */ +static bool gsp_phys_rmcall_gr_shader_exc = true; +module_param_named(sema_gl_shader_exc, gsp_phys_rmcall_gr_shader_exc, + bool, 0644); +MODULE_PARM_DESC(sema_gl_shader_exc, + "Emit SET_SHADER_EXCEPTIONS (0x1528) <- 0 after SET_OBJECT like _nv3dInitChannelTuring (default 1)"); + +/* + * SESSION 154-1: encoding of the emitted SET_SHADER_EXCEPTIONS write. + * + * The client (and NVIDIA's _nv3dInitChannelTuring, which uses + * nvPushImmedVal) writes this method in the one-word immd13 form: header + * 0x80000000 | (value << 16) | (method >> 2), value 0, no data word - the + * captured init pushraw line is `off=0x5a4 op=4 addr=0x1528 cnt=0 v0=0`. + * The booter historically emitted the two-word INC form + * `0x2001054a 0x00000000`. Both encode the same register write, but the + * INC form is the method the FE's trapped-method register names + * (r1483 `trap=80001528`, mthd=0x1528) on every replay, while the + * client's own immd13 write is accepted (the X window's later 0x22c is + * what the trap advances to). Default 1 emits the client's exact form; + * A/B with 0 restores the legacy INC pair. + */ +static bool gsp_phys_rmcall_gr_shader_exc_immd = true; +module_param_named(sema_gl_shader_exc_immd, + gsp_phys_rmcall_gr_shader_exc_immd, bool, 0644); +MODULE_PARM_DESC(sema_gl_shader_exc_immd, + "Emit SET_SHADER_EXCEPTIONS (0x1528) in the client's immd13 form (0x8000054a, one word) instead of the legacy INC pair (default 1)"); + +/* + * SESSION 154-1: do not duplicate a window-carried SET_SHADER_EXCEPTIONS. + * The X channel's captured per-frame window carries the client's own + * 0x1528 run (the full channel-init block is inside it), so emitting the + * booter's copy again just adds an extra method ahead of the client's + * init. When this is on and the replayed window already contains a 0x1528 + * write, the emission is skipped (logged). Default 1; A/B with 0 keeps + * the unconditional emission. + */ +static bool gsp_phys_rmcall_gr_shader_exc_skip = true; +module_param_named(sema_gl_shader_exc_skip, + gsp_phys_rmcall_gr_shader_exc_skip, bool, 0644); +MODULE_PARM_DESC(sema_gl_shader_exc_skip, + "Skip the emitted SET_SHADER_EXCEPTIONS when the captured window already carries a 0x1528 write (default 1)"); + +/* + * SESSION 153-CONTINUED-91: clear DS_HWW_REPORT_MASK/_2 (0x405844/0x40584c) + * ahead of every replayed 3D window (host write + in-stream MME PRI R/W + * macro 5) so a DS SPH validation miss latches without poisoning the PRI + * block and the GSP-RM exception handler killing the channel. See the + * emission-site comment. Default 1; A/B with 0 restores the old behavior + * (every SPH miss = priv-error poison + channel wedge). + */ +static bool gsp_phys_rmcall_gr_ds_mask_clear = true; +module_param_named(sema_ds_mask_clear, gsp_phys_rmcall_gr_ds_mask_clear, + bool, 0644); +MODULE_PARM_DESC(sema_ds_mask_clear, + "Clear DS_HWW_REPORT_MASK/_2 ahead of replayed 3D windows (default 1; 0 = leave the all-report mask)"); + +/* + * SESSION 153-CONTINUED-91: emit the Turing program-region pair (the MME + * shadow scratch 0x78/0x79 = NVC397_SET_MME_PROGRAM_REGION) with the + * launch's own program family page as the shader-heap base, so the DS's + * SPH validation has a region to accept the fetched SPHs against. See the + * emission-site comment. Default 1; A/B with 0. + */ +static bool gsp_phys_rmcall_gr_sph_region = true; +module_param_named(sema_gl_sph_region, gsp_phys_rmcall_gr_sph_region, + bool, 0644); +MODULE_PARM_DESC(sema_gl_sph_region, + "Emit the Volta+ program-region pair (SET_MME_SHADOW_SCRATCH 0x78/0x79 <- program family page) ahead of replayed 3D windows (default 1)"); + +/* + * Runtime-selectable program-region base. 0 = auto (the launch's own + * program family page, the guest-VA heap base the real driver would use); + * a nonzero value forces that base - the engine-side shadow-VA space needs + * its own experiment (the SPH_INFO snapshot shows the DS fetched through + * shadow VAs 0x620900/0x620200/0x620000, so a guest-VA base may be the + * mismatch). Writable at runtime for a live A/B. + */ +static u64 gsp_phys_rmcall_gr_sph_region_base; +module_param_named(sema_sph_region_base, gsp_phys_rmcall_gr_sph_region_base, + ullong, 0644); +MODULE_PARM_DESC(sema_sph_region_base, + "Program-region base override (0 = launch program family page; nonzero = explicit engine-side base)"); + +/* + * SESSION 154-1: prefer the bound program's own engine-visible VA page + * as the program region base. The reference driver sets the region to + * the program's GPU address (nv3dGetProgramGpuAddress); the replay's + * program is fetched at the relocated shadow VA, so the region must be + * that VA's 4K page. The forced sema_sph_region_base (0x1500000 in the + * campaign args) is the shadow pool's base: the X program's shadow + * 0x15c0000 lands inside it, but the beef program's 0x19a0000 does not, + * and the DS rejects the fetched SPH (r1484 `GR-STATE sph` stages 1/2/5 + * = 0x19a0f00/0x19a0200/0x19a0000 together with `ds=86040646`). With + * this on, the builder scans the already-relocated window for the first + * SET_PIPELINE_PROGRAM_ADDRESS pair and uses that VA's page as the base + * (beef: 0x19a0000), falling back to the override/auto only when the + * window carries no program bind. Default 1; A/B with 0 uses the old + * forced/auto base. + */ +static bool gsp_phys_rmcall_gr_sph_region_bind = true; +module_param_named(sema_gl_sph_region_bind, + gsp_phys_rmcall_gr_sph_region_bind, bool, 0644); +MODULE_PARM_DESC(sema_gl_sph_region_bind, + "Use the first bound program's relocated shadow page as the program-region base (default 1; 0 = forced/auto base)"); + +/* + * SESSION 153-CONTINUED-101: MME fixed-constant-buffer scratch seed + * (SET_MME_SHADOW_SCRATCH(0x18), method 0x3460). + * + * mmeCodeOGL_C597_SETUP_FIXED_CONST (macro slot 0) reads shadow scratch + * 0x18/0xd18 via STATE and unconditionally emits its high/low halves as + * SET_CONSTANT_BUFFER_SELECTOR_B/C (the compiled C597 form carries NO + * bug2548370 branch: the config table says no972630). With the replay + * context's scratch at 0 the macro emits selector B/C = 0 and the draw's + * constant fetch has no address. The client's own write (gf100Init- + * ConstantBuffers: (u32)(fixed CB VA >> 8)) is a once-per-channel event + * the ladder never receives, so the builder seeds it ahead of the + * window. Value source: sema_gl_mme_seed_val when nonzero, else the + * client's own captured write (gsp_sema_mme_scratch18_get). With + * sema_gl_mme_seed_xlat (default 1) the page is translated through the + * launch's own page set / shadow mapping first (the same relocator the + * window operand pairs use) so the emitted selector resolves in the + * engine's address space; on MISS the raw value is emitted (the A/B + * control). Default off; live-tunable. + */ +static bool gsp_phys_rmcall_gr_mme_seed; +module_param_named(sema_gl_mme_seed, gsp_phys_rmcall_gr_mme_seed, bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_seed, + "Seed MME shadow scratch 0x18 (method 0x3460) with the fixed-CB base>>8 ahead of replayed 3D windows (default 0)"); +static u32 gsp_phys_rmcall_gr_mme_seed_val; +module_param_named(sema_gl_mme_seed_val, gsp_phys_rmcall_gr_mme_seed_val, + uint, 0644); +MODULE_PARM_DESC(sema_gl_mme_seed_val, + "Explicit scratch 0x18 value (fixed CB VA>>8); 0 = use the client's captured 0x3460 write"); +static bool gsp_phys_rmcall_gr_mme_seed_xlat = true; +module_param_named(sema_gl_mme_seed_xlat, gsp_phys_rmcall_gr_mme_seed_xlat, + bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_seed_xlat, + "Translate the seed value's page to the engine-side shadow/identity VA before emitting (default 1; 0 = raw)"); + +/* + * SESSION 153-CONTINUED-105: golden MME runtime state seed, STATE + * 0xD26..0xD2A (SET_MME_SHADOW_SCRATCH 0x26..0x2a, methods 0x3498/0x349c/ + * 0x34a0/0x34a4/0x34a8). The bug2548370 loadConstantBuffer macro (slot 57 / + * macro address 0x39) branches at its entry on these five words: + * + * 0xD26 BUG2548370_PAGE_ALLOCATION_COUNT + * 0xD27 BUG2548370_ZERO_VIEW_OFFSET + * 0xD28 BUG2548370_LAST_UPDATED_CONSTANT_BUFFER_SELECTOR_B_C + * 0xD29 BUG2548370_LOAD_CONSTANT_BUFFER_OFFSET + * 0xD2A BUG2548370_LAST_UPDATED_LOAD_CONSTANT_BUFFER_OFFSET + * + * (verified against the driver's generated macro source: the entry reads + * exactly 0xd28/0xd19/0xd29/0xd2a/0xd26/0xd27 and the WAR path compares + * 0xd26 against 30 and 0xd27 against zero). The client's own prelude + * writes only 0xd2a <- 0x80000000 / 0xd29 <- 0 ("reset tracking after each + * draw") and the per-channel 0x3464/0x3460 writes; 0xd26/0xd27/0xd28 carry + * over from the client's own context and never appear in the captured + * stream. The replayed context therefore reads whatever the replay left + * there. Emit the five writes ahead of every replayed 3D window (values + * from sema_mme_state_d26..d2a; all-zero default = the zero-valid form). + * Default off; live-tunable A/B. + */ +static bool gsp_phys_rmcall_gr_mme_state_seed; +module_param_named(sema_gl_mme_state_seed, gsp_phys_rmcall_gr_mme_state_seed, + bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_state_seed, + "Seed MME runtime state 0xD26..0xD2A (methods 0x3498..0x34a8) ahead of replayed 3D windows (default 0)"); +static u32 gsp_phys_rmcall_gr_mme_state_d26; +module_param_named(sema_mme_state_d26, gsp_phys_rmcall_gr_mme_state_d26, + uint, 0644); +MODULE_PARM_DESC(sema_mme_state_d26, "Seed value for STATE 0xD26 PAGE_ALLOCATION_COUNT (method 0x3498)"); +static u32 gsp_phys_rmcall_gr_mme_state_d27; +module_param_named(sema_mme_state_d27, gsp_phys_rmcall_gr_mme_state_d27, + uint, 0644); +MODULE_PARM_DESC(sema_mme_state_d27, "Seed value for STATE 0xD27 ZERO_VIEW_OFFSET (method 0x349c)"); +static u32 gsp_phys_rmcall_gr_mme_state_d28; +module_param_named(sema_mme_state_d28, gsp_phys_rmcall_gr_mme_state_d28, + uint, 0644); +MODULE_PARM_DESC(sema_mme_state_d28, "Seed value for STATE 0xD28 LAST_UPDATED_CB_SELECTOR_B_C (method 0x34a0)"); +static u32 gsp_phys_rmcall_gr_mme_state_d29; +module_param_named(sema_mme_state_d29, gsp_phys_rmcall_gr_mme_state_d29, + uint, 0644); +MODULE_PARM_DESC(sema_mme_state_d29, "Seed value for STATE 0xD29 LOAD_CONSTANT_BUFFER_OFFSET (method 0x34a4)"); +static u32 gsp_phys_rmcall_gr_mme_state_d2a; +module_param_named(sema_mme_state_d2a, gsp_phys_rmcall_gr_mme_state_d2a, + uint, 0644); +MODULE_PARM_DESC(sema_mme_state_d2a, "Seed value for STATE 0xD2A LAST_UPDATED_LOAD_CONSTANT_BUFFER_OFFSET (method 0x34a8)"); +static u32 gsp_phys_rmcall_gr_mme_state_d19; +module_param_named(sema_mme_state_d19, gsp_phys_rmcall_gr_mme_state_d19, + uint, 0644); +MODULE_PARM_DESC(sema_mme_state_d19, "Seed value for STATE 0xD19 CONSTANT_BUFFER_SELECTOR_B_C (method 0x3464, CB VA>>8); 0 = use the client's captured 0x3464 write"); +static u32 gsp_phys_rmcall_gr_mme_state_d24; +module_param_named(sema_mme_state_d24, gsp_phys_rmcall_gr_mme_state_d24, + uint, 0644); +MODULE_PARM_DESC(sema_mme_state_d24, "Seed value for STATE 0xD24 slow-path input (method 0x3490); 0 = use the client's captured 0x3490 write"); +static u32 gsp_phys_rmcall_gr_mme_state_d25; +module_param_named(sema_mme_state_d25, gsp_phys_rmcall_gr_mme_state_d25, + uint, 0644); +MODULE_PARM_DESC(sema_mme_state_d25, "Seed value for STATE 0xD25 BUG2548370_CONSTANT_BUFFER_SELECTOR_A / CB size (method 0x3494); 0 = use the client's captured 0x3494 write"); + +/* + * SESSION 155-CONT-5: engine-visible constant-buffer selector seed. + * + * The STATE seed above only writes MME *shadow scratch* (0x3460/0x3464/ + * 0x3498..0x34a8). The bug2548370 loadConstantBuffer macro (slot 57) + * only *reads* that scratch to track page changes; it never emits + * SET_CONSTANT_BUFFER_SELECTOR_A/B/C itself. The selector is programmed + * by SETUP_FIXED_CONST (slot 0) from scratch 0x18. When a replayed + * window carries a CALL macro 57 run but no CALL macro 0 (the driver + * legitimately elides it while the selector is already bound, e.g. the + * r1541 68-word tail), the engine executes LOAD_CONSTANT_BUFFER with no + * valid selector and the C597 class raises + * "Offset 00002390, Data bf800000, ErrorCode 00000003" (the r1456/ + * r1485/r1541 poison). With this knob the builder emits the same + * 4-method group macro 0 would emit - SET_CONSTANT_BUFFER_SELECTOR_A + * (size = MME_VPC_CONSTANT_BUFFER_SIZE 0x7000), B/C (the fixed-CB + * base>>8, translated through the launch's page set exactly like the + * scratch seed) and LOAD_CONSTANT_BUFFER_OFFSET (sema_mme_state_d29) - + * ahead of every replayed 3D window. Idempotent when macro 0 is also in + * the stream (it re-programs the same selector). Default off; requires + * sema_gl_mme_state_seed; legacy bytes unchanged when off. + */ +static bool gsp_phys_rmcall_gr_mme_selector_seed; +module_param_named(sema_gl_mme_selector_seed, + gsp_phys_rmcall_gr_mme_selector_seed, bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_selector_seed, + "Emit SET_CONSTANT_BUFFER_SELECTOR_A/B/C + LOAD_CONSTANT_BUFFER_OFFSET (0x2380..0x238c) from the fixed-CB scratch ahead of replayed 3D windows (default 0; requires sema_gl_mme_state_seed)"); + +/* + * SESSION 153-CONTINUED-105: split the packed MME prelude emission across + * launches. The CONT-99b pack rebuilds the partitioned IR and emits it as + * ONE contiguous ONEINC run; for the current 55-slot capture that stream is + * 2 + maxa + 3*npair = ~2966 words (maxa = 2799, npair = 55), over the + * 2048-word per-launch block cap, so the pack silently falls back to the + * raw capture-order drain (which is only correct when the capture's runs + * do not overlap). With this knob the packed stream is drained with a + * generation-keyed cursor: the SA table goes out once, the IR goes out as + * whole-instruction chunks in consecutive launches (each chunk = one + * ONEINC 0x114 run with its own instruction-address pointer), and the + * prelude latch is set only when the last chunk is emitted. Calls stay + * held across the chunks (hold_calls + the FE-idle witness). Default 0. + */ +static bool gsp_phys_rmcall_gr_mme_pack_split = true; +module_param_named(sema_gl_mme_pack_split, gsp_phys_rmcall_gr_mme_pack_split, + bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_pack_split, + "Split the packed MME prelude (SA table + IR chunks) across launches when it exceeds the block cap (default 1; 0 = all-or-nothing CONT-99b pack)"); + +/* + * SESSION 153-CONTINUED-93: additionally emit the legacy program-region + * pair (0x1608/0x160c) beside the MME shadow pair (see the emission site). + * Default 0 (A/B; the DS snapshot registers share the legacy pair's names). + */ +static bool gsp_phys_rmcall_gr_sph_region_legacy; +module_param_named(sema_sph_region_legacy, gsp_phys_rmcall_gr_sph_region_legacy, + bool, 0644); +MODULE_PARM_DESC(sema_sph_region_legacy, + "Also emit the legacy SET_PROGRAM_REGION_A/B (0x1608/0x160c) pair with the same base (default 0)"); + +/* + * SESSION 153-CONTINUED-87: the missing TESS_CONTROL/PIXEL program binds. + * The client's real channel state binds every pipeline stage once per + * context; the captured init-push image (guest RAM scrape) carries + * `20020825 [0, 04400200]` (j=2 TESS_CONTROL) and + * `20020855 [0, 04400000]` (j=5 PIXEL) beside the SPH version pair - + * but the captured per-frame window carries only the j=1 (VTG) pair + * (0x4400900/0x4400300). The DS validates the SPH of every pipeline + * stage the draw activates, so the unbound stages latch their header + * errors (DS_HWW_ESR 0xa2040a00 = SPH9|SPH11|SPH18) while the V3 blob + * passes wherever a stage IS bound. Emit both binds ahead of the + * window at the window's own program-family page (all four VAs share + * one 4K page: 0x4400000 j=5, 0x4400200 j=2, 0x4400300/0x4400900 j=1), + * pushed through the same relocation pass as the window - a MISS skips + * the bind entirely (never a raw client VA). Beef client channels only + * (the family layout is the client's). Default 1; A/B with 0. + */ +static bool gsp_phys_rmcall_gr_sph_bind = true; +module_param_named(sema_gl_sph_bind, gsp_phys_rmcall_gr_sph_bind, bool, 0644); +MODULE_PARM_DESC(sema_gl_sph_bind, + "Emit the client's missing TESS_CONTROL (j=2) and PIXEL (j=5) pipeline program binds ahead of replayed 3D windows (program-family page of the window's prog VA; default 1)"); + +static u32 gsp_phys_rmcall_gr_sph_bind_mask; /* bit j = synth slot j */ +module_param_named(sema_sph_bind_mask, gsp_phys_rmcall_gr_sph_bind_mask, + uint, 0644); +MODULE_PARM_DESC(sema_sph_bind_mask, + "Synth SPH-BIND slot mask (bit j = slot j; 0x20 = PS only, 0x01/0x10 = VSA/GS VTG image, 0x25 = VSA+PS) used when sema_gl_sph_bind=0"); + +bool gsp_phys_rmcall_gr_sph_bind_value(void) +{ + return READ_ONCE(gsp_phys_rmcall_gr_sph_bind); +} + +/* + * SESSION 155 CONT 25: the synth slot mask must be readable by the sema + * service so it authors exactly the stage images the booter binds. With + * sema_gl_sph_bind=0 + sema_sph_bind_mask=0x35 the booter emits j0/j2/j5, + * but the service's PS/TCS authoring loop was gated on the all-slots bool, + * so the PS image could be left stale/zero at draw time (r1575). + */ +u32 gsp_phys_rmcall_gr_sph_bind_mask_value(void) +{ + return READ_ONCE(gsp_phys_rmcall_gr_sph_bind_mask); +} + +/* + * SESSION 154c (viewport wall): the replayed 3D window carries no + * viewport family state - the colour target, raster enable, shader + * binds, attributes, index buffer and draws are present, but the + * 0x0a00 scale/offset block, the 0x1924/0x192c/0x193c pixel and clip + * controls and the 0x0c00 clip rect are not (the client's captured + * push never writes them; the slot-31 viewport macro exists in the + * capture but is never called). In the fresh replayed context a + * missing viewport clips all geometry, so the engine executes the + * DRAW but stores no fragments. This knob synthesizes the missing + * state as a preamble ahead of each replayed 3D window. + * + * Words verified against the window decoder (op = hdr[31:29], + * count = hdr[28:16], method = hdr[12:0] << 2): + * 0x20060280 INC cnt=6 method 0x0a00 -> SET_VIEWPORT_SCALE_X/ + * Y/Z(0) and SET_VIEWPORT_OFFSET_X/Y/Z(0) + * 0x80000649 IMM data=0 method 0x1924 -> SET_VIEWPORT_PIXEL + * 0x8001064b IMM data=1 method 0x192c -> SET_VIEWPORT_SCALE_OFFSET + * 0x8001064f IMM data=1 method 0x193c -> SET_VIEWPORT_CLIP_CONTROL + * 0x20040300 INC cnt=4 method 0x0c00 -> SET_VIEWPORT_CLIP_ + * HORIZONTAL/VERTICAL/MIN_Z/MAX_Z(0) + * Mode 1 = scale/offset block; mode 2 adds the immediate controls; + * mode 3 adds the 1280x720 clip rect. Mode 4 is the same words with + * the clip-rect encoding verified against the class definitions + * (X0/Y0 in bits 15:0, WIDTH/HEIGHT in bits 31:16, integer pixel + * units) and the producer's (size << 16) | origin packing, so a + * 1280x720 target at a top-left origin emits 0x05000000, + * 0x02d00000, 0.0f (0x00000000) and 1.0f (0x3f800000); min_z/max_z + * are IEEE-754 floats. None of the words is an address operand, so + * the sequence is emitted verbatim (no relocation pass). Default 0 + * keeps the legacy window bytes. + * + * Mode 5 is the driver's own viewport macro path: CALL_MME_MACRO(33) + * (0x3908) with (x,y,w,h) = (0,0,1280,720) followed by the macro's + * appended state. Mode 6 is the raw equivalent without the CALL + * (scale/offset included, scaleY = +360). Header check against the + * decoder above: 0xa0040e42 = op 5 ONE_INC, cnt 4, method 0x3908 + * (four data words follow); 0x8001064b = op 4 IMM, data 1 in the + * count field, method 0x192c (single word, consumes no data); + * 0x2001064f + 0x00021800 = op 1 INC, cnt 1, method 0x193c (one + * data word). + */ +#define GSP_PHYS_GR_VP_PRE_WORDS 15u +static const u32 gsp_phys_gr_vp_pre[GSP_PHYS_GR_VP_PRE_WORDS] = { + /* mode 1: viewport scale/offset */ + 0x20060280u, 0x44200000u, 0xc3b40000u, 0x3f000000u, + 0x44200000u, 0x43b40000u, 0x3f000000u, + /* mode 2: pixel centre, scale/offset enable, clip control */ + 0x80000649u, 0x8001064bu, 0x8001064fu, + /* mode 3: clip rect (x0=0, w=1280, y0=0, h=720, z 0..1) */ + 0x20040300u, 0x05000000u, 0x02d00000u, 0x00000000u, + 0x3f800000u, +}; + +#define GSP_PHYS_GR_VP_PRE_M5_WORDS 17u +static const u32 gsp_phys_gr_vp_pre_m5[GSP_PHYS_GR_VP_PRE_M5_WORDS] = { + 0xa0040e42u, 0x00000000u, 0x00000000u, 0x00000500u, 0x000002d0u, + 0x20010282u, 0x3f000000u, 0x20010285u, 0x3f000000u, + 0x20020302u, 0x00000000u, 0x3f800000u, + 0x20010649u, 0x00000000u, 0x8001064bu, + 0x2001064fu, 0x00021800u, +}; + +#define GSP_PHYS_GR_VP_PRE_M6_WORDS 19u +static const u32 gsp_phys_gr_vp_pre_m6[GSP_PHYS_GR_VP_PRE_M6_WORDS] = { + 0x20020280u, 0x44200000u, 0x43b40000u, + 0x20040282u, 0x3f000000u, 0x44200000u, 0x43b40000u, 0x3f000000u, + 0x20020300u, 0x05000000u, 0x02d00000u, + 0x20020302u, 0x00000000u, 0x3f800000u, + 0x20010649u, 0x00000000u, 0x8001064bu, + 0x2001064fu, 0x00021800u, +}; + +/* + * SESSION 154 r1506: mode 7 = the CLIENT'S OWN window viewport (800x600 at + * the root origin) instead of the captured RT's 1280x720 screen viewport. + * The glxgears window is 800x600+0+0 on the 1280x720 root; its own GL + * viewport is (0,0,800,600) -> scale (400,300), offsets (400,300), clip + * 800x600. Mode 6 maps the geometry across the whole X screen; if the + * client's real draw args are window-relative, mode 7 is the faithful + * transform. Same encoding as mode 6 (scale/offset block, then clip, + * z range, pixel centre, scale-offset enable, clip control). + */ +#define GSP_PHYS_GR_VP_PRE_M7_WORDS 19u +static const u32 gsp_phys_gr_vp_pre_m7[GSP_PHYS_GR_VP_PRE_M7_WORDS] = { + 0x20020280u, 0x43c80000u, 0x43960000u, /* scale 400, 300 */ + 0x20040282u, 0x3f000000u, 0x43c80000u, 0x43960000u, 0x3f000000u, + 0x20020300u, 0x03200000u, 0x02580000u, /* clip 800x600 at 0,0 */ + 0x20020302u, 0x00000000u, 0x3f800000u, + 0x20010649u, 0x00000000u, 0x8001064bu, + 0x2001064fu, 0x00021800u, +}; + +static u32 gsp_phys_gr_vp_pre_len(int mode) +{ + switch (mode) { + case 1: + return 7u; + case 2: + return 10u; + case 3: + case 4: + return GSP_PHYS_GR_VP_PRE_WORDS; + case 5: + return GSP_PHYS_GR_VP_PRE_M5_WORDS; + case 6: + return GSP_PHYS_GR_VP_PRE_M6_WORDS; + case 7: + return GSP_PHYS_GR_VP_PRE_M7_WORDS; + default: + return 0u; + } +} + +static const u32 *gsp_phys_gr_vp_pre_src(int mode) +{ + switch (mode) { + case 5: + return gsp_phys_gr_vp_pre_m5; + case 6: + return gsp_phys_gr_vp_pre_m6; + case 7: + return gsp_phys_gr_vp_pre_m7; + default: + return gsp_phys_gr_vp_pre; + } +} + +/* + * Insertion word offset for the viewport prelude inside an emitted 3D + * window. The walk is the same run grammar as the MME-WINRUN census and + * the CALL drop pass (op = hdr[31:29], count = hdr[28:16] for op 1/3/5, + * run method = hdr[11:0] << 2; multi-word INC/NON_INCR runs cover + * m + 4k, IMM/ONE_INCR entries cover only their own method). Runs are + * never split. pos 1 stops at the first run touching the CALL_MME + * family 0x3800..0x39fc (after the window's state binds, ahead of its + * macro calls); pos 2 stops at the first run touching 0x3910. A window + * without the requested run returns n so the caller can skip it + * (non-draw window). + */ +static u32 gsp_phys_gr_vp_insert_off(const u32 *win, u32 n, int pos) +{ + u32 r = 0; + + if (pos != 1 && pos != 2) + return n; + while (r < n) { + u32 h = win[r]; + u32 op = (h >> 29) & 7u; + u32 cnt = (h >> 16) & 0x1fffu; + u32 span = 1u, k; + + if ((op == 1u || op == 3u || op == 5u) && cnt && + r + 1u + cnt <= n) + span = 1u + cnt; + for (k = 0; k < span; k++) { + u32 m = ((h & 0xfffu) << 2) + 4u * k; + + if (k && op != 1u && op != 3u) + break; + if (pos == 1) { + if (m >= 0x3800u && m <= 0x39fcu) + return r; + } else if (m == 0x3910u) { + return r; + } + } + r += span; + } + return n; +} + +/* + * First-run offset of a method in a window, using the same run grammar + * as the scanners above (op 1/3/5 carry 1+cnt data words; op 5 writes + * only its header method, op 4 IMM carries data in the count field and + * is a single word). Returns n when no run touches the method. + */ +static u32 gsp_phys_gr_method_run_off(const u32 *win, u32 n, u32 method) +{ + u32 r = 0; + + while (r < n) { + u32 h = win[r]; + u32 op = (h >> 29) & 7u; + u32 cnt = (h >> 16) & 0x1fffu; + u32 span = 1u, k; + + if ((op == 1u || op == 3u || op == 5u) && cnt && + r + 1u + cnt <= n) + span = 1u + cnt; + for (k = 0; k < span; k++) { + u32 m = ((h & 0xfffu) << 2) + 4u * k; + + if (k && op != 1u && op != 3u) + break; + if (m == method) + return r; + } + r += span; + } + return n; +} + +/* + * The client's pixel shader has MRT_ENABLE=0 and writes RT0 (0x800), + * while its init push selects CT target0=0 (0x121c = 0x0fac6881) and + * the replayed draw window binds only RT1 (0x840) - so the colour + * write has no enabled target. When sema_gl_rt0_bind=1 and the window + * does not write 0x800 itself, insert the missing RT0 bind (same + * 1280x720 X8R8G8B8 target at B=0x0a000000) plus a count-2 CT select + * (target0=0, target1=1) ahead of the window's first 0x840 run, and + * only when the window carries a CALL run (0x3800..0x39fc); a non-draw + * window is left unchanged. + * Constant words, no relocation. Default 0 is a strict no-op. + */ +#define GSP_PHYS_GR_RT0_BIND_WORDS 9u +static const u32 gsp_phys_gr_rt0_bind[GSP_PHYS_GR_RT0_BIND_WORDS] = { + 0x20060200u, 0x00000000u, 0x0a000000u, 0x00000500u, 0x000002d0u, + 0x000000e6u, 0x00000040u, 0x20010487u, 0x0fac6882u, +}; + +/* + * SESSION 155-CONT-6: the synthesized RT0 bind above writes + * SET_COLOR_TARGET_A..MEMORY (0x800..0x814) but not + * SET_COLOR_TARGET_THIRD_DIMENSION (0x818). The client's own full + * colour-target group is the 8-method form (gf100pushbuf.c / the + * gf100commandList.c 6-word push from WIDTH through ARRAY_PITCH, which + * includes THIRD_DIMENSION and ARRAY_PITCH); those load-only windows are + * not replayed, so the channel's 0x818 stays 0 while 0x814 carries + * THIRD_DIMENSION_CONTROL = DEFINES_ARRAY_SIZE. On the first native + * DRAW_INDEX_BUFFER (0x17e0) the C597 class then reports + * "ErrorCode 000000d2" = + * NV_PGRAPH_CLASS_ERROR_CODE_ERROR_ILLEGAL_COLOR_TARGET0_LAYER_OFFSET_VS_THIRD_DIMENSION + * (tu104/dev_graphics_nobundle.h:563): layer offset 0 is not below the + * array size 0. The driver's layer-mode ONE path writes depth 1 with + * ARRAY_SIZE control (gf100commandList.c:1232-1235), so widen the + * inserted group to the 8-method form with THIRD_DIMENSION(0)=1 and + * ARRAY_PITCH(0)=0. Default off; legacy bytes and count unchanged. + */ +#define GSP_PHYS_GR_RT0_BIND_EXT_WORDS 11u +static const u32 gsp_phys_gr_rt0_bind_ext[GSP_PHYS_GR_RT0_BIND_EXT_WORDS] = { + 0x20080200u, 0x00000000u, 0x0a000000u, 0x00000500u, 0x000002d0u, + 0x000000e6u, 0x00000040u, 0x00000001u, 0x00000000u, + 0x20010487u, 0x0fac6882u, +}; + +static bool gsp_phys_rmcall_gr_rt0_third_dim; +module_param_named(sema_gl_rt0_third_dim, + gsp_phys_rmcall_gr_rt0_third_dim, bool, 0644); +MODULE_PARM_DESC(sema_gl_rt0_third_dim, + "Extend the synthesized RT0 bind with SET_COLOR_TARGET_THIRD_DIMENSION(0)=1 and ARRAY_PITCH(0)=0 (0x818/0x81c) so DRAW_INDEX_BUFFER does not raise class error 0xd2 (default 0)"); + + +static int gsp_phys_rmcall_gr_rt0_bind; +module_param_named(sema_gl_rt0_bind, gsp_phys_rmcall_gr_rt0_bind, int, + 0644); +MODULE_PARM_DESC(sema_gl_rt0_bind, + "Insert the missing RT0 colour-target bind and count-2 SET_CT_SELECT ahead of a replayed window's first 0x840 (RT1) run when the window does not bind 0x800 and carries a CALL run (non-draw windows are left unchanged; default 0 = off)"); + +static int gsp_phys_rmcall_gr_vp_prelude; +module_param_named(sema_gl_vp_prelude, gsp_phys_rmcall_gr_vp_prelude, + int, 0644); +MODULE_PARM_DESC(sema_gl_vp_prelude, + "Synthesize the missing viewport state ahead of replayed 3D windows (0 = off (default); 1 = scale/offset; 2 = + pixel centre and clip control; 3 = + clip rect; 4 = same clip rect with the verified pixel-unit encoding; 5 = driver viewport macro CALL(33) 0x3908 + appended state; 6 = raw mode-5 state without the CALL; 7 = mode-6 encoding for the client window viewport 800x600 at 0,0)"); + +static int gsp_phys_rmcall_gr_vp_prelude_pos = 1; +module_param_named(sema_gl_vp_prelude_pos, gsp_phys_rmcall_gr_vp_prelude_pos, + int, 0644); +MODULE_PARM_DESC(sema_gl_vp_prelude_pos, + "Insertion point of the synthesized viewport prelude in a replayed 3D window (0 = window head (legacy); 1 = before the first CALL run 0x3800..0x39fc, after the state binds (default); 2 = before the first 0x3910 run); pos 1/2 leave a window without the requested run unchanged (non-draw window), only 0 inserts without a CALL run)"); + +/* + * r1512 diagnostic export: the sema service's draw-coverage receipt needs + * the viewport state the replay will inject for the current launch. Pure + * read of the module parameter + the literal table (no side effects). + */ +void gsp_phys_gr_vp_state_get(int *mode, u32 *words, u32 *n) +{ + int m = READ_ONCE(gsp_phys_rmcall_gr_vp_prelude); + u32 len = gsp_phys_gr_vp_pre_len(m); + const u32 *src = gsp_phys_gr_vp_pre_src(m); + + if (mode) + *mode = m; + if (n) + *n = len; + if (words && len && src) + memcpy(words, src, (size_t)len * 4u); +} + +/* + * CONT-119 (r1450): the BINDING words of the synthesized stage-2/5 + * REGISTER_COUNT runs. CONT-90 emitted each as a 2-word + * REGISTER_COUNT/BINDING run (`20020823 0000001a 00000000` for j=2, the + * j=5 twin on 0x214c) so the DS SPH validation saw a full stage-1-shaped + * pair. The FE walking our replayed window then rejects the synthesized + * `0x2090 <- 0` write with Class Error EC4 - it is the only 0x2090 writer + * in the stream (the client never emits a stage-2/5 BINDING), and it + * wedges the slot-57 macro at mpc 899/900. Default 0 drops the BINDING + * data words and emits only the 1-word REG_COUNT runs + * (`20010823 0000001a`, `20010853 0000001a`); 1 keeps the legacy CONT-90 + * bytes for a same-boot A/B. + */ +static bool gsp_phys_rmcall_gr_sph_bind_binding; +module_param_named(sema_gl_mme_sph_bind_binding, + gsp_phys_rmcall_gr_sph_bind_binding, bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_sph_bind_binding, + "1 = legacy emit BINDING(2)/(5)=0 which the FE rejects EC4 @0x2090 when walking the window; 0 = drop the BINDING writes, keep the REG_COUNT runs (default 0)"); + +/* + * SESSION 153-CONTINUED-92: leave program-family operands at their ORIGINAL + * guest VAs when the operand's page is resolved and identity-mappable. + * + * The DS I-fetch reaches the delivered images through the shadow VAs the + * window pass picks (r1298-r1306: stages fetched at 0x620900/0x620200/ + * 0x620000) and rejects them with the bytes provably correct at those VAs + * (INLINE-RB rb=020a0481; canary proves the fetch follows the mapped page's + * content). The one variable left is the address space the DS validates in: + * the client programmed its own guest-VA program heap, and the shadow VAs + * (0x2xxxxx..0x6xxxxx pools) sit outside any region the client's state + * describes - a region/space mismatch rejects all bound stages regardless + * of content (rr=0/0, ds=a400e20 at every retire). With this knob the + * window pass first tries the Tier-1 identity mapper on the operand's own + * guest VA (launch page set first, the sema resolver else): when the guest + * VA is mapped to the resolved HPA, the operand is left verbatim, so the + * DS fetches and validates the SPH at the same VA space the client's heap + * uses. A page without an identity mapping keeps the shadow rewrite exactly + * as before (fail-open), and the identity-kept pairs still count toward the + * pass's relocated census so the SPH-BIND guard (orel == nrun) is stable. + * r1307 RESULT (experiment executed, negative): with the knob on, the + * beef lane's program pairs stayed at their guest VAs (0x4350900/0x4350f00/ + * 0x4350200/0x4350000, PROG-GUEST-VA receipts) and the engine faulted + * `MMU Fault: ENGINE GRAPHICS HUBCLIENT_PD faulted @ 0x0_04350000. FAULT_PTE + * ACCESS_TYPE_VIRT_READ` within 2.5 s of every such launch - the Tier-1 + * identity map installs (the PDE window is walked: FAULT_PTE, not + * FAULT_PDE) yet the engine still cannot fetch at the client's own VA. + * This reconfirms the r929 v2 lesson: guest VAs are not engine-fetchable on + * this replay channel, so the "validate in guest VA space" hypothesis is + * not testable by operand rewriting (the fetch dies before any DS check). + * Default 0 (baseline shadow rewrite); 1 reproduces the FAULT_PTE experiment. + */ +static bool gsp_phys_rmcall_gr_prog_guest_va; +module_param_named(sema_gl_prog_guest_va, gsp_phys_rmcall_gr_prog_guest_va, + bool, 0644); +MODULE_PARM_DESC(sema_gl_prog_guest_va, + "Leave program-family (0x2014-family) operands at their original guest VAs when identity-mappable (default 0; 1 = the r1307 FAULT_PTE experiment - guest VAs are not engine-fetchable, do not keep)"); + +/* + * SESSION 153-CONTINUED-84: SET_MME_SHADOW_RAM_CONTROL mode before the + * window. The vendor comment (mesh shader bind path) is explicit: + * "MME filtering swallows them all if old and new state values are the + * same" - METHOD_TRACK_WITH_FILTER (1) suppresses every method write whose + * value equals the shadowed one. Our replay repeats the same window every + * launch, so after the first replay every redundant write - including the + * CALL_MME_MACRO/CALL_MME_DATA parameter writes - matches the shadow and + * is swallowed; the macro call never reaches the MME (r1236-r1250: calls + * in the emitted window, all MME registers idle, some launches park the + * front-end). The mode-bearing writes live in the captured PRELUDE + * (0x124 x4 in the client's setup push), which the win_patch window sweep + * does not touch; emit METHOD_TRACK (0 = filtering OFF, semantics + * unchanged: the write simply applies) right before the SA re-arm and the + * window. Value = sema_gl_mme_track (0 = TRACK; 1/2/3 = other modes; -1 = + * do not emit). + */ +static int gsp_phys_rmcall_gr_mme_track; +module_param_named(sema_gl_mme_track, gsp_phys_rmcall_gr_mme_track, + int, 0644); +MODULE_PARM_DESC(sema_gl_mme_track, + "SET_MME_SHADOW_RAM_CONTROL mode emitted before each replayed window (0=TRACK filtering off (default); 1=TRACK_WITH_FILTER; 2=PASSTHROUGH; 3=REPLAY; -1=off)"); + +/* + * SESSION 153-CONTINUED-85: A/B knob for the SA re-arm (the r1237 fix). + * Before CONT-84 the calls in the replayed window resolved to the SA state + * the prelude left behind (its last captured SA write re-clobbers the slot; + * r1231: SA[34]=0 lands last, every draw call resolved to macro 0 and + * emitted nothing - and the r1228-phase launches retired). With the re-arm + * the calls resolve to the captured IR entries; every call-bearing launch + * since (r1245/r1250/r1265/r1270) parks or poisons the front-end. This knob + * restores the pre-arm resolution in one boot so the two states can be A/B'd + * without a rebuild (0 = do not re-point; default 1 = re-arm). + */ +static bool gsp_phys_rmcall_gr_mme_sa_rearm = true; +module_param_named(sema_gl_mme_sa_rearm, gsp_phys_rmcall_gr_mme_sa_rearm, + bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_sa_rearm, + "Re-emit LOAD_MME_START_ADDRESS_RAM pairs before each replayed window so CALL_MME_MACRO resolves to the captured SA entries (default 1; 0 = pre-arm resolution, A/B)"); + +/* + * The same stubbed-slot predicate the launch builder applies (see the + * sa_stub emit decision): a slot is stubbed when stubbing is on and either + * the all-slots knob is set, the capture holds no SA entry, or the IR + * instruction the entry names never entered the replay. A kept slot + * (sema_gl_mme_keep_mask) is never stubbed - unless the capture cannot + * serve it at all (keep_clamp, see the CONT-113 param comment). + */ +static bool gsp_phys_gr_slot_kept(u32 j, u64 sa_mask, const u32 *sa_ent) +{ + if (!(READ_ONCE(gsp_phys_rmcall_gr_keep_mask) & (1ull << j))) + return false; + if (READ_ONCE(gsp_phys_rmcall_gr_nop_drop) & (1ull << j)) + return false; + /* + * r1448 STRATEGY 3.3: golden-only means the kept-slot exemption + * cannot resurrect a body outside the golden corpus (fail-closed). + */ + if (gsp_phys_gr_slot_golden_blank(j)) + return false; + if (!READ_ONCE(gsp_phys_rmcall_gr_keep_clamp)) + return true; + if ((sa_mask & (1ull << j)) && gsp_sema_mme_ir_covered(sa_ent[j])) + return true; + /* + * SESSION 153-CONTINUED-114: a keep_mask slot the golden library + * serves is kept without capture coverage - the pack overlay + * carries its code (see the include comment). + */ + { + u32 gent, gn; + const u32 *gw; + + if (gsp_phys_gr_golden_slot(j, &gent, &gw, &gn)) + return true; + } + if (gsp_phys_gr_keep_clamp_log < 16u) { + gsp_phys_gr_keep_clamp_log++; + pr_warn("vgsp-gr: MME-KEEP-CLAMP slot=%u (keep_mask set but capture has no SA entry/coverage; stubbed + calls dropped, never run) entry=0x%x covered=%u\n", + j, sa_ent[j], + (unsigned)gsp_sema_mme_ir_covered(sa_ent[j])); + } + return false; +} + +static bool gsp_phys_gr_slot_stubbed(u32 j, u64 sa_mask, const u32 *sa_ent) +{ + if (j >= 64u || !gsp_sema_gr_sa_stub_value()) + return false; + /* + * SESSION 153-CONTINUED-80: a nop-drop slot is force-stubbed even + * when it is also kept or covered - the r1186 evidence shows a kept + * slot 57 loses its SA end-noop stub (the stub builder skips kept + * slots) and any call-run encoding the drop filter misses then + * executes the real macro 57 (the park class). Dropping the runs and + * stubbing SA[57] together is the safe combination. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_nop_drop) & (1ull << j)) + return true; + /* + * r1448 STRATEGY 3.3: no golden body, no execution - force-stub the + * slot so its captured body can never run (its CALL runs are then + * dropped and its SA pair points at the end-noop). + */ + if (gsp_phys_gr_slot_golden_blank(j)) + return true; + if (gsp_phys_gr_slot_kept(j, sa_mask, sa_ent)) + return false; + /* + * SESSION 153-CONTINUED-114: a slot the golden library serves is + * never stubbed (its CALL runs stay in the window and the pack + * carries its code; see the include comment). + */ + { + u32 gent, gn; + const u32 *gw; + + if (gsp_phys_gr_golden_slot(j, &gent, &gw, &gn)) + return false; + } + if (READ_ONCE(gsp_phys_rmcall_gr_sa_stub_all)) + return true; + if (!(sa_mask & (1ull << j))) + return true; + return !gsp_sema_mme_ir_covered(sa_ent[j]); +} + +/* + * In-place filter for the relocated 3D window: remove every run that + * writes the CALL_MME_MACRO(j)/CALL_MME_DATA(j) method pair of a stubbed + * slot. Pure run-level removal (runs are self-contained: op 1/3/5 carry + * 1+cnt data words; 0/2/4/6 are single words), so the surviving method + * sequence is unchanged and both the F2A window walk and the engine see + * the same stream shape with the calls elided. + */ +static u32 gsp_phys_gr_mme_drop_calls(u32 *win, u32 n, u64 sa_mask, + const u32 *sa_ent, u64 *dropped_mask) +{ + u32 r = 0, w = 0, dropped = 0, transform = 0; + u64 dmask = 0; + + while (r < n) { + u32 h = win[r]; + u32 op = (h >> 29) & 7u; + u32 cnt = (h >> 16) & 0xfffu; + u32 span = 1u, k, m; + bool iscall = false; + + if ((op == 1u || op == 3u || op == 5u) && cnt && + r + 1u + cnt <= n) + span = 1u + cnt; + /* + * Decode the touched methods the way the engine walks them: + * multi-word runs land on m + 4k; single-word entries carry + * their method in the low bits (IMMD and the SDM family both + * put it at 11:0 for our purposes). + */ + for (k = 0; k < span; k++) { + m = ((h & 0xfffu) << 2) + 4u * k; + if (k && op != 1u && op != 3u) + break; + if (m >= 0x3800u && m <= 0x39fcu) { + u32 j = (m - 0x3800u) / 8u; + + if ((READ_ONCE(gsp_phys_rmcall_gr_keep_mask) & + (1ull << j)) && + (READ_ONCE(gsp_phys_rmcall_gr_nop_param_mask) & + (1ull << j))) { + if (READ_ONCE(gsp_phys_rmcall_gr_nop_drop)) { + /* + * SESSION 153-CONTINUED-80: the + * retarget parks the PBDMA + * live (see the + * sema_gl_mme_nop_drop + * comment) - drop the whole + * run instead. + */ + iscall = true; + dmask |= 1ull << j; + break; + } + /* + * SESSION 153-CONTINUED-79: kept + + * nop-param slot - keep the run but + * retarget its header to the DATA + * port so the params still land + * (see the sema_gl_mme_nop_param_mask + * comment). + */ + u32 nm = 0x3804u + 8u * j; + + win[r] = (h & ~0xfffu) | + ((nm >> 2) & 0xfffu); + dmask |= 1ull << j; + transform++; + } else if (gsp_phys_gr_slot_stubbed(j, sa_mask, + sa_ent)) { + iscall = true; + dmask |= 1ull << j; + break; + } + } + } + if (iscall) { + dropped += span; + r += span; + continue; + } + if (w != r) + memmove(&win[w], &win[r], (size_t)span * 4u); + w += span; + r += span; + } + if (transform) + pr_info("vgsp-gr: MME-CALL-NOP %u call run(s) retargeted to the CALL_MME_DATA port (params conserved)\n", + transform); + if (dropped_mask) + *dropped_mask = dmask; + return dropped; +} + +/* + * SESSION 153-CONTINUED-61: generation of the MME prelude already emitted + * into the engine context (0 = none). The r1110 evidence: the prelude was + * emitted once on the first beef launch (gen=1, 512 words), that launch + * priv-errored (grfifo=0xbadf1201) and WEDGE-RECOVER ran TSG preempt + + * resched - which resets the engine context and with it the resident MME + * program - yet the latch kept suppressing re-emission, so every later + * window replay ran its macros against an unloaded MME (mme_pc=513 class + * stall, no retire). The recovery clears the latch so the next launch + * re-preloads the context. + * + * SESSION 153-CONTINUED-68: the latch is keyed by (generation, channel). + * A bare generation latch is SHARED across channels: the X channel's + * first post-maturity launch consumed the emission for the whole system + * and every beef launch then shipped without the program (r1140: only + * h=0x1000013 logged "MME-PRELUDE emitted gen=3", every beef launch + * words=400 prelude-less). Re-emitting per channel is idempotent. + */ +static u32 gsp_phys_gr_mme_prelude_gen; +static u32 gsp_phys_gr_mme_prelude_h; +static u32 gsp_phys_gr_mme_pre_full_dump; + +/* + * CONT-118: one-launch-delay release for the real CALL runs. The known + * poison class is the first calls-kept launch after a fresh split pack + * completes (the r1359 half-loaded-IR race): the drain-completing carrier + * holds itself (the r1222 clause), but the very next window would run the + * real calls against a program whose carrier only just executed. With this + * knob that first post-latch launch is held too - it arms the release - so + * the real calls resume one full launch AFTER the completing carrier in + * every case (the r1443b first-generation shape, reproduced for recreated + * generations too). Default 1; A/B with 0 to release on the first + * post-latch launch (legacy behavior). + */ +static u32 gsp_phys_gr_mme_release_gen; +static u32 gsp_phys_gr_mme_release_h; +static bool gsp_phys_rmcall_gr_mme_hold_settle = true; +module_param_named(sema_gl_mme_hold_settle, + gsp_phys_rmcall_gr_mme_hold_settle, bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_hold_settle, + "CONT-118: hold real CALL runs for one extra launch after a generation's split prelude latches (default 1; 0 = release on the first post-latch launch)"); + +/* + * r1445 3b: strict variant of the CONT-118 settled release. With this on, + * the armed release additionally requires a launch that carried NO prelude + * words, an idle front-end (0x400504 == 1) and the carrier's FETCH-WIT + * promotion (live_gen == pl_gen), so the real calls resume only after the + * completing carrier has provably executed against a settled MME. The + * rg/rh gen-h match machinery is unchanged; default 1, A/B with 0. + */ +static bool gsp_phys_rmcall_gr_mme_hold_strict = true; +module_param_named(sema_gl_mme_hold_strict, + gsp_phys_rmcall_gr_mme_hold_strict, bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_hold_strict, + "CONT-118 strict: release the held CALL runs only on a no-emission launch with the front-end idle and the carrier's generation proven live (default 1; 0 = the plain settled release)"); + +/* + * r1447: the first post-latch ("arming") launch has poisoned in every boot + * that reached it (r1444 103322, r1446 #4, r1446c #4) while window+calls- + * kept launches ran 22x clean (r1443b). The arming launch carries the + * window with its CALL runs elided - the one shape that never retired. Hold + * the WINDOW on the arming launch too (the calls are already held by the + * settle clause); the first window execution then is the proven + * calls+window-kept shape, one release later. The hold is keyed by + * generation+channel and persists until the release actually fires, so a + * strict-starved release cannot ride the window against elided CALL runs + * either; it is cleared at every residency reset (gate fail, wedge + * recovery, park recovery) and when the release fires. Bounded: the set + * happens only while the release is not yet armed, so at most one arming + * window-drop per generation. Default 1; 0 = legacy arming launch carries + * the calls-held window. + */ +static u32 gsp_phys_gr_mme_window_hold_gen; +static u32 gsp_phys_gr_mme_window_hold_h; +static bool gsp_phys_rmcall_gr_mme_hold_window = true; +module_param_named(sema_gl_mme_hold_window, + gsp_phys_rmcall_gr_mme_hold_window, bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_hold_window, + "r1447: drop the 3D window on the one arming launch per generation until the held CALL runs release (default 1; 0 = legacy arming launch carries the calls-held window)"); + +/* + * r1447: bounded fail-open for the strict release. The strict clause + * requires gsp_phys_gr_fe_idle() == 0x00000001 exactly, a value the GR + * FIFO status does not produce under client traffic (observed settled: + * 0x0e0e0001-class), so with strict=1 the armed release can starve the + * calls forever. After this many strict-held launches in one generation + * the FE term fails open (the release proceeds) and logs why. + * Default 8; 0 = legacy never-fail-open (the strict release stays strictly + * gated on 0x400504 == 1 and can starve). + */ +static u32 gsp_phys_gr_mme_strict_holds; +static u32 gsp_phys_rmcall_gr_mme_strict_grace = 8u; +module_param_named(sema_gl_mme_strict_grace, + gsp_phys_rmcall_gr_mme_strict_grace, uint, 0644); +MODULE_PARM_DESC(sema_gl_mme_strict_grace, + "r1447: fail the strict release's FE-idle term open after this many strict-held launches per generation (default 8; 0 = legacy never-fail-open)"); + +/* + * SESSION 153-CONTINUED-107: readback-failure counter + give-up bound for + * the load-only starvation (see the mme_load_only comment). 0 = unlimited; + * default 24. + */ +static u32 gsp_phys_gr_mme_iram_gate_fails; +static u32 gsp_phys_rmcall_gr_mme_iram_gate_max = 24u; + +/* + * SESSION 153-CONTINUED-107: gate decision shared by both IRAM-gate call + * sites (see gsp_phys_gr_mme_iram_gate). The readback failed, so hold the + * calls and owe a full re-emission. Clearing the prelude latch makes the + * next launch emit again (the builder's emit condition); rewinding the pack + * and drain cursors makes that re-emission restart from the first word. The + * pending-gen latch is cleared so the poll break cannot promote the + * generation to "resident". + */ +static void gsp_phys_gr_mme_iram_gate_fail(void) +{ + gsp_phys_gr_mme_iram_gate_fails++; + WRITE_ONCE(gsp_phys_gr_mme_pending_gen, 0); + WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, 0); + WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_sa, false); + /* + * CONT-118: residency is unproven again, so a settled release armed + * for this generation is void; the re-emission re-earns it. + */ + WRITE_ONCE(gsp_phys_gr_mme_release_gen, 0); + WRITE_ONCE(gsp_phys_gr_mme_release_h, 0); + /* + * r1447: same for the arming window-hold and the strict grace budget. + */ + WRITE_ONCE(gsp_phys_gr_mme_window_hold_gen, 0); + WRITE_ONCE(gsp_phys_gr_mme_window_hold_h, 0); + WRITE_ONCE(gsp_phys_gr_mme_strict_holds, 0); +} + +module_param_named(sema_gl_mme_iram_gate_max, + gsp_phys_rmcall_gr_mme_iram_gate_max, uint, 0644); +MODULE_PARM_DESC(sema_gl_mme_iram_gate_max, + "Give up the IRAM-gate load-only starvation after this many readback failures (0 = never; default 24)"); + +static bool gsp_phys_gr_mme_iram_gate_giveup(void) +{ + u32 m = READ_ONCE(gsp_phys_rmcall_gr_mme_iram_gate_max); + + return m && READ_ONCE(gsp_phys_gr_mme_iram_gate_fails) >= m; +} + +/* + * SESSION 153-CONTINUED-57: attr/pipeline state prelude. The r1091 + * full-MME-prelude boot cleared MME (mme_esr=80000000) but the first beef + * window then latched a draw-shader program-header error + * (NV_PGRAPH_PRI_DS_HWW_ESR 0x405840 = 0xa2040a00: SPH9|SPH11|SPH18, + * M_PIPE_STAGE = VSB|PS) and a class error at SET_VERTEX_ATTRIBUTE_A(2) + * (0x1168, INVALID_ENUM): the replayed window is a mid-stream state cut - + * the attribute/pipeline record the client set in EARLIER windows (the + * missing stages' SET_PIPELINE_PROGRAM_ADDRESS binds and the attribute + * format state) was never replayed, so the DS validated program headers + * that were never bound. With this knob the launch builder preplays the + * captured runs from the service's attr prelude buffer (see + * svc_attr_prelude_capture), relocated exactly like the window itself + * (the patched copy is computed by the caller - it needs the launch page + * set and the live shadow counter), run-aligned, drained across the first + * launches of a capture generation. Declared before build_gr_launch (its + * reader); the matching capture-side knob is sema_attr_prelude_cap. + * Default 1; A/B with 0. + */ +static bool gsp_phys_rmcall_gr_attr_prelude = true; +module_param_named(sema_gl_attr_prelude, gsp_phys_rmcall_gr_attr_prelude, + bool, 0644); +MODULE_PARM_DESC(sema_gl_attr_prelude, + "Preplay the captured client attr/pipeline runs (0x1100..0x11fc, 0x2000..0x205c, relocation applied) ahead of the first 3D window replays of a capture generation (default 1)"); + +/* + * CONT-120 (r1451): drop the attr-prelude's SET_PIPELINE_SHADER(2) runs + * (run header method field encoding method 0x2080) from the attr copy. + * The r1450c wall #8 proved the writer of `Offset 0x2080 Data 0x20 + * ErrorCode 4` is the guest's own captured one-shot attr emission + * (`20010820 00000020` = SET_PIPELINE_SHADER(2) <- 0x20, a cold disable + * word) replayed on a windowless LOAD-ONLY carrier: with no SPH-BIND + * stage-2 enable ahead of it the FE refuses the write contextually. The + * same word rides window-bearing launches clean because our SPH-BIND + * block (0x2080 <- 0x21) is prepended. This knob drops only those runs + * (ops 1/3/5, cnt != 0, method == 0x2080), cursor advancing exactly like + * the SDM drop (no raw fallback, dropped for the generation). Default 1; + * A/B with 0 = legacy byte-for-byte. + */ +static bool gsp_phys_rmcall_gr_attr_sph_drop = true; +module_param_named(sema_gl_attr_sph_drop, gsp_phys_rmcall_gr_attr_sph_drop, + bool, 0644); +MODULE_PARM_DESC(sema_gl_attr_sph_drop, + "Drop the attr-prelude SET_PIPELINE_SHADER(2) runs (method 0x2080, the guest's cold 0x20 disable word) the FE refuses with EC4 on a windowless LOAD-ONLY carrier (default 1; 0 = legacy)"); +static u32 gsp_phys_gr_attr_sph_drop_count; /* runs dropped (all gens) */ +static u32 gsp_phys_gr_attr_sph_drop_gen; /* gen of the last drop log */ + +/* + * The patched (relocation-applied) attr prelude for the current capture + * generation, built by the submit path just before build_gr_launch. Single + * caller (the ordered single-inflight sema work item), so plain statics. + */ +#define GSP_PHYS_GR_ATTR_PRE_WORDS 512u +static u32 gsp_phys_gr_attr_pre[GSP_PHYS_GR_ATTR_PRE_WORDS]; +static u32 gsp_phys_gr_attr_pre_n; +static u32 gsp_phys_gr_attr_pre_gen; /* capture gen the patch is from */ +static u32 gsp_phys_gr_attr_emit_gen; /* gen whose runs are draining */ +static u32 gsp_phys_gr_attr_emit_off; /* word cursor into attr_pre */ + +/* + * SESSION 153-CONTINUED-62: surface (render-target) prelude. The beef + * client's per-frame draw windows are mid-stream cuts: the colour-target + * bind is sticky context state (the client and the X server write it once + * per context), so the replayed windows carry none - and every replayed + * draw rasterized into an unbound target and stored nothing (the observed + * 0/N retire). The service captures the last C597 colour-target run from + * any trusted push (svc_surf_prelude_capture; the observed carrier is the + * X server's 1280x720 screen bind) and this site replays it - aliased and + * relocated exactly like the window - ahead of any window that carries no + * usable target bind of its own. Emitted per capture generation (the + * patched buffer is reused while the captured run is unchanged, and the + * alias VA is stable per boot); a fresh generation re-patches. A/B via + * the sema_gl_surf_prelude knob (default 1). + */ +#define GSP_PHYS_GR_SURF_PRE_WORDS 32u +static u32 gsp_phys_gr_surf_pre[GSP_PHYS_GR_SURF_PRE_WORDS]; /* patched */ +static u32 gsp_phys_gr_surf_pre_in[GSP_PHYS_GR_SURF_PRE_WORDS]; +static u32 gsp_phys_gr_surf_pre_n; /* patched words, 0 = none */ +static u32 gsp_phys_gr_surf_pre_gen; /* capture gen the patch is from */ +static bool gsp_phys_rmcall_gr_surf_prelude = true; +module_param_named(sema_gl_surf_prelude, gsp_phys_rmcall_gr_surf_prelude, + bool, 0644); +MODULE_PARM_DESC(sema_gl_surf_prelude, + "Replay the captured C597 colour-target bind ahead of 3D windows that carry no render target of their own (default 1)"); + +/* + * SESSION 153-CONTINUED-62: does this window already carry a usable + * colour-target bind? A run with va == 0 (the all-zero degenerate bind + * seen on the client's 0x402094c push) does not count - the surface + * prelude is injected for it too. + */ +static bool gsp_phys_gr_window_has_surface(const u32 *win, u32 n){ + u32 off = 0; + + while (off < n) { + u32 hdr = win[off]; + u32 op = (hdr >> 29) & 7u; + u32 cnt = (hdr >> 16) & 0xfffu; + u32 addr = (hdr & 0x1fffu) << 2; + u32 span = 1u; + + if ((op == 1u || op == 3u || op == 5u) && cnt && + off + 1u + cnt <= n) + span = 1u + cnt; + else { + off++; + continue; + } + if (op == 1u && addr >= 0x800u && addr < 0xc00u && + (addr & 0x3fu) == 0u && cnt >= 2u) { + u64 va = ((u64)(win[off + 1] & 0xffu) << 32) | + (u64)(win[off + 2] & 0xfffff000u); + + if (va) + return true; + } + off += span; + } + return false; +} + + +/* * GR launch method block: the inline QMD run at its original shape and the * channel REPORT_SEMAPHORE release. The stock guest push does NOT emit * SET_OBJECT before the inline run (the compute class is bound on the * channel long before the launch), so the replay must not either. * Returns the block size in dwords. */ +/* SESSION 153-CONTINUED-101: forward declaration - the seed emission in + * gsp_phys_rmcall_build_gr_launch (below) translates the seed page through + * the same relocator the window operand pairs use (defined further down). */ +static bool gsp_phys_rmcall_gr_3d_reloc(u32 *up_w, u32 *lo_w, + const struct gsp_phys_gr_submit *req, + u32 *n_shadow); + +/* + * SESSION 153-CONTINUED-104: FE idle witness for the calls-kept gate. + * The r1358/r1359 A/B (identical config; only the fixed-CB seed value + * differed) exposes the r1222-class race directly: + * - r1358: the calls-kept launch was submitted with the FE settled + * (MME-FAST begin grfifo=0x00000001) and the whole real-call window + * executed - the macro emissions landed (the wall became the CB read + * FAULT_PTE at fixed-CB+0x4000, i.e. the raw constant path now works); + * - r1359: the same launch was submitted while the FE still showed + * grfifo=0x0d0d1602 (the previous prelude-carrying launch's SA/IR + * method runs still draining) and the first macro execution poisoned + * with a REAL MME ILLEGAL_OPCODE (ESR 0x404490=0x80000004). + * The MME consumes the prelude's SA/IR port runs asynchronously, so a + * CALL_MME_MACRO decoded while those writes are in flight fetches + * half-loaded IR. Hold the real CALL runs until 0x400504 reads the idle + * value (0x00000001): the held launches still retire (the proven-safe + * drop shape) and a later window retries; once idle the calls run + * against a settled program. Fail-open when BAR0 is unavailable. + */ +static bool gsp_phys_gr_fe_idle(void) +{ + void __iomem *bar0 = gsp_phys_gpu_bar0(); + + return !bar0 || ioread32(bar0 + 0x00400504u) == 0x00000001u; +} + +/* + * r1447: raw GR_FIFO_STATUS word for the strict-grace log (0 when BAR0 is + * unavailable). Read-only diagnostic; the release decision still uses + * gsp_phys_gr_fe_idle(). + */ +static u32 gsp_phys_gr_fe_status(void) +{ + void __iomem *bar0 = gsp_phys_gpu_bar0(); + + return bar0 ? ioread32(bar0 + 0x00400504u) : 0u; +} + +/* + * r1445 5c: submit-side FE-busy deferral of the load-only carrier. The + * r1444 first-poison class submitted its first load-only carrier while the + * front-end was mid-fetch and the packed-program fetch poisoned the lane. + * This predicate is evaluated at the service's request gate BEFORE any + * emission/submission: when the launch would become a load-only carrier + * (the sema_gl_mme_iram_gate decision in build_gr_launch) and the FE is NOT + * idle, the request is busy-deferred (candidate survives, pump retries; + * nothing is emitted and no release is authored) and the carrier goes out + * once the FE has settled. Bounded: at most GSP_PHYS_GR_MME_FE_DEFER_MAX + * consecutive defers per prelude generation - the next attempt submits the + * carrier as before, so a never-idle FE cannot starve the lane. The caller + * passes a 3D stream launch (h_object / win3d_n); knob 0 = no-op. + */ +#define GSP_PHYS_GR_MME_FE_DEFER_MAX 4u +static bool gsp_phys_rmcall_gr_mme_fe_defer = true; +module_param_named(sema_gl_mme_fe_defer, + gsp_phys_rmcall_gr_mme_fe_defer, bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_fe_defer, + "Busy-defer a 3D launch that would be a load-only MME carrier while the GR front-end is not idle, so the packed program is not loaded mid-fetch (default 1; 0 = submit the carrier as before)"); +static u32 gsp_phys_gr_mme_fe_defer_gen; +static u32 gsp_phys_gr_mme_fe_defer_n; +static u32 gsp_phys_gr_mme_fe_defer_giveup_gen; + +bool gsp_phys_gr_mme_fe_defer_wanted(u32 h_object, u32 win3d_n) +{ + u32 pl_n = 0, pl_gen = 0; + bool latch_now, live_now; + + if (!READ_ONCE(gsp_phys_rmcall_gr_mme_fe_defer) || + !gsp_phys_rmcall_gr_3d) + return false; + if (!win3d_n || win3d_n + 13u > GSP_PHYS_RMCALL_GR_METHOD_WORDS) + return false; + if (!READ_ONCE(gsp_phys_rmcall_gr_mme_prelude) || + !READ_ONCE(gsp_phys_rmcall_gr_mme_iram_gate) || + !READ_ONCE(gsp_phys_rmcall_gr_mme_prelude_pack) || + !READ_ONCE(gsp_phys_rmcall_gr_mme_hold_calls)) + return false; + if (gsp_phys_gr_mme_iram_gate_giveup()) + return false; + if (!gsp_sema_mme_prelude_get(&pl_n, &pl_gen) || !pl_n || !pl_gen) + return false; + if (gsp_phys_gr_mme_fe_defer_gen != pl_gen) { + gsp_phys_gr_mme_fe_defer_gen = pl_gen; + gsp_phys_gr_mme_fe_defer_n = 0; + } + /* + * Same (post-emission-equivalent) load-only predicate as + * build_gr_launch: a launch is a load-only carrier until its + * generation is both latched and proven live. + */ + latch_now = (pl_gen == READ_ONCE(gsp_phys_gr_mme_prelude_gen) && + h_object == READ_ONCE(gsp_phys_gr_mme_prelude_h)); + live_now = (READ_ONCE(gsp_phys_rmcall_gr_mme_live_skip) && + READ_ONCE(gsp_phys_gr_mme_live_gen) == pl_gen); + if (latch_now && live_now) { + gsp_phys_gr_mme_fe_defer_n = 0; + return false; /* calls-bearing launch, no carrier */ + } + if (gsp_phys_gr_fe_idle()) { + gsp_phys_gr_mme_fe_defer_n = 0; + return false; /* FE settled: the carrier may go out now */ + } + if (gsp_phys_gr_mme_fe_defer_n >= GSP_PHYS_GR_MME_FE_DEFER_MAX) { + if (gsp_phys_gr_mme_fe_defer_giveup_gen != pl_gen) { + gsp_phys_gr_mme_fe_defer_giveup_gen = pl_gen; + pr_warn("vgsp-gr: MME-FE-DEFER-GIVEUP gen=%u h=0x%x n=%u (front-end still busy; loading anyway)\n", + pl_gen, h_object, gsp_phys_gr_mme_fe_defer_n); + } + return false; /* bounded: historical carrier stands */ + } + gsp_phys_gr_mme_fe_defer_n++; + pr_info_ratelimited("vgsp-gr: MME-FE-DEFER gen=%u h=%#x n=%u\n", + pl_gen, h_object, gsp_phys_gr_mme_fe_defer_n); + return true; +} + +/* + * r1487 STREAM-DAMAGE: a client 3D section is sliced into method-aligned + * chunks replayed as separate launches (svc_gl_stream_chunks). The channel + * state accumulates across those launches, so a chunk that is *dropped* + * (MME-LOAD-ONLY carrier while the prelude is not resident, or the r1447 + * arming window-hold) permanently removes that slice of the method stream. + * A later chunk of the same push then executes CALL_MME_MACRO runs against a + * channel missing the dropped state: r1485's 81-word tail called + * LOAD_CONSTANT_BUFFER_BUG2548370 after chunks 1..3 had been dropped, the + * macro emitted method 0x2390 (LOAD_CONSTANT_BUFFER(0)) without a valid + * constant-buffer selector, the 3D class raised + * "Offset 00002390, Data bf800000, ErrorCode 00000003", and the FECS + * priv-error sentinel 0xbadf1201 wedged the lane (pc=513/mac=0xf1 are + * artifacts of that sentinel: 0xbadf1201 & 0xfff == 0x201). + * + * Fail closed per stream: remember the lowest dropped stream-relative offset + * for a push; while a later chunk of that push arrives, force the calls-held + * emission (CALL runs elided, the proven-safe shape) so no macro can run + * against the incomplete state. A chunk at stream_rel 0 is the start of a + * fresh client section and clears the record, so the next frame replays its + * whole window in order and draws normally. + */ +#define GSP_PHYS_GR_STREAM_DAMAGE_SLOTS 8 +struct gsp_phys_gr_stream_damage { + u64 push_va; + u32 rel; + bool valid; +}; +static struct gsp_phys_gr_stream_damage + gsp_phys_gr_stream_damage[GSP_PHYS_GR_STREAM_DAMAGE_SLOTS]; +static u32 gsp_phys_gr_stream_damage_next; +static u32 gsp_phys_gr_stream_damage_hits; +static u32 gsp_phys_gr_stream_damage_notes; + +static bool gsp_phys_gr_stream_damaged(u64 push_va, u32 rel) +{ + u32 i; + + for (i = 0; i < GSP_PHYS_GR_STREAM_DAMAGE_SLOTS; i++) { + struct gsp_phys_gr_stream_damage *d = + &gsp_phys_gr_stream_damage[i]; + + if (d->valid && d->push_va == push_va) + return rel > d->rel; + } + return false; +} + +static void gsp_phys_gr_stream_damage_note(u64 push_va, u32 rel) +{ + u32 i; + + for (i = 0; i < GSP_PHYS_GR_STREAM_DAMAGE_SLOTS; i++) { + struct gsp_phys_gr_stream_damage *d = + &gsp_phys_gr_stream_damage[i]; + + if (d->valid && d->push_va == push_va) { + if (rel < d->rel) + d->rel = rel; + return; + } + } + i = gsp_phys_gr_stream_damage_next++ % + GSP_PHYS_GR_STREAM_DAMAGE_SLOTS; + gsp_phys_gr_stream_damage[i].push_va = push_va; + gsp_phys_gr_stream_damage[i].rel = rel; + gsp_phys_gr_stream_damage[i].valid = true; +} + +static void gsp_phys_gr_stream_damage_clear(u64 push_va) +{ + u32 i; + + for (i = 0; i < GSP_PHYS_GR_STREAM_DAMAGE_SLOTS; i++) { + struct gsp_phys_gr_stream_damage *d = + &gsp_phys_gr_stream_damage[i]; + + if (d->valid && d->push_va == push_va) + d->valid = false; + } +} + static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, const struct gsp_phys_gr_submit *req, const u32 *qmd, u64 qmd_va, - u64 sem_va, u32 sem_payload) + u64 sem_va, u32 sem_payload, + const u32 *win3d, u32 win3d_n, + const u32 *attr_pre, + u32 attr_pre_n) { u32 *p = pb; int prelude = gsp_phys_rmcall_gr_prelude_value(); @@ -16644,6 +20886,2332 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, int two_step = gsp_phys_rmcall_gr_pcas_two_step_value(); /* + * r929 (Path A): a captured 3D method window replays verbatim. Bind + * TURING_A (0xC597) on the channel subchannel (the same bind the + * window itself carries - belt and braces, a rebind is harmless), + * then the window words (address operands relocated to shadow-pool + * VAs by gsp_phys_rmcall_gr_3d_patch_window), then the channel + * release tail. No compute init, no QMD, no PCAS: the engine walks + * real 3D methods (SET_OBJECT C597, state binds, LINE_LENGTH_IN + + * LAUNCH_DMA inline uploads, program/tex binds). Completion is the + * trailing REPORT_SEMAPHORE alone (the submit path leaves qmd_rel + * off for this mode). + */ + if (gsp_phys_rmcall_gr_3d && req && win3d && win3d_n && + win3d_n + 2u + 3u + 8u <= GSP_PHYS_RMCALL_GR_METHOD_WORDS) { + /* + * SESSION 153-CONTINUED-53: MME context-setup prelude (see the + * sema_gl_mme_prelude knob above). Preplay the captured client + * setup runs once per capture generation, right after the + * SET_OBJECT C597 bind and ahead of the window, so + * CALL_MME_MACRO() runs execute against a loaded macro + * program instead of pc=0 -> ILLEGAL_OPCODE. The read is + * seqchecked (generation before/after): a concurrent append + * only defers emission to the next launch. A window too large + * to also carry the prelude replays without it (logged) - the + * emission then happens on the next launch that fits. + */ + bool emit_prelude = false; + bool pre_done = false; + bool mme_hold_calls = false; + bool mme_hold_window = false; + bool mme_hold_fe_busy = false; + bool mme_load_only = false; + bool latch_before = false; + bool stream_damaged = false; + u32 pre_words_out = 0; + u32 win_emit_off = 0; + const u32 *pl = NULL; + u32 pl_n = 0, pl_gen = 0, pl_room = 0; + u32 block_cap = READ_ONCE(gsp_phys_rmcall_gr_block_cap); + + if (block_cap < 64u || + block_cap > GSP_PHYS_RMCALL_GR_METHOD_WORDS) + block_cap = GSP_PHYS_RMCALL_GR_METHOD_WORDS; + /* + * SESSION 153-CONTINUED-86: harden the block cap against the + * emissions the block-cap arithmetic below does not fully + * count (the SA re-arm, 3 words per slot, up to 32 slots - + * see the MME-SA-REARM site). Clamping the cap keeps every + * combination inside the 4096-word method buffer; with the + * default 2048 cap this changes nothing. + */ + if (block_cap > GSP_PHYS_RMCALL_GR_METHOD_WORDS - 128u) + block_cap = GSP_PHYS_RMCALL_GR_METHOD_WORDS - 128u; + /* + * r1487 STREAM-DAMAGE: stream_rel 0 starts a fresh client + * section, so any dropped-chunk record for this push is + * stale; a later chunk keeps the record and is emitted + * calls-held below (see the table comment). + */ + if (req->stream_rel == 0u) + gsp_phys_gr_stream_damage_clear(req->push_va); + stream_damaged = gsp_phys_gr_stream_damaged(req->push_va, + req->stream_rel); + /* + * SESSION 153-CONTINUED-63: MME START-ADDRESS RAM completion. + * The prelude capture is capped at 512 words, so the client's + * SA table (macro slot -> IR address) may be incomplete; on a + * fresh context the missing slots read 0 and CALL_MME_MACRO(j) + * jumps to IR 0 - the slot-0 macro - which with the caller's + * args emitted LOAD_CONSTANT_BUFFER_OFFSET <- 0x20020 and + * poisoned the lane (r1116; verified offline with the MME64 + * simulator). Stubbing every CALLED slot the capture's SA + * table does not cover with a harmless end-noop makes the + * unknown call a no-op instead of a lane-killer. The called + * slots come from the window being emitted (IMMD writes to + * 0x3800+j*8 and INC/ONEINC runs writing any 0x3800..0x39f8 + * method, the .Macro half of each pair). + */ + u32 sa_ent[64]; + u64 sa_mask = 0; + u32 sa_stub[40]; + u32 sa_stub_ent[40]; + u32 sa_cov[40]; + u32 sa_cov_ent[40]; + u32 sa_nstub = 0, sa_ncov = 0, sa_block = 0; + + if (gsp_sema_gr_sa_stub_value() && win3d && win3d_n) { + u32 so = 0; + + sa_mask = gsp_sema_mme_sa_get(sa_ent, 64); + while (so < win3d_n && (sa_nstub < 40u || sa_ncov < 40u)) { + u32 h = win3d[so]; + u32 sop = (h >> 29) & 7u; + u32 scnt = (h >> 16) & 0x1fffu; + u32 sspan = 1u; + u32 k, slo, shi; + + if ((sop == 1u || sop == 3u || sop == 5u) && + scnt && so + 1u + scnt <= win3d_n) { + sspan = 1u + scnt; + slo = 0; + /* + * SESSION 153-CONTINUED-68: ONE_INC + * (op 5) writes only data[0] to the + * header method; the rest go to + * method+4 (the data port, see + * nv3dLoadConstantsHeader). Walking + * mthd+4k for op=5 invented ghost + * CALL_MME_MACRO slots (r1134/1135 + * list). + */ + shi = (sop == 5u) ? 0u : scnt - 1u; + } else if (sop == 4u) { + slo = 0; + shi = 0; + } else { + so++; + continue; + } + for (k = slo; k <= shi; k++) { + u32 mthd = ((h & 0xfffu) << 2) + 4u * k; + u32 j, t; + bool have = false; + + if (mthd < 0x3800u || mthd >= 0x3a00u || + (mthd & 7u)) + continue; + j = (mthd - 0x3800u) / 8u; + if (j >= 64u) + continue; + /* + * SESSION 153-CONTINUED-70: a kept slot + * (sema_gl_mme_keep_mask) is never + * stubbed - its real macro must run + * against the context exactly as the + * client's own SA/IR uploads left it. + * Logged in the covered list for the + * census; no stub pair is emitted. + * SESSION 153-CONTINUED-113: unless + * the capture cannot serve it at all + * (keep_clamp) - then it falls into + * the stub + call-drop path below. + */ +if (gsp_phys_gr_slot_kept(j, sa_mask, + sa_ent)) { + for (t = 0; t < sa_ncov; t++) + if (sa_cov[t] == j) + have = true; + if (!have && sa_ncov < 40u) { + sa_cov[sa_ncov] = j; + sa_cov_ent[sa_ncov] = + sa_ent[j]; + sa_ncov++; + } + continue; + } + /* + * SESSION 153-CONTINUED-114: a called + * slot outside the capture that the + * golden library serves counts as + * covered - the pack overlay emits its + * SA pair + code, so no stub pair is + * emitted and the CALL runs survive + * the drop pass (see slot_stubbed). + */ + { + u32 gent, gn; + const u32 *gw; + + if (gsp_phys_gr_golden_slot( + j, &gent, &gw, + &gn)) { + for (t = 0; t < sa_ncov; + t++) + if (sa_cov[t] == j) + have = true; + if (!have && + sa_ncov < 40u) { + sa_cov[sa_ncov] = j; + sa_cov_ent[sa_ncov] = + gent; + sa_ncov++; + } + continue; + } + } + /* + * SESSION 153-CONTINUED-68: a slot is only + * usable when the capture holds its SA entry + * AND the IR instruction that entry names + * actually entered the replay (see + * gsp_sema_mme_ir_covered); a last-wins/ + * truncated mismatch makes the engine fetch + * macro data that is not there + * (MISSING_MACRO_DATA). Called-and-covered + * slots are logged too (r1134 ILLEGAL_OPCODE + * attribution: which macros actually run). + */ + if (sa_mask & (1ull << j)) { + if (!READ_ONCE(gsp_phys_rmcall_gr_sa_stub_all) && + gsp_sema_mme_ir_covered(sa_ent[j]) && + !gsp_phys_gr_slot_golden_blank(j) && + !(READ_ONCE(gsp_phys_rmcall_gr_nop_drop) & + (1ull << j))) { + for (t = 0; t < sa_ncov; t++) + if (sa_cov[t] == j) + have = true; + if (!have && sa_ncov < 40u) { + sa_cov[sa_ncov] = j; + sa_cov_ent[sa_ncov] = + sa_ent[j]; + sa_ncov++; + } + continue; + } + } + for (t = 0; t < sa_nstub; t++) + if (sa_stub[t] == j) + have = true; + if (!have && sa_nstub < 40u) { + sa_stub[sa_nstub] = j; + sa_stub_ent[sa_nstub] = + (sa_mask & (1ull << j)) ? + sa_ent[j] : 0u; + sa_nstub++; + } + } + so += sspan; + } + if (sa_nstub) + sa_block = 8u + 3u * sa_nstub; + } + + /* + * SESSION 153-CONTINUED-68: full run-header census of the 3D + * window being replayed (bounded; first 4 windows). The + * r1134 ILLEGAL_OPCODE attribution needs the real call + * structure: which methods the window writes, how many data + * words each run carries, and the CALL_MME_MACRO/DATA + * pattern (op 1 INC vs op 5 ONE_INC) around them. + */ + { + static u32 win_dump_n; + static u32 win_dump_beef; + + /* + * SESSION 153-CONTINUED-69: the first-4 budget was + * consumed by the X channel's early windows in every + * boot, so the beef draw-class window (the one that + * faults) never got its run-header census. Two extra + * dumps are reserved for beef-family windows. + */ + if (win3d && win3d_n && + (win_dump_n < 4u || + (((req->h_object & 0xfffff000u) == 0xbeef0000u) && + win_dump_beef < 2u))) { + u32 wo = 0, wr = 0; + + win_dump_n++; + if ((req->h_object & 0xfffff000u) == 0xbeef0000u) + win_dump_beef++; + pr_info("vgsp-gr: MME-WINRUN dump h=0x%x push=0x%llx window=%u\n", + req->h_object, req->push_va, win3d_n); + while (wo < win3d_n && wr < 200u) { + u32 h = win3d[wo]; + u32 op = (h >> 29) & 7u; + u32 cnt = (h >> 16) & 0x1fffu; + u32 addr = (h & 0xfffu) << 2; + u32 span = 1u; + u32 d0 = (wo + 1u < win3d_n) ? + win3d[wo + 1u] : 0u; + u32 d1 = (wo + 2u < win3d_n) ? + win3d[wo + 2u] : 0u; + u32 d2 = (wo + 3u < win3d_n) ? + win3d[wo + 3u] : 0u; + u32 d3 = (wo + 4u < win3d_n) ? + win3d[wo + 4u] : 0u; + + if ((op == 1u || op == 3u || op == 5u) && + cnt && wo + 1u + cnt <= win3d_n) + span = 1u + cnt; + pr_info("vgsp-gr: MME-WINRUN off=%x op=%u cnt=%u mthd=%x d0=%x d1=%x d2=%x d3=%x\n", + wo * 4u, op, cnt, addr, d0, d1, + d2, d3); + wo += span; + wr++; + } + } + } + + if (READ_ONCE(gsp_phys_rmcall_gr_mme_prelude)) { + pl = gsp_sema_mme_prelude_get(&pl_n, &pl_gen); + /* + * r1453 PAIR-CHAIN: remember the generation this + * launch was built against; the submit's success + * edge promotes it to gsp_phys_gr_mme_pairs_gen + * once a pair-bearing window retires (the held + * elision extension's install proof). + */ + gsp_phys_gr_mme_launch_gen = pl_gen; + /* + * SESSION 153-CONTINUED-72: with keep_mask bits set a + * kept macro can execute, and it needs the MME program + * resident. The r1152 evidence: the prelude was carried + * by one launch of the generation only; when that launch + * did not complete (engine fetch NONE / timeout), the + * later kept-macro launches ran an UNLOADED program and + * priv-errored (ILLEGAL_OPCODE -> poison). Force + * re-emission on every launch while any slot is kept + * (idempotent per the emission contract). + * + * SESSION 153-CONTINUED-79: the emission is drained in + * per-launch parts capped at block_cap words (see the + * sema_gl_mme_block_cap comment; oversized blocks never + * start). The latch (gsp_phys_gr_mme_prelude_gen/h) is + * set only when the final part has gone out. + */ + if (pl && pl_n && pl_gen) { + /* + * Fixed overhead: SET_OBJECT bind (2), SA-stub + * block, window, and the 4x2-word release tail. + */ + u32 fixed = 2u + sa_block + win3d_n + 8u; + + /* + * SESSION 153-CONTINUED-82: latch state BEFORE + * this launch's emission (see the hold decision + * below). A launch that first uploads a + * generation's MME code must not run real CALL + * runs in the same block (the r1222 evidence: + * the drain-completing launch bundled part-2's + * IR uploads with the window's real calls and + * hit ILLEGAL_OPCODE even though the latch was + * set by its own emission step). + */ + latch_before = + (pl_gen == READ_ONCE( + gsp_phys_gr_mme_prelude_gen) && + req->h_object == READ_ONCE( + gsp_phys_gr_mme_prelude_h)); + + pl_room = (fixed + 24u <= block_cap) ? + block_cap - fixed : 0u; + if (pl_room > pl_n) + pl_room = pl_n; + /* + * (Re)start the drain on a new generation/ + * channel, or when the request is forced (kept + * slots; a context-wiping wedge recovery clears + * the latch) and nothing is left to drain. + */ + if (pl_gen != + READ_ONCE(gsp_phys_gr_mme_pre_emit_gen) || + req->h_object != + READ_ONCE(gsp_phys_gr_mme_pre_emit_h) || + (READ_ONCE(gsp_phys_gr_mme_pre_emit_off) >= + pl_n && + ((READ_ONCE(gsp_phys_rmcall_gr_keep_mask) && + !(READ_ONCE( + gsp_phys_rmcall_gr_mme_live_skip) && + READ_ONCE(gsp_phys_gr_mme_live_gen) == + pl_gen)) || + pl_gen != READ_ONCE( + gsp_phys_gr_mme_prelude_gen)))) { + /* + * r1447: an actual generation change + * restarts the strict grace budget + * with the new residency attempt. + */ + if (pl_gen != + READ_ONCE( + gsp_phys_gr_mme_pre_emit_gen)) + WRITE_ONCE( + gsp_phys_gr_mme_strict_holds, + 0); + WRITE_ONCE(gsp_phys_gr_mme_pre_emit_gen, + pl_gen); + WRITE_ONCE(gsp_phys_gr_mme_pre_emit_h, + req->h_object); + WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0); + /* + * CONT-105: the packed split shares this + * restart edge; reset its cursor too so a + * wedged-recover latch clear replays the + * whole packed stream. + */ + WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_sa, + false); + } + if (pl_room >= 24u) { + emit_prelude = + (pl_gen != READ_ONCE( + gsp_phys_gr_mme_prelude_gen) || + req->h_object != READ_ONCE( + gsp_phys_gr_mme_prelude_h) || + READ_ONCE( + gsp_phys_gr_mme_pre_emit_off) < + pl_n); + } else { + static u32 pre_defer_log; + + if (pre_defer_log < 8) { + pre_defer_log++; + pr_info("vgsp-gr: MME-PRELUDE deferred (window=%u prelude=%u sa=%u cap=%u) h=0x%x\n", + win3d_n, pl_n, sa_block, + block_cap, + req->h_object); + } + } + } + } + + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | (1u << 16) | + (0u << 13) | + (GSP_PHYS_RMCALL_GR_SET_OBJECT >> 2); + *p++ = GSP_PHYS_RMCALL_CLASS_GR_3D; + /* + * SESSION 153-CONTINUED-90: SET_SHADER_EXCEPTIONS (0x1528) + * <- 0, the first write of NVIDIA's own Turing 3D channel + * init (see the sema_gl_shader_exc knob): with shader + * exceptions enabled the DS raises "Shader Program Header + * 5/9/10/11" on the draws whose SPHs it would otherwise + * tolerate, and the raised exception poisons the channel + * before any store retires. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_shader_exc)) { + static u32 shader_exc_log; + bool in_window = + READ_ONCE(gsp_phys_rmcall_gr_shader_exc_skip) && + gsp_phys_gr_method_present(win3d, win3d_n, + 0x1528u); + + if (in_window) { + if (shader_exc_log < 8u) { + shader_exc_log++; + pr_info("vgsp-gr: SHADER-EXC skipped (window carries 0x1528) h=0x%x push=0x%llx\n", + req->h_object, + req->push_va); + } + } else if (READ_ONCE( + gsp_phys_rmcall_gr_shader_exc_immd)) { + /* + * SESSION 154-1: the client's own + * encoding (nvPushImmedVal): immd13 + * header with the 13-bit value in bits + * 16..28 and NO following data word. + * Value 0 = ENABLE FALSE. + */ + *p++ = 0x80000000u | (0x1528u >> 2); + if (shader_exc_log < 8u) { + shader_exc_log++; + pr_info("vgsp-gr: SHADER-EXC emitted 0x1528 <- 0 (1 word immd13 %08x; pos=%u h=0x%x push=0x%llx)\n", + *(p - 1), (u32)(p - pb), + req->h_object, req->push_va); + } + } else { + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM + << 29) | + (1u << 16) | (0x1528u >> 2); + *p++ = 0u; + if (shader_exc_log < 8u) { + shader_exc_log++; + pr_info("vgsp-gr: SHADER-EXC emitted 0x1528 <- 0 (2 words INC; pos=%u h=0x%x push=0x%llx)\n", + (u32)(p - pb), + req->h_object, + req->push_va); + } + } + } + /* + * SESSION 153-CONTINUED-91: DS HWW report suppression. + * + * The DS validates the SPH of every bound pipeline program at + * draw time and reports failures through DS_HWW_REPORT_MASK + * (0x405844, observed live = 0x00ffffff = every SPH<n>_ERR + * reported; mask_2 = 0x77). A reported failure poisons the + * whole PRI block (reads return 0xbadf1201 - the priv-error + * marker already documented for the MME latch) and the GSP-RM + * exception handler kills the channel: every draw attempt + * wedges at the first SPH error storm even when the SPH bytes + * at the bound VAs are the client's own real V4 images (r1298: + * INLINE-RB rb=020a0481 at the bind VA, DS still 5/9/10/11). + * The reference driver's own diagnostic path (Aftermath) + * explicitly ENABLES these reports to capture them, i.e. the + * reports are a diagnostic tap, not the shader-execute gate. + * Clear the mask so a SPH miss latches (readable between + * faults) without poisoning the channel. Host write only: + * an in-stream CALL_MME_MACRO(5) (the NVN PRI R/W macro) + * faulted with MME ILLEGAL_OPCODE (pc=513 mac=0xf1) on + * r1300 - the Turing MME's loaded macro program does not + * carry that GF100-era slot - so the register is written + * directly (it persists: the next beef launch reads 0/0). + * A/B with sema_ds_mask_clear=0. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_ds_mask_clear)) { + static u32 ds_mask_log; + void __iomem *ds_bar0 = gsp_phys_gpu_bar0(); + u32 m0 = 0, m2 = 0; + + if (ds_bar0) { + m0 = ioread32(ds_bar0 + 0x00405844u); + m2 = ioread32(ds_bar0 + 0x0040584cu); + if (m0 || m2) { + iowrite32(0u, ds_bar0 + 0x00405844u); + iowrite32(0u, ds_bar0 + 0x0040584cu); + } + /* + * SESSION 153-CONTINUED-92: clear the DS HWW + * ESR latch (W1C) ahead of every window so the + * retire readback names EXACTLY this launch's + * failed SPH checks instead of the union of + * every prior draw's. The latch was observed + * sticky across launches (r1308: a6000e20 + * constant over 25 s of draws), which made + * "did this window's checks change?" unreadable. + * Log the value being cleared (the previous + * window's set) so the A/B sequence is in the + * ring either way. + */ + { + u32 esr = ioread32(ds_bar0 + 0x00405840u); + + if (esr) + iowrite32(esr | 0x40000000u, + ds_bar0 + 0x00405840u); + if (ds_mask_log < 24u) { + ds_mask_log++; + pr_info("vgsp-gr: DS-ESR pre-clear=%08x (prev window's latch; this window starts clean) h=0x%x push=0x%llx\n", + esr, req->h_object, + req->push_va); + } + } + } + if (ds_mask_log < 24u) { + ds_mask_log++; + pr_info("vgsp-gr: DS-MASK clear mask=%08x/%08x rb=%08x/%08x h=0x%x push=0x%llx\n", + m0, m2, + ds_bar0 ? ioread32(ds_bar0 + 0x00405844u) : 0u, + ds_bar0 ? ioread32(ds_bar0 + 0x0040584cu) : 0u, + req->h_object, req->push_va); + } + } + /* + * SESSION 153-CONTINUED-86: the Turing channel-init SPH version + * pair (see the sema_gl_sph_version knob). Emitted on EVERY + * replayed 3D window (3 words, idempotent): the DS validates + * each bound program's SPH version against this window at draw + * time, and the replay context never received the client's + * once-per-channel write. Emitted before the prelude-room + * computations below so the live (p - pb) accounting covers + * it exactly. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_sph_version)) { + /* + * SESSION 153-CONTINUED-87: the window's accepted + * version range must COVER the images the engine + * will fetch. The client's own init push writes + * 4|(4<<16), but our authored (V3) images are + * rejected by a 4/4 window (SPH9|11|18) while the + * r950-era acceptance - same V3 image - ran with + * NO version window. Emit oldest=V(blob), + * current=max(V(blob),4) so both the V3 image and + * a future V4 re-tag stay inside the range. + */ + u32 bv = (u32)gsp_sema_gl_sph_blob_version(); + u32 val = 0x00040004u; + u64 ov = READ_ONCE(gsp_phys_rmcall_gr_sph_version_val); + + if (ov) + val = (u32)ov; + else if (bv >= 1u && bv <= 15u) + val = ((bv < 4u ? 4u : bv) << 16) | bv; + + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + (2u << 16) | (0x16a4u >> 2); + *p++ = val; + *p++ = val; + } + /* + * SESSION 153-CONTINUED-91: the Turing program region. + * + * The DS's SPH validation checks each fetched SPH against the + * channel's PROGRAM REGION. On pre-Volta that was + * SET_PROGRAM_REGION_A/B (0x1608/0x160c); on Volta+ those + * methods are obsolete and the driver writes the same 64-bit + * shader-code-heap base through the MME shadow scratch pair + * (nvTuring: NVC397_SET_MME_PROGRAM_REGION = SET_MME_SHADOW_ + * SCRATCH(MME_SCRATCH_PROGRAM_REGION_A=0x78/B=0x79), + * gv100ResendProgramAddresses/`gf100Init`; method 0x3400 + + * i*4 -> 0x35e0/0x35e4). Observed live at the faulting draw: + * DS_HWW_ESR_SPH_INFO_PROGRAM_REGION_A/B = 0/0 and DS latches + * SPH errors for every bound stage whose bytes are provably + * correct (r1298 INLINE-RB rb=020a0481 at the bind VA; + * r1303 GR-STATE sph stages 1/2/5 = 0x620900/0x620200/0x620000 + * with ds=a4000e20 and rr=0/0) - a region mismatch rejects all + * of them regardless of content. Emit the pair (A = upper + * 7:0, B = lower 31:0) with the launch's own program family + * page as the heap base. A/B with sema_gl_sph_region=0. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_sph_region) && req->prog_va) { + static u32 sph_reg_log; + u64 base = READ_ONCE(gsp_phys_rmcall_gr_sph_region_base); + u64 bind_va = 0; + bool bind_base = + READ_ONCE(gsp_phys_rmcall_gr_sph_region_bind) && + gsp_phys_gr_first_prog_bind(win3d, win3d_n, + &bind_va); + + /* + * SESSION 154-1: the DS compares the fetched SPH + * address against this region. The engine fetches + * the program through its relocated shadow VA, so + * the correct base is the bound pair's own shadow + * page; the forced override/shadow-pool base is + * only the fallback when the window binds nothing. + */ + if (bind_base) + base = bind_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + else if (!base) + base = req->prog_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + (1u << 16) | (0x35e0u >> 2); + *p++ = (u32)(base >> 32) & 0xffu; + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + (1u << 16) | (0x35e4u >> 2); + *p++ = (u32)base; + if (sph_reg_log < 16u) { + sph_reg_log++; + pr_info("vgsp-gr: SPH-REGION emitted base=0x%llx (scratch 0x78/0x79; src=%s bind=0x%llx) h=0x%x push=0x%llx\n", + base, bind_base ? "bind-shadow" : + "override/auto", bind_va, + req->h_object, req->push_va); + } + } + /* + * SESSION 153-CONTINUED-101: MME fixed-constant-buffer scratch + * seed (see the sema_gl_mme_seed* knobs). mmeCodeOGL_C597_ + * SETUP_FIXED_CONST (slot 0) emits SET_CONSTANT_BUFFER_ + * SELECTOR_B/C from shadow scratch 0x18/0xd18; the replay + * context never received the client's once-per-channel write + * (gf100InitConstantBuffers pushes method 0x3460), so the + * macro read 0 and the selector went empty. Emit the seed + * ahead of the window; the value is the fixed CB address>>8, + * page-translated to the launch's shadow/identity mapping when + * sema_gl_mme_seed_xlat is set (see gsp_phys_rmcall_gr_3d_ + * reloc - same machinery as the operand pairs). + */ + if (READ_ONCE(gsp_phys_rmcall_gr_mme_seed)) { + static u32 mme_seed_log; + u32 sv = READ_ONCE(gsp_phys_rmcall_gr_mme_seed_val); + bool have = sv != 0u; + + if (!have) + have = gsp_sema_mme_scratch18_get(&sv); + if (have && sv && + (u32)(p - pb) + 8u < GSP_PHYS_RMCALL_GR_METHOD_WORDS) { + u64 va = (u64)sv << 8; + bool xl = false; + + if (READ_ONCE(gsp_phys_rmcall_gr_mme_seed_xlat) + && va) { + u32 up_w = (u32)((va >> 32) & 0xffu); + u32 lo_w = (u32)va & 0xfffffffcu; + u32 n_sh = 0; + + if (gsp_phys_rmcall_gr_3d_reloc( + &up_w, &lo_w, req, &n_sh)) { + va = ((u64)(up_w & 0xffu) + << 32) | + (u64)(lo_w & 0xfffffffcu); + sv = (u32)(va >> 8); + xl = true; + } + } + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + (1u << 16) | (0x3460u >> 2); + *p++ = sv; + if (mme_seed_log < 64u) { + mme_seed_log++; + pr_info("vgsp-gr: MME-SEED scratch 0x18 <- 0x%08x (va=0x%llx xlat=%u) h=0x%x push=0x%llx\n", + sv, va, xl, req->h_object, + req->push_va); + } + } else if (mme_seed_log < 64u) { + mme_seed_log++; + pr_info("vgsp-gr: MME-SEED no value (client 0x3460 not seen yet; set sema_gl_mme_seed_val) h=0x%x\n", + req->h_object); + } + } + /* + * SESSION 153-CONTINUED-115: STATE 0xD19 seed (shadow scratch + * 0x19, method 0x3464 = SET_MME_SHADOW_SCRATCH(0x19): the + * constant-buffer selector B/C address>>8). The bug2548370 + * loadConstantBuffer macro's in-band update emits the selector + * from d19 once the fast path is refused (see the state seed + * below); the replay context never received the client's + * once-per-channel 0x3464 write, so d19 read 0 and the + * consumer fetched constants from address 0. Value: + * sema_mme_state_d19 when nonzero, else the captured write + * (0x3464); fail-closed - a missed shadow translation skips + * the emission instead of writing an untranslated selector. + * Gated by sema_gl_mme_state_seed (STATE class). + */ + if (READ_ONCE(gsp_phys_rmcall_gr_mme_state_seed)) { + static u32 mme_state_d19_log; + u32 sv = READ_ONCE(gsp_phys_rmcall_gr_mme_state_d19); + bool have = sv != 0u; + + if (!have) + have = gsp_sema_mme_scratch19_get(&sv); + if (have && sv && + (u32)(p - pb) + 8u < GSP_PHYS_RMCALL_GR_METHOD_WORDS) { + u64 va = (u64)sv << 8; + bool xl = false; + bool ok = true; + + if (READ_ONCE(gsp_phys_rmcall_gr_mme_seed_xlat) + && va) { + u32 up_w = (u32)((va >> 32) & 0xffu); + u32 lo_w = (u32)va & 0xfffffffcu; + u32 n_sh = 0; + + if (gsp_phys_rmcall_gr_3d_reloc( + &up_w, &lo_w, req, + &n_sh)) { + va = ((u64)(up_w & 0xffu) + << 32) | + (u64)(lo_w & 0xfffffffcu); + sv = (u32)(va >> 8); + xl = true; + } else { + ok = false; + } + } + if (ok) { + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM + << 29) | + (1u << 16) | (0x3464u >> 2); + *p++ = sv; + if (mme_state_d19_log < 64u) { + mme_state_d19_log++; + pr_info("vgsp-gr: MME-STATE-SEED d19=0x%08x (va=0x%llx xlat=%u) h=0x%x push=0x%llx\n", + sv, va, xl, + req->h_object, + req->push_va); + } + } + } + } + /* + * SESSION 153-CONTINUED-116: STATE 0xD24 seed (shadow scratch + * 0x24, method 0x3490 = SET_MME_SHADOW_SCRATCH(0x24)). The + * bug2548370 slow path reads the 0x349x family beside the d19 + * selector; the replay context never received the client's + * once-per-channel 0x3490 write, so d24 read whatever the + * replay left (r1440: ILLEGAL_OPCODE in the same post-slot-57 + * window). Value: sema_mme_state_d24 when nonzero, else the + * captured write (0x3490); fail-closed - a missed shadow + * translation skips the emission instead of writing an + * untranslated value. Gated by sema_gl_mme_state_seed. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_mme_state_seed)) { + static u32 mme_state_d24_log; + u32 sv = READ_ONCE(gsp_phys_rmcall_gr_mme_state_d24); + bool have = sv != 0u; + + if (!have) + have = gsp_sema_mme_scratch24_get(&sv); + if (have && sv && + (u32)(p - pb) + 8u < GSP_PHYS_RMCALL_GR_METHOD_WORDS) { + u64 va = (u64)sv << 8; + bool xl = false; + bool ok = true; + + if (READ_ONCE(gsp_phys_rmcall_gr_mme_seed_xlat) + && va) { + u32 up_w = (u32)((va >> 32) & 0xffu); + u32 lo_w = (u32)va & 0xfffffffcu; + u32 n_sh = 0; + + if (gsp_phys_rmcall_gr_3d_reloc( + &up_w, &lo_w, req, + &n_sh)) { + va = ((u64)(up_w & 0xffu) + << 32) | + (u64)(lo_w & 0xfffffffcu); + sv = (u32)(va >> 8); + xl = true; + } else { + ok = false; + } + } + if (ok) { + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM + << 29) | + (1u << 16) | (0x3490u >> 2); + *p++ = sv; + if (mme_state_d24_log < 64u) { + mme_state_d24_log++; + pr_info("vgsp-gr: MME-STATE-SEED d24=0x%08x (va=0x%llx xlat=%u) h=0x%x push=0x%llx\n", + sv, va, xl, + req->h_object, + req->push_va); + } + } + } + } + /* + * SESSION 153-CONTINUED-116: STATE 0xD25 seed (shadow scratch + * 0x25, method 0x3494 = SET_MME_SHADOW_SCRATCH(0x25)), the + * next unseeded input on the same slow path; same value + * policy, translation and fail-closed rule as d24 above. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_mme_state_seed)) { + static u32 mme_state_d25_log; + u32 sv = READ_ONCE(gsp_phys_rmcall_gr_mme_state_d25); + bool have = sv != 0u; + + if (!have) + have = gsp_sema_mme_scratch25_get(&sv); + if (have && sv && + (u32)(p - pb) + 8u < GSP_PHYS_RMCALL_GR_METHOD_WORDS) { + u64 va = (u64)sv << 8; + bool xl = false; + bool ok = true; + + if (READ_ONCE(gsp_phys_rmcall_gr_mme_seed_xlat) + && va) { + u32 up_w = (u32)((va >> 32) & 0xffu); + u32 lo_w = (u32)va & 0xfffffffcu; + u32 n_sh = 0; + + if (gsp_phys_rmcall_gr_3d_reloc( + &up_w, &lo_w, req, + &n_sh)) { + va = ((u64)(up_w & 0xffu) + << 32) | + (u64)(lo_w & 0xfffffffcu); + sv = (u32)(va >> 8); + xl = true; + } else { + ok = false; + } + } + if (ok) { + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM + << 29) | + (1u << 16) | (0x3494u >> 2); + *p++ = sv; + if (mme_state_d25_log < 64u) { + mme_state_d25_log++; + pr_info("vgsp-gr: MME-STATE-SEED d25=0x%08x (va=0x%llx xlat=%u) h=0x%x push=0x%llx\n", + sv, va, xl, + req->h_object, + req->push_va); + } + } + } + } + /* + * SESSION 153-CONTINUED-105: golden MME runtime state seed + * (see the sema_gl_mme_state_seed comment). Five idempotent + * shadow-scratch writes ahead of the window so the + * bug2548370 loadConstantBuffer macro (slot 57) branches on + * the seeded values instead of whatever the replay context + * happens to hold (the captured stream never carries + * 0xd26/0xd27/0xd28). + */ + if (READ_ONCE(gsp_phys_rmcall_gr_mme_state_seed)) { + static u32 mme_state_seed_log; + u32 mv[5]; + u32 mi; + + mv[0] = READ_ONCE(gsp_phys_rmcall_gr_mme_state_d26); + mv[1] = READ_ONCE(gsp_phys_rmcall_gr_mme_state_d27); + mv[2] = READ_ONCE(gsp_phys_rmcall_gr_mme_state_d28); + mv[3] = READ_ONCE(gsp_phys_rmcall_gr_mme_state_d29); + mv[4] = READ_ONCE(gsp_phys_rmcall_gr_mme_state_d2a); + if ((u32)(p - pb) + 10u < + GSP_PHYS_RMCALL_GR_METHOD_WORDS) { + for (mi = 0; mi < 5u; mi++) { + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM + << 29) | + (1u << 16) | + ((0x3498u + 4u * mi) >> 2); + *p++ = mv[mi]; + } + if (mme_state_seed_log < 64u) { + mme_state_seed_log++; + pr_info("vgsp-gr: MME-STATE-SEED d26=%08x d27=%08x d28=%08x d29=%08x d2a=%08x h=0x%x push=0x%llx\n", + mv[0], mv[1], mv[2], mv[3], + mv[4], req->h_object, + req->push_va); + } + } + } + /* + * SESSION 155-CONT-5: engine-visible CB selector seed (see + * the sema_gl_mme_selector_seed comment). Emits the exact + * 4-method group SETUP_FIXED_CONST would emit, so a window + * that calls macro 57 without a preceding macro 0 has a + * valid selector instead of poisoning at 0x2390. Size is + * MME_VPC_CONSTANT_BUFFER_SIZE (0x7000, gf100mme.h); B/C + * carry the same fixed-CB base>>8 and translation as the + * 0x3460 scratch seed; the offset is the seeded d29. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_mme_selector_seed) && + READ_ONCE(gsp_phys_rmcall_gr_mme_state_seed)) { + static u32 mme_sel_seed_log; + u32 sv = READ_ONCE(gsp_phys_rmcall_gr_mme_seed_val); + bool have = sv != 0u; + + if (!have) + have = gsp_sema_mme_scratch18_get(&sv); + if (have && sv && + (u32)(p - pb) + 8u < + GSP_PHYS_RMCALL_GR_METHOD_WORDS) { + u64 va = (u64)sv << 8; + bool xl = false; + u32 off; + + if (READ_ONCE(gsp_phys_rmcall_gr_mme_seed_xlat) + && va) { + u32 up_w = (u32)((va >> 32) & 0xffu); + u32 lo_w = (u32)va & 0xfffffffcu; + u32 n_sh = 0; + + if (gsp_phys_rmcall_gr_3d_reloc( + &up_w, &lo_w, req, &n_sh)) { + va = ((u64)(up_w & 0xffu) + << 32) | + (u64)(lo_w & 0xfffffffcu); + sv = (u32)(va >> 8); + xl = true; + } + } + off = READ_ONCE(gsp_phys_rmcall_gr_mme_state_d29); + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + (1u << 16) | (0x2380u >> 2); + *p++ = 0x7000u; + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + (1u << 16) | (0x2384u >> 2); + *p++ = (sv >> 24) & 0xffu; + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + (1u << 16) | (0x2388u >> 2); + *p++ = sv & 0xffffffu; + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + (1u << 16) | (0x238cu >> 2); + *p++ = off & 0xffffu; + if (mme_sel_seed_log < 64u) { + mme_sel_seed_log++; + pr_info("vgsp-gr: MME-SELECTOR-SEED size=0x7000 addr=0x%06x off=0x%04x (va=0x%llx xlat=%u) h=0x%x push=0x%llx\n", + sv & 0xffffffu, off & 0xffffu, + va, xl, req->h_object, + req->push_va); + } + } + } + /* + * SESSION 153-CONTINUED-93: the LEGACY program-region pair + * (0x1608/0x160c = the pre-Volta SET_PROGRAM_REGION_A/B whose + * names the DS's own snapshot registers + * DS_HWW_ESR_SPH_INFO_PROGRAM_REGION_A/B carry). Those + * snapshots read 0/0 on every SPH-faulting window while the + * MME shadow pair (0x78/0x79) is provably emitted, so the + * DS's validation region may be the legacy pair's state, not + * the MME scratch. The methods are "obsolete" on Volta+ (not + * in the c597 class table), so this is an A/B: a PBDMA that + * refuses them faults the launch visibly, an accepting one + * tells us via the latch. Knob sema_sph_region_legacy + * (default 0). Emitted beside the MME pair with the same + * base. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_sph_region) && req->prog_va && + READ_ONCE(gsp_phys_rmcall_gr_sph_region_legacy)) { + static u32 sph_reg_leg_log; + u64 base = READ_ONCE(gsp_phys_rmcall_gr_sph_region_base); + u64 bind_va = 0; + bool bind_base = + READ_ONCE(gsp_phys_rmcall_gr_sph_region_bind) && + gsp_phys_gr_first_prog_bind(win3d, win3d_n, + &bind_va); + + if (bind_base) + base = bind_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + else if (!base) + base = req->prog_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + (1u << 16) | (0x1608u >> 2); + *p++ = (u32)(base >> 32) & 0xffu; + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + (1u << 16) | (0x160cu >> 2); + *p++ = (u32)base; + if (sph_reg_leg_log < 16u) { + sph_reg_leg_log++; + pr_info("vgsp-gr: SPH-REGION-LEGACY emitted base=0x%llx (0x1608/0x160c; src=%s bind=0x%llx) h=0x%x push=0x%llx\n", + base, bind_base ? "bind-shadow" : + "override/auto", bind_va, + req->h_object, req->push_va); + } + } + /* + * SESSION 153-CONTINUED-84: SET_MME_VERSION=1 (0x234). + * + * The Turing channel-init defaults push + * `nvcMethodPusher: NVC597_SET_MME_VERSION <- 1` once per + * channel (class c597, TURING_OR_NEWER), and the replayed + * context otherwise never sees it: the captured MME setup + * families carry IR/SA/shadow-scratch loads but no 0x234 run + * (verified in the injected full-SA stream too). With the + * version register unset the MME never starts a macro - the + * r1245 evidence: call-bearing launches (real {0,2,34,39,40}, + * full 55-slot prelude, SA re-armed before the window) hang + * the front-end (`MME-FAST ... events=0` -> `GR LAUNCH FAILED: + * timeout after 2500 ms` with qmd_rel=0/ch_rel=0) while every + * MME register stays idle (`pc_valid=0`, `gen0=0`, + * `mme_esr=80000000`): the FE waits forever for a macro the + * halted MME cannot run. Emitted with the prelude step (the + * resident program is (re)loaded right after), ahead of the + * window, so the version select has landed before the first + * CALL_MME_MACRO run. Knob: sema_gl_mme_version (default 1; + * A/B with 0 reproduces the parked launches). + */ + if (emit_prelude && READ_ONCE(gsp_phys_rmcall_gr_mme_version)) { + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + (1u << 16) | + (0x234u >> 2); + *p++ = 1u; + } + if (emit_prelude) { + u32 g0 = pl_gen, g1 = 0, sdrop = 0, pwords = 0; + u32 doff = READ_ONCE(gsp_phys_gr_mme_pre_emit_off); + u32 room = (block_cap > (u32)(p - pb) + win3d_n + 8u + + sa_block) ? + block_cap - (u32)(p - pb) - win3d_n - 8u - + sa_block : 0u; + + /* + * SESSION 153-CONTINUED-79: run-aligned drain at the + * generation cursor; whole runs only, SDM-family + * single-word entries skipped exactly like the legacy + * sanitize (see gsp_phys_rmcall_gr_sdm_drop). + */ + /* + * SESSION 153-CONTINUED-99b: packed emission (see the + * sema_gl_mme_prelude_pack comment). Rebuild the + * partitioned IR, then emit the SA table + one + * contiguous 0x114 run; the drain loop is skipped. + */ + bool packed = false; + + if (READ_ONCE(gsp_phys_rmcall_gr_mme_prelude_pack)) { + u32 sa_ent[64]; + u64 sa_mask = gsp_sema_mme_sa_get(sa_ent, 64); + u32 maxa = 0, r = 0, nruns = 0, npair = 0; + u32 need, kk; + u32 *pp; + + memset(gsp_phys_gr_mme_ir_pack, 0, + sizeof(gsp_phys_gr_mme_ir_pack)); + while (r < pl_n) { + u32 h = pl[r]; + u32 op = (h >> 29) & 7u; + u32 cnt = (h >> 16) & 0xfffu; + u32 span = ((op == 1u || op == 3u || + op == 5u) && cnt && + r + 1u + cnt <= pl_n) ? + 1u + cnt : 1u; + + if (r + span > pl_n) + span = pl_n - r; + if (op == 5u && + ((h & 0x1fffu) << 2) == 0x114u && + cnt >= 2u) { + u32 ptr = pl[r + 1u]; + u32 blen = cnt - 1u; + u32 nxt = 0, k; + + for (kk = 0; kk < 64u; kk++) + if ((sa_mask & (1ull << kk)) && + sa_ent[kk] > ptr && + (!nxt || + sa_ent[kk] < nxt)) + nxt = sa_ent[kk]; + /* + * CONT-100 units fix: `ptr` is an + * MME INSTRUCTION address (the + * driver advances codeOffset by + * codeSize/3, mmeDownload.cpp), + * and each instruction is 3 IR + * words stored at 3*ptr..3*ptr+2. + * Indexing the contiguous word + * image at ptr+k (the pre-CONT-100 + * form) misplaced every macro and + * left 2/3 of the program + * unreachable (r1343/r1344 poison, + * only ~312 of 933 instructions + * covered). Clamp the tail at the + * next SA entry in INSTRUCTIONS. + */ + if (nxt && + ptr + (blen + 2u) / 3u > nxt) + blen = 3u * (nxt - ptr); + for (k = 0; k < blen && + ptr * 3u + k < 4096u; k++) + gsp_phys_gr_mme_ir_pack[ptr * 3u + k] = + pl[r + 2u + k]; + if (ptr * 3u + blen > maxa) + maxa = ptr * 3u + blen; + nruns++; + } + r += span; + } + /* + * SESSION 153-CONTINUED-114: golden overlay. + * After the capture fill, merge the golden + * library's words into the packed image and + * the local SA pair table for every slot the + * capture cannot serve (mode 1), or every + * slot (mode 2 / force). Written after the + * capture fill so the golden words win any + * overlap; a complete capture is untouched + * in mode 1 (the overlay then only logs). + */ + if (READ_ONCE(gsp_phys_rmcall_gr_mme_golden)) { + u32 gj, goldn = 0, ginsp = 0; + u32 gblank = 0; + + for (gj = 0; gj < 64u; gj++) { + u32 gent, gn, q; + const u32 *gw; + + if (!gsp_phys_gr_golden_slot( + gj, &gent, &gw, + &gn)) + continue; + ginsp++; + if (!READ_ONCE(gsp_phys_rmcall_gr_mme_golden_force)) { + /* + * Mode 1: only slots the + * capture cannot serve + * (no SA entry, or the + * entry's IR never + * entered the replay). + */ + if ((sa_mask & + (1ull << gj)) && + sa_ent[gj] && + gsp_sema_mme_ir_covered( + sa_ent[gj])) + continue; + } + for (q = 0; + q < gn && + gent * 3u + q < 4096u; + q++) + gsp_phys_gr_mme_ir_pack[gent * 3u + q] = + gw[q]; + if (gent * 3u + gn > maxa) + maxa = gent * 3u + gn; + sa_mask |= 1ull << gj; + sa_ent[gj] = gent; + goldn++; + } + /* + * r1448 STRATEGY 3.3: force-blank + * the captured bodies of every slot + * the golden library cannot serve so + * no capture word can survive - only + * golden bodies may execute + * (fail-closed). The zero range is + * bounded by the next captured SA + * entry, exactly the bound the + * capture fill above applied. + * Clearing the local sa_mask/sa_ent + * pair also unpublishes the slot + * from the SA re-arm; its CALL runs + * are dropped via the stubbed-slot + * predicate (slot_stubbed -> + * golden_blank). + */ + if (READ_ONCE(gsp_phys_rmcall_gr_mme_golden_force) && + READ_ONCE(gsp_phys_rmcall_gr_mme_golden_only)) { + for (gj = 0; gj < 64u; gj++) { + u32 gent, gn, q, kk2; + const u32 *gw; + u32 bstart, bend; + u64 nmask; + + if (!(sa_mask & (1ull << gj)) || + !sa_ent[gj]) + continue; + if (gsp_phys_gr_golden_slot( + gj, &gent, &gw, + &gn)) + continue; + bstart = 3u * sa_ent[gj]; + bend = 4096u; + nmask = sa_mask & + ~(1ull << gj); + for (kk2 = 0; kk2 < 64u; kk2++) { + if (!(nmask & + (1ull << kk2)) || + !sa_ent[kk2]) + continue; + if (3u * sa_ent[kk2] > bstart && + 3u * sa_ent[kk2] < bend) + bend = 3u * sa_ent[kk2]; + } + for (q = bstart; q < bend; q++) + gsp_phys_gr_mme_ir_pack[q] = 0; + sa_mask &= ~(1ull << gj); + sa_ent[gj] = 0; + gblank++; + } + } + if (goldn && gsp_phys_gr_golden_log < 8u) { + gsp_phys_gr_golden_log++; + pr_info("vgsp-gr: MME-GOLDEN overlay %u slot(s) inspected=%u blank=%u maxa=%u h=0x%x push=0x%llx\n", + goldn, ginsp, gblank, + maxa, + req->h_object, + req->push_va); + } + } + /* + * CONT-112: single-word A/B override of the + * packed program (see the sema_gl_mme_irpatch + * comment). Applied after the image is built so + * the override is what the macro executes and + * what the IRAM gate compares against. + */ + if (gsp_phys_gr_mme_irpatch_off < maxa) { + pr_info("vgsp-gr: MME-IRPATCH word[%u] = 0x%08x (was 0x%08x, maxa=%u)\n", + gsp_phys_gr_mme_irpatch_off, + gsp_phys_gr_mme_irpatch_val, + gsp_phys_gr_mme_ir_pack[ + gsp_phys_gr_mme_irpatch_off], + maxa); + gsp_phys_gr_mme_ir_pack[ + gsp_phys_gr_mme_irpatch_off] = + gsp_phys_gr_mme_irpatch_val; + } + for (kk = 0; kk < 64u; kk++) + if ((sa_mask & (1ull << kk)) && + sa_ent[kk]) + npair++; + need = 2u + maxa + 3u * npair; + /* + * CONT-105: chunked emission. The CONT-99b + * all-or-nothing form only fires when the + * whole packed stream fits one block (need <= + * room); the current 55-slot capture needs + * ~2966 words against ~1600 available, so it + * silently fell back to the raw drain. The + * split form drains SA (once) + whole- + * instruction IR chunks across consecutive + * launches with the generation cursor reset + * alongside the drain cursor (see + * gsp_phys_gr_mme_pack_off). + */ + { + bool split = READ_ONCE( + gsp_phys_rmcall_gr_mme_pack_split); + u32 poff = READ_ONCE( + gsp_phys_gr_mme_pack_off); + bool sa_done = READ_ONCE( + gsp_phys_gr_mme_pack_sa); + u32 left, chunk; + bool can; + + /* + * Reserve the window's SA re-arm + * budget (<= 32 entries * 3 words) so + * the block stays under block_cap. + */ + left = (split && room > 96u) ? + room - 96u : (split ? 0u : room); + can = maxa >= 3u && poff < maxa && + (split ? left >= 5u : + need <= room); + if (can && split && !sa_done && + 3u * npair > left) + can = false; + if (can) { + pp = p; + if (!sa_done) { + for (kk = 0; kk < 64u; + kk++) { + if (!(sa_mask & + (1ull << + kk)) || + !sa_ent[kk]) + continue; + *pp++ = ((u32) + GSP_PHYS_RMCALL_METHOD_IMM + << 29) | + (2u << 16) | + (0x11cu >> 2); + *pp++ = kk; + *pp++ = + sa_ent[kk]; + } + left -= 3u * npair; + WRITE_ONCE( + gsp_phys_gr_mme_pack_sa, + true); + } + chunk = maxa - poff; + if (chunk > left - 2u) + chunk = left - 2u; + chunk -= chunk % 3u; + if (chunk >= 3u) { + *pp++ = ((u32) + GSP_PHYS_RMCALL_METHOD_ONEINC + << 29) | + ((chunk + 1u) << 16) | + (0x114u >> 2); + *pp++ = poff / 3u; + memcpy(pp, + &gsp_phys_gr_mme_ir_pack[poff], + (size_t)chunk * 4u); + pp += chunk; + poff += chunk; + WRITE_ONCE( + gsp_phys_gr_mme_pack_off, + poff); + } + pwords = (u32)(pp - p); + p = pp; + packed = true; + if (poff >= maxa) { + doff = pl_n; + pr_info("vgsp-gr: MME-PRELUDE pack ir_words=%u runs=%u sa=%u h=0x%x push=0x%llx\n", + maxa, nruns, + npair, + req->h_object, + req->push_va); + } else { + pr_info("vgsp-gr: MME-PRELUDE pack-part ir_words=%u/%u sa=%u split=%u h=0x%x push=0x%llx\n", + poff, maxa, + npair, + split ? 1u : 0u, + req->h_object, + req->push_va); + } + } else if (split && maxa >= 3u) { + /* + * Split mode never falls back + * to the raw drain: an + * un-emittable chunk simply + * waits for a launch with + * room (the cursor is + * untouched; the prelude + * latch stays unset so the + * calls stay held). + */ + packed = true; + } + } + } + while (!packed && doff < pl_n && + pwords + 1u <= room) { + u32 h = pl[doff]; + u32 op = (h >> 29) & 7u; + u32 cnt = (h >> 16) & 0xfffu; + u32 span = ((op == 1u || op == 3u || + op == 5u) && cnt && + doff + 1u + cnt <= pl_n) ? + 1u + cnt : 1u; + + if (doff + span > pl_n) + span = pl_n - doff; + if (READ_ONCE(gsp_phys_rmcall_gr_sdm_drop) && + h && (op == 0u || op == 2u || op == 6u)) { + sdrop += span; + doff += span; + continue; + } + /* + * SESSION 153-CONTINUED-99: IR-only drain (see + * the sema_gl_mme_prelude_ir_only comment). + * Keep only the runs aimed at the MME SA/IR + * ports; skip everything else with the same + * run-aligned accounting as the SDM drop. + */ + if (READ_ONCE( + gsp_phys_rmcall_gr_mme_prelude_ir_only)) { + u32 ma = (h & 0x1fffu) << 2; + + if (ma != 0x114u && ma != 0x118u && + ma != 0x11cu && ma != 0x120u) { + sdrop += span; + doff += span; + continue; + } + } + if (pwords + span > room) + break; + memcpy(p, &pl[doff], (size_t)span * 4u); + p += span; + pwords += span; + doff += span; + } + WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, doff); + pre_done = (doff >= pl_n); + pre_words_out = pwords; + /* seqcheck: a mid-copy append must not be recorded as + * emitted (the copy may be a torn prefix). */ + gsp_sema_mme_prelude_get(NULL, &g1); + if (pre_done && g1 == g0) { + WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, g0); + WRITE_ONCE(gsp_phys_gr_mme_prelude_h, + req->h_object); + /* CONT-103: this launch carries the final + * part; its poll outcome (executed or not) + * decides whether the program is live. */ + WRITE_ONCE(gsp_phys_gr_mme_pending_gen, g0); + pr_info("vgsp-gr: MME-PRELUDE emitted gen=%u words=%u window=%u h=0x%x push=0x%llx sdm_drop=%u cap=%u\n", + g0, pl_n, win3d_n, req->h_object, + req->push_va, sdrop, block_cap); + /* + * SESSION 153-CONTINUED-70: one full dump of the + * replayed MME context-setup program per boot + * (the offline MME64 sim needs the same words the + * engine will execute). Also list the SA table + * with per-entry IR coverage so the slot-by-slot + * macro re-enable has its proof in one place. + */ + if (!gsp_phys_gr_mme_pre_full_dump) { + u32 k, sj; + u32 sa_ent[64]; + u64 sa_mask; + u32 cov_n = 0; + + gsp_phys_gr_mme_pre_full_dump = 1; + sa_mask = gsp_sema_mme_sa_get(sa_ent, 64); + for (sj = 0; sj < 64u; sj++) + if ((sa_mask & (1ull << sj)) && + gsp_sema_mme_ir_covered(sa_ent[sj])) + cov_n++; + pr_info("vgsp-gr: MME-PRE-DUMP words=%u gen=%u h=0x%x ir_end=0x%x sa_mask=0x%llx covered=%u code_words=%u keep_mask=0x%llx\n", + pl_n, g0, req->h_object, + gsp_sema_mme_ir_end_get(), + (unsigned long long)sa_mask, + cov_n, gsp_sema_mme_code_words_get(), + (unsigned long long)READ_ONCE(gsp_phys_rmcall_gr_keep_mask)); + for (k = 0; k + 4u <= pl_n; k += 4u) + pr_info("vgsp-gr: MME-PRE[%03u] %08x %08x %08x %08x\n", + k, pl[k], pl[k + 1], + pl[k + 2], pl[k + 3]); + for (sj = 0; sj < 64u; sj++) + if (sa_mask & (1ull << sj)) + pr_info("vgsp-gr: MME-PRE-SA[%u] entry=0x%x cov=%u\n", + sj, sa_ent[sj], + gsp_sema_mme_ir_covered(sa_ent[sj])); + } + } else if (pre_done) { + pr_info("vgsp-gr: MME-PRELUDE copy raced an append (gen %u->%u); will re-emit next launch\n", + g0, g1); + } else { + pr_info("vgsp-gr: MME-PRELUDE part gen=%u off=%u words=%u/%u window=%u h=0x%x push=0x%llx sdm_drop=%u cap=%u (continuation next launch)\n", + g0, doff, pwords, pl_n, win3d_n, + req->h_object, req->push_va, sdrop, + block_cap); + } + } + /* + * SESSION 153-CONTINUED-107: prelude-only LOAD carriers while + * the IRAM gate has not verified the packed program. + * + * r1372's first instruction (decisive): the calls-held pack + * carrier still poisoned (`DS-FAST ... ds a2040a00->badf1201 + * ... MME-FAST end launch events=1 (poison)` -> WEDGE-RECOVER + * wipes the context) because the block also carried the WINDOW + * - the window's own SPH state trips the DS validation and the + * FECS poison episode interrupts the 0x114/0x118 IRAM load + * mid-flight. Holding the calls is not enough; the LOAD must + * ride a carrier whose methods cannot fault, so while the + * generation is not verified live (live_gen != pl_gen) emit + * the pack/SA/release WITHOUT the window. The carrier retires, + * the poll-break gate reads the IRAM back; on a pass live_gen + * is promoted and the very next launch carries the window + * again (latch && live). Bounded by + * sema_gl_mme_iram_gate_max so a broken readback cannot starve + * the lane forever. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_mme_iram_gate) && + READ_ONCE(gsp_phys_rmcall_gr_mme_prelude_pack) && + READ_ONCE(gsp_phys_rmcall_gr_mme_hold_calls) && + pl && pl_n && pl_gen && + !gsp_phys_gr_mme_iram_gate_giveup()) { + bool latch_now = + (pl_gen == + READ_ONCE(gsp_phys_gr_mme_prelude_gen) && + req->h_object == + READ_ONCE(gsp_phys_gr_mme_prelude_h)); + bool live_now = + (READ_ONCE(gsp_phys_rmcall_gr_mme_live_skip) && + READ_ONCE(gsp_phys_gr_mme_live_gen) == pl_gen); + + if (!(latch_now && live_now)) + mme_load_only = true; + } + /* + * SESSION 153-CONTINUED-82: hold real CALL runs until the + * prelude latch for this generation+channel is set (see the + * sema_gl_mme_hold_calls comment). Evaluated AFTER the + * emission step above, so a launch that completed the drain + * in this same block has just written the latch. Real calls + * stay held when EITHER + * - the latch for this generation+channel is not valid after + * this launch (partial drain, raced append, or a fresh + * generation whose code is not fully resident), or + * - this launch uploaded generation code for the FIRST time + * (emitted words with the latch invalid before the step) - + * the r1222 evidence: the drain-completing launch bundled + * part-2's IR uploads with the window's real calls and hit + * ILLEGAL_OPCODE even though its own emission set the + * latch. A later launch (latch already valid, only the + * idempotent keep-forced re-emission of the prefix) runs + * the real calls against the now-resident program. + */ + if (pl && pl_n && pl_gen && + READ_ONCE(gsp_phys_rmcall_gr_mme_prelude) && + READ_ONCE(gsp_phys_rmcall_gr_mme_hold_calls)) { + u32 lg = READ_ONCE(gsp_phys_gr_mme_prelude_gen); + u32 lh = READ_ONCE(gsp_phys_gr_mme_prelude_h); + bool latch_after = (pl_gen == lg && + req->h_object == lh); + + { + u32 rg = READ_ONCE(gsp_phys_gr_mme_release_gen); + u32 rh = READ_ONCE(gsp_phys_gr_mme_release_h); + bool settle = READ_ONCE( + gsp_phys_rmcall_gr_mme_hold_settle); + bool strict = READ_ONCE( + gsp_phys_rmcall_gr_mme_hold_strict); + bool fe_idle_now = gsp_phys_gr_fe_idle(); + /* + * r1447: the strict FE-idle term fails open + * after sema_gl_mme_strict_grace held launches + * of this generation (0 = never fail open). + */ + bool grace_used = + (READ_ONCE( + gsp_phys_rmcall_gr_mme_strict_grace) != 0u && + READ_ONCE(gsp_phys_gr_mme_strict_holds) >= + READ_ONCE( + gsp_phys_rmcall_gr_mme_strict_grace)); + bool strict_ok = (!strict || + (pre_words_out == 0 && + READ_ONCE( + gsp_phys_gr_mme_live_gen) == + pl_gen && + (fe_idle_now || grace_used))); + + /* + * r1447: an armed window-hold for this + * generation+channel stays effective until the + * release actually fires (keyed exactly like + * the CONT-118 release statics; cleared at the + * residency resets and at release - see the + * sema_gl_mme_hold_window comment). + */ + mme_hold_window = + (READ_ONCE( + gsp_phys_gr_mme_window_hold_gen) == + pl_gen && + READ_ONCE( + gsp_phys_gr_mme_window_hold_h) == + req->h_object); + + /* + * r1445 3b: the strict variant strengthens the + * settled release - it only releases on a launch + * that emitted nothing, with the FE idle and the + * generation proven live (FETCH-WIT). strict=0 + * reduces to the exact rg/rh clause. + */ + mme_hold_calls = !latch_after || + (pre_words_out && !latch_before) || + (settle && + !(rg == pl_gen && + rh == req->h_object && + strict_ok)); + /* + * r1447: the arming launch - latch valid, no + * emission, release not yet armed - also drops + * the window (the calls are held by the settle + * clause above). The arm below writes rg/rh, so + * this set can fire at most once per generation; + * the first window execution is then the proven + * calls+window-kept shape. When the param is 0 + * this block is a no-op (legacy shape). + */ + if (settle && latch_after && !pre_words_out && + !(rg == pl_gen && rh == req->h_object) && + !stream_damaged && + READ_ONCE( + gsp_phys_rmcall_gr_mme_hold_window)) { + WARN_ON_ONCE(mme_load_only); + mme_hold_window = true; + WRITE_ONCE( + gsp_phys_gr_mme_window_hold_gen, + pl_gen); + WRITE_ONCE( + gsp_phys_gr_mme_window_hold_h, + req->h_object); + } + /* + * r1447: bounded fail-open for the strict + * release - count the launches the exact + * 0x400504 == 1 predicate held after the + * release was armed; at the grace bound the + * FE term fails open on the next launch. + */ + if (strict && settle && latch_after && + mme_hold_calls && + rg == pl_gen && rh == req->h_object && + !strict_ok) { + WRITE_ONCE( + gsp_phys_gr_mme_strict_holds, + gsp_phys_gr_mme_strict_holds + 1u); + if (READ_ONCE( + gsp_phys_rmcall_gr_mme_strict_grace) && + READ_ONCE( + gsp_phys_gr_mme_strict_holds) == + READ_ONCE( + gsp_phys_rmcall_gr_mme_strict_grace)) + pr_warn("vgsp-gr: STRICT-GRACE gen=%u n=%u h=%#x 0x400504=%08x (FE never read idle; releasing on the next launch) push=0x%llx\n", + pl_gen, + READ_ONCE( + gsp_phys_gr_mme_strict_holds), + req->h_object, + gsp_phys_gr_fe_status(), + req->push_va); + } + /* + * CONT-118: a launch that carries no prelude + * words but sees a valid latch is the first + * settled launch; arm the release so the NEXT + * launch runs the real calls. The completing + * carrier (pre_words_out > 0) must not arm it + * (the r1222 clause already holds it). + */ + if (settle && latch_after && !pre_words_out) { + WRITE_ONCE( + gsp_phys_gr_mme_release_gen, + pl_gen); + WRITE_ONCE( + gsp_phys_gr_mme_release_h, + req->h_object); + } + } + /* + * CONT-104: also hold while the front-end is not + * settled (the r1359 MME ILLEGAL_OPCODE race; see + * gsp_phys_gr_fe_idle). The held launch retires and + * a later window retries once 0x400504 reads idle. + * SESSION 153-CONTINUED-107: separately knob-gated - + * with the IRAM gate proving residency, the residual + * value of this clause is only the r1359 race; r1373 + * left the FE at 0x07071602 for minutes and starved + * every calls-kept window, so allow an explicit A/B + * (sema_gl_mme_hold_fe_idle=0). + */ + if (!mme_hold_calls && + READ_ONCE(gsp_phys_rmcall_gr_mme_hold_fe_idle) && + !gsp_phys_gr_fe_idle()) { + mme_hold_calls = true; + mme_hold_fe_busy = true; + } + } + /* + * r1487 STREAM-DAMAGE: an earlier chunk of this same push was + * dropped (LOAD-ONLY carrier or the arming window hold), so + * the channel no longer holds the state that chunk carried. + * Run this chunk's CALL_MME_MACRO/DATA runs held (elided) - + * the same proven-safe shape as the prelude-hold path - so no + * macro can emit LOAD_CONSTANT_BUFFER against the incomplete + * state and class-error into the FECS priv-error poison. The + * window's non-CALL state methods still execute, and the next + * client section (stream_rel 0) clears the record. + */ + if (stream_damaged && !mme_load_only) { + if (!mme_hold_calls && gsp_phys_gr_stream_damage_hits < 32u) { + gsp_phys_gr_stream_damage_hits++; + pr_info("vgsp-gr: STREAM-DAMAGE h=0x%x push=0x%llx rel=%u window=%u (calls held: earlier chunk dropped)\n", + req->h_object, req->push_va, + req->stream_rel, win3d_n); + } + mme_hold_calls = true; + } + /* + * SESSION 153-CONTINUED-107: IRAM readback gate (see the + * gsp_phys_gr_mme_iram_gate comment). The calls would run this + * launch; verify the live IRAM actually holds the packed image + * at the SA entries first. On a mismatch (the CONT-106 + * all-zero fetch) hold the calls and owe a full re-emission - + * the r1368 evidence says the real CALL runs against a + * non-resident program poison the lane (ILLEGAL_OPCODE at the + * macro entry) instead of drawing. Only meaningful on the + * packed path (the pack image is the comparator). + */ + if (!mme_hold_calls && pl && pl_n && pl_gen && + READ_ONCE(gsp_phys_rmcall_gr_mme_prelude) && + READ_ONCE(gsp_phys_rmcall_gr_mme_hold_calls) && + READ_ONCE(gsp_phys_rmcall_gr_mme_prelude_pack) && + READ_ONCE(gsp_phys_rmcall_gr_mme_iram_gate) && + READ_ONCE(gsp_phys_rmcall_gr_mme_iram_gate_build)) { + void __iomem *gbar0 = gsp_phys_gpu_bar0(); + + if (gbar0 && + !gsp_phys_gr_mme_iram_gate(gbar0, req->h_object, + pl_gen, "build")) { + mme_hold_calls = true; + gsp_phys_gr_mme_iram_gate_fail(); + } + } + /* + * SESSION 153-CONTINUED-63: emit the SA-RAM completion computed + * above - a one-instruction end-noop stub in the IR plus one + * ONE_INC 0x11c run loading [slot, stub] for every called slot + * the capture's SA table does not cover (see the sa_stub + * comment at the emit decision). Emitted with the prelude (same + * generation latch), so it lands once per capture generation + * and the context keeps it for every later window. + */ + if (emit_prelude && sa_block) { + u32 t, stub_pc = gsp_sema_mme_ir_end_get(); + + /* + * SESSION 153-CONTINUED-68: the stub PC must be past the + * replayed program's last instruction - the window also + * calls COVERED slots, and a fixed PC inside a captured + * macro body would clobber its code. Fall back to 0xC0 + * when the capture reports nothing yet. + */ + if (stub_pc < GSP_PHYS_RMCALL_GR_MME_STUB_PC) + stub_pc = GSP_PHYS_RMCALL_GR_MME_STUB_PC; + /* + * SESSION 153-CONTINUED-69: explicit stub-PC override + * (sema_gl_mme_stub_pc). The ir_end-derived PC sits at + * the first instruction past the replayed program; if + * the MME's IR allocation/ONEINC pointer semantics + * differ (instruction vs word units, allocation + * boundary), the stub body may land where the engine + * does not look. This knob moves the stub (e.g. to + * 0xf0/0x100, or back to the 0xC0 floor) so the + * landing can be A/B-tested without a rebuild. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_mme_stub_pc)) + stub_pc = READ_ONCE(gsp_phys_rmcall_gr_mme_stub_pc); + /* + * The IR upload MUST use the client's own SEC_OP (5, + * ONE_INC): with op=1 the method auto-increment walked + * the data words onto 0x11c/0x120 and clobbered SA[0] + * while writing no usable stub code (r1131). count = 7 + * = 1 pointer + 6 code words (two 3-word NOPs; see the + * gsp_phys_gr_mme_stub_code comment). + */ + *p++ = (GSP_PHYS_RMCALL_METHOD_ONEINC << 29) | + (7u << 16) | (0x114u >> 2); + *p++ = stub_pc; + *p++ = gsp_phys_gr_mme_stub_code[0]; + *p++ = gsp_phys_gr_mme_stub_code[1]; + *p++ = gsp_phys_gr_mme_stub_code[2]; + *p++ = gsp_phys_gr_mme_stub_code[3]; + *p++ = gsp_phys_gr_mme_stub_code[4]; + *p++ = gsp_phys_gr_mme_stub_code[5]; + /* + * Each [slot, stub] pair is its own INC 0x11c run: the + * pair methods are re-written per slot (a single longer + * run would walk onto 0x124 SetMmeShadowRamControl and + * beyond). + */ + for (t = 0; t < sa_nstub; t++) { + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + (2u << 16) | (0x11cu >> 2); + *p++ = sa_stub[t]; + *p++ = stub_pc; + } + pr_info("vgsp-gr: MME-SA-STUB slots=%u words=%u pc=0x%x h=0x%x push=0x%llx\n", + sa_nstub, sa_block, stub_pc, req->h_object, + req->push_va); + { + char sb[192]; + u32 k, o = 0; + + sb[0] = '\0'; + for (k = 0; k < sa_nstub && o + 8u < sizeof(sb); k++) + o += scnprintf(sb + o, sizeof(sb) - o, + "%s%u:%x", k ? "," : "", + sa_stub[k], + sa_stub_ent[k]); + pr_info("vgsp-gr: MME-SA-STUB list=[%s]\n", sb); + } + { + char sb[192]; + u32 k, o = 0; + + sb[0] = '\0'; + for (k = 0; k < sa_ncov && o + 8u < sizeof(sb); k++) + o += scnprintf(sb + o, sizeof(sb) - o, + "%s%u:%x", k ? "," : "", + sa_cov[k], + sa_cov_ent[k]); + pr_info("vgsp-gr: MME-SA-COVERED list=[%s]\n", + sb); + } + } + /* + * SESSION 153-CONTINUED-57: attr/pipeline state prelude (see the + * sema_gl_attr_prelude knob above). Emit the patched captured + * runs after the MME prelude and ahead of the window, + * run-aligned, draining across launches when the remaining + * block budget cannot take the whole remainder. The cursor + * resets when a new capture generation is patched. A window + * with no room at all defers the whole step to the next launch + * (logged by the patch site). + */ + if (attr_pre && attr_pre_n && + READ_ONCE(gsp_phys_rmcall_gr_attr_prelude)) { + u32 avail = block_cap - + (u32)(p - pb) - (win3d_n + 2u + 8u); + u32 off, copied = 0; + + /* + * SESSION 153-CONTINUED-79: the attr prelude shares + * the same per-launch block cap (see + * sema_gl_mme_block_cap); unsigned underflow would + * hand it the whole module buffer on an oversized + * window. + */ + if ((u32)(p - pb) + win3d_n + 2u + 8u >= block_cap) + avail = 0; + + if (gsp_phys_gr_attr_emit_gen != + READ_ONCE(gsp_phys_gr_attr_pre_gen)) { + WRITE_ONCE(gsp_phys_gr_attr_emit_gen, + READ_ONCE(gsp_phys_gr_attr_pre_gen)); + WRITE_ONCE(gsp_phys_gr_attr_emit_off, 0); + } + off = READ_ONCE(gsp_phys_gr_attr_emit_off); + while (off < attr_pre_n) { + u32 hdr = attr_pre[off]; + u32 op = (hdr >> 29) & 7u; + u32 cnt = (hdr >> 16) & 0xfffu; + u32 span = ((op == 1u || op == 3u || + op == 5u) && cnt) ? + 1u + cnt : 1u; + + if (off + span > attr_pre_n) + span = attr_pre_n - off; + /* + * SESSION 153-CONTINUED-59: the same SDM + * sanitize as the MME prelude (see + * gsp_phys_rmcall_gr_sdm_drop); the cursor + * still advances so the drop is permanent. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_sdm_drop) && + hdr && (op == 0u || op == 2u || + op == 6u)) { + off += span; + continue; + } + /* + * CONT-120 (r1451): drop the attr-prelude + * SET_PIPELINE_SHADER(2) run(s) - the guest's + * cold `0x2080 <- 0x20` word is refused by the + * FE with Class Error EC4 when its one-shot + * attr emission rides a windowless LOAD-ONLY + * carrier (no SPH-BIND enable ahead of it; see + * sema_gl_attr_sph_drop). Cursor advances like + * the SDM drop: no raw fallback. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_attr_sph_drop) && + (op == 1u || op == 3u || op == 5u) && cnt && + ((hdr & 0xfffu) << 2) == 0x2080u) { + gsp_phys_gr_attr_sph_drop_count++; + if (gsp_phys_gr_attr_sph_drop_gen != + READ_ONCE(gsp_phys_gr_attr_pre_gen)) { + gsp_phys_gr_attr_sph_drop_gen = + READ_ONCE(gsp_phys_gr_attr_pre_gen); + pr_info("vgsp-gr: ATTR-PRELUDE dropped SET_PIPELINE gen=%u mthd=0x%x drops=%u\n", + gsp_phys_gr_attr_sph_drop_gen, + (u32)((hdr & 0xfffu) << 2), + gsp_phys_gr_attr_sph_drop_count); + } + off += span; + continue; + } + if (copied + span > avail) + break; + memcpy(&p[copied], &attr_pre[off], + (size_t)span * 4u); + copied += span; + off += span; + } + if (copied) { + p += copied; + WRITE_ONCE(gsp_phys_gr_attr_emit_off, off); + pr_info("vgsp-gr: ATTR-PRELUDE emitted gen=%u words=%u/%u window=%u h=0x%x push=0x%llx%s\n", + READ_ONCE(gsp_phys_gr_attr_pre_gen), + off, attr_pre_n, win3d_n, + req->h_object, req->push_va, + off < attr_pre_n ? + " (continuation next launch)" : ""); + } else if (off < attr_pre_n) { + static u32 attr_defer_log; + + if (attr_defer_log < 8) { + attr_defer_log++; + pr_info("vgsp-gr: ATTR-PRELUDE deferred (no room: window=%u avail=%u) h=0x%x\n", + win3d_n, avail, + req->h_object); + } + } + } + /* + * SESSION 153-CONTINUED-84: shadow-RAM mode before the window + * (see the sema_gl_mme_track comment). METHOD_TRACK (0) is + * emitted by default so the repeated CALL/DATA parameter + * writes are not swallowed by the redundant-method filter. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_mme_track) >= 0 && + p + 2u <= pb + GSP_PHYS_RMCALL_GR_METHOD_WORDS) { + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + (1u << 16) | (0x124u >> 2); + *p++ = (u32)READ_ONCE(gsp_phys_rmcall_gr_mme_track); + } + /* + * SESSION 153-CONTINUED-84: SA re-arm BEFORE the window. + * The r1236 evidence: this block was emitted AFTER the + * window copy (contradicting its own "before the window" + * log), so the window's CALL_MME_MACRO(j) runs executed + * against the SA table state left by the prelude - whose + * last captured SA write re-clobbers the slot (r1231: + * SA[34]=0 lands last, every draw call resolved to macro + * 0 and emitted nothing). Because the prelude is + * re-emitted on every launch while any slot is kept, the + * late re-arm was re-clobbered before the next window + * anyway; the re-point only ever held between a window + * and the next prelude re-emission. Emitting the re-arm + * ahead of the window makes every launch's calls resolve + * to the captured SA entry. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_mme_sa_rearm)) { + u32 sa_ent[64]; + u64 sa_mask = gsp_sema_mme_sa_get(sa_ent, 64); + u32 r = 0, rearm = 0; + + while (r < win3d_n && rearm < 32u) { + u32 h = win3d[r]; + u32 op = (h >> 29) & 7u; + u32 cnt = (h >> 16) & 0xfffu; + u32 span = ((op == 1u || op == 3u || + op == 5u) && cnt && + r + 1u + cnt <= win3d_n) ? + 1u + cnt : 1u; + u32 mb = (h & 0xfffu) << 2; + + if (mb >= 0x3800u && mb <= 0x39fcu) { + u32 j = (mb - 0x3800u) / 8u; + + if (j < 64u && + (sa_mask & (1ull << j)) && + sa_ent[j]) { + *p++ = ((u32) + GSP_PHYS_RMCALL_METHOD_IMM + << 29) | + (2u << 16) | + (0x11cu >> 2); + *p++ = j; + *p++ = sa_ent[j]; + rearm++; + } + } + r += span; + } + if (rearm) + pr_info("vgsp-gr: MME-SA-REARM %u slot(s) re-pointed before the window (h=0x%x push=0x%llx)\n", + rearm, req->h_object, req->push_va); + } + /* + * r1447: the release actually fires on this launch (the calls + * are kept) - the arming window-hold is consumed here, so the + * release launch carries the CALL runs and the window together + * (the r1443b shape); the strict grace budget restarts with the + * next generation cycle. + */ + if (mme_hold_window && !mme_load_only && + (!mme_hold_calls || stream_damaged)) { + WRITE_ONCE(gsp_phys_gr_mme_window_hold_gen, 0); + WRITE_ONCE(gsp_phys_gr_mme_window_hold_h, 0); + WRITE_ONCE(gsp_phys_gr_mme_strict_holds, 0); + mme_hold_window = false; + } + /* + * r1453 PAIR-CHAIN: the window was already drained in the + * submit path (after the SPH-BIND/SURF-PRELUDE prepends, + * once per launch - see gsp_phys_gr_pair_chain_fix). The + * emitted-pair proof is recomputed from the final branch + * output below. + */ + win_emit_off = (u32)(p - pb); + if (mme_load_only || mme_hold_window) { + /* + * r1487 STREAM-DAMAGE: this window slice never reaches + * the engine. Record the push/offset so the remaining + * chunks of the same client section hold their CALL + * runs instead of executing a macro against the lost + * state (see the damage table comment). + */ + gsp_phys_gr_stream_damage_note(req->push_va, + req->stream_rel); + if (gsp_phys_gr_stream_damage_notes < 32u) { + gsp_phys_gr_stream_damage_notes++; + pr_info("vgsp-gr: STREAM-DAMAGE-NOTE h=0x%x push=0x%llx rel=%u window=%u (%s)\n", + req->h_object, req->push_va, + req->stream_rel, win3d_n, + mme_hold_window ? "window-hold" : + "load-only"); + } + /* + * r1465 PAIR-ORPHAN-DROP: the FE consumed none of + * this window, so the pair projection is a no-op: + * restore the launch-entry snapshot the fixer took + * before the walk (MME-WINDOW-HOLD and MME-LOAD-ONLY + * both land here; the held-call branch still emits + * the window, so its pairs are FE-visible). + */ + if (READ_ONCE(gsp_phys_rmcall_gr_pair_orphan_drop)) { + WRITE_ONCE(gsp_phys_gr_mme_pair_open, + READ_ONCE(gsp_phys_gr_mme_pair_open_entry)); + WRITE_ONCE(gsp_phys_gr_mme_pair_open_split, + READ_ONCE(gsp_phys_gr_mme_pair_open_entry_split)); + } + if (mme_hold_window && !mme_load_only) { + pr_info_ratelimited("vgsp-gr: MME-WINDOW-HOLD gen=%u h=%#x window=%u rel=%u dropped (r1447 arming hold) push=0x%llx\n", + pl_gen, req->h_object, win3d_n, + req->stream_rel, req->push_va); + } else { + /* + * SESSION 153-CONTINUED-107: no window on a + * LOAD carrier (see the mme_load_only + * comment). The block is SET_OBJECT + + * prelude/SA/pack + release only; the engine + * can retire it without any DS/shader state + * to validate, so the IRAM load cannot be + * interrupted by the SPH poison. + */ + static u32 load_only_log; + + load_only_log++; + if (load_only_log <= 24u) + pr_info("vgsp-gr: MME-LOAD-ONLY carrier h=0x%x gen=%u window=%u rel=%u dropped until the IRAM gate verifies (log %u) push=0x%llx\n", + req->h_object, pl_gen, + win3d_n, req->stream_rel, + load_only_log, + req->push_va); + } + } else if (mme_hold_calls) { + /* + * SESSION 153-CONTINUED-82: the prelude latch for + * this generation+channel is not set - elide every + * remaining CALL_MME_MACRO/DATA run of the window + * (the proven-safe drop shape; see the + * sema_gl_mme_hold_calls comment). Runs touching + * the MME macro method space 0x3800..0x39fc are the + * ones that execute a macro; the drop decision is + * the same walk gsp_phys_gr_mme_drop_calls uses. + * + * r1453 PAIR-CHAIN: the held window is the 0x2058 + * "other writer" (r1453 strategy §3.2) - it copies + * everything except CALL runs, including the + * program-address pair runs, and the r1452b #33 + * launch then traps 0x2058 at Data 0x19b0900 / + * EC 0x0d. Extend the sema_gl_mme_nop_pair elision + * here ONLY once a pair-bearing window of this + * same generation has retired (pairs_gen == pl_gen, + * set on the submit success edge): eliding before + * that starves the binds (r1452 D1, HUBCLIENT_PD + * @ 0x0). The elision still requires the nop_pair + * opt-in, so the default held shape is unchanged. + */ + u32 r = 0, w = 0, held = 0, pelided = 0; + bool pair_installed = + READ_ONCE(gsp_phys_rmcall_gr_mme_nop_pair) && + READ_ONCE(gsp_phys_rmcall_gr_pair_chain) && + pl_gen && READ_ONCE(gsp_phys_gr_mme_pairs_gen) == + pl_gen; + + while (r < win3d_n) { + u32 h = win3d[r]; + u32 op = (h >> 29) & 7u; + u32 cnt = (h >> 16) & 0xfffu; + u32 span = 1u; + bool iscall = false, ispair = false; + u32 k, m0, mend; + + if ((op == 1u || op == 3u || op == 5u) && + cnt && r + 1u + cnt <= win3d_n) + span = 1u + cnt; + m0 = (h & 0xfffu) << 2; + mend = m0 + 4u * (span - 1u); + if (pair_installed && + (op == 0u || op == 1u || op == 3u || + op == 4u || op == 5u) && + m0 <= 0x2158u && mend >= 0x2054u) + ispair = true; + for (k = 0; k < span && !ispair; k++) { + u32 m = ((h & 0xfffu) << 2) + + 4u * k; + + if (k && op != 1u && op != 3u) + break; + if (m >= 0x3800u && m <= 0x39fcu) { + iscall = true; + break; + } + } + if (iscall || ispair) { + if (iscall) + held += span; + if (ispair) + pelided += span; + r += span; + continue; + } + memcpy(&p[w], &win3d[r], (size_t)span * 4u); + w += span; + r += span; + } + if (held) + pr_info("vgsp-gr: MME-CALL-HOLD %u word(s) of real CALL runs held (prelude gen=%u not latched for h=0x%x; window=%u->%u) push=0x%llx%s\n", + held, pl_gen, req->h_object, + win3d_n, w, req->push_va, + mme_hold_fe_busy ? + " [FE-not-idle (CONT-104)]" : ""); + if (pelided) + pr_info_ratelimited("vgsp-gr: PAIR-CHAIN held-elide %u word(s) of program-pair runs dropped (pairs installed gen=%u h=0x%x; 0x2058 other-writer) push=0x%llx\n", + pelided, pl_gen, req->h_object, + req->push_va); + p += w; + } else if (READ_ONCE(gsp_phys_rmcall_gr_mme_nop_pair)) { + /* + * CONT-103: the calls-kept launch now starts (the + * prelude-live skip shrank it under the ~2050-word + * wall) and the engine decodes the window, but the + * FE then TRAPS on the relocated program-address + * pair write (trap=0x80002058 data=<shadow VA>, + * CONT-101: the relocated 0x2054-pair operand) and + * parks the front-end (grfifo 00000001). The binds + * persist on the channel from the earlier + * calls-held launches of the same capture, so + * eliding the pair re-binds removes the trapping + * write while keeping the program bound. Drops any + * method run overlapping the program-pair space + * [0x2054, 0x2158] (0x2054/0x2058, 0x2094/0x2098, + * 0x2154/0x2158 - the client window's own pairs and + * the synthesized SPH binds alike; this walk runs on + * the final window, prepends included). + */ + u32 r = 0, w = 0, elided = 0; + + while (r < win3d_n) { + u32 h = win3d[r]; + u32 op = (h >> 29) & 7u; + u32 cnt = (h >> 16) & 0xfffu; + u32 span = 1u, m0, mend; + + if ((op == 1u || op == 3u || op == 5u) && + cnt && r + 1u + cnt <= win3d_n) + span = 1u + cnt; + m0 = (h & 0xfffu) << 2; + mend = m0 + 4u * (span - 1u); + if ((op == 0u || op == 1u || op == 3u || + op == 4u || op == 5u) && + m0 <= 0x2158u && mend >= 0x2054u) { + elided += span; + r += span; + continue; + } + memcpy(&p[w], &win3d[r], (size_t)span * 4u); + w += span; + r += span; + } + if (elided) + pr_info("vgsp-gr: MME-PAIR-DROP %u word(s) of program-pair runs elided (calls-kept launch; binds persist from the held-call launches) h=0x%x push=0x%llx window=%u->%u\n", + elided, req->h_object, req->push_va, + win3d_n, w); + p += w; + } else { + memcpy(p, win3d, (size_t)win3d_n * 4); + p += win3d_n; + } + /* + * r1454 PAIR-CHAIN-ENTRY-ELIDE: a window that ENTERS with a + * BEGIN..END pair already open (carried from the previous + * emitted window) must not re-write the program-address pair + * runs - the FE refuses the first state method with EC0x0d + * BEGIN_END_ACTIVE (r1453b #11: 0x2058 <- shadow VA at words + * 18-20, before the orphan END that closes the carried pair). + * Re-walk the FINAL emitted window with the same run math as + * the nop_pair branch and compact away every run overlapping + * [0x2054, 0x2158]. Fail closed: requires the pair_chain + * knob, the entry-open snapshot, and the non-zero pairs_gen + * retire proof (set only when a pair-bearing window retires), + * so launch #1 still writes its pairs (prev_open=0) and D1 + * stays structurally dead. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_pair_chain) && + READ_ONCE(gsp_phys_gr_mme_pair_open_entry) && + READ_ONCE(gsp_phys_gr_mme_pairs_gen) != 0u && + (u32)(p - pb) > win_emit_off) { + u32 *ww = &pb[win_emit_off]; + u32 wn = (u32)(p - ww); + u32 r = 0, w = 0, elided = 0; + + while (r < wn) { + u32 h = ww[r]; + u32 op = (h >> 29) & 7u; + u32 cnt = (h >> 16) & 0xfffu; + u32 span = 1u, m0, mend; + + if ((op == 1u || op == 3u || op == 5u) && + cnt && r + 1u + cnt <= wn) + span = 1u + cnt; + m0 = (h & 0xfffu) << 2; + mend = m0 + 4u * (span - 1u); + if ((op == 0u || op == 1u || op == 3u || + op == 4u || op == 5u) && + m0 <= 0x2158u && mend >= 0x2054u) { + elided += span; + r += span; + continue; + } + if (w != r) + memmove(&ww[w], &ww[r], + (size_t)span * 4u); + w += span; + r += span; + } + if (elided) { + gsp_phys_gr_mme_pair_entry_elide_n++; + p -= elided; + pr_info_ratelimited("vgsp-gr: PAIR-CHAIN-ENTRY-ELIDE gen=%u n=%u total=%u (pair open at launch start; window=%u->%u)\n", + READ_ONCE(gsp_phys_gr_mme_pairs_gen), + elided, + gsp_phys_gr_mme_pair_entry_elide_n, + wn, w); + } + } + /* + * r1453 PAIR-CHAIN: the install proof for the held-call + * elision extension. Recomputed from the FINAL emitted window + * (after the CALL/pair elision branches above), so a launch + * that dropped every pair never marks its generation as + * pair-installed. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_pair_chain) && + (u32)(p - pb) > win_emit_off) + gsp_phys_gr_mme_pairs_emitted = + gsp_phys_gr_pair_present(&pb[win_emit_off], + (u32)(p - pb) - win_emit_off); + /* + * SESSION 153-CONTINUED-84: the SA re-arm block used to be + * emitted HERE (after the window). It was moved ahead of the + * window above; see that block's comment for the r1236 + * evidence. An SA[slot]=entry re-point behind the call that + * needs it cannot help that call, and the per-launch + * prelude re-emission re-clobbers it before the next window. + */ + p = gsp_phys_rmcall_push_gr(p, + GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_A, + (u32)(sem_va >> 32)); + p = gsp_phys_rmcall_push_gr(p, + GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_B, + (u32)sem_va); + p = gsp_phys_rmcall_push_gr(p, + GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_C, + sem_payload); + p = gsp_phys_rmcall_push_gr(p, + GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_D, + GSP_PHYS_RMCALL_GR_SEM_RELEASE_D); + { + static u32 gr3d_log; + + if (gr3d_log < 16) { + gr3d_log++; + pr_info("vgsp-gr: 3D STREAM replay h=0x%x push=0x%llx class=0xc597 window=%u words=%u verbatim (no prelude/QMD)\n", + req->h_object, req->push_va, + win3d_n, (u32)(p - pb)); + } + /* + * SESSION 153-CONTINUED-45 diag: the exact block the + * PBDMA fetches (first/last window words). The beef + * family's replay schedule-stalls (grfifo frozen) while + * the X family's retires; this dump is the content A/B + * behind that divergence. Bounded per boot. + */ + if (gr3d_log < 16) { + u32 wi; + + pr_info("vgsp-gr: 3DSTREAM hdr=%08x %08x win0..9=", + pb[0], pb[1]); + for (wi = 0; wi < 10 && wi < win3d_n; wi++) + pr_cont("%08x ", win3d[wi]); + pr_cont("| tail="); + for (wi = (win3d_n > 10 ? win3d_n - 10 : 0); + wi < win3d_n; wi++) + pr_cont("%08x ", win3d[wi]); + pr_cont("\n"); + } + } + return (u32)(p - pb); + } + + /* * r425 A/B: with gsp_phys_rmcall_gr_init_once the compute init preamble * is emitted only on the first forwarded launch (the stock driver's * per-channel shape); subsequent launches start at the inline-QMD @@ -16659,6 +23227,25 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, GSP_PHYS_RMCALL_GR_SET_QMD_VIRT_CONTROL, GSP_PHYS_RMCALL_GR_QMD_VIRT_IQ2M_SEND_PCAS); /* + * r597: the prelude's acquire groups (if any) go before the QMD run + * so the engine cannot read the launch's data pages until the copy + * channel's semaphore has advanced. + */ + { + u32 w = gsp_phys_rmcall_gr_emit_acquires(p, req, + 128u); + static u32 acq_log; + + if (w) { + p += w; + if (acq_log < 16) { + acq_log++; + pr_info("vgsp-gr: prelude acquire groups emitted words=%u h=0x%x push=0x%llx\n", + w, req->h_object, req->push_va); + } + } + } + /* * Verbatim replay: copy the captured GRCE prelude and the guest's * own inline-QMD run, keeping the method headers, and relocate only * the ADDRESS_A/B data words and the 64 QMD words exactly like the @@ -16833,7 +23420,7 @@ static int gsp_phys_rmcall_gr_map_va_locked(void __iomem *bar0, u64 guest_va, pt_va = gsp_phys_chunk_page_va(pt, 0); if (!pt_va) return -EIO; - put_unaligned_le64(gsp_phys_rmcall_pte(hpa), + put_unaligned_le64(gsp_phys_rmcall_gr_pte_leaf(hpa), pt_va + (size_t)pte_idx * 8); dma_wmb(); boot_stage.rmcall_gr_win[slot].pte[i].idx = pte_idx; @@ -16843,6 +23430,58 @@ static int gsp_phys_rmcall_gr_map_va_locked(void __iomem *bar0, u64 guest_va, return 0; } +/* + * r1497: install a VID-aperture PTE at the guest VA so the engine reads + * the real FBMEM page (fb_off) instead of an empty shadow page. Same + * never-rewrite discipline; the cache key is tagged so a VID entry can + * never be mistaken for a host-PA alias. + */ +static int gsp_phys_rmcall_gr_map_va_vid_locked(void __iomem *bar0, + u64 guest_va, u64 fb_off, + u32 epoch) +{ + u64 page = guest_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + u32 pde_index = (u32)(page >> 21); + u32 pte_idx = (u32)((page >> 12) & 0x1ffu); + u32 line = pte_idx / GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES; + u64 key = (fb_off & ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) | + (1ULL << 63); + struct gsp_phys_boot_chunk *pt; + u8 *pt_va; + u32 slot, i; + int ret; + + ret = gsp_phys_rmcall_gr_win_slot_locked(bar0, pde_index, &slot); + if (ret) + return ret; + for (i = 0; i < boot_stage.rmcall_gr_win[slot].pte_count; i++) { + if (boot_stage.rmcall_gr_win[slot].pte[i].idx == pte_idx) { + if (boot_stage.rmcall_gr_win[slot].pte[i].hpa == key) + return 0; + return -EALREADY; + } + if (boot_stage.rmcall_gr_win[slot].pte[i].epoch != epoch && + boot_stage.rmcall_gr_win[slot].pte[i].idx / + GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES == line) + return -EBUSY; + } + if (boot_stage.rmcall_gr_win[slot].pte_count >= + GSP_PHYS_RMCALL_GR_PT_ENTRIES) + return -ENOSPC; + pt = &boot_stage.rmcall_gr_pt[boot_stage.rmcall_gr_win[slot].pt_slot]; + pt_va = gsp_phys_chunk_page_va(pt, 0); + if (!pt_va) + return -EIO; + put_unaligned_le64(gsp_phys_rmcall_gr_pte_leaf_vid(fb_off), + pt_va + (size_t)pte_idx * 8); + dma_wmb(); + boot_stage.rmcall_gr_win[slot].pte[i].idx = pte_idx; + boot_stage.rmcall_gr_win[slot].pte[i].epoch = epoch; + boot_stage.rmcall_gr_win[slot].pte[i].hpa = key; + boot_stage.rmcall_gr_win[slot].pte_count++; + return 0; +} + static void __iomem *gsp_phys_gr_scratch_bar0; /* @@ -17001,7 +23640,7 @@ static int gsp_phys_rmcall_gr_map_shadow_locked(u32 slot, u64 hpa, pt = gsp_phys_rmcall_guest_pt_page_va(page); if (!pt) return -EIO; - put_unaligned_le64(gsp_phys_rmcall_pte(hpa), + put_unaligned_le64(gsp_phys_rmcall_gr_pte_leaf(hpa), pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * line_in_page) * 8); dma_wmb(); @@ -17048,7 +23687,7 @@ static int gsp_phys_rmcall_gr_map_shadow_contig_locked(u32 line, u32 pte, pt = gsp_phys_rmcall_guest_pt_page_va(page); if (!pt) return -EIO; - put_unaligned_le64(gsp_phys_rmcall_pte(hpa), + put_unaligned_le64(gsp_phys_rmcall_gr_pte_leaf(hpa), pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * line_in_page + pte) * 8); dma_wmb(); @@ -17389,6 +24028,54 @@ static struct gsp_phys_boot_chunk *gsp_phys_gr_data_chunk(void) } /* + * r452: scan the whole active GR data-bounce chunk for an 8-byte pattern + * (write "0x<pattern>" to /sys/module/vgsp/parameters/gsp_phys_gr_scan). + * Verifies by content where a lost engine store actually landed: the chunk + * is the only bulk memory the operand window PTEs map, so a store that + * executed but is absent from the writer's tracked pages must be findable + * here (a stale-PTE landing) or nowhere (a dropped store). + */ +static int gsp_phys_gr_scan_set(const char *val, + const struct kernel_param *kp) +{ + struct gsp_phys_boot_chunk *c = gsp_phys_gr_data_chunk(); + u64 pattern = 0; + u32 i, hits = 0; + + if (kstrtoull(val, 0, &pattern)) + return -EINVAL; + if (!c->pages) + return -ENODEV; + for (i = 0; i < c->pages; i++) { + const u8 *p = gsp_phys_chunk_page_va(c, i); + u32 off; + + if (!p) + continue; + for (off = 0; off + 8 <= GSP_PHYS_PAGE_SIZE; off += 8) { + if (get_unaligned_le64(p + off) == pattern) { + if (hits < 32) + pr_info("vgsp-gr: CHUNK-SCAN found 0x%llx at bi=%u off=0x%x (va=0x%llx)\n", + pattern, i, off, + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 + + ((u64)i << 12) + off); + hits++; + } + } + } + pr_info("vgsp-gr: CHUNK-SCAN 0x%llx done: %u hit(s) over %u pages\n", + pattern, hits, c->pages); + return 0; +} + +static const struct kernel_param_ops gsp_phys_gr_scan_ops = { + .set = gsp_phys_gr_scan_set, +}; +module_param_cb(gsp_phys_gr_scan, &gsp_phys_gr_scan_ops, NULL, 0200); +MODULE_PARM_DESC(gsp_phys_gr_scan, + "Scan the GR data-bounce chunk for an 8-byte pattern (hex)"); + +/* * B-2: page index of a relocated operand VA inside the engine-visible data * region, replacing the five inlined (reloc_va - 0x110000) >> 12 derivations * and their MAIL_SLOT bounds. Grown region: VAs from @@ -17434,6 +24121,45 @@ static u32 gsp_phys_gr_data_bi(u64 reloc_va) * gsp-phys-work item is the only caller, reset per submission exactly like * bounce_csum_in. */ +/* + * r564: minimum data records for a forwardable launch. Healthy vadd-class + * launches resolve 968-969 records (the client's whole arena); a launch + * with only a handful after the fill's drops has an unresolved argument + * set and must not be handed to the engine (see the submit guard). + * + * SESSION 127: the Vulkan inline-QMD launch (the C5C0 translator) is a + * compute dispatch whose operand set is one descriptor page and one + * parameter bank - a handful of records is healthy for it. Tunable module + * parameter (default 16 preserves the CUDA-era threshold); the translator + * cycles pass gsp_phys_gr_min_data=2. + */ +#define GSP_PHYS_GR_MIN_DATA_RECS 16u +static unsigned int gsp_phys_gr_min_data_param = GSP_PHYS_GR_MIN_DATA_RECS; +module_param_named(gsp_phys_gr_min_data, gsp_phys_gr_min_data_param, uint, 0644); +MODULE_PARM_DESC(gsp_phys_gr_min_data, + "GR submit minimum data records (degenerate-set refusal threshold, 0..4096; 0 = explicit bypass for the no-external-data launch class; default 16)"); + +/* + * r801 (SESSION 143): 0 is an explicit BYPASS, not a clamp to 1. The + * Vulkan launch's single QMD pointer pair names its own QMD window and its + * params are scalars, so the launch has no external data by construction; + * the bare n_data>=min gate wrongly classifies it degenerate. The bypass is + * safe now that gsp_phys_chunk_page_va answers NULL for an unallocated + * chunk (r800): before that guard the empty-set post-mortem print + * dereferenced a garbage pointer and oopsed at + * gsp_phys_backend_guest_gr_submit+0x4394. The proper predicate (all + * candidates launch-internal-filtered, none unresolved => not degenerate) + * replaces the bare threshold next; until then 0 stays an explicit, + * opt-in debug/test value and the default remains 16. + */ +static unsigned int gsp_phys_gr_min_data_value(void) +{ + unsigned int v = READ_ONCE(gsp_phys_gr_min_data_param); + + if (v > 4096) + v = 4096; + return v; +} #define GSP_PHYS_GR_GAP_MAX 4096u /* total late pages per launch */ /* * relfix PART 3 (r217 span containment): the late path authors ONLY the @@ -17838,7 +24564,7 @@ static bool gsp_phys_gr_reloc_fix_word(const struct gsp_phys_gr_submit *req, */ static noinline u32 gsp_phys_gr_scan_pages(const struct gsp_phys_gr_submit *req, - const u32 *csum_in, u32 *n_tracked, u64 *hash_out) + const u64 *csum_in, u32 *n_tracked, u64 *hash_out) { static u32 checked[GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES]; struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); @@ -17884,15 +24610,24 @@ gsp_phys_gr_scan_pages(const struct gsp_phys_gr_submit *req, *n_tracked = n_all; for (j = 0; j < n_all; j++) { const u8 *src = gsp_phys_chunk_page_va(dchunk, checked[j]); - u32 ci, csum = 0; + u64 fp; if (!src) continue; - for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) - csum += get_unaligned_le32(src + ci); - if (csum != csum_in[checked[j]]) + /* + * r452: FNV-1a per page, not a 32-bit additive sum. The + * add-sum was PROVABLY blind to a power-of-two in-place + * scale: doubling every word of a 4 KB page shifts each + * exponent by +1, i.e. +2^23 per word, and 1024 words give + * 2^33 == 0 (mod 2^32) - so c[i] *= 2.0f left the checksum + * bit-identical, the page classified "unchanged", and the + * client read the pre-scale values (the deterministic + * streams2/scale class). FNV-1a has no such structure. + */ + fp = gsp_phys_gr_fnv1a64(src, GSP_PHYS_PAGE_SIZE); + if (fp != csum_in[checked[j]]) changed++; - hash ^= csum; + hash ^= fp; hash *= 1099511628211ULL; } if (hash_out) @@ -17928,7 +24663,7 @@ static void gsp_phys_gr_qmd64_post(const u8 *deposit, const char *tag) static noinline u32 gsp_phys_gr_output_retire_wait(const struct gsp_phys_gr_submit *req, - const u32 *csum_in, u32 *n_tracked) + const u64 *csum_in, u32 *n_tracked) { u32 n_all = 0, iter; u32 changed = 0; @@ -18082,7 +24817,7 @@ static u32 gsp_phys_gr_idem_n; static u64 gsp_phys_gr_launch_fp(const u32 *qmd, const struct gsp_phys_gr_submit *req, - const u32 *csum_in) + const u64 *csum_in) { u64 h = 1469598103934665603ULL; u32 i; @@ -18160,6 +24895,22 @@ static void gsp_phys_gr_page_spot_check(const u8 *dst, const u8 *src, } /* + * r452: the legacy 32-bit additive page sum, kept ONLY for the all-zero + * filter. It is never a change-detection fingerprint any more: doubling every + * word of a 4 KB page leaves the sum bit-identical (1024 x 2^23 == 2^33 == 0 + * mod 2^32), which silently classified an in-place x2 scale as "no engine + * store" and shipped stale values (the deterministic streams2 class). + */ +static inline u32 gsp_phys_gr_page_sum32(const u8 *p) +{ + u32 ci, csum = 0; + + for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) + csum += get_unaligned_le32(p + ci); + return csum; +} + +/* * COPYWIT r226 zero filter: an all-zero page is never a meaningful alias * partner. A fresh client's fill stages long runs of zero pages * (uninitialized cudaMalloc buffers), and consecutive zero pages produced @@ -18205,7 +24956,7 @@ static u32 gsp_phys_gr_in_census_log; * again). Returns the number of pages staged. */ static u32 gsp_phys_gr_fill_bounce(const struct gsp_phys_gr_submit *req, - u32 *csum_in, u64 *staged_hpa) + u64 *csum_in, u64 *staged_hpa) { struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); struct { @@ -18242,7 +24993,9 @@ static u32 gsp_phys_gr_fill_bounce(const struct gsp_phys_gr_submit *req, "gr-bounce-copy-in"); for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) csum += get_unaligned_le32(dst + ci); - csum_in[bi] = csum; + /* r452: the change baseline is the FNV-1a fingerprint (the + * add-sum above survives only as the all-zero test). */ + csum_in[bi] = gsp_phys_gr_fnv1a64(dst, GSP_PHYS_PAGE_SIZE); zero = gsp_phys_gr_page_all_zero(csum, dst); if (!zero && gsp_phys_gr_copywit_enabled()) { u32 j; @@ -18291,7 +25044,7 @@ static u32 gsp_phys_gr_fill_bounce(const struct gsp_phys_gr_submit *req, * retry, so a retried launch starts from the guest inputs in both sets. * Returns the number of pages staged. */ -static u32 gsp_phys_gr_fill_gap_bounce(u32 *csum_in, u64 *staged_hpa) +static u32 gsp_phys_gr_fill_gap_bounce(u64 *csum_in, u64 *staged_hpa) { struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); u32 gi, copied = 0; @@ -18299,7 +25052,6 @@ static u32 gsp_phys_gr_fill_gap_bounce(u32 *csum_in, u64 *staged_hpa) for (gi = 0; gi < gsp_phys_gr_gap_n; gi++) { const u8 *src = (const u8 *)__va(gsp_phys_gr_gap[gi].hpa); u8 *dst = gsp_phys_chunk_page_va(dchunk, gsp_phys_gr_gap[gi].bi); - u32 ci, csum = 0; if (!src || !dst) continue; @@ -18309,9 +25061,9 @@ static u32 gsp_phys_gr_fill_gap_bounce(u32 *csum_in, u64 *staged_hpa) staged_hpa[gsp_phys_gr_gap[gi].bi] = gsp_phys_gr_gap[gi].hpa; gsp_phys_gr_page_spot_check(dst, src, gsp_phys_gr_gap[gi].va, "gr-bounce-copy-in"); - for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) - csum += get_unaligned_le32(dst + ci); - csum_in[gsp_phys_gr_gap[gi].bi] = csum; + /* r452: FNV-1a baseline (see gsp_phys_gr_scan_pages). */ + csum_in[gsp_phys_gr_gap[gi].bi] = + gsp_phys_gr_fnv1a64(dst, GSP_PHYS_PAGE_SIZE); copied++; } dma_wmb(); @@ -18408,7 +25160,8 @@ static bool gsp_phys_gr_win0_retry_rearm(u32 *qmd, */ static noinline void gsp_phys_gr_poll_break_log(u32 w0, u32 want_qmd, u32 w1, u32 since_us, - const struct gsp_phys_rmcall_ce_submit *sub) + const struct gsp_phys_rmcall_ce_submit *sub, + u32 h_object) { struct gsp_phys_boot_chunk *ud_c = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD]; @@ -18416,16 +25169,180 @@ gsp_phys_gr_poll_break_log(u32 w0, u32 want_qmd, u32 w1, u32 since_us, u32 usr_gpget = ud ? get_unaligned_le32(ud + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPGET_OFF) : 0; + /* + * SESSION 153-CONTINUED-47: re-read the ring slot the launch + * published (the entry AS THE ENGINE SEES IT) after the wait, so an + * X-vs-beef diff on one boot compares the exact e0/e1 the PBDMA + * fetched, plus the doorbell-visible GP_GET/GP_PUT movement. + */ + struct gsp_phys_boot_chunk *r_c = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO]; + u8 *ring = r_c->pages ? gsp_phys_chunk_page_va(r_c, 0) : NULL; + u32 re0 = 0, re1 = 0; + + if (ring && sub->slot < GSP_PHYS_RMCALL_GPFIFO_ENTRIES) { + re0 = get_unaligned_le32(ring + sub->slot * 8); + re1 = get_unaligned_le32(ring + sub->slot * 8 + 4); + } - pr_info("vgsp-gr: poll break w0=0x%x want=0x%x ch=0x%x after %u us usr_gpget=0x%x gpget=0x%x gpput=0x%x\n", + pr_info("vgsp-gr: poll break w0=0x%x want=0x%x ch=0x%x after %u us usr_gpget=0x%x gpget=0x%x gpput=0x%x ring[%u]=%08x %08x (pub %08x %08x)\n", w0, want_qmd, w1, (u32)ktime_to_us(ktime_get()) - since_us, - usr_gpget, sub->gpget, sub->gpput); + usr_gpget, sub->gpget, sub->gpput, + sub->slot, re0, re1, sub->e0, sub->e1); + /* + * CONT-103: engine-fetch witness + prelude-liveness bookkeeping. + * GP_GET advancing past the published slot is the PBDMA consuming + * this launch's GPFIFO entry - the fetch witness the QMD sentinel + * cannot provide on the stream3d lane (no QMD deposit there). A + * break with GP_GET frozen is a stale/software-authored release. + */ + pr_info("vgsp-gr: FETCH-WIT h=0x%x slot=%u gpget %u->%u %s\n", + h_object, sub->slot, sub->gpget, usr_gpget, + usr_gpget != sub->gpget ? "engine-fetched" : + "GP_GET-frozen (software ack)"); + if (usr_gpget != sub->gpget && + READ_ONCE(gsp_phys_gr_mme_pending_gen)) { + WRITE_ONCE(gsp_phys_gr_mme_live_gen, + READ_ONCE(gsp_phys_gr_mme_pending_gen)); + WRITE_ONCE(gsp_phys_gr_mme_pending_gen, 0); + } /* r425: the healthy-launch SKED/FECS baseline for the wedge compare. */ gsp_phys_gr_sked_wit(gsp_phys_gpu_bar0(), "post-poll"); } /* + * CONT-103: fetch-witness for a launch that did NOT break - GP_GET frozen + * means the PBDMA never consumed the entry (the >2050-word fetch wall / + * parked front-end class), so the carrier's generation is not live and the + * next launch must re-emit the prelude. noinline: keeps the submit frame. + */ +static noinline void +gsp_phys_gr_fetch_miss_log(u32 h_object, + const struct gsp_phys_rmcall_ce_submit *sub, + const char *tag) +{ + struct gsp_phys_boot_chunk *ud_c = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD]; + u8 *ud = ud_c->pages ? gsp_phys_chunk_page_va(ud_c, 0) : NULL; + u32 usr_gpget = ud ? + get_unaligned_le32(ud + gsp_phys_rmcall_userd_ctl_off() + + GSP_PHYS_RMCALL_USERD_GPGET_OFF) : 0; + + pr_info("vgsp-gr: FETCH-WIT %s h=0x%x slot=%u gpget %u->%u %s\n", + tag, h_object, sub->slot, sub->gpget, usr_gpget, + usr_gpget != sub->gpget ? "engine-fetched" : + "GP_GET-frozen (NOT fetched: parked or over-cap block)"); + /* + * SESSION 153-CONTINUED-109: readback-free live promotion on the + * timeout path. The poll-break path promotes the generation whose + * final prelude part was carried when GP_GET advanced (the engine + * consumed the entry). A launch can also time out the w0/w1 release + * wait AFTER the engine fetched it (r1377: `FETCH-WIT timeout ... + * gpget 5->6 engine-fetched` on the load-only carriers - their + * releases only matched ~10 s later through the ladder), and the + * historical code cleared pending_gen here, permanently losing the + * promotion: live_gen never caught pl_gen, every later launch stayed + * window-elided, and the MME never got a calls-bearing launch + * (`pc_valid=0` on all samples). Promote on the same GP_GET witness + * the break path trusts; only a genuinely unfetched carrier clears. + */ + if (usr_gpget != sub->gpget && + READ_ONCE(gsp_phys_gr_mme_pending_gen)) { + pr_info("vgsp-gr: MME-LIVE promote via GP_GET advance (%s) gen=%u h=0x%x\n", + tag, READ_ONCE(gsp_phys_gr_mme_pending_gen), + h_object); + WRITE_ONCE(gsp_phys_gr_mme_live_gen, + READ_ONCE(gsp_phys_gr_mme_pending_gen)); + WRITE_ONCE(gsp_phys_gr_mme_pending_gen, 0); + } else { + WRITE_ONCE(gsp_phys_gr_mme_pending_gen, 0); + } +} + +/* + * SESSION 153-CONTINUED-72: GR-STATE retire readback. The keep-mask A/B + * must answer "does the kept macro execute at all" before chasing the + * 0/903 stores. The FE decode-state EX records (0x40a630..3c) name the + * last MME-GENERATED method write: bit31 GENERATED, 21:12 macro address, + * 11:0 pc - gen0=1 with the kept slot's macro address at the retire is + * the live witness that the engine fetched and ran the kept macro. The + * MME HWW latch (0x404490 + INFO/IR), the trapped method (0x40704 offset + * + bit20 mmegen, data 0x40708) and the DS latch catch the write when it + * faulted. One line per retire; bounded per boot (beef up to 48, others + * 4) so a battery boot cannot flood. + */ +static noinline void gsp_phys_gr_state_rb_log(u32 h_object) +{ + static u32 gs_beef_n, gs_other_n; + void __iomem *bar0 = gsp_phys_gpu_bar0(); + bool beef = (h_object & 0xfffff000u) == 0xbeef0000u; + u32 fed0, fed1, fed2, fed3, esr, info, ir2, trap, data, ds, gf; + /* + * SESSION 153-CONTINUED-91: with the DS HWW report mask cleared + * (sema_ds_mask_clear) a SPH validation miss latches in DS_HWW_ESR + * without the GSP-RM fault handler poisoning the PRI block, so the + * per-stage SPH_INFO snapshot is finally readable: STAGE0..5 START + * (0x405c00+si*4) / UPPER (0x405c24+si*4) name the exact SPH VA each + * pipeline stage fetched, and PROGRAM_REGION_A/B (0x405c1c/c20) the + * region the DS validated against (the r1277+ wall's missing datum). + */ + u32 sph[6], sphu[6], rr_a, rr_b; + u32 si; + + if (!bar0) + return; + if (beef ? gs_beef_n >= 48u : gs_other_n >= 4u) + return; + if (beef) + gs_beef_n++; + else + gs_other_n++; + for (si = 0; si < 6u; si++) { + sph[si] = ioread32(bar0 + 0x00405c00u + si * 4u); + sphu[si] = ioread32(bar0 + 0x00405c24u + si * 4u); + } + rr_a = ioread32(bar0 + 0x00405c1cu); + rr_b = ioread32(bar0 + 0x00405c20u); + fed0 = ioread32(bar0 + 0x0040a630u); + fed1 = ioread32(bar0 + 0x0040a634u); + fed2 = ioread32(bar0 + 0x0040a638u); + fed3 = ioread32(bar0 + 0x0040a63cu); + esr = ioread32(bar0 + 0x00404490u); + info = ioread32(bar0 + 0x00404494u); + /* SESSION 153-CONTINUED-106: INFO2 is 0x40449c (0x404498 is CONFIG). */ + ir2 = ioread32(bar0 + 0x0040449cu); + trap = ioread32(bar0 + 0x00400704u); + data = ioread32(bar0 + 0x00400708u); + ds = ioread32(bar0 + 0x00405840u); + gf = ioread32(bar0 + 0x00400504u); + pr_info("vgsp-gr: GR-STATE retire h=0x%x fed=%08x,%08x,%08x,%08x (pc0=%u mac0=0x%x gen0=%u) mme_esr=%08x info=%08x ir=%08x trap=%08x(mthd=0x%x mmegen=%u) data=%08x ds=%08x grfifo=%08x\n", + h_object, fed0, fed1, fed2, fed3, + fed0 & 0xfffu, (fed0 >> 12) & 0x3ffu, (fed0 >> 31) & 1u, + esr, info, ir2, trap, trap & 0x3ffcu, (trap >> 20) & 1u, + data, ds, gf); + pr_info("vgsp-gr: GR-STATE sph h=0x%x ds2=%08x sph=%08x,%08x,%08x,%08x,%08x,%08x sphU=%02x,%02x,%02x,%02x,%02x,%02x rr=%08x,%08x\n", + h_object, ioread32(bar0 + 0x00405848u), + sph[0], sph[1], sph[2], sph[3], sph[4], sph[5], + sphu[0] & 0xffu, sphu[1] & 0xffu, sphu[2] & 0xffu, + sphu[3] & 0xffu, sphu[4] & 0xffu, sphu[5] & 0xffu, + rr_a, rr_b); + /* + * SESSION 153-CONTINUED-91: did the emitted program-region pair + * (SET_MME_SHADOW_SCRATCH 0x78/0x79 = method 0x35e0/0x35e4 through + * class 0xc597) actually land in the MME shadow? The shadow read + * port is the same one the store-side probe uses; a zero readback + * means the write never took effect and every region experiment so + * far was a no-op. + */ + if (beef) + pr_info("vgsp-gr: GR-STATE region-shadow h=0x%x scratch[0x78]=%08x scratch[0x79]=%08x\n", + h_object, + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x35e0u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x35e4u)); +} + +/* * F4-b: one-shot deposit readback at the first timeout (noinline for the same * frame reason as above). Reads the qmdwin/shadow destination (or the guest * QMD page) with the booter's existing unaligned host readback; w63 is the @@ -18574,6 +25491,7 @@ static u32 gsp_phys_m91_fault_start; static u32 gsp_phys_m91_cap_used; static bool gsp_phys_m91_cap_suppressed; static bool gsp_phys_m91_icache_log; +static bool gsp_phys_r449_ccache_log; static bool gsp_phys_gr_rel_membar_log; static noinline void gsp_phys_gr_m91_fault_cap(const struct gsp_phys_gr_submit *req, @@ -18663,7 +25581,7 @@ gsp_phys_gr_m91_fault_cap(const struct gsp_phys_gr_submit *req, fn = &gsp_phys_fault_notes[(gsp_phys_fault_note_next + GSP_PHYS_FAULT_NOTE_MAX - 1u) % GSP_PHYS_FAULT_NOTE_MAX]; - pr_info("vgsp-m91: FAULT-CAP ok=%u faults=%u retry=%s attempts=%u qmd=0x%llx qmd64=%s prog_w48=0x%08x w49=0x%08x prog_va=0x%llx prog_pa=0x%llx code_src=%s off=0x%x code64=%s fault_et=0x%x fault_chid=%u fault_err=\"%.48s\"\n", + pr_info("vgsp-m91: FAULT-CAP ok=%u faults=%u retry=%s attempts=%u qmd=0x%llx qmd64=%s prog_w48=0x%08x w49=0x%08x prog_va=0x%llx prog_pa=0x%llx code_src=%s off=0x%x code64=%s fault_et=0x%x fault_chid=%u fault_err=\"%.200s\"\n", ok ? 1u : 0u, faults, fast ? "fast-rearm" : (rearmed ? "resched/fresh-entry" : "none"), attempts, qmd_dst, qhex, qmd[48], qmd[49], pva, ppa, src, off, @@ -18857,6 +25775,2457 @@ static void gsp_phys_gr_stall_log(const char *reason, u64 push_va, reason, push_va, jiffies_to_msecs(jiffies - start), action); } +/* + * SESSION 153-CONTINUED-27: big-surface alias bookkeeping. A replayed + * 3D window's colour target (C597 INC 0x800+j*0x40 [upper, lower, w, h, + * fmt] and its TWOD twin INC 0x218 [w, h, upper, lower]) can be a surface + * far larger than the launch's 16-page set (the X screen 0x120400000, + * 1280x720x4 = 900 pages). Two earlier routes are dead ends for it: + * - one-page shadow relocation (r992: `pair addr=0x840 va=0x120400000 + * -> MISS` then FAULT_PDE @ +0xa000 - the raster walk crosses the + * single alias page); + * - Tier-1 windows at the surface's own VAs (CONT.27 r993: PDE_UPDATE + * for pdeIndex 2306/2307 status=0x0, yet GPC2/GPC5 still fault + * FAULT_PDE @ +0x4000/+0xa000 - same class as the r929 v2 FE faults; + * the GPC engines only see the scratch-PT pool). + * The surviving route is the proven one: a CONTIGUOUS run of never-used + * scratch-PT fetch lines (one line = 16 PTEs = 64 KB VA; lines are + * monotonic and consecutive lines are VA-contiguous inside each pool + * block), one PTE per surface page, so the engine reaches all 900 pages + * through one contiguous alias VA and the raster walk never crosses into + * an unmapped page. The pair operands are rewritten to <alias>|<offset> + * by gsp_phys_rmcall_gr_3d_reloc. The table is per-boot: the alias PTEs + * are never rewritten (fetch-line rule) and the screen's HPAs are fixed + * by its single allocation, so every later launch is a pure rewrite. + */ +struct gsp_phys_gr_surf_keep { + u64 va; /* guest surface base VA (page aligned) */ + u64 alias; /* engine-visible contiguous shadow alias base */ + u64 npages; /* pages aliased (declared + slack tail) */ +}; +static struct gsp_phys_gr_surf_keep gsp_phys_gr_surf[4]; +static u32 gsp_phys_gr_surf_n; + +/* + * r1490: the effective engine target of the captured surface ON THIS LAUNCH. + * The captured-surface rec set (gsp_sema_gr_surf_recs) is global - it is the + * last C597 colour-target bind a trusted channel emitted - while a launch's + * own data recs only cover it when the fill resolved the WHOLE span through + * the captured bind's VAS. Several clients share the same 0x1200xxxxx VAs: + * the beef client holds a ~10-page SYS allocation at 0x120400000, the X + * server's 1280x720 screen of the same VA is a 900-page FB allocation. A + * partial run must never win the rec-over-alias preference: the engine would + * be pointed at the client's code/operand buffer, its stores would land in + * (and its copy-in would read) method/SASS words instead of the screen, and + * the surface would look like it received the method stream. + * + * Complete rec run (contiguous base..last reloc VA) -> its bounce base, the + * path that lets the retire wait and copy-out see the pixels. Otherwise the + * per-boot contiguous alias built from the captured bind's VAS pages is the + * target: the engine writes the guest screen HPAs directly, exactly as the + * pre-r1483 alias path did. 0 = neither is available (caller keeps the + * per-page path). + */ +static u64 gsp_phys_gr_surf_launch_target(const struct gsp_phys_gr_submit *req) +{ + u64 sva = 0, last_va, base = 0, last = 0; + u32 sn = 0, di; + bool have_base = false, have_last = false; + + if (!req) + return 0; + if (gsp_sema_gr_surf_recs(&sva, &sn) && sva && sn >= 2u) { + for (di = 0; di < req->n_data; di++) { + if (req->data[di].va == sva && + req->data[di].reloc_va) { + base = req->data[di].reloc_va; + have_base = true; + break; + } + } + last_va = sva + ((u64)(sn - 1u) << 12); + for (di = 0; di < req->n_data; di++) { + if (req->data[di].va == last_va && + req->data[di].reloc_va) { + last = req->data[di].reloc_va; + have_last = true; + break; + } + } + if (have_base && have_last && + last == base + ((u64)(sn - 1u) << 12)) + return base; + } + if (gsp_sema_surf_va_get(&sva, &sn)) { + for (di = 0; di < gsp_phys_gr_surf_n; di++) + if (gsp_phys_gr_surf[di].va == sva) + return gsp_phys_gr_surf[di].alias; + } + return 0; +} + +/* + * SESSION 153-CONTINUED-27: slack tail for the alias. r994 measured the + * replay's colour writes landing past the declared w*h*4 footprint + * (1280x720x4 = 0x384000) by up to +0xE000 (11.2 rows) - FAULT_PTE on + * the last fetch line's unwritten PTE slots (0x5d6000..0x5e2000 with the + * alias at 0x250000). The tail pages are aliased too: a page that still + * resolves (inside the guest allocation's own padding - exactly where + * native overdraw would land) maps the real HPA; one that does not + * resolves to a dedicated zeroed module page so the stray writes vanish + * instead of faulting. The chunk is allocated lazily and lives until + * module unload (4 KB). + */ +#define GSP_PHYS_GR_SURF_SLACK_PAGES 32u +static struct gsp_phys_boot_chunk gsp_phys_gr_surf_dummy; + +static u64 gsp_phys_gr_surf_dummy_dma_locked(void) +{ + struct gsp_phys_boot_chunk *d = &gsp_phys_gr_surf_dummy; + + if (!d->pages) { + if (gsp_phys_chunk_alloc(boot_stage.pdev, d, 1u)) + return 0; + gsp_phys_chunk_zero(d); + } + return (u64)gsp_phys_chunk_page_dma(d, 0); +} + +/* + * r929 v3 (Path A): relocate one 3D-window address operand to a shadow-pool + * VA. The guest VAs the 3D methods name (uploads 0x1_214f8f00.., vertex, + * program, tex pools) are installed in the ladder page tree by the Tier-1 + * mapper yet the engine's FE/PD walks still fault FAULT_PDE on them (r929 + * v2 evidence: "ENGINE GRAPHICS HUBCLIENT_FE faulted @ 0x1_214f8000 ... + * FAULT_PDE"); the shadow pool lines (map_shadow_locked) ARE the proven + * engine-visible region (every CUDA data/page relocation lives there). The + * page offset is preserved; a VA whose 4 KB page has no resolved hpa stays + * verbatim (it faults later and is named in the drain). Called with + * gsp_phys_rm_lock held. + */ +/* + * SESSION 153-CONTINUED-59: window/attr operand-miss state for the + * pre-doorbell abort (see gsp_phys_rmcall_gr_3d_reloc's level-2 fallback + * and the MISS-ABORT site in gsp_phys_backend_guest_gr_submit). Reset at + * the start of each 3D-window patch pass; a nonzero count at the submit + * point means the emitted block still carries a raw client VA the physical + * engine cannot resolve, which is the r1104/r1105 lane-killing class. + * sema_gl_miss_abort=0 restores the historical submit-anyway behaviour. + */ +static u32 gsp_phys_rmcall_gr_3d_miss_n; +static u64 gsp_phys_rmcall_gr_3d_junk_n; +static bool gsp_phys_rmcall_gr_miss_abort = true; +static u64 gsp_phys_gr_miss_abort_n; +module_param_named(sema_gl_miss_abort, gsp_phys_rmcall_gr_miss_abort, bool, + 0644); +MODULE_PARM_DESC(sema_gl_miss_abort, + "Abort a 3D-window launch before the doorbell while a window/attr operand relocation MISSes (page demanded via the pin path; retried on the next forward; fail-open after 3000 aborts). Default 1"); + +/* + * SESSION 153-CONTINUED-73: direct-write witness over the launch's OWN + * relocated operand destinations. The retirement scan only sees stores into + * the data-rec bounce copies; a pair rewritten to a shadow or surface-alias + * VA makes the engine store STRAIGHT into the guest page behind it (the + * 3d_reloc alias comment calls this class "direct guest writes, invisible + * to the witness"). Every such rewrite records its destination guest page + * here (with the source VA for the report); the submit path fingerprints + * the set right before the doorbell and the retirement wait re-fingerprints + * it, so one healthy run answers: do the draw's stores land in the bounce + * (counted today), in guest pages through a relocated operand (this + * witness), or in NEITHER (the draw rasterized nothing and the missing + * pipeline/vertex state is the wall). The MME-family payload rewrites + * (sema_gl_mme_reloc -> mme-reloc-rsv) feed the same set, so the MME + * shadow-scratch destinations are covered too. Static scratch: the only + * caller is the single-inflight ordered gsp-phys-work item. + */ +#define GSP_PHYS_GR_WIT_MAX 4096u +static u64 gsp_phys_gr_wit_pa[GSP_PHYS_GR_WIT_MAX]; +static u64 gsp_phys_gr_wit_va[GSP_PHYS_GR_WIT_MAX]; +static u64 gsp_phys_gr_wit_csum[GSP_PHYS_GR_WIT_MAX]; +static u64 gsp_phys_gr_wit_first8[GSP_PHYS_GR_WIT_MAX]; +static u32 gsp_phys_gr_wit_n; +static u32 gsp_phys_gr_wit_rep_log; + +/* + * CONT-111: the changed-page line budget was a hardcoded 4; a store-rich + * launch (r1378 reached 305/903 relocated-operand pages) needs the whole + * list to find the render target among the operand pages. 0 keeps the + * summary line only. + */ +static u32 gsp_phys_gr_wit_shown = 4; +module_param_named(sema_gl_wit_shown, gsp_phys_gr_wit_shown, uint, 0644); +MODULE_PARM_DESC(sema_gl_wit_shown, + "Max GUEST-WIT changed va/pa lines per report (default 4; CONT-111 RT hunt)"); + +static void gsp_phys_gr_wit_reset(void) +{ + gsp_phys_gr_wit_n = 0; +} + +static void gsp_phys_gr_wit_note(u64 va, u64 hpa) +{ + u32 k; + + if (!hpa) + return; + hpa &= ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + for (k = 0; k < gsp_phys_gr_wit_n; k++) + if (gsp_phys_gr_wit_pa[k] == hpa) + return; + if (gsp_phys_gr_wit_n >= GSP_PHYS_GR_WIT_MAX) + return; + gsp_phys_gr_wit_pa[gsp_phys_gr_wit_n] = hpa; + gsp_phys_gr_wit_va[gsp_phys_gr_wit_n] = + va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + gsp_phys_gr_wit_n++; +} + +static void gsp_phys_gr_wit_capture(const struct gsp_phys_gr_submit *req) +{ + u32 k; + + /* + * Also witness the guest pages behind the launch's tracked data + * records (and the late-resolved gap pages): a store through a + * surface ALIAS VA lands in the surface's own guest HPA - invisible + * to the bounce scan even when the same page is a tracked record - + * so the direct-write pass needs these HPAs in the set. The + * pre-doorbell timing (after every module-side writer) makes the + * baseline the guest content the engine starts from. + */ + for (k = 0; req && k < req->n_data; k++) + gsp_phys_gr_wit_note(req->data[k].va, req->data[k].hpa); + for (k = 0; k < gsp_phys_gr_gap_n; k++) + gsp_phys_gr_wit_note(gsp_phys_gr_gap[k].va, + gsp_phys_gr_gap[k].hpa); + for (k = 0; k < gsp_phys_gr_wit_n; k++) { + const u8 *p = (const u8 *)__va(gsp_phys_gr_wit_pa[k]); + + gsp_phys_gr_wit_first8[k] = p ? + get_unaligned_le64(p) : 0; + gsp_phys_gr_wit_csum[k] = p ? + gsp_phys_gr_fnv1a64(p, GSP_PHYS_PAGE_SIZE) : 0; + } +} + +static u32 gsp_phys_gr_wit_report(const char *where) +{ + u32 k, changed = 0, shown = 0; + u64 first_va = 0, first_pa = 0; + + if (!gsp_phys_gr_wit_n) + return 0; + for (k = 0; k < gsp_phys_gr_wit_n; k++) { + const u8 *p = (const u8 *)__va(gsp_phys_gr_wit_pa[k]); + u64 fp = p ? + gsp_phys_gr_fnv1a64(p, GSP_PHYS_PAGE_SIZE) : 0; + + if (fp != gsp_phys_gr_wit_csum[k]) { + changed++; + if (!first_pa) { + first_pa = gsp_phys_gr_wit_pa[k]; + first_va = gsp_phys_gr_wit_va[k]; + } + /* + * CONT-73 attribution: the pre/post first word of up + * to four changed pages. A QMD/descriptor word or an + * MME-scratch datum names the writer; freshly built + * vertex/float data means the guest CPU rewrote its + * buffer while the engine ran. + */ + if (shown < gsp_phys_gr_wit_shown) { + shown++; + pr_info("vgsp-gr: GUEST-WIT %s changed va=0x%llx pa=0x%llx pre=0x%016llx post=0x%016llx\n", + where, + gsp_phys_gr_wit_va[k], + gsp_phys_gr_wit_pa[k], + gsp_phys_gr_wit_first8[k], + p ? get_unaligned_le64(p) : 0); + } + } + } + /* + * One line per launch while the report budget lasts (a full boot can + * carry hundreds of X-channel launches; the beef rows are the ones + * being chased). + */ + if (gsp_phys_gr_wit_rep_log < 512u) { + gsp_phys_gr_wit_rep_log++; + pr_info("vgsp-gr: GUEST-WIT %s: %u/%u relocated-operand guest page(s) changed (direct shadow/alias stores%s)\n", + where, changed, gsp_phys_gr_wit_n, + changed ? "" : " - NONE (no direct guest writes)"); + if (changed) + pr_info("vgsp-gr: GUEST-WIT %s first changed va=0x%llx pa=0x%llx\n", + where, first_va, first_pa); + } + /* + * CONT-73 settle control: the engine is retired by now, so a second + * scan after a short quiet delay separates engine stores (one-shot, + * will not grow) from LIVE guest-CPU rewrites (still growing). A + * nonzero "late" count here means the changed rows above are guest + * noise, not engine writes. + */ + if (changed && gsp_phys_gr_wit_rep_log <= 128u) { + u32 late = 0; + + msleep(150); + for (k = 0; k < gsp_phys_gr_wit_n; k++) { + const u8 *p = + (const u8 *)__va(gsp_phys_gr_wit_pa[k]); + u64 fp = p ? + gsp_phys_gr_fnv1a64(p, + GSP_PHYS_PAGE_SIZE) : 0; + + if (fp != gsp_phys_gr_wit_csum[k]) + late++; + } + pr_info("vgsp-gr: GUEST-WIT %s settle: %u page(s) changed at +150 ms (was %u)%s\n", + where, late, changed, + late > changed ? + " - GUEST-ACTIVE (growing; noise)" : + " - quiescent (engine-final writes)"); + } + return changed; +} + +/* + * SESSION 153-CONTINUED-90: SPH-alias receipt. The DS I-fetch of a bound + * stage reaches the program-family VA through the shadow VA the relocation + * picked, and the PTE of that shadow VA names the physical page the engine + * reads. Log the mapping choice (launch page set vs trusted resolver), the + * shadow VA and the mapped HPA, then read the fetch words back through the + * host direct map of that very page (for a RAM-backed HPA the direct map is + * content-identical to the engine's PTE-based read), so "which physical page + * does the DS I-fetch actually read" is provable from the log alone. Bounded + * per boot; foreign (non-RAM) HPAs are named but not dereferenced. + */ +static void gsp_phys_gr_sph_alias_receipt(u64 page, u64 va, u64 sh, u64 hpa, + const char *via) +{ + static u32 sph_alias_n; + u32 w[8] = { 0 }; + u32 c[4] = { 0 }; + u32 off = (u32)(va & (GSP_PHYS_PAGE_SIZE - 1u)); + u32 coff = off + 0x80u; /* SPH code starts 0x80 B into the image */ + bool rd = false; + + if (sph_alias_n >= 48u) + return; + sph_alias_n++; + if (pfn_valid((unsigned long)(hpa >> PAGE_SHIFT))) { + const u8 *p = (const u8 *)__va(hpa + off); + u32 i; + + for (i = 0; i < 8u; i++) + w[i] = get_unaligned_le32(p + (size_t)i * 4u); + if (coff + sizeof(c) <= GSP_PHYS_PAGE_SIZE) { + const u8 *cp = (const u8 *)__va(hpa + coff); + + for (i = 0; i < 4u; i++) + c[i] = get_unaligned_le32(cp + (size_t)i * 4u); + } + rd = true; + } + pr_info("vgsp-gr: SPH-ALIAS-%s page=0x%llx va=0x%llx off=0x%x shadow=0x%llx hpa=0x%llx rd=%u w8=%08x %08x %08x %08x %08x %08x %08x %08x code4=%08x %08x %08x %08x\n", + via, page, va, off, sh, hpa, rd ? 1u : 0u, + w[0], w[1], w[2], w[3], w[4], w[5], w[6], w[7], + c[0], c[1], c[2], c[3]); +} + +/* + * SESSION 153-CONTINUED-90: A/B canary for the DS fetch path. When set, + * every program-family operand the window relocates maps a dedicated canary + * page (a deterministic 0xcc00cc00|idx pattern) instead of the resolved + * guest page. The DS error set either changes (the DS does fetch through + * this shadow VA) or stays byte-identical (something else supplies the + * stage start, e.g. the QMD program pointer path). Live-flippable. + */ +static bool gsp_phys_gr_sph_canary; +module_param_named(sema_gl_sph_canary, gsp_phys_gr_sph_canary, bool, 0644); +MODULE_PARM_DESC(sema_gl_sph_canary, + "Map program-family shadow VAs to a canary page (SPH fetch-path A/B)"); +static struct gsp_phys_boot_chunk sph_canary_chunk; +static bool sph_canary_ready; + +static u64 gsp_phys_gr_sph_canary_hpa(void) +{ + if (!sph_canary_ready) { + u8 *p; + u32 i; + + if (!boot_stage.pdev || + gsp_phys_chunk_alloc(boot_stage.pdev, &sph_canary_chunk, 1)) + return 0; + p = gsp_phys_chunk_page_va(&sph_canary_chunk, 0); + if (!p) + return 0; + for (i = 0; i < GSP_PHYS_PAGE_SIZE / 4u; i++) + ((u32 *)p)[i] = 0xcc00cc00u | (i & 0xffu); + sph_canary_ready = true; + } + return (u64)gsp_phys_chunk_page_dma(&sph_canary_chunk, 0); +} + +/* + * SESSION 153-CONTINUED-93: 8K program-pair shadow line (mission task 2 + * fallback). The DS validates a bound program through its fetched SPH; a + * per-page shadow line maps exactly one 4 KB page at PTE0 and leaves the + * line's PTE1 invalid, so any read past the first 4 KB of the program page + * (the SASS tail of a bind whose delivered length crosses 0x1000, or any + * DS full-program walk) leaves the authored bytes behind. Map the program + * family page (PTE0) and family+0x1000 (PTE1) into ONE fresh 128 B fetch + * line BEFORE the engine can fetch it (a line's contents are frozen once + * fetched), so a contiguous 8 KB is engine-visible from its first window + * on. The line is cached per hpa0 (pair cache) so every later window reuses + * the identical never-rewritten mapping; the alias cache is re-pointed at + * the pair line so every other consumer (program binds, code-fetch + * operands) sees the same 8 KB view. Live A/B via sema_gl_sph_8k. Called + * with gsp_phys_rm_lock held. + */ +static bool gsp_phys_rmcall_gr_sph_8k; +module_param_named(sema_gl_sph_8k, gsp_phys_rmcall_gr_sph_8k, bool, 0644); +MODULE_PARM_DESC(sema_gl_sph_8k, + "Map the program family page and family+0x1000 in one contiguous 8K shadow line (fresh fetch line; default 0)"); +#define GSP_PHYS_RMCALL_GR_PAIR_CACHE 64u +static u64 gsp_phys_gr_pair_hpa[GSP_PHYS_RMCALL_GR_PAIR_CACHE]; +static u64 gsp_phys_gr_pair_va[GSP_PHYS_RMCALL_GR_PAIR_CACHE]; +static u32 gsp_phys_gr_pair_n; + +/* + * Resolve the HPA of `page0 + 4K` through the same sources the relocation + * chain uses: the launch's own page set (the pages the copy-in authored), + * the tracked data recs, the sema service's trusted resolver, then the + * late-resolved gap pages. 0 = not resolvable. + */ +static u64 gsp_phys_rmcall_gr_prog_next_hpa_locked( + const struct gsp_phys_gr_submit *req, u64 page0) +{ + u64 page1 = page0 + GSP_PHYS_PAGE_SIZE; + u32 i; + + for (i = 0; i < req->n_pages; i++) + if ((req->pages[i].guest_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) == page1 && + req->pages[i].hpa) + return req->pages[i].hpa; + for (i = 0; i < req->n_data; i++) + if ((req->data[i].va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) == page1 && + req->data[i].hpa) + return req->data[i].hpa; + { + struct gsp_sema_gr_reloc r; + + if (!gsp_sema_gr_reloc_resolve(page1, &r) && r.hpa) + return r.hpa; + } + for (i = 0; i < gsp_phys_gr_gap_n; i++) + if ((gsp_phys_gr_gap[i].va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) == page1 && + gsp_phys_gr_gap[i].hpa) + return gsp_phys_gr_gap[i].hpa; + return 0; +} + +/* + * r1494: resolve one guest page through the same chain the relocation pass + * uses: the launch's own page set (copy-in authored pages), the tracked + * data recs, the GR ledger/pin/walk resolver, then the late-resolved gap + * pages. 0 = not resolvable. Unlike prog_next_hpa this takes any page, not + * just page0+4K. + */ +static u64 gsp_phys_rmcall_gr_stream_page_hpa_locked( + const struct gsp_phys_gr_submit *req, u64 page) +{ + u32 i; + + /* + * r1497: a vertex/index fetch page can resolve to an FB page whose + * emulated shadow is empty while the same VA has a sysmem mapping + * with the real bytes (r1496: the active indexed draw's 0x44fea00 + * stream read all-zero). Populate the empty shadow page before the + * stream line is built. Non-empty pages and SYS answers are no-ops. + */ + gsp_sema_gr_fetch_backing(page); + for (i = 0; i < req->n_pages; i++) + if ((req->pages[i].guest_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) == page && + req->pages[i].hpa) + return req->pages[i].hpa; + for (i = 0; i < req->n_data; i++) + if ((req->data[i].va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) == page && + req->data[i].hpa) + return req->data[i].hpa; + { + struct gsp_sema_gr_reloc r; + + if (!gsp_sema_gr_reloc_resolve(page, &r) && r.hpa) + return r.hpa; + } + for (i = 0; i < gsp_phys_gr_gap_n; i++) + if ((gsp_phys_gr_gap[i].va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) == page && + gsp_phys_gr_gap[i].hpa) + return gsp_phys_gr_gap[i].hpa; + return 0; +} + +static void gsp_phys_rmcall_gr_alias_cache_set_locked(u64 hpa, u64 va) +{ + u32 i; + + for (i = 0; i < boot_stage.rmcall_gr_alias_count; i++) + if (boot_stage.rmcall_gr_alias_hpa[i] == hpa) { + boot_stage.rmcall_gr_alias_va[i] = va; + return; + } + if (boot_stage.rmcall_gr_alias_count < GSP_PHYS_RMCALL_GR_ALIAS_CACHE) { + u32 ci = boot_stage.rmcall_gr_alias_count; + + boot_stage.rmcall_gr_alias_hpa[ci] = hpa; + boot_stage.rmcall_gr_alias_va[ci] = va; + boot_stage.rmcall_gr_alias_count = ci + 1u; + } +} + +/* + * r1494: a vertex stream in the MME payload is + * [stream, end_hi, end_lo, base_hi, base_lo]; END and BASE are both client + * VAs and the MME derives the stream length from end - base. The generic + * per-word payload pass relocated each end independently to its own + * single-page shadow fetch line, so any stream that crosses a 4 KB + * boundary (the client's vertex buffers here are 0x600..0x2d2c bytes and + * several cross) came out with its two ends in unrelated shadow windows: + * end - base went negative, and the VTG's fetch past the base page hit an + * invalid PTE (the 0xbadf1201 miss=1 priv-error seen once the real VS + * started executing). Map the whole [base, end) range into ONE fresh + * fetch line (up to 16 x 4 KB) and rewrite BOTH words from that line, so + * the size the MME computes is the client's real size and every page of + * the stream is engine-visible at base+offset. A page that cannot be + * resolved maps the module zero page: a zero attribute is a degenerate + * vertex, never a fault. Returns the line base VA; caller consumes one + * shadow slot. + */ +static u64 gsp_phys_rmcall_gr_stream_line_locked( + const struct gsp_phys_gr_submit *req, u64 base_va, u64 end_va, + u32 slot) +{ + struct gsp_phys_boot_chunk *sh = &boot_stage.rmcall_gr_shadow; + u64 base_page = base_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + u64 end_page = (end_va - 1u) & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + u64 hpa[GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES]; + u64 pte[GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES]; + u64 fb_off[GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES]; + bool real[GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES]; + u64 va0, dummy = 0; + u32 npages, i, line, page, line_in_page, dummy_n = 0, vid_n = 0; + u8 *pt; + int ret; + + if (end_va <= base_va) + return 0; + npages = (u32)((end_page - base_page) >> 12) + 1u; + if (npages > GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES) + return 0; + for (i = 0; i < npages; i++) { + u64 pa = base_page + ((u64)i << 12); + + real[i] = false; + fb_off[i] = 0; + /* + * r1497: the active draw's vertex/index page resolves to a + * vidmem page whose emulated shadow is empty. Point the + * fetch line at the real FBMEM page (the guest's own page + * table names the offset) instead of delivering zeros. + */ + if (gsp_phys_gr_fetch_vid_off_locked(pa, &fb_off[i])) { + pte[i] = gsp_phys_rmcall_gr_pte_leaf_vid(fb_off[i]); + hpa[i] = 0; + real[i] = true; + vid_n++; + continue; + } + hpa[i] = gsp_phys_rmcall_gr_stream_page_hpa_locked(req, pa); + if (!hpa[i]) { + if (!dummy) + dummy = gsp_phys_gr_surf_dummy_dma_locked(); + if (!dummy) + return 0; + hpa[i] = dummy; + dummy_n++; + } + pte[i] = gsp_phys_rmcall_gr_pte_leaf(hpa[i]); + } + if (!sh->pages) { + ret = gsp_phys_chunk_alloc(boot_stage.pdev, sh, + GSP_PHYS_RMCALL_GR_SHADOW_PAGES); + if (ret) + return 0; + gsp_phys_chunk_zero(sh); + } + if (slot >= sh->pages || !boot_stage.rmcall_va_guest_pt) + return 0; + line = boot_stage.rmcall_guest_pt_next_line; + if (line >= GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES) + return 0; + page = line / GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; + line_in_page = line % GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; + ret = gsp_phys_rmcall_gr_scratch_win_install_locked(page); + if (ret) + return 0; + pt = gsp_phys_rmcall_guest_pt_page_va(page); + if (!pt) + return 0; + for (i = 0; i < npages; i++) + put_unaligned_le64(pte[i], pt + (size_t)( + GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * + line_in_page + i) * 8); + dma_wmb(); + boot_stage.rmcall_guest_pt_next_line = line + 1u; + va0 = gsp_phys_rmcall_guest_pt_win_va(page) + + ((u64)line_in_page << 16); + for (i = 0; i < npages; i++) + if (!real[i]) + gsp_phys_rmcall_gr_alias_cache_set_locked(hpa[i], + va0 + ((u64)i << 12)); + { + static u32 stream_line_log; + u32 w[8] = { 0 }, k; + u64 hpa0 = hpa[0]; + + if (!real[0] && + pfn_valid((unsigned long)(hpa0 >> PAGE_SHIFT))) { + const u8 *p = (const u8 *)__va( + hpa0 + (base_va & (GSP_PHYS_PAGE_SIZE - 1))); + + for (k = 0; k < 8u; k++) + w[k] = get_unaligned_le32(p + (size_t)k * 4u); + } + if (stream_line_log < 32u) { + stream_line_log++; + pr_info("vgsp-gr: MME-VSTREAM-LINE h=0x%x base=0x%llx end=0x%llx pages=%u dummy=%u vid=%u fb0=0x%llx line=0x%llx hpa0=0x%llx first8=%08x %08x %08x %08x %08x %08x %08x %08x\n", + req ? req->h_object : 0u, base_va, end_va, + npages, dummy_n, vid_n, fb_off[0], va0, hpa0, + w[0], w[1], w[2], w[3], w[4], w[5], w[6], + w[7]); + } + } + return va0; +} + +/* + * Allocate (or reuse) the pair line. Returns the line base VA on success, + * 0 when the second half cannot be resolved or the pool is exhausted (the + * caller then falls through to the single-page path). The fresh line is + * allocated on first use for an hpa0; the pair cache reuses it afterwards + * (the PTEs are identical on every window by construction). + */ +static u64 gsp_phys_rmcall_gr_sph_pair_locked( + const struct gsp_phys_gr_submit *req, u64 page0, u64 hpa0, u32 slot) +{ + struct gsp_phys_boot_chunk *sh = &boot_stage.rmcall_gr_shadow; + u32 i, line, page, line_in_page; + u64 hpa1, va0; + u8 *pt; + int ret; + + for (i = 0; i < gsp_phys_gr_pair_n; i++) + if (gsp_phys_gr_pair_hpa[i] == hpa0) + return gsp_phys_gr_pair_va[i]; + hpa1 = gsp_phys_rmcall_gr_prog_next_hpa_locked(req, page0); + if (!hpa1) + return 0; + if (!sh->pages) { + ret = gsp_phys_chunk_alloc(boot_stage.pdev, sh, + GSP_PHYS_RMCALL_GR_SHADOW_PAGES); + if (ret) + return 0; + gsp_phys_chunk_zero(sh); + } + if (slot >= sh->pages || !boot_stage.rmcall_va_guest_pt) + return 0; + line = boot_stage.rmcall_guest_pt_next_line; + if (line >= GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES) + return 0; + page = line / GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; + line_in_page = line % GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; + if (gsp_phys_rmcall_gr_scratch_win_install_locked(page)) + return 0; + pt = gsp_phys_rmcall_guest_pt_page_va(page); + if (!pt) + return 0; + put_unaligned_le64(gsp_phys_rmcall_gr_pte_leaf(hpa0), + pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * + line_in_page) * 8); + put_unaligned_le64(gsp_phys_rmcall_gr_pte_leaf(hpa1), + pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * + line_in_page + 1u) * 8); + dma_wmb(); + boot_stage.rmcall_guest_pt_next_line = line + 1u; + va0 = gsp_phys_rmcall_guest_pt_win_va(page) + + ((u64)line_in_page << 16); + gsp_phys_rmcall_gr_alias_cache_set_locked(hpa0, va0); + gsp_phys_rmcall_gr_alias_cache_set_locked(hpa1, + va0 + GSP_PHYS_PAGE_SIZE); + if (gsp_phys_gr_pair_n < GSP_PHYS_RMCALL_GR_PAIR_CACHE) { + gsp_phys_gr_pair_hpa[gsp_phys_gr_pair_n] = hpa0; + gsp_phys_gr_pair_va[gsp_phys_gr_pair_n] = va0; + gsp_phys_gr_pair_n++; + } + pr_info("vgsp-gr: SPH-PAIR line=0x%llx hpa0=0x%llx hpa1=0x%llx (8K contiguous program page) h=0x%x\n", + va0, hpa0, hpa1, req->h_object); + return va0; +} + +/* + * SESSION 153-CONTINUED-92: identity-fetch keep test (see the + * sema_gl_prog_guest_va knob above). Resolve the operand's page (the + * launch's own page set first - the pages the copy-in authored - then the + * sema service's trusted resolver), install the Tier-1 identity PTE at the + * guest VA itself and report whether the operand may stay verbatim. The + * receipt names the exact VA the DS will now fetch through (the guest VA, + * shadow column = the guest VA) and reads the fetch words back through the + * host direct map of the mapped HPA, so "the DS reads the authored page at + * its own heap VA" is provable from the log alone. Anything the identity + * mapper refuses (no window slot, stale fetch line, unresolved HPA) returns + * false and the caller falls back to the shadow rewrite. Called with + * gsp_phys_rm_lock held. + */ +static bool gsp_phys_rmcall_gr_prog_keep_locked(void __iomem *bar0, + u32 epoch, u32 up_w, u32 lo_w, + const struct gsp_phys_gr_submit *req) +{ + u64 va = ((u64)(up_w & 0xffu) << 32) | (u64)(lo_w & 0xfffffffcu); + u64 page = va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + u64 hpa = 0; + u32 i; + + if (!va || !page) + return false; + for (i = 0; i < req->n_pages; i++) { + if ((req->pages[i].guest_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) == page && + req->pages[i].hpa) { + hpa = req->pages[i].hpa; + break; + } + } + if (!hpa) { + struct gsp_sema_gr_reloc r; + + if (!gsp_sema_gr_reloc_resolve(page, &r) && r.hpa) + hpa = r.hpa; + } + if (!hpa) + return false; + if (gsp_phys_rmcall_gr_map_va_locked(bar0, page, hpa, epoch)) + return false; + gsp_phys_gr_sph_alias_receipt(page, va, va, hpa, "prog-guest"); + return true; +} + +static bool gsp_phys_rmcall_gr_3d_reloc(u32 *up_w, u32 *lo_w, + const struct gsp_phys_gr_submit *req, + u32 *n_shadow) +{ + u64 va = ((u64)(*up_w & 0xffu) << 32) | (u64)(*lo_w & 0xfffffffcu); + u64 page = va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + u64 sh = 0; + u32 line = 0, i; + + if (!va || !page) + return false; + /* + * SESSION 153-CONTINUED-90: must run before every resolution stage so + * the canary cannot be shadowed by an earlier (surd) alias hit. + */ + if (READ_ONCE(gsp_phys_gr_sph_canary) && req->prog_va && + page == (req->prog_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1))) { + u64 chpa = gsp_phys_gr_sph_canary_hpa(); + u32 line3 = 0; + + if (chpa && + *n_shadow < GSP_PHYS_RMCALL_GR_SHADOW_PAGES && + !gsp_phys_rmcall_gr_map_shadow_locked(*n_shadow, chpa, + &sh, &line3)) { + if (line3 != 0xffffffffu) + (*n_shadow)++; + sh |= va & (GSP_PHYS_PAGE_SIZE - 1); + gsp_phys_gr_sph_alias_receipt(page, va, sh, chpa, + "canary"); + *lo_w = (*lo_w & 0x3u) | ((u32)sh & 0xfffffffcu); + *up_w = (*up_w & ~0xffu) | + ((u32)(sh >> 32) & 0xffu); + return true; + } + } + /* + * SESSION 153-CONTINUED-63: rec-over-alias preference. When the fill + * tracked the render target through the bounce lane (every page + * resolved into contiguous data recs), rewrite the operand to the + * rec's bounce VA so the engine's stores land where the retire wait + * can count them and the copy-out can ship them back to the guest's + * own pages. The completeness gate (gsp_sema_gr_surf_recs) makes a + * partial run impossible to point at. Runs before the alias so the + * identity alias (direct guest writes, invisible to the witness) + * cannot win for a tracked target. + */ + { + u64 sva = 0; + u32 sn = 0; + + if (gsp_sema_gr_surf_recs(&sva, &sn) && + va >= sva && va < sva + ((u64)sn << 12)) { + for (i = 0; i < req->n_data; i++) { + if (req->data[i].va != page || + !req->data[i].reloc_va) + continue; + sh = req->data[i].reloc_va | + (va & (GSP_PHYS_PAGE_SIZE - 1)); + *lo_w = (*lo_w & 0x3u) | + ((u32)sh & 0xfffffffcu); + *up_w = (*up_w & ~0xffu) | + ((u32)(sh >> 32) & 0xffu); + return true; + } + } + } + /* + * SESSION 153-CONTINUED-27: a big surface (the X screen) is aliased + * as ONE contiguous run by the pre-pass; rewrite the operand to the + * alias base | offset. A one-page shadow relocation of it would + * break the raster walk at the first page crossing (r992), and the + * raw VA has no PTE chain the GPC engines can see (r993). + */ + for (i = 0; i < gsp_phys_gr_surf_n; i++) { + if (va >= gsp_phys_gr_surf[i].va && + va < gsp_phys_gr_surf[i].va + + (gsp_phys_gr_surf[i].npages << 12)) { + /* + * CONT-73: the alias VA maps the surface's guest + * HPAs 1:1 (the alias PTEs were built from + * req->pages[] first, the trusted resolver else), so + * a store through it lands straight in that guest + * page - record it for the direct-write witness. + */ + { + u32 pi; + + for (pi = 0; pi < req->n_pages; pi++) { + if ((req->pages[pi].guest_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) != + page || + !req->pages[pi].hpa) + continue; + gsp_phys_gr_wit_note(page, + req->pages[pi].hpa); + break; + } + if (pi == req->n_pages) { + struct gsp_sema_gr_reloc ra; + + if (!gsp_sema_gr_reloc_resolve(page, + &ra) && + ra.hpa) + gsp_phys_gr_wit_note(page, + ra.hpa); + } + } + sh = gsp_phys_gr_surf[i].alias + + (va - gsp_phys_gr_surf[i].va); + *lo_w = (*lo_w & 0x3u) | ((u32)sh & 0xfffffffcu); + *up_w = (*up_w & ~0xffu) | + ((u32)(sh >> 32) & 0xffu); + return true; + } + } + for (i = 0; i < req->n_pages; i++) { + if ((req->pages[i].guest_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) != page || + !req->pages[i].hpa) + continue; + if (*n_shadow >= GSP_PHYS_RMCALL_GR_SHADOW_PAGES) + return false; + /* + * SESSION 153-CONTINUED-93: 8K program-pair line (see the + * pair helper). Only the program family page rides this + * path; an unresolvable second half falls through to the + * single-page map below. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_sph_8k) && req->prog_va && + page == (req->prog_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1))) { + u64 pv = gsp_phys_rmcall_gr_sph_pair_locked( + req, page, req->pages[i].hpa, *n_shadow); + + if (pv) { + (*n_shadow)++; + sh = pv | (va & + (GSP_PHYS_PAGE_SIZE - 1)); + gsp_phys_gr_wit_note(page, req->pages[i].hpa); + gsp_phys_gr_sph_alias_receipt(page, va, pv, + req->pages[i].hpa, "pair"); + *lo_w = (*lo_w & 0x3u) | + ((u32)sh & 0xfffffffcu); + *up_w = (*up_w & ~0xffu) | + ((u32)(sh >> 32) & 0xffu); + return true; + } + } + if (gsp_phys_rmcall_gr_map_shadow_locked(*n_shadow, + req->pages[i].hpa, + &sh, &line)) + return false; + if (line != 0xffffffffu) + (*n_shadow)++; /* alias hit consumes no slot */ + /* CONT-73: shadow VA maps this guest HPA 1:1. */ + gsp_phys_gr_wit_note(page, req->pages[i].hpa); + sh |= va & (GSP_PHYS_PAGE_SIZE - 1); + if (req->prog_va && + page == (req->prog_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1))) + gsp_phys_gr_sph_alias_receipt(page, va, sh, + req->pages[i].hpa, + "pages"); + *lo_w = (*lo_w & 0x3u) | ((u32)sh & 0xfffffffcu); + *up_w = (*up_w & ~0xffu) | ((u32)(sh >> 32) & 0xffu); + return true; + } + /* + * SESSION 153-CONTINUED-59: level-2 late resolution, the same trusted + * chain the MME payload branch uses (ipin/pin/lat/map/walk then shadow + * map). The r1104 first beat window MISSed the program pair 0x2054 + * (va=0x4350900) and the r1105 windows MISSed the index-buffer pairs + * (0x17c8, va=0x44fedf0...) whose pages were not in the launch page + * set at build time; the raw guest VA then let the engine fault the + * MMU (HUBCLIENT_PD) and the guest driver reset the channel. Resolve + * the page here if any stage knows it, so a MISS only survives when + * every source fails; a surviving MISS is recorded for the pin demand + * set and counts toward the pre-doorbell abort. + */ + { + struct gsp_sema_gr_reloc r; + u32 line2 = 0; + + if (!gsp_sema_gr_reloc_resolve(va, &r) && r.hpa && + *n_shadow < GSP_PHYS_RMCALL_GR_SHADOW_PAGES && + !gsp_phys_rmcall_gr_map_shadow_locked(*n_shadow, r.hpa, + &sh, &line2)) { + if (line2 != 0xffffffffu) + (*n_shadow)++; + /* CONT-73: shadow VA maps this guest HPA 1:1. */ + gsp_phys_gr_wit_note(page, r.hpa); + sh |= va & (GSP_PHYS_PAGE_SIZE - 1); + if (req->prog_va && + page == + (req->prog_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1))) + gsp_phys_gr_sph_alias_receipt(page, va, sh, + r.hpa, + "resolve"); + *lo_w = (*lo_w & 0x3u) | ((u32)sh & 0xfffffffcu); + *up_w = (*up_w & ~0xffu) | ((u32)(sh >> 32) & 0xffu); + return true; + } + } + /* + * SESSION 153-CONTINUED-61: junk-operand filter. The window + * recognizer is byte-shape based, so float/immediate DATA words in + * the method stream also parse as address pairs. The observed junk + * class sits in the 256 MB..4 GB numeric range (0x3d000000..0xc2...... + * float bit patterns: 0x3f800000 = 1.0f, 0xbf800000 = -1.0f, ...) + * and can NEVER resolve - r1109 measured miss=20 constant across + * every attempt, all of it this class, which pinned the abort census + * nonzero forever and blocked every submit. Real client VAs of this + * driver live below 0x5000000 (the 64-80 MB GL/heap range, e.g. + * 0x4250000/0x4350000/0x462fc40) or at/above 0x1_00000000 (the 4G+ + * push-buffer/UVM ranges). A miss outside both is a data-word + * misidentification, not a raw client VA the engine would read as an + * address: it is neither demanded (the demand set was 60 % junk by + * r1109, diverting the pin cycle) nor counted toward the pre-doorbell + * abort. Witnessed per class, bounded. + */ + if (page >= 0x10000000ULL && page < 0x100000000ULL) { + static u32 gr_missjunk_log; + + gsp_phys_rmcall_gr_3d_junk_n++; + if (gr_missjunk_log < 24 || + !(gsp_phys_rmcall_gr_3d_junk_n % 512u)) { + gr_missjunk_log++; + pr_info("vgsp-gr: MISS-JUNK va=0x%llx (data-word class; not demanded, not counted toward the abort) n=%llu\n", + page, (u64)gsp_phys_rmcall_gr_3d_junk_n); + } + return false; + } + /* + * Unresolvable: demand the page for the pin path and count the miss + * (the submit path aborts pre-doorbell while the count is nonzero). + */ + gsp_phys_rmcall_gr_3d_miss_n++; + gsp_sema_demand_note(page); + return false; +} + +/* + * r929 v3: copy the 3D window and relocate every address operand a 3D-class + * method run carries (same recognizers as the sema service's svc_gl_field: + * LINE_LENGTH_IN 0x180 OFFSET_OUT pair, SET_VERTEX_STREAM_SUBSTITUTE 0xf84, + * SET_TEX_SAMPLER_POOL 0x155c, SET_TEX_HEADER_POOL 0x1574 and the + * SET_PIPELINE_PROGRAM_ADDRESS 0x2014/0x2018+j*0x40 pairs). Returns the + * number of relocated pair(s). Run widths follow the service's decoder: + * op 1/3/5 are 1+count dwords, everything else resyncs one dword. + */ +/* + * SESSION 153-CONTINUED-13 (draw-window strip experiment): comma-separated + * hex NVC597 method byte-addresses to REMOVE from a captured 3D window + * before the verbatim replay (whole methods - a dropped run takes its + * count data words with it). Answers whether a DRAW window's non-retire + * lives in the draw content (BEGIN/END/immediate vertices/DRAW_ARRAY) or + * in the state binds/method-stream shape. Empty = off (historical + * behaviour); live-tunable via sysfs so one boot can sweep the variants. + */ +/* + * SESSION 153-CONTINUED-47 (A/B knob): gates the newly recognised pair + * classes (SET_INDEX_BUFFER_A/B 0x17c8 and the vertex-stream banks + * 0x1c20..0x1cf0) in the verbatim-replay relocation pass. Default 0 keeps + * the r1035-r1041 baseline relocation set exactly; flipping it live + * (`echo Y > /sys/module/vgsp/parameters/sema_gl_idx_reloc`) rewrites the + * next launches' index/vertex operands to shadow VAs, so one boot can A/B + * the effect instead of comparing across boots. + */ +static bool gsp_phys_rmcall_gr_idx_reloc; +module_param_named(sema_gl_idx_reloc, gsp_phys_rmcall_gr_idx_reloc, bool, 0644); +MODULE_PARM_DESC(sema_gl_idx_reloc, + "Relocate SET_INDEX_BUFFER (0x17c8) and vertex-stream banks 0x1c20..0x1cf0 operands to shadow VAs in 3D replays (default 0 = baseline set only)"); + +/* + * SESSION 153-CONTINUED-48: MME-family payload relocation. The C597 class + * carries CALL_MME_MACRO(j) = 0x3800+j*8 and CALL_MME_DATA(j) = 0x3804+j*8 + * (class/clc597.h); the GLX client's macro(2) run (observed at byte + * address 0x3810, cnt=5) inlines raw 32-bit client VAs as macro arguments + * (`... 0450f800 ... 0450f000` in the first beef window of r1042/r1043). + * Those words are the last raw-operand class left in the faulting window + * after the index/vertex relocation: the engine walks the macro data + * through its own fetch path, and a raw client VA has no PTE chain the + * GPC engines can see (the r993/r1043 FAULT_PDE->PRI-fault class). The + * pass rewrites any payload word of a run whose byte address lies in + * [0x3800, 0x3a00) through the same vetted shadow mapping the pair + * classes use: only a word whose page the launch's own page set (or the + * big-surface alias) resolves is rewritten (MISS stays verbatim), the + * rewrite maps the SAME HPA at a shadow VA so the value is + * content-equivalent, and a shadow VA that would need an upper byte (a + * 64-bit alias in a 32-bit slot) is refused. Default 0 (baseline windows + * byte-identical); live-tunable so one boot can A/B. + */ +static bool gsp_phys_rmcall_gr_mme_reloc; +module_param_named(sema_gl_mme_reloc, gsp_phys_rmcall_gr_mme_reloc, bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_reloc, + "Relocate CALL_MME_MACRO/DATA(j) run payload words whose page resolves in the launch page set to shadow VAs in 3D replays (default 0 = off)"); + +/* + * SESSION 153-CONTINUED-58: torn-read zero guard for the vertex-attribute + * families. The r1098/r1099 first-launch class error + * (`Class Error: ChId 0001, Class 0000c597, Offset 00001168, + * Data 00000000, ErrorCode 00000005` = SET_VERTEX_ATTRIBUTE_A(2) written + * with NUMERICAL_TYPE=0, the header's UNUSED_ENUM_DO_NOT_USE value -> + * INVALID_ENUM) is written by a replayed window run + * `20030458 38400000 38200040 00000000`: a cnt=3 INC run at method 0x1160 + * whose THIRD data word is exactly zero and is immediately followed by a + * 52-word zero region - the read caught the client's push buffer after + * part of it had been reused/cleared. The same client writes valid A(2) + * values in the other windows of the same push (0x38200040/0x38200600/ + * 0x38400600), so the zero is a torn read, not client content; and + * NUMERICAL_TYPE=0 is never a legitimate client write. Guard: for INC + * runs in the SET_VERTEX_ATTRIBUTE_A/B families (0x1160..0x11dc) drop a + * data word that is exactly zero, splitting the run so the remaining + * words keep their methods (the attribute then keeps the value the + * ATTR-PRELUDE or the previous frame wrote). Default 1; live-tunable. + */ +static u32 gsp_phys_rmcall_gr_attr_zero_fix = 1; +module_param_named(sema_gl_attr_zero_fix, gsp_phys_rmcall_gr_attr_zero_fix, uint, 0644); +MODULE_PARM_DESC(sema_gl_attr_zero_fix, + "Drop torn-read zero data words in the SET_VERTEX_ATTRIBUTE_A/B runs (0x1160..0x11dc) of 3D replays, splitting the run (default 1)"); + +/* (SESSION 153-CONTINUED-53 MME prelude knob is declared before + * gsp_phys_rmcall_build_gr_launch, its reader.) */ + +/* + * SESSION 153-CONTINUED-48: ladder-wedge recovery. CONT-47 measured that + * the FIRST beef launch per boot whose window carries an unresolvable raw + * operand PRI-faults the GR: every SKED/FIFO register then reads the + * priv-error poison (cp. kernel_falcon_ga102.c `privErrVal = 0xbadf0000`) + * and the ladder channel stays wedged for the whole boot (engine GP_GET + * stuck; every later publish unconsumed, even an emptied window) - which + * makes content bisection a one-shot per boot. When enabled, the launch + * poll watches the GR FIFO status (0x400504, the sked recorder's + * `grfifo`) for the 0xbadf-tagged poison and performs ONE bounded + * recovery per launch: the r445 hard re-arm (TSG PREEMPT bWait=1 + the + * m155 lane-enable tail + GPFIFO/TSG schedules + doorbell) followed by + * the warm GR semaphore probe, exactly the mechanism behind + * gsp_phys_gr_wedge_rearm. Bounded: one detection per launch, poison + * gated, all RPCs keep their own timeouts. Default 1 (the poisoned state + * is already a guaranteed-dead boot; the recovery only adds the chance + * that later launches in the same boot still execute). + */ +static bool gsp_phys_gr_wedge_recover = true; +module_param_named(sema_gr_wedge_recover, gsp_phys_gr_wedge_recover, bool, + 0644); +MODULE_PARM_DESC(sema_gr_wedge_recover, + "On the 0xbadf priv-error poison in the GR FIFO status, run one bounded TSG-preempt+resched hard re-arm per launch (default 1)"); + +/* + * SESSION 153-CONTINUED-93: parked-engine recovery on the warm-sem ladder + * gate. The r1311-r1313 class: after the first few DS-validated draws the + * front-end stops fetching (grfifo=0x00000001, no poison, no trap) and the + * warm GR semaphore probe times out on every later launch - the ladder + * then fails fast and the whole boot is dead for bisection. When this knob + * is set, a wedged probe triggers ONE bounded hard re-arm (the proven + * r445 TSG-preempt bWait=1 + resched + probe) before giving up; an ALIVE + * probe after the re-arm lets the fresh-entry retries resume, and the MME + * prelude latch is cleared so the re-preload is owed (the preempt reset + * the context). Bounded: one per launch; all RPCs keep their timeouts. + * Default 1; A/B with 0 restores the fail-fast ladder. + */ +static bool gsp_phys_gr_park_recover = true; +module_param_named(sema_gr_park_recover, gsp_phys_gr_park_recover, bool, + 0644); +MODULE_PARM_DESC(sema_gr_park_recover, + "On a wedged warm-sem probe in the launch timeout ladder, run one bounded TSG-preempt+resched hard re-arm per launch and retry (default 1)"); + +static char *gsp_phys_rmcall_gr_win_drop; +module_param_named(sema_gl_win_drop, gsp_phys_rmcall_gr_win_drop, charp, 0644); +MODULE_PARM_DESC(sema_gl_win_drop, + "Hex method byte-addresses (comma-separated) to strip from a captured 3D window before replay (draw isolation; default empty)"); + +static bool gsp_phys_rmcall_gr_win_drop_match(u32 addr) +{ + const char *p = gsp_phys_rmcall_gr_win_drop; + u32 n = 0; + + if (!p || !*p) + return false; + while (*p && n < 64) { + u32 v = 0; + u32 digits = 0; + + while (*p == ' ' || *p == ',' || *p == ':' || *p == ';') + p++; + /* + * SESSION 153-CONTINUED-46: accept an optional 0x/0X prefix. + * The original parser scanned hex digits only, so the natural + * "0x3810,0x39c8" input parsed as 0 (the 'x' ended the digit + * run after the leading '0') and silently never matched - + * cost a full bisect round. + */ + if (p[0] == '0' && (p[1] == 'x' || p[1] == 'X')) + p += 2; + for (;;) { + char c = *p; + u32 d; + + if (c >= '0' && c <= '9') + d = c - '0'; + else if (c >= 'a' && c <= 'f') + d = c - 'a' + 10; + else if (c >= 'A' && c <= 'F') + d = c - 'A' + 10; + else + break; + if (v <= 0xfffffffu) + v = (v << 4) | d; + p++; + digits++; + } + if (!digits) + break; + /* + * SESSION 153-CONTINUED-47: `lo-hi` drops every method + * byte-address in the closed range (both ends accept the + * 0x prefix). The content bisect needs whole families + * (`0x3800-0x3a00` MME, `0x3400-0x3500` MME scratch) and + * the emptied-window control (`0-0x7ffc`): with the window + * stripped whole, a launch that still retires proves the + * SET_OBJECT + release tail path is healthy and that the + * wedge lives in the stripped content. + */ + if (*p == '-') { + const char *q = p + 1; + u32 hi = 0; + u32 hd = 0; + + if (q[0] == '0' && (q[1] == 'x' || q[1] == 'X')) + q += 2; + for (;;) { + char c = *q; + u32 h; + + if (c >= '0' && c <= '9') + h = c - '0'; + else if (c >= 'a' && c <= 'f') + h = c - 'a' + 10; + else if (c >= 'A' && c <= 'F') + h = c - 'A' + 10; + else + break; + if (hi <= 0xfffffffu) + hi = (hi << 4) | h; + q++; + hd++; + } + if (hd) { + p = q; + n++; + if (addr >= v && addr <= hi) + return true; + continue; + } + } + n++; + if (v == addr) + return true; + } + return false; +} + +/* + * SESSION 153-CONTINUED-27: auto-drop of the software-emulated lanes from + * every verbatim 3D replay. Three method triggers in a captured window + * cause the physical engine to repeat work the sema service already + * executes in software - and, on the replay channel, with the WRONG + * context state (no previous-window sticky registers): + * - 0x1b0 LAUNCH_DMA (C597 CE lane): triggers both the CE copy and the + * LOAD_INLINE_DATA upload. The service authors every LID payload into + * the guest pages itself (svc_gl_lid/svc_grce_inline_write) and + * emulates the CE strips (svc_op2d); physically re-running them with + * an unset source/context walks unbounded memory (r992-r996: the + * "march" - faults at whatever page the alias cap ended, e.g. + * 0x5d6000/0x5f4000, after having written every mapped page). + * - 0x5e0 (TWOD RECTS fill trigger): the second point pair of the + * client's screen clear. The service's FERMI_TWOD executor runs every + * fill (r986: 46x `sema-twod-fill ... done=3686400 rc=0`); the + * physical re-run uses the replay channel's unset dst PITCH and + * marches (same fault class). + * - 0x10c NOTIFY: the completion write; without the preceding + * SET_NOTIFY_A/B (which live in other pushes) the engine writes + * address 0 (r996: `faulted @ 0x0_00000000 FAULT_PTE VIRT_WRITE`). + * The kick-notify lane + the backend's release tail author every + * completion the client actually waits on. + * All three are pure triggers - their setup methods stay in the window, + * so dropping the run can never misalign display state. The emulated + * lanes are the live ones (the r986 pixel proof: the screen paints via + * svc_twod fills, and every CE strip retires rc=0 in software). Default + * on; set 0 to replay the raw window unmodified. + */ +static bool gsp_phys_gr_win_drop_auto = 1; +module_param_named(sema_gl_win_drop_auto, gsp_phys_gr_win_drop_auto, bool, + 0644); +MODULE_PARM_DESC(sema_gl_win_drop_auto, + "Drop the software-emulated trigger methods (0x10c NOTIFY, 0x1b0 LAUNCH_DMA, 0x5e0 TWOD fill) from verbatim 3D replays (default 1)"); + +static bool gsp_phys_rmcall_gr_win_drop_auto_match(u32 addr) +{ + if (!READ_ONCE(gsp_phys_gr_win_drop_auto)) + return false; + /* + * SESSION 153-CONTINUED-48: NVC597_SET_REPORT_SEMAPHORE_A=0x1b00 - + * the C597 twin of the 0x10c NOTIFY trigger. Every glcore draw + * window ends with 4-word runs [A=upper(7:0), B=lower(31:0), + * C=payload, D=flags] whose B word is a raw client VA (observed + * 0x41a5bb0/0x759e870/0x251800; payloads = frame counters 0xb4f3+). + * The replayed engine executing them writes a client VA the GPC + * engines cannot reach - the VIRT_WRITE fault class (r1061: the + * kept [0,0x2000) window whose only fetch/write triggers besides + * the index binds are these raw 0x1b00 writes is exactly the one + * that still faulted with grfifo=0xbadf1201; r1062: the full window + * stalls the front-end right after the PBDMA consumes it and + * fecs_cur never loads). Their page cannot join the launch page set + * (n_ptr is capped at 7 by the QMD pointer region and the draw's + * program/index/vertex/MME pages already fill it), so relocation is + * impossible; dropping is the same call the 0x10c NOTIFY drop + * already makes: the module appends its own SET_REPORT_SEMAPHORE + * tail and the host-side completion lanes author every completion + * the client actually waits on (the r1061 keep-prefix window + * retired with ch_rel=0xc0ffee02 and the client kept pushing + * frames without any 0x1b00 in the replay). + */ + return addr == 0x010cu || addr == 0x01b0u || addr == 0x05e0u || + addr == 0x1b00u; +} + +/* + * SESSION 153-CONTINUED-14 (FECS residency / draw-stall isolation): patch + * the DATA of one method inside a replayed 3D window before it reaches the + * engine. Format: comma-separated `addr:data` hex pairs, addr = method + * byte-address (e.g. 1618 = BEGIN, 1614 = END), data = the replacement + * value. Header-only methods (IMMD) get their 13-bit data field rewritten + * in place; count-carrying methods (INC/NON_INC/INC_ONCE) get their FIRST + * payload word replaced. Live-tunable via sysfs so one boot can sweep + * variants (BEGIN OP=7 QUADS -> TRIANGLES etc.) against the draw-stall. + */ +static char *gsp_phys_rmcall_gr_win_patch; +module_param_named(sema_gl_win_patch, gsp_phys_rmcall_gr_win_patch, charp, + 0644); +MODULE_PARM_DESC(sema_gl_win_patch, + "Hex addr:data pairs (comma-separated) patched into a captured 3D window method before replay (default empty)"); + +static bool gsp_phys_rmcall_gr_win_patch_find(u32 addr, u32 *out_data) +{ + const char *p = gsp_phys_rmcall_gr_win_patch; + u32 n = 0; + + if (!p || !*p) + return false; + while (*p && n < 64) { + u32 v = 0, d = 0; + u32 digits = 0; + bool have_data = false; + + while (*p == ' ' || *p == ',' || *p == ':' || *p == ';') + p++; + for (;;) { + char c = *p; + u32 h; + + if (c >= '0' && c <= '9') + h = c - '0'; + else if (c >= 'a' && c <= 'f') + h = c - 'a' + 10; + else if (c >= 'A' && c <= 'F') + h = c - 'A' + 10; + else + break; + if (v <= 0xfffffffu) + v = (v << 4) | h; + p++; + digits++; + } + if (!digits) + break; + if (*p == ':' || *p == '=') { + p++; + for (;;) { + char c = *p; + u32 h; + + if (c >= '0' && c <= '9') + h = c - '0'; + else if (c >= 'a' && c <= 'f') + h = c - 'a' + 10; + else if (c >= 'A' && c <= 'F') + h = c - 'A' + 10; + else + break; + if (d <= 0xfffffffu) + d = (d << 4) | h; + p++; + have_data = true; + } + } + n++; + if (have_data && v == addr) { + *out_data = d; + return true; + } + } + return false; +} + +static u32 gsp_phys_rmcall_gr_3d_last_subdrop; +static u32 gsp_phys_rmcall_gr_3d_last_sdmdrop; + +/* + * SESSION 153-CONTINUED-62: an MME-family payload word that neither the + * launch page set nor the level-2 resolver could translate has to stay + * verbatim in the emitted stream (never guessed) - but a raw client VA in + * an 0x3810 macro payload is exactly the operand the engine faults on + * (`MMU Fault HUBCLIENT_PD`). Until this helper the word was left silent: + * the abort census stayed 0 and the launch went out to fault seconds + * later (r1115: fault @0x0_04350000, the 0x4350900 program-page payload). + * Now the surviving miss counts toward the pre-doorbell MISS-ABORT and + * joins the pin demand set, so the launch is held (fail-closed, bounded) + * until the page resolves, exactly like a window pair miss. The float/ + * data-word class (0x10000000..0x100000000) is excluded - it can never + * resolve and must not pin the census nonzero (the r1109 lesson). + */ +static void gsp_phys_gr_3d_mme_miss(u32 wv) +{ + u64 page = (u64)wv & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + + if (page >= 0x10000000ULL && page < 0x100000000ULL) + return; + gsp_phys_rmcall_gr_3d_miss_n++; + gsp_sema_demand_note(page); +} + + +static u32 gsp_phys_rmcall_gr_3d_last_progkeep; +static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out, + u32 *out_n, u32 *n_dropped, + const struct gsp_phys_gr_submit *req, + u32 *n_shadow, + void __iomem *bar0, u32 epoch) +{ + u32 off = 0, wout = 0, reloc = 0, dropped = 0, subdrop = 0, sdm_drop = 0; + u32 wp_match = 0, wp_lowrun = 0, prog_keep = 0; + + *out_n = n; + *n_dropped = 0; + while (off < n) { + u32 hdr = in[off]; + u32 op = (hdr >> 29) & 0x7u; + u32 cnt = (hdr >> 16) & 0xfffu; + u32 addr = (hdr & 0x1fffu) << 2; + u32 sub = (hdr >> 13) & 0x7u; + u32 span = 1u; + + if ((op == 1 || op == 3 || op == 5) && cnt && + off + 1u + cnt <= n) + span = 1u + cnt; + /* + * SESSION 153-CONTINUED-52: drop METHOD WRITES aimed at a + * non-zero subchannel. The glcore draw window ends with + * `2001c00d 70000000/68000000` = INC m=0x34 cnt=1 SUBCHANNEL 6 + * (channel-class MEM_OP_D lane) two-word runs. The ladder + * channel has nothing bound on subchannel 6, so the physical + * PBDMA hands those methods to its host-software lane and + * raises NV_PPBDMA_INTR_0_METHOD (bit 21): the r1075 relay + * records `Channel ID 00000004 intr 00200000` at every launch, + * and the real host driver's software-method handler answers + * "Invalid host sofware method received" -> rcRequired -> + * robust-channel error recovery kills the channel mid-window + * (fifo_gk104.c). The methods are channel-level barriers whose + * effects the module's own release/fence lanes already author, + * so dropping them from the replay is the same call the 0x10c + * NOTIFY / 0x1b0 LAUNCH_DMA auto-drops make. Only count- + * carrying writes (op 1/3/5) are candidates: the single-word + * TERT encodings (op 0/2/4/7) reuse bits 15:13 as mask/type + * payload. + */ + if ((op == 1u || op == 3u || op == 5u) && sub != 0u) { + dropped++; + subdrop++; + off += span; + continue; + } + /* + * SESSION 153-CONTINUED-59: single-word SDM-type entries + * (SEC_OP 0/2/6, see gsp_phys_rmcall_gr_sdm_drop). The + * retiring X-family windows never yield one; this guards a + * window that carries the client's SDM markers (the + * 0x1fff0-class marker words or the prelude 0x046x6dXX + * family), which the physical PBDMA refuses with PBENTRY. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_sdm_drop) && + hdr && (op == 0u || op == 2u || op == 6u)) { + /* + * SESSION 153-CONTINUED-86: name every dropped SDM + * entry (bounded) so the A/B arm can be audited: the + * "keep the drop only for entries proven unrelated" + * decision needs the actual words, not the count. + */ + static u32 sdm_log; + + if (sdm_log < 24u) { + sdm_log++; + pr_info("vgsp-gr: SDM-DROP in[%u]=%08x op=%u addr=%#x sub=%u cnt/val=%#x (span=%u)\n", + off, hdr, op, addr, sub, + (cnt ? cnt : ((hdr >> 16) & 0x1fffu)), + span); + } + dropped++; + sdm_drop++; + off += span; + continue; + } + if (gsp_phys_rmcall_gr_win_drop_match(addr) || + gsp_phys_rmcall_gr_win_drop_auto_match(addr)) { + dropped++; + off += span; + continue; + } + memcpy(&out[wout], &in[off], (size_t)span * 4); + /* + * SESSION 153-CONTINUED-58: torn-read zero guard - see the + * sema_gl_attr_zero_fix comment. Only INC runs (op 1) whose + * data words advance one method each; the attribute runs are + * exactly this shape. Rebuild the run in place, skipping + * exactly-zero words and re-targeting the surviving words' + * methods (a zero word's method is simply never written, so + * the attribute keeps its prelude/previous-frame value). + */ + if (READ_ONCE(gsp_phys_rmcall_gr_attr_zero_fix) && + op == 1u && cnt && addr >= 0x1160u && addr <= 0x11dcu) { + u32 k, k0 = 0, L = 0, wr = wout; + u32 subch = (hdr >> 13) & 0x7u; + + for (k = 0; k <= cnt; k++) { + u32 wv = (k < cnt) ? out[wout + 1u + k] : 0u; + /* + * SESSION 153-CONTINUED-62: the bad predicate + * also covers a torn ACTIVE attribute whose + * COMPONENT_BIT_WIDTHS enum is 0 (no such + * encoding exists): the r1113 setup-push + * capture carried + * `20030458 38400000 38200040 000081a4` + * (A(2)=0x81a4, widths=0) and the replayed + * prelude write trapped the engine - + * `Class Error: Class 0000c597, Offset + * 00001168, Data 000081a4, ErrorCode 0xc` + * -> grfifo=0xbadf1201 poison the moment the + * surf prelude let the draw execute. An + * inactive attribute (SOURCE=1, bit 6) keeps + * its word; only the invalid active encoding + * is dropped, so the attribute keeps its + * prelude/previous value until the window's + * own (valid) write lands. + */ + bool bad = (k < cnt) && + (wv == 0u || + (((wv >> 21) & 0x3fu) == 0u && + !(wv & 0x40u))); + + if (!bad && k < cnt) { + if (!L) + k0 = k; + out[wr + 1u] = out[wout + 1u + k]; + wr++; + L++; + continue; + } + if (L) { + u32 naddr = addr + 4u * k0; + u32 nhdr = (1u << 29) | + (L << 16) | + ((naddr >> 2) & 0x1fffu) | + (subch << 13); + + out[wr - L] = nhdr; + L = 0; + } + if (bad) { + static u32 az_log; + + if (az_log < 64) { + az_log++; + pr_info("vgsp-gr: ATTR-ZERO addr=0x%x word=%u/%u off=%u dropped (torn zero/invalid-width; attribute keeps its prelude value)\n", + addr + 4u * k, k, + cnt, off); + } + } + } + /* +1: the rebuilt run still needs its (new) header word */ + span = (wr - wout) + 1u; + } + { + /* + * SESSION 153-CONTINUED-14: live data patch (BEGIN + * OP sweep etc.). Header-only forms rewrite the 13-bit + * data field; count-carrying forms rewrite the first + * payload word. + */ + u32 pv; + + if (gsp_phys_rmcall_gr_win_patch_find(addr, &pv)) { + static u32 patch_log; + u32 old; + + wp_match++; + if (span > 1u) { + old = out[wout + 1]; + out[wout + 1] = pv; + } else { + old = (out[wout] >> 16) & 0x1fffu; + out[wout] = (out[wout] & ~0x1fff0000u) | + ((pv & 0x1fffu) << 16); + } + if (patch_log < 32) { + patch_log++; + pr_info("vgsp-gr: 3D win-patch addr=0x%x off=%u data 0x%x -> 0x%x\n", + addr, off, old, pv); + } + } else if (addr >= 0x100u && addr < 0x200u) { + /* + * SESSION 153-CONTINUED-84: the win_patch + * knob (sema_gl_win_patch=124:1) reaches + * this pass only if the window carries a + * run at the patch address; log the low-page + * runs so a decode mismatch is visible + * instead of silent. + */ + wp_lowrun++; + if (wp_lowrun <= 4u && + gsp_phys_rmcall_gr_win_patch && + *gsp_phys_rmcall_gr_win_patch) + pr_info("vgsp-gr: WIN-PATCH lowrun addr=0x%x op=%u cnt=%u hdr=%08x\n", + addr, op, cnt, hdr); + } + } + if ((op == 1 || op == 3 || op == 5) && cnt && + off + 1u + cnt <= n) { + u32 *d = &out[wout + 1]; + u32 lo_i = 0; + bool pair = false; + + if (op == 1 && addr == 0x0180 && cnt >= 4) { + lo_i = 2; /* OFFSET_OUT is v2/v3 */ + pair = true; + } else if (op == 1 && addr == 0x0f84 && cnt >= 2) { + pair = true; + } else if (op == 1 && addr == 0x155c && cnt >= 3) { + pair = true; + } else if (op == 1 && addr == 0x1574 && cnt >= 3) { + pair = true; + } else if (op == 1 && addr >= 0x2014 && + addr < 0x4000 && (addr & 0x3fu) == 0x14 && + cnt >= 2) { + pair = true; + } else if (op == 1 && (addr == 0x1c00 || addr == 0x1c10) && + cnt >= 3) { + /* SESSION 153-CONTINUED-10: SET_VERTEX_STREAM_ + * A_FORMAT(j)+BASE(j) run [fmt, upper, lower] + * (j*0x10 stride) - the vertex data the draw + * reads. Observed [00001000 1 2150b790]. */ + lo_i = 1; + pair = true; + } else if (READ_ONCE(gsp_phys_rmcall_gr_idx_reloc) && + op == 1 && addr >= 0x1c20 && addr <= 0x1cf0 && + (addr & 0xfu) == 0u && cnt >= 3) { + /* SESSION 153-CONTINUED-47 (A/B knob): widen + * the vertex-stream banks from j=0/1 to every + * FORMAT slot (0x1c20..0x1cf0). The GLX + * client's 3D window binds streams j>=2 that + * svc_gl_field already collects into the page + * set but the window never had rewritten. */ + lo_i = 1; + pair = true; + } else if (READ_ONCE(gsp_phys_rmcall_gr_idx_reloc) && + op == 1 && addr == 0x17c8 && cnt >= 2) { + /* SESSION 153-CONTINUED-47 (A/B knob): SET_ + * INDEX_BUFFER_A/B [ADDRESS_UPPER(7:0), + * ADDRESS_LOWER(31:0)] - the indexed draw + * fetches its indices through this VA. The + * r1042 boot showed an X window carrying + * index binds PRI-stalling once relocated + * (relocation of exactly this pair is the + * only kernel delta vs the r1035 baseline + * where index-bearing windows did not exist + * and every X window retired), so the + * rewrite is gated off by default and A/B-ed + * live per boot. Observed [00000000 + * 0450fcc8] -> VA 0x450fcc8. */ + lo_i = 0; + pair = true; + } else if (READ_ONCE(gsp_phys_rmcall_gr_idx_reloc) && + op == 1 && addr == 0x1b00 && cnt >= 2) { + /* SESSION 153-CONTINUED-48: NVC597_SET_REPORT_ + * SEMAPHORE_A/B..D (0x1b00) - the engine's own + * completion write [upper(7:0), lower(31:0), + * payload, flags] (observed [0, 0x4181bb0, + * 0x4f9f6, 0x1000f010] at the end of every + * glcore window). With the VA raw the replayed + * engine writes a client VA the GPC engines + * cannot reach - the VIRT_WRITE fault class + * (r1061: the kept [0,0x2000) window that + * faults is exactly the one still carrying the + * raw 0x1b00 VAs). Rewrite the pair to the + * shadow VA (same HPA, page collected by + * svc_gl_field 0x1b00), so the write lands in + * the client's own page - its wait then sees a + * real hardware completion. MISS stays + * verbatim like every other pair. */ + lo_i = 0; + pair = true; + } else if (op == 1 && addr == 0x2380 && cnt >= 3) { + /* SET_CONSTANT_BUFFER_SELECTOR_A/B: + * [size, upper, lower]; observed + * [00010000 1 2150a500]. */ + lo_i = 1; + pair = true; + } else if (op == 1 && addr >= 0x0800 && addr < 0x0c00 && + (addr & 0x3fu) == 0u && cnt >= 2) { + /* SET_COLOR_TARGET_A/B(j) offset pair: + * [upper, lower, w, h, fmt]; the surface the + * draw rasterizes into (must live in guest + * RAM for the present lane to see pixels). + * Observed [1 222fa200 40 4 f7 0]. */ + lo_i = 0; + pair = true; + } else if (op == 1 && addr == 0x218 && cnt >= 4 && + (d[2] & ~0xffu) == 0u && + (d[3] & 0xfffu) == 0u) { + /* SESSION 153-CONTINUED-27: the TWOD twin + * of the colour target - INC 0x218 cnt>=4 + * covers 0x218..0x224 = [w, h, upper, + * lower] (the emulator's SET_DST sequence; + * the glxgears client window fills the X + * screen through it). Shape-checked so a + * non-surface run can never be rewritten. */ + lo_i = 2; + pair = true; + } + if (pair) { + u64 pva = ((u64)(d[lo_i] & 0xffu) << 32) | + (u64)(d[lo_i + 1] & 0xfffffffcu); + static u32 pair_log; + bool ok = false, kept = false; + + /* + * SESSION 153-CONTINUED-92: the program-family + * space experiment (see sema_gl_prog_guest_va). + * The DS rejects every bound stage whose SPH it + * fetches through a relocated shadow VA while + * the bytes at those VAs are provably correct; + * leave the operand at its ORIGINAL guest VA + * whenever the page resolves + identity-maps, + * so the DS fetches/validates in the same VA + * space the client's program heap describes. + * Kept pairs count as relocated so the SPH-BIND + * guard (orel == nrun) is unaffected; a page the + * identity mapper refuses falls through to the + * shadow rewrite exactly as before. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_prog_guest_va) && + bar0 && req->prog_va && + addr >= 0x2014u && addr < 0x4000u && + (addr & 0x3fu) == 0x14u && + gsp_phys_rmcall_gr_prog_keep_locked( + bar0, epoch, d[lo_i], d[lo_i + 1], + req)) { + static u32 progva_log; + + kept = true; + prog_keep++; + if (progva_log < 24u) { + progva_log++; + pr_info("vgsp-gr: PROG-GUEST-VA keep addr=0x%x va=0x%llx (Tier-1 identity; operand left verbatim) h=0x%x\n", + addr, pva, + req->h_object); + } + } else { + ok = gsp_phys_rmcall_gr_3d_reloc( + &d[lo_i], &d[lo_i + 1], req, + n_shadow); + } + if (ok || kept) + reloc++; + if (!kept && pair_log < 48) { + pair_log++; + pr_info("vgsp-gr: pair addr=0x%x va=0x%llx -> %s0x%llx\n", + addr, pva, ok ? "" : + "MISS ", + ((u64)(d[lo_i] & 0xffu) << 32) | + (u64)(d[lo_i + 1] & 0xfffffffcu)); + } + } + /* + * SESSION 153-CONTINUED-48: MME-family payload + * relocation (see the sema_gl_mme_reloc param). The + * C597 MME macro/data family spans 0x3800..0x3a00; + * its runs carry the macro arguments as raw payload + * words, which include 32-bit client VAs + * (0x450f800/0x450f000 observed in the beef macro(2) + * run at 0x3810). Rewrite only a word whose page the + * launch's own page set resolves (the same vetted + * shadow mapping the pairs use); a MISS, an + * unaligned/too-small word, or a shadow VA that + * cannot fit the 32-bit slot stays verbatim. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_mme_reloc) && + addr >= 0x3800u && addr < 0x3a00u) { + u32 wi; + bool stream_pair = false; + + /* + * r1494: VERTEX_STREAM macro(2) carries + * [stream, end_hi, end_lo, base_hi, base_lo] + * where END and BASE are client VAs (the MME + * computes the stream length as end - base). + * Relocating the two words independently put + * a page-crossing stream's ends in unrelated + * shadow windows: end - base went negative + * and the VTG fetch past the base page read + * an invalid PTE -> 0xbadf1201 miss=1 once + * the real VS ran. Map the whole range into + * one fresh contiguous line and rewrite both + * words from it; fall back to the historical + * per-word pass only when the range cannot be + * mapped. + */ + if (addr == 0x3810u && cnt >= 5u && + !d[1] && !d[3]) { + u64 bva = (u64)d[4]; + u64 eva = (u64)d[2]; + u64 lva; + + if (bva >= 0x10000u && eva > bva && + (eva - bva) <= 0x10000u && + !(bva & 3u) && !(eva & 3u) && + *n_shadow < + GSP_PHYS_RMCALL_GR_SHADOW_PAGES && + (lva = gsp_phys_rmcall_gr_stream_line_locked( + req, bva, eva, + *n_shadow)) != 0) { + u64 bpage = bva & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + + d[2] = (u32)(lva + (eva - bpage)); + d[4] = (u32)(lva + (bva & + (GSP_PHYS_PAGE_SIZE - 1))); + (*n_shadow)++; + stream_pair = true; + reloc += 2; + } + } + for (wi = 0; wi < cnt; wi++) { + u32 wv = d[wi]; + u32 up = 0, lo = wv; + static u32 mme_log; + + if (stream_pair && + (wi == 2u || wi == 4u)) + continue; + if (wv < 0x10000u || (wv & 3u)) + continue; + if (!gsp_phys_rmcall_gr_3d_reloc( + &up, &lo, req, n_shadow)) { + /* + * SESSION 153-CONTINUED-51: + * the payload page missed the + * launch's page set (the QMD + * pointer region caps the set + * far below a full window's + * operand count - r1071: the + * macro payload pages + * 0x44fd200..0x44fde00 were + * ~12 distinct pages while the + * set held 7; the words stayed + * RAW and the engine fetched an + * untranslatable guest VA on + * every full window - the + * 0xbadf1201 priv-error + * poison). Resolve the page + * through the SAME trusted + * ledger/pin/walk chain the + * launch pages use (the + * RELOC-FIXED resolver) and + * shadow-map it exactly like a + * matched page. A resolve/map + * miss or a shadow VA that + * cannot fit the 32-bit slot + * stays verbatim (never + * guessed) - and, SESSION + * 153-CONTINUED-62, it now + * COUNTS toward the abort + * census and joins the pin + * demand set: r1115's residual + * fault was exactly this class + * (`MMU Fault HUBCLIENT_PD @ + * 0x0_04350000` 2.5 s after a + * launch whose census read 0 + * because an 0x3810 payload + * word 0x4350900 missed here + * silently). + */ + struct gsp_sema_gr_reloc r; + u64 va2 = (u64)wv; + u64 sh2 = 0; + u32 line2 = 0; + + if (gsp_sema_gr_reloc_resolve( + va2, &r) || + !r.hpa) { + gsp_phys_gr_3d_mme_miss( + wv); + continue; + } + if (*n_shadow >= + GSP_PHYS_RMCALL_GR_SHADOW_PAGES) { + gsp_phys_gr_3d_mme_miss( + wv); + continue; + } + if (gsp_phys_rmcall_gr_map_shadow_locked( + *n_shadow, r.hpa, + &sh2, &line2)) { + gsp_phys_gr_3d_mme_miss( + wv); + continue; + } + if (line2 != 0xffffffffu) + (*n_shadow)++; + /* CONT-73: MME shadow-scratch + * destination maps this guest + * HPA 1:1; record it so a store + * through the MME path is + * witnessed too. */ + gsp_phys_gr_wit_note( + va2 & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1), + r.hpa); + sh2 |= va2 & + (GSP_PHYS_PAGE_SIZE - 1); + up = (u32)(sh2 >> 32) & 0xffu; + lo = (wv & 0x3u) | + ((u32)sh2 & 0xfffffffcu); + if (up != 0) { + gsp_phys_gr_3d_mme_miss( + wv); + continue; + } + d[wi] = lo; + reloc++; + if (mme_log < 32) { + mme_log++; + pr_info("vgsp-gr: mme-reloc-rsv addr=0x%x wi=%u w=0x%x -> 0x%x\n", + addr, wi, wv, + lo); + } + continue; + } + if (up != 0) { + gsp_phys_gr_3d_mme_miss(wv); + continue; + } + d[wi] = lo; + reloc++; + if (mme_log < 32) { + mme_log++; + pr_info("vgsp-gr: mme-reloc addr=0x%x wi=%u w=0x%x -> 0x%x\n", + addr, wi, wv, lo); + } + } + /* + * r1491: the VERTEX_STREAM macro + * (CALL_MME_MACRO(2)=0x3810) carries + * [stream, limit_hi, limit_lo, addr_hi, + * addr_lo]; log the post-relocation words so + * the VTG stream base the engine will program + * is provable from the log. Bounded. + */ + if (addr == 0x3810u && cnt >= 5u) { + static u32 vstream_log; + + if (vstream_log < 24u) { + vstream_log++; + pr_info("vgsp-gr: MME-VSTREAM h=0x%x stream=%u end=%08x:%08x base=%08x:%08x (post-reloc)\n", + req ? req->h_object : 0u, + d[0], d[1], d[2], + d[3], d[4]); + } + } + } + wout += span; + off += 1u + cnt; + continue; + } + off++; /* header-only / resync, like the service decoder */ + wout++; + } + /* + * SESSION 153-CONTINUED-84: prove the win_patch knob reached the + * pass (the r1236 "never logged" was a late write, not a decode + * gap - this line makes the distinction explicit on every boot). + */ + if (gsp_phys_rmcall_gr_win_patch && *gsp_phys_rmcall_gr_win_patch) { + static u32 wp_dbg; + + if (wp_dbg < 24u) { + wp_dbg++; + pr_info("vgsp-gr: WIN-PATCH sweep knob=%s runs=%u matched=%u lowruns=%u h=0x%x\n", + gsp_phys_rmcall_gr_win_patch, n, wp_match, + wp_lowrun, + req ? req->h_object : 0u); + } + } + *out_n = wout; + *n_dropped = dropped; + gsp_phys_rmcall_gr_3d_last_subdrop = subdrop; + gsp_phys_rmcall_gr_3d_last_sdmdrop = sdm_drop; + gsp_phys_rmcall_gr_3d_last_progkeep = prog_keep; + return reloc; +} + +/* + * SESSION 153-CONTINUED-27: alias-build for a big colour target. Consumes + * ceil(npages/16) never-used scratch-PT fetch lines and writes one PTE per + * surface page, so the whole surface is reachable through one CONTIGUOUS + * alias VA (lines are monotonic; consecutive lines are VA-contiguous + * inside a pool block - both pool blocks are checked). Every page's HPA + * comes from the launch's own page set when present, else from the + * service's trusted resolver - the identical vetted translation the + * shadow relocation uses. Lines are consumed even on failure (they are + * never reused, so a half-written line can never be shadowed later); + * the caller then falls back to the existing per-page path. Called with + * gsp_phys_rm_lock held. + */ +static bool gsp_phys_gr_surface_alias_locked(const struct gsp_phys_gr_submit *req, + u64 va, u64 npages, u64 *alias_out) +{ + u64 map_pages = npages + GSP_PHYS_GR_SURF_SLACK_PAGES; + u32 nlines; + u32 base_line; + u32 i, j; + u64 first_va, last_va; + + if (map_pages > 4096u) + map_pages = 4096u; + nlines = (u32)((map_pages + + GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES - 1) / + GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES); + base_line = boot_stage.rmcall_guest_pt_next_line; + if (!nlines || base_line + nlines > GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES) + return false; + first_va = gsp_phys_rmcall_guest_pt_win_va( + base_line / GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE) + + ((u64)(base_line % + GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE) << 16); + last_va = gsp_phys_rmcall_guest_pt_win_va( + (base_line + nlines - 1u) / + GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE) + + ((u64)((base_line + nlines - 1u) % + GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE) << 16); + /* one contiguous run: never straddle a pool block boundary */ + if (last_va != first_va + ((u64)(nlines - 1u) << 16)) + return false; + /* + * Consume the lines before writing: a failure mid-way leaves the + * partially written lines consumed (never reused), so the monotonic + * line invariant - a cached line can never shadow a later mapping - + * holds unconditionally. + */ + boot_stage.rmcall_guest_pt_next_line = base_line + nlines; + for (i = 0; i < nlines; i++) { + u32 line = base_line + i; + u32 page = line / GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; + u32 lip = line % GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; + u8 *pt; + + if (gsp_phys_rmcall_gr_scratch_win_install_locked(page)) + return false; + pt = gsp_phys_rmcall_guest_pt_page_va(page); + if (!pt) + return false; + for (j = 0; j < GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES; j++) { + u64 k = (u64)i * GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES + + j; + u64 pv, hpa = 0; + u32 pi; + + if (k >= map_pages) + break; + pv = va + (k << 12); + for (pi = 0; pi < req->n_pages; pi++) { + if ((req->pages[pi].guest_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) == pv && + req->pages[pi].hpa) { + hpa = req->pages[pi].hpa; + break; + } + } + if (!hpa) { + struct gsp_sema_gr_reloc r; + + if (gsp_sema_gr_reloc_resolve(pv, &r) || + !r.hpa) { + /* + * A declared page must resolve; a + * slack-tail page past the guest + * allocation gets the dummy so the + * engine's overdraw vanishes. + */ + if (k < npages) + return false; + hpa = gsp_phys_gr_surf_dummy_dma_locked(); + if (!hpa) + return false; + } else { + hpa = r.hpa; + } + } + put_unaligned_le64( + gsp_phys_rmcall_pte(hpa), + pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * + lip + + j) * + 8); + } + dma_wmb(); + } + *alias_out = first_va; + return true; +} + +/* + * SESSION 153-CONTINUED-27: pre-pass over a raw 3D window. Scans the + * colour-target runs (C597 INC 0x800+j*0x40 [upper, lower, w, h, fmt, + * blksz] and - same surface, shape seen on the TWOD subchannel - INC + * 0x218 cnt>=4 [w, h, upper, lower]) and aliases each big surface once + * per boot so the relocation pass can rewrite its operands to the alias + * (r992/r993 evidence: the X screen 0x120400000, 1280x720 fmt 0xe6, is + * the draw's render target AND the TWOD fill's destination). Small + * surfaces keep the historical one-pair shadow relocation. Called with + * gsp_phys_rm_lock held. + */ +static void gsp_phys_gr_surface_alias_scan_locked( + const u32 *win, u32 n, const struct gsp_phys_gr_submit *req) +{ + u32 off = 0; + + while (off < n) { + u32 hdr = win[off]; + u32 op = (hdr >> 29) & 0x7u; + u32 cnt = (hdr >> 16) & 0xfffu; + u32 addr = (hdr & 0x1fffu) << 2; + u32 span = 1u; + + if ((op == 1 || op == 3 || op == 5) && cnt && + off + 1u + cnt <= n) { + span = 1u + cnt; + } else { + off++; + continue; + } + if (op == 1 && ((addr >= 0x0800 && addr < 0x0c00 && + (addr & 0x3fu) == 0u && cnt >= 4) || + (addr == 0x218 && cnt >= 4))) { + const u32 *d = &win[off + 1u]; + u64 va; + u32 w, h, fmt; + u64 bytes, npages; + u32 sdi; + u64 alias = 0; + + if (addr == 0x218) { + va = ((u64)(d[2] & 0xffu) << 32) | + (u64)(d[3] & 0xfffff000u); + w = d[0]; + h = d[1]; + fmt = 0xe6u; /* TWOD run carries fmt on + * its own IMMD; the surface + * class here is 32bpp */ + } else { + va = ((u64)(d[0] & 0xffu) << 32) | + (u64)(d[1] & 0xfffff000u); + w = d[2]; + h = d[3]; + fmt = d[4]; + } + if (!va || (va & 0xfffu) || fmt != 0xe6u || + !w || !h || w > 8192u || h > 8192u) + goto next; + bytes = (u64)w * 4u * (u64)h; + npages = (bytes + GSP_PHYS_PAGE_SIZE - 1) >> 12; + if (npages < 2u || npages > 4096u) + goto next; + for (sdi = 0; sdi < gsp_phys_gr_surf_n; sdi++) + if (gsp_phys_gr_surf[sdi].va == va) { + /* r1510: content-gated screen + * code-heap quarantine (no-op when + * the surface is clean). */ + (void)gsp_sema_surf_clean_restore( + va, npages); + goto next; /* already aliased */ + } + if (gsp_phys_gr_surface_alias_locked(req, va, npages, + &alias)) { + if (gsp_phys_gr_surf_n < + ARRAY_SIZE(gsp_phys_gr_surf)) { + gsp_phys_gr_surf[gsp_phys_gr_surf_n].va = + va; + gsp_phys_gr_surf[gsp_phys_gr_surf_n] + .alias = alias; + gsp_phys_gr_surf[gsp_phys_gr_surf_n] + .npages = npages; + gsp_phys_gr_surf_n++; + } + pr_info("vgsp-gr: SURFACE alias va=0x%llx pages=%llu (%ux%u fmt=0x%x) -> 0x%llx (contiguous shadow run; operands rewritten)\n", + va, npages, w, h, fmt, alias); + } else { + pr_info("vgsp-gr: SURFACE alias FAILED va=0x%llx (%ux%u fmt=0x%x): operands fall to the per-page path\n", + va, w, h, fmt); + } + } +next: + off += span; + } +} + +/* + * SESSION 153-CONTINUED-66: write a built method block into the module + * pushbuffer. The block may exceed one 4 KB page (GSP_PHYS_RMCALL_GR_METHOD_ + * WORDS is 2048), so copy page-wise: page 0 is the registered BUF_PB chunk, + * page 1 is the PTE-planted rmcall_gr_pb2 page at VA_PB2. The two pages are + * contiguous in GPU VA (VA_PB..VA_PB+0x2000), so the GPFIFO entry's single + * fetch address + length walks both. A short/absent page-2 plant leaves the + * tail unwritten and the caller's bound check keeps such launches <= 4 KB. + */ +static void gsp_phys_gr_pb_copy(const u32 *methods, u32 words) +{ + struct gsp_phys_boot_chunk *pb = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB]; + u32 per = GSP_PHYS_PAGE_SIZE / 4u; + u32 done = 0; + + while (done < words) { + struct gsp_phys_boot_chunk *c; + u32 page = done / per; + u32 off = done % per; + u32 n = words - done; + u8 *dst; + + if (n > per - off) + n = per - off; + if (!page) + c = pb; + else if (page == 1u) + c = &boot_stage.rmcall_gr_pb2; + else + return; + dst = gsp_phys_chunk_page_va(c, 0); + if (page == 2u || page == 3u) { + /* + * SESSION 153-CONTINUED-77: pages 1 and 2 of the pb2 + * chunk are the third/fourth pushbuffer pages + * (VA_PB3 slot 0x103, VA_PB4 slot 0x104). + */ + if (boot_stage.rmcall_gr_pb2.pages < page) + return; + dst = gsp_phys_chunk_page_va(&boot_stage.rmcall_gr_pb2, + page - 1u); + } + if (!dst) + return; + memcpy(dst + (size_t)off * 4u, methods + done, (size_t)n * 4u); + done += n; + } +} + +/* + * SESSION 153-CONTINUED-95: CTX-BIND launch receipt - is the client's + * promoted GR context buffer engine-visible on the replay channel? + * + * The guest driver promotes its per-channel context buffers through + * NV2080_CTRL_CMD_GPU_PROMOTE_CTX entries (MAIN bufferId 0 and PATCH + * bufferId 2 carry bInitialize=1 on a fresh context). The emulated GSP-RM + * answers SUCCESS and records the statements (gsp_sema_service_ctx_ + * promote_note), but it does not install them anywhere: the replay + * channel's engine context is the buffer the kernel's own ladder built + * (server FBMEM MAIN, mapped at GSP_PHYS_RMCALL_VA_GR_CTX and committed by + * the mode-35 lane promote). This receipt logs, at the first launches of a + * boot, the exact promote statements next to the engine's binding so the + * "context state never replayed" hypothesis is decided by bytes: the + * client's MAIN/PATCH pa/va/size/bInitialize, and the first words of the + * client MAIN buffer read through the GPA->HPA map (a zeroed buffer is the + * expected state before a server-side ctx init that the emulation never + * performs). + */ +static void gsp_phys_ctx_bind_receipt(void) +{ + static u32 receipt_n; + u64 cpa = 0, cva = 0, csz = 0, ppa = 0, pva = 0, psz = 0; + u32 cattr = 0, cinit = 0, pattr = 0, pinit = 0; + bool have_main, have_patch; + + if (receipt_n >= 4u) + return; + receipt_n++; + have_main = gsp_sema_service_ctx_promote_get(GSP_PHYS_RMCALL_ENGINE_GR0, + 0u, &cpa, &cva, &csz, + &cattr, &cinit) == 0; + have_patch = gsp_sema_service_ctx_promote_get(GSP_PHYS_RMCALL_ENGINE_GR0, + 2u, &ppa, &pva, &psz, + &pattr, &pinit) == 0; + pr_info("vgsp-gr: CTX-BIND client-main=%s pa=0x%llx va=0x%llx size=0x%llx attr=0x%x binit=%u; client-patch=%s pa=0x%llx va=0x%llx size=0x%llx binit=%u; replay-ctx pa=0x%llx size=0x%x fbmem=%u va=0x%llx srv_ctxva=%u; installed=0 (engine ctx is the ladder's buffer, NOT the client promote)\n", + have_main ? "seen" : "none", cpa, cva, csz, cattr, cinit, + have_patch ? "seen" : "none", ppa, pva, psz, pinit, + boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_size, + boot_stage.rmcall_ctx_fbmem ? 1u : 0u, + boot_stage.rmcall_ctx_va, boot_stage.rmcall_srv_ctxva ? 1u : 0u); + if (have_main && cpa) { + u64 hpa = 0; + + if (gsp_phys_gpa_to_hpa_vgpu(NULL, cpa & ~(u64)(GSP_PHYS_PAGE_SIZE - 1), &hpa) == 0 && + pfn_valid(hpa >> PAGE_SHIFT)) { + const u32 *w = (const u32 *)__va(hpa); + + pr_info("vgsp-gr: CTX-BIND client-main content[0..7]=%08x %08x %08x %08x %08x %08x %08x %08x (gpa=0x%llx hpa=0x%llx)\n", + get_unaligned_le32((const u8 *)&w[0]), + get_unaligned_le32((const u8 *)&w[1]), + get_unaligned_le32((const u8 *)&w[2]), + get_unaligned_le32((const u8 *)&w[3]), + get_unaligned_le32((const u8 *)&w[4]), + get_unaligned_le32((const u8 *)&w[5]), + get_unaligned_le32((const u8 *)&w[6]), + get_unaligned_le32((const u8 *)&w[7]), + cpa, hpa); + } else { + pr_info("vgsp-gr: CTX-BIND client-main content: GPA 0x%llx -> HPA translation unavailable (no guest RAM registration yet)\n", + cpa); + } + } +} + +/* + * SESSION 153-CONTINUED-95: engine context readback (gsp-phys-ctx-dump). + * + * What the DS validates SPHs against is the *engine's* context buffer, not + * this host's view of anything the client promoted. The replay channel's + * context is the ladder's buffer (server FBMEM MAIN) mapped into the + * channel VAS at GSP_PHYS_RMCALL_VA_GR_CTX with VIDMEM PTEs. The host has + * no aperture to FBMEM (no BAR1 window), so the copy engine reads it: one + * 4 KB CE copy per page from the ctx VA to a host page placed on a fresh + * never-reused scratch fetch line, polled on the module semaphore exactly + * like the P6a guest copies. Runs once per boot at the first launch (the + * channel is idle: single-inflight work item), while the ladder's channel + * is not server-memory and the identity DMA domain holds. Bounded: stops + * at the first refused/timed-out page and records how many bytes landed. + */ +#define GSP_PHYS_CTX_RB_BYTES 0x140000u /* MAIN 0x133000 + PATCH 0x5000 + slack */ + +static u8 *gsp_phys_ctx_rb_buf; +static u32 gsp_phys_ctx_rb_len; +static bool gsp_phys_ctx_rb_tried; + +static const u8 *gsp_phys_ctx_readback_get(u32 *len) +{ + if (len) + *len = READ_ONCE(gsp_phys_ctx_rb_len); + return gsp_phys_ctx_rb_buf; +} + +/* + * SESSION 153-CONTINUED-96: re-arm the one-shot readback. A write to the + * gsp-phys-ctx-read debugfs file clears the tried flag (and the length), so + * the next forwarded launch re-runs the copy with the current server state. + * Safe from any context: the readback itself only runs in the single-inflight + * launch path and re-checks every gate. + */ +static void gsp_phys_ctx_readback_rearm(void) +{ + WRITE_ONCE(gsp_phys_ctx_rb_len, 0); + WRITE_ONCE(gsp_phys_ctx_rb_tried, false); +} + +static void gsp_phys_ctx_readback_try(void) +{ + struct gsp_phys_boot_chunk *sem = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM]; + void __iomem *bar0; + struct page *pg; + u64 hpa; + u32 i, bytes = 0; + u32 ctx_pages, patch_pages = 0; + + if (READ_ONCE(gsp_phys_ctx_rb_tried)) + return; + if (!boot_stage.rmcall_ctx_valid || !boot_stage.rmcall_ctx_pa) + return; + /* + * SESSION 153-CONTINUED-96: the copies run on the secondary COPY0 + * lane (gsp_phys_rmcall_ce_lane_bringup). CE class methods are never + * legal on the GR0 ladder channel, so the lane's readiness (alloc + + * BIND/schedules + its own token) is the hard requirement here; the + * GR channel pieces are not used by the copy. Gate on the pieces the + * lane submission needs: claimed, RM init-done, not unloading. + */ + if (!boot_stage.rmcall_ce_lane_valid || !boot_stage.rmcall_ce_token) + return; + if (gsp_phys_backend_state() != GSP_PHYS_BACKEND_READY) + return; + if (!gsp_phys_gpu_claimed() || !boot_stage.rm_init_done || + boot_stage.rm_unload_submitted) + return; + if (gsp_phys_dma_domain_identity() != 1) + return; + bar0 = gsp_phys_gpu_bar0(); + if (!bar0) + return; + if (!gsp_phys_ctx_rb_buf) { + gsp_phys_ctx_rb_buf = vmalloc(GSP_PHYS_CTX_RB_BYTES); + if (!gsp_phys_ctx_rb_buf) + return; + } + /* + * SESSION 153-CONTINUED-96: copy the WHOLE MAIN context (the golden + * register image the engine restores) and the PATCH buffer (the + * context patch area the DS's SPH region may live in), in that order, + * after the 64 KB r1323/r1324 first run only proved the head. The + * dump file serves MAIN first, then PATCH; lengths are logged. + */ + ctx_pages = DIV_ROUND_UP(boot_stage.rmcall_ctx_size, + GSP_PHYS_PAGE_SIZE); + if (ctx_pages > (GSP_PHYS_CTX_RB_BYTES / GSP_PHYS_PAGE_SIZE)) + ctx_pages = GSP_PHYS_CTX_RB_BYTES / GSP_PHYS_PAGE_SIZE; + if (boot_stage.rmcall_golden_fb_aux && boot_stage.rmcall_patch_size) { + patch_pages = DIV_ROUND_UP(boot_stage.rmcall_patch_size, + GSP_PHYS_PAGE_SIZE); + if (ctx_pages + patch_pages > + GSP_PHYS_CTX_RB_BYTES / GSP_PHYS_PAGE_SIZE) + patch_pages = GSP_PHYS_CTX_RB_BYTES / + GSP_PHYS_PAGE_SIZE - ctx_pages; + } + pg = alloc_page(GFP_KERNEL); + if (!pg) + return; + hpa = (u64)page_to_phys(pg); + for (i = 0; i < ctx_pages + patch_pages; i++) { + u64 src_base = (i < ctx_pages) ? + GSP_PHYS_RMCALL_VA_GR_CTX : + GSP_PHYS_RMCALL_VA_GR_PATCH; + u64 page_off = (i < ctx_pages) ? + (u64)i * GSP_PHYS_PAGE_SIZE : + (u64)(i - ctx_pages) * GSP_PHYS_PAGE_SIZE; + u64 copy_src_va = src_base + page_off; + u64 map_src_va = 0, dst_va = 0, sem_va = 0; + u32 poll = 0; + unsigned long deadline; + int ret; + + mutex_lock(&gsp_phys_rm_lock); + ret = gsp_phys_rmcall_guest_map_locked(hpa, hpa, 0, &map_src_va, + &dst_va, &sem_va); + if (ret) { + mutex_unlock(&gsp_phys_rm_lock); + break; + } + memset((u8 *)__va(hpa), 0xab, GSP_PHYS_PAGE_SIZE); + ret = gsp_phys_rmcall_ce_lane_submit_locked(bar0, + copy_src_va, dst_va, GSP_PHYS_RMCALL_VA_SEM, + GSP_PHYS_PAGE_SIZE, + GSP_PHYS_RMCALL_SEM_PAYLOAD, NULL); + mutex_unlock(&gsp_phys_rm_lock); + if (ret) { + pr_warn("vgsp-gr: CTX-READBACK page %u submit refused (%d) after %u bytes; stopping\n", + i, ret, bytes); + break; + } + deadline = jiffies + msecs_to_jiffies(250); + for (;;) { + poll = READ_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0)); + if (poll == GSP_PHYS_RMCALL_SEM_PAYLOAD) + break; + if (time_after_eq(jiffies, deadline)) { + poll = 0; + break; + } + gsp_phys_rmcall_wait_quantum(); + } + if (poll != GSP_PHYS_RMCALL_SEM_PAYLOAD) { + pr_warn("vgsp-gr: CTX-READBACK page %u (src VA 0x%llx) no CE release within 250 ms after %u bytes; stopping\n", + i, copy_src_va, bytes); + break; + } + memcpy(gsp_phys_ctx_rb_buf + (size_t)i * GSP_PHYS_PAGE_SIZE, + (const u8 *)__va(hpa), GSP_PHYS_PAGE_SIZE); + bytes += GSP_PHYS_PAGE_SIZE; + } + __free_page(pg); + WRITE_ONCE(gsp_phys_ctx_rb_len, bytes); + WRITE_ONCE(gsp_phys_ctx_rb_tried, true); + pr_info("vgsp-gr: CTX-READBACK engine ctx pa=0x%llx copied=%u bytes on the CE lane (MAIN %u pages then PATCH %u pages; server/own buffer, not the client promote; WPR2 untouched)\n", + boot_stage.rmcall_ctx_pa, bytes, ctx_pages, patch_pages); +} + int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, struct gsp_phys_backend_copy_result *out) { @@ -18874,6 +28243,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * under the warning bound. */ static u32 methods[GSP_PHYS_RMCALL_GR_METHOD_WORDS]; + static u32 stream_patch[GSP_PHYS_RMCALL_GR_METHOD_WORDS]; static u32 qmd[GSP_PHYS_GR_QMD_WORDS]; static struct gsp_phys_fault_info fault; unsigned long start = jiffies; @@ -18884,6 +28254,34 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, const u8 *guest_param_pg; u32 want_qmd = 0, want_ch = 0; bool qmd_rel = false, ch_rel = false; + /* + * r929 (Path A): this submission is a raw 3D method window. The QMD + * machinery (qmdwin relocation, QMD prewrite, release0 redirect, + * PCAS) does not apply: the block is SET_OBJECT C597 + the window + + * the channel release tail, and completion is the channel release + * alone (qmd_rel stays off so the poll does not wait for a QMD + * release word no engine will write). + */ + bool stream3d = gsp_phys_rmcall_gr_3d && req && req->stream_n; + /* + * CONT-73: fresh direct-write witness for this submission. The + * window/attr patch passes below record every relocated operand + * destination's guest page; the pre-doorbell capture and the retire + * report read exactly this launch's set (single-inflight, statics). + */ + gsp_phys_gr_wit_reset(); + const u32 *win3d = NULL; + u32 win3d_n = 0; + /* + * r1483: the SURF prelude emitted for THIS launch. The RT0 bind + * (sema_gl_rt0_bind) used to carry a constant 0xa000000 target; + * that is only correct when this launch's data-rec base is the 0xa + * half. The PS writes RT0 while the client binds RT1, so RT0 must + * name the same surface as the SURF prelude or half the launches + * store into the other half's (untracked) bounce window. When set, + * the RT0 run copies the emitted SURF run's target words. + */ + bool surf_pre_emitted = false; bool qmd_rel_redirected = false; /* r420b: release0 -> module sem */ u64 qmd_rel_hpa = 0, ch_rel_hpa = 0; u64 ch_sem_va = GSP_PHYS_RMCALL_VA_SEM + 4; @@ -18891,9 +28289,10 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, u8 *qmd_dst_mem = NULL; u32 qmd_rel_off = 0, ch_rel_off = 0; u32 poll = 0, w0 = 0, w1 = 0, grpoll_n = 0; + static u32 post_spin_n; bool sem_gated = false; - static u32 sked_prev[11]; - static u32 sked_sample[11]; + static u32 sked_prev[28]; + static u32 sked_sample[28]; static u32 sked_tick; static bool sked_first; /* @@ -18906,7 +28305,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * stack). Safe as static: the only caller is the single-inflight * ordered gsp-phys-work item. */ - static u32 bounce_csum_in[GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES]; + static u64 bounce_csum_in[GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES]; /* * r378: which guest page's content each bounce slot holds this * launch (set by the copy-in fills, cleared per submission). The @@ -18920,6 +28319,14 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, sked_tick = 0; sked_first = true; guest_param_pg = NULL; + /* + * SESSION 153-CONTINUED-95: prove the client's promoted GR context + * buffer binding at the first launches, then read the engine's own + * context buffer back for byte-level inspection (both bounded, both + * diagnostic; no behavior change to the launch itself). + */ + gsp_phys_ctx_bind_receipt(); + gsp_phys_ctx_readback_try(); u32 qmd_win_pde = 0, qmd_win_installed = 0; int qmd_win_ret = 0; bool qmd_win_diag = false; @@ -18929,6 +28336,22 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, bool rearmed = false; u32 attempts = 0; /* + * SESSION 153-CONTINUED-48: one bounded priv-error recovery attempt + * per launch (sema_gr_wedge_recover), with a per-boot tally for the + * report. Static like the other submit scratch: the single-inflight + * ordered work item is the only caller. + */ + bool wedge_recover_done = false; + static u32 wedge_recoveries; + /* + * SESSION 153-CONTINUED-72: the MME-FAST pre-reset spin can observe + * the 0xbadf poison and then the FECS clears it before the poll + * loop's first recovery check reads grfifo (the r1149 race: recovery + * missed -> prelude latch stuck -> engine wedged for the boot). The + * observation is latched here and OR-ed into the WEDGE-RECOVER gate. + */ + bool fast_poison = false; + /* * F2/F4 state: the fast first-timeout re-arm runs once per launch; the * deposit readback is one-shot per launch (the first timeout is the * evidence window - the old r198 post-mortem only ran after attempt 3, @@ -18953,6 +28376,17 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, fast_rearmed = false; deposit_rb_done = false; + gsp_phys_gr_mme_pairs_emitted = false; /* r1453 PAIR-CHAIN */ + gsp_phys_gr_mme_launch_gen = 0; + /* + * r1454 PAIR-CHAIN-ENTRY-ELIDE: per-launch scratch only - the + * cross-launch tracker (pair_open/split) must survive this site + * (it is reset at the two context wipes instead). Clearing the + * entry snapshot here is fail-closed: a launch whose window never + * reaches gsp_phys_gr_pair_chain_fix cannot elide. + */ + gsp_phys_gr_mme_pair_open_entry = false; + gsp_phys_gr_mme_pair_open_entry_split = 0; gsp_phys_gr_codehash_done = false; /* CODEHASH: once per launch */ /* * M91 FAULT-CAP: snapshot the fault epoch per launch, so the capture @@ -19004,6 +28438,26 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, gsp_phys_gr_refused++; return -EINVAL; } + /* + * r564: refuse a degenerate data set BEFORE the engine ever sees it. + * r565 evidence: the fill dropped 4 argument bases (GRFILL-DROP), + * RDW showed nrecs=2 where healthy launches run 968-969, the launch + * was still submitted and the backend timed out after ~19 s + * ("GR forward failed rc=-110"), after which the tail authored the + * software release and the client kept spinning on a completion for a + * kernel that never ran - the worst of both (20 s of backend + * occupancy plus a false completion). Refusing up front fails fast + * into the same software-completion fallback without touching the + * engine, and cannot wedge it for the next client. + */ + if (req->n_data < gsp_phys_gr_min_data_value()) { + gsp_phys_gr_refused++; + pr_err_ratelimited("vgsp-gr: GR submit refused: degenerate data set n_data=%u (min %u) launch_va=0x%llx\n", + req->n_data, + gsp_phys_gr_min_data_value(), + req->push_va); + return -ENODATA; + } if (!gsp_phys_backend_gr_ready() || !gsp_phys_guest_mem_enabled() || gsp_phys_dma_domain_identity() != 1 || !gsp_phys_guest_ram_active()) { gsp_phys_gr_refused++; @@ -19023,6 +28477,41 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, ret = -EIO; goto out; } + /* + * SESSION 153-CONTINUED-60: all-zero 3D-window skip. The client's push + * page is cyclically zeroed/rewritten (CONT-45); a capture that lands + * on the zero phase still forwards an all-zero method window. Live + * r1107: such a window launched (3D raw dump words=252, every word 0), + * the engine trapped SET_VERTEX_ATTRIBUTE_A(2) (trap=00001168 + * data_lo=0x50) and escalated to a priv-error poison (grfifo= + * 0xbadf1201, mme_pc=513) that wedged the GR lane for the rest of the + * boot; r1031 had already seen the same class fault the PBDMA on the + * zero lead (PB-FAULT-DUMP P0). A zero window carries no method to + * execute and nothing to store: skip the hardware handoff entirely - + * no shadow relocation, no engine contact, no wedge - and return the + * distinct rc -EBADMSG so the service authors the software release and + * suppresses the synthetic fault (a frame with no content deserves no + * client-visible engine error). + */ + if (stream3d) { + u32 zwi; + bool zerowin = req->stream_n != 0; + + for (zwi = 0; zwi < req->stream_n; zwi++) { + if (req->stream[zwi]) { + zerowin = false; + break; + } + } + if (zerowin) { + gsp_phys_gr_zerowin_skips++; + pr_warn_ratelimited("vgsp-gr: ZEROWIN-SKIP h=0x%x push=0x%llx window=%u all-zero (no engine submit; software release) n=%llu\n", + req->h_object, req->push_va, req->stream_n, + gsp_phys_gr_zerowin_skips); + ret = -EBADMSG; + goto out; + } + } gsp_phys_gr_submitted++; gsp_phys_gr_override_apply_param(); @@ -19084,6 +28573,32 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, } } /* + * r449 SHADER-CONSTANT-CACHE invalidate (gsp_phys_gr_ccache_inv; default + * 0 = off): the same cache-control word's bit 31 (the V01_06-layout + * SHADER_CONSTANT_CACHE position) is never set by the captured guest + * QMDs. The forwarded param page is rewritten in place between launches + * (the two-bank win0 pool), so a stale constant-cache line can serve + * the PREVIOUS launch's parameters and the current launch's output + * pages stay untouched - the post-dispatch no-store signature. Mode 1 + * sets bit 31; mode 2 forces the whole invalidate set (bits 24..31). + * Only when the QMD carries the driver's standard cache-control + * signature (bits 26..29) so an unknown QMD revision is never touched. + */ + if (gsp_phys_gr_ccache_inv_value() && + (qmd[5] & 0x3c000000u) == 0x3c000000u) { + u32 mode = gsp_phys_gr_ccache_inv_value(); + + if (mode == 2) + qmd[5] |= 0xff000000u; + else + qmd[5] |= 1u << 31; + if (!gsp_phys_r449_ccache_log) { + gsp_phys_r449_ccache_log = true; + pr_info("vgsp-gr: r449 constant-cache invalidate mode=%u applied (w5=0x%08x)\n", + mode, qmd[5]); + } + } + /* * r433 RELEASE_MEMBAR_TYPE A/B (gsp_phys_gr_rel_membar): force the * forwarded QMD's RELEASE_MEMBAR_TYPE to FE_SYSMEMBAR (QMDV00_06 * word 11 bit 14) so the front-end flushes its pending system-memory @@ -19117,13 +28632,16 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * a fresh shadow line so the guest observes its own QMD release; else * point release0 at the module semaphore so the host observes engine * completion. A reduction release is never redirected (its arithmetic - * would change). + * would change). r929: skipped entirely for a 3D stream replay - + * there is no QMD, so there is no release0 to point anywhere. */ - if (((qmd[4] >> 10) & 1)) { + if (!stream3d && ((qmd[4] >> 10) & 1)) { u32 red_en = (qmd[24] >> 26) & 1; u32 one_word = (qmd[24] >> 31) & 1; bool rel_mod = gsp_phys_rmcall_gr_rel_mod_value() != 0; + int rel_map_ret = 0; + if (!rel_mod && req->rel_hpa && one_word && !red_en && n_shadow < GSP_PHYS_RMCALL_GR_SHADOW_PAGES) { u64 sh_va = 0; @@ -19143,6 +28661,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, 0xffu); qmd[25] = req->rel_payload; } else { + rel_map_ret = ret; ret = 0; } } @@ -19171,6 +28690,23 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * completed on the guest word (the r414 saxpy hang). */ qmd_rel_redirected = true; + /* + * r633 diag: name the exact redirect cause. rel_hpa==0 + * means the caller could not resolve the release page + * (capture resolution or PA->HPA); map_ret!=0 means the + * page was resolved but its shadow mapping failed. + */ + { + static u32 qmdrel_redir_log; + + if (qmdrel_redir_log < 32) { + qmdrel_redir_log++; + pr_info("vgsp-gr: qmd-rel redirect: rel_hpa=0x%llx map_ret=%d one_word=%u red_en=%u rel_mod=%d n_shadow=%u pay=%u\n", + req->rel_hpa, rel_map_ret, + one_word, red_en, rel_mod ? 1 : 0, + n_shadow, req->rel_payload); + } + } } } @@ -19268,7 +28804,53 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, pg_va = pg->guest_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); pr_info("vgsp-gr: page[%u] va=0x%llx hpa=0x%llx qmd_word=%u\n", i, pg->guest_va, pg->hpa, pg->qmd_word); - if (pg->hpa) { + /* + * r1497/r1498: an FB *fetch* page (a QMD pointer pair, not + * the data-fill surface) whose shadow is empty is the active + * indexed draw's vertex/index class. Map the real FBMEM page + * through a VID leaf instead of the empty shadow. The + * render-target/surface pages (QMD word NONE) must keep + * their shadow identity: the engine's stores have to land in + * the emulated page so the copy-out can ship them back to + * the guest (r1498: mapping the surface to real FB made the + * frame invisible to the guest). + */ + if (pg->hpa && pg->qmd_word != 48u && + pg->qmd_word != GSP_PHYS_GR_QMD_WORD_NONE) { + u64 fb_off = 0; + + if (gsp_phys_gr_fetch_vid_off_locked(pg_va, + &fb_off)) { + ret = gsp_phys_rmcall_gr_map_va_vid_locked( + bar0, pg_va, fb_off, epoch); + if (!ret) { + mapped = true; + gsp_phys_gr_vid_map_pages++; + if (gsp_phys_gr_vid_log < 32u) { + gsp_phys_gr_vid_log++; + pr_info("vgsp-gr: FETCH-VID-MAP va=0x%llx fb=0x%llx qmd=%u h=0x%x push=0x%llx\n", + pg_va, fb_off, + pg->qmd_word, + req->h_object, + req->push_va); + } + } else if (ret == -ENOSPC) { + gsp_phys_gr_pool_exhausted++; + } + } + } + /* + * SESSION 149 R15 (M2-S1): with the window-0 shadow path + * enabled, force the program page (QMD word 48) OFF the + * post-commit Tier-1 install. r801b proved that install is + * ineffective for a late GR page (the engine kept reading + * the empty guest page: CODEHASH h0=0, FAULT-CAP + * code_src=guest-hpa); skipping it lets the page fall to + * the win0 shadow copy + w48/49 rewrite line, which is the + * proven one. Every other page keeps its Tier-1 route. + */ + if (!mapped && pg->hpa && !(pg->qmd_word == 48u && + gsp_phys_rmcall_gr_shadow_win0_value())) { ret = gsp_phys_rmcall_gr_map_va_locked(bar0, pg_va, pg->hpa, epoch); if (!ret) { @@ -19375,6 +28957,80 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, pg->guest_va, sh_va, slot); } } + /* + * SESSION 153-CONTINUED-89: inline records that land on a + * qmd_word-NONE page. The record-apply loop below belongs to + * the QMD-word shadow-*copy* path and never runs for these + * pages; they are resolved to the guest HPA directly (Tier-1 + * identity, contiguous shadow-line extension, or the alias + * above), so authoring the record into pg->hpa is exactly + * what the engine will read. The concrete case: the real + * captured upload at 0x4350f00 (1408 B) crosses its 4 KB + * page - the head (256 B) lands on the program page 0x4350000 + * and applies, but the tail page 0x4351000 is qmd_word=NONE + * and was silently dropped, leaving the draw's winning j=1 + * program (the later bind) truncated at 256 B at the engine + * (the DS rejects the SASS it cannot fetch: SPH 9/11). + */ + if (mapped && pg->hpa && + pg->qmd_word == GSP_PHYS_GR_QMD_WORD_NONE) { + u32 jx; + + for (jx = 0; jx < req->n_inl; jx++) { + const struct gsp_phys_gr_inline *inl = + &req->inl[jx]; + u32 len; + + if ((inl->dst_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) != pg_va) + continue; + len = inl->len; + if (len > GSP_PHYS_GR_INLINE_BYTES) + len = GSP_PHYS_GR_INLINE_BYTES; + if ((u64)(inl->dst_va & + (GSP_PHYS_PAGE_SIZE - 1)) + len > + GSP_PHYS_PAGE_SIZE) + continue; + memcpy((u8 *)__va(pg->hpa) + + (inl->dst_va & + (GSP_PHYS_PAGE_SIZE - 1)), + inl->data, len); + { + static u32 inl_alias_log; + static u32 inl_alias_writer_log; + + if (inl_alias_log < 12u) { + inl_alias_log++; + pr_info("vgsp-gr: INLINE-APPLY-ALIAS dst=0x%llx len=%u pa=0x%llx first=%08x (qmd_word=NONE page authored)\n", + inl->dst_va, len, + pg->hpa, + len >= 4 ? + get_unaligned_le32(inl->data) : + 0u); + } + if (inl_alias_writer_log < 16u && + len >= 128u) { + inl_alias_writer_log++; + pr_info("vgsp-gr: INLINE-WRITER-ALIAS h=0x%x push=0x%llx dst=0x%llx len=%u pg_va=0x%llx pa=0x%llx ap=%u fb=0x%llx hpa=0x%llx first8=%08x %08x %08x %08x\n", + req->h_object, req->push_va, + inl->dst_va, len, pg_va, + pg->reserved & + 0x3fffffffuLL, + (pg->reserved >> 31) ? + 2u : 1u, + (pg->reserved >> 31) ? + (u64)(pg->reserved & + 0x3fffffffu) : + 0ULL, + pg->hpa, + len >= 4 ? get_unaligned_le32(inl->data) : 0u, + len >= 8 ? get_unaligned_le32(inl->data + 4) : 0u, + len >= 12 ? get_unaligned_le32(inl->data + 8) : 0u, + len >= 16 ? get_unaligned_le32(inl->data + 12) : 0u); + } + } + } + } if (!mapped && pg->qmd_word != GSP_PHYS_GR_QMD_WORD_NONE && pg->qmd_word + 1 < GSP_PHYS_GR_QMD_WORDS) { struct gsp_phys_boot_chunk *sh = @@ -19556,6 +29212,118 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, memcpy(sh_page + (inl->dst_va & (GSP_PHYS_PAGE_SIZE - 1)), inl->data, len); + /* + * SESSION 153-CONTINUED-10: a stream3d (3D + * method-window) submission maps the engine's + * VAs straight at the GUEST page HPA + * (gsp_phys_rmcall_gr_map_shadow_locked + * installs the guest PTE - there is no + * shadow-chunk copy in that path), so the + * engine's I-fetch reads the guest page, not + * sh_page. The GLX program page is never + * written by the guest itself (the driver's + * uploads never ran), and the r945 evidence is + * exactly that: "Graphics Exception: Shader + * Program Header 11/18 Error" on the draw + * whose inline prog-hex blob had only reached + * this shadow copy. Author the inline bytes + * into the guest page too, emulating the + * upload that never executed. + */ + /* + * SESSION 154: a program page whose CODESRC + * verdict keeps it at the guest VA is read by + * the engine directly (code_keep_guest sets + * mapped without a QMD rewrite). The inline + * image MUST land in the guest HPA there even + * when the window was not classified + * stream3d: the shadow copy is then unused and + * the engine fetches the empty guest page. + * r1470: the complete 0x41c0000 window's VS + * applied guest=0 into the shadow while the + * guest page stayed zero (the win0 shadow copy + * then restored the zero page as well). + */ + if ((stream3d || code_keep_guest) && pg->hpa) + memcpy((u8 *)__va(pg->hpa) + + (inl->dst_va & + (GSP_PHYS_PAGE_SIZE - 1)), + inl->data, len); + /* + * SESSION 153-CONTINUED-11: bounded witness of + * every inline record applied (which program + * page received which image), so the SPH + * delivery is auditable from the log. + */ + { + static u32 inl_apply_log; + static u32 inl_writer_log; + + if (inl_apply_log < 24) { + inl_apply_log++; + pr_info("vgsp-gr: INLINE-APPLY dst=0x%llx len=%u pg_va=0x%llx guest=%u first=%08x %08x\n", + inl->dst_va, len, pg_va, + (stream3d || code_keep_guest) && + pg->hpa ? 1u : 0u, + len >= 4 ? get_unaligned_le32(inl->data) : 0u, + len >= 8 ? get_unaligned_le32(inl->data + 4) : 0u); + } + /* + * r1503 (mission task 1): decisive + * writer receipt for every image-sized + * inline delivery - names the emitting + * launch, the VA, the resolved PA / + * aperture and the first payload words, + * so a delivery that lands in a + * scanout/window-buffer page is + * provable from the log alone. + */ + if (inl_writer_log < 32u && len >= 128u) { + inl_writer_log++; + pr_info("vgsp-gr: INLINE-WRITER h=0x%x push=0x%llx dst=0x%llx len=%u pg_va=0x%llx pa=0x%llx ap=%u fb=0x%llx hpa=0x%llx first8=%08x %08x %08x %08x\n", + req->h_object, req->push_va, + inl->dst_va, len, pg_va, + pg->reserved & + 0x3fffffffuLL, + (pg->reserved >> 31) ? + 2u : 1u, + (pg->reserved >> 31) ? + (u64)(pg->reserved & + 0x3fffffffu) : + 0ULL, + pg->hpa, + len >= 4 ? get_unaligned_le32(inl->data) : 0u, + len >= 8 ? get_unaligned_le32(inl->data + 4) : 0u, + len >= 12 ? get_unaligned_le32(inl->data + 8) : 0u, + len >= 16 ? get_unaligned_le32(inl->data + 12) : 0u); + } + } + /* + * SESSION 153-CONTINUED-87: readback witness. + * For a stream3d window the engine's I-fetch + * walks the GUEST page (the inline bytes were + * just authored there above); read the exact + * engine-visible location back so the fetch + * content is provable from the log - if the DS + * still rejects the header, the bytes are known + * present at the VA it fetched. + */ + if ((stream3d || code_keep_guest) && pg->hpa) { + static u32 inl_rb_log; + const u32 *gw = (const u32 *) + ((u8 *)__va(pg->hpa) + + (inl->dst_va & + (GSP_PHYS_PAGE_SIZE - 1))); + + if (inl_rb_log < 12u) { + inl_rb_log++; + pr_info("vgsp-gr: INLINE-RB dst=0x%llx pa=0x%llx rb=%08x %08x %08x %08x %08x %08x %08x %08x\n", + inl->dst_va, pg->hpa, + gw[0], gw[1], gw[2], + gw[3], gw[4], gw[5], + gw[6], gw[7]); + } + } copied++; } /* @@ -19822,8 +29590,9 @@ gr_page_unmapped: * GPFIFO/PB/REPORT_SEM legs already use. The 2 MiB window is installed * by the ladder itself, so no window ensure is sent for it (a * pdeIndex-0 Tier-1 install would touch the ladder's own PDE). + * r929: a 3D stream replay has no inline QMD at all (skipped). */ - if (gsp_phys_rmcall_gr_qmdwin_value()) { + if (!stream3d && gsp_phys_rmcall_gr_qmdwin_value()) { u8 *qw = boot_stage.rmcall_gr_qmdwin.pages ? gsp_phys_chunk_page_va(&boot_stage.rmcall_gr_qmdwin, 0) : NULL; @@ -19850,9 +29619,9 @@ gr_page_unmapped: * use it as the destination when the caller resolved it; a Tier-1 * install keeps the guest VA (the guest sees its own QMD), a Tier-2 * alias keeps the launch runnable, and an unresolved page keeps the - * module scratch VA. + * module scratch VA. r929: skipped for a 3D stream replay. */ - if (!qmdwin_on && req->qmd_hpa) { + if (!stream3d && !qmdwin_on && req->qmd_hpa) { u64 qmd_page = req->qmd_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); u32 qmd_off = (u32)(req->qmd_va & (GSP_PHYS_PAGE_SIZE - 1)); bool dst_ok = false; @@ -19970,10 +29739,1254 @@ gr_page_unmapped: GSP_PHYS_GR_QMD_SENTINEL); } + /* + * r929 v3 (Path A): relocate the 3D window's address operands to + * shadow-pool VAs the engine walkers can reach (see + * gsp_phys_rmcall_gr_3d_patch_window; the r929 v2 FE faulted + * FAULT_PDE on a Tier-1-mapped guest VA). Runs with the rm lock + * held, before the method block is built. + */ + if (stream3d) { + u32 win_n = req->stream_n; + u32 ndrop = 0; + u32 nrel; + + /* + * SESSION 153-CONTINUED-27: a big colour target (the X + * screen) is aliased as ONE contiguous shadow run BEFORE + * the relocation pass runs, so the pass rewrites its + * operands to the alias (see the table above). + */ + gsp_phys_gr_surface_alias_scan_locked(req->stream, + req->stream_n, req); + /* SESSION 153-CONTINUED-59: fresh miss census for this pass + * (window + attr patch below both feed it). */ + gsp_phys_rmcall_gr_3d_miss_n = 0; + nrel = gsp_phys_rmcall_gr_3d_patch_window( + req->stream, req->stream_n, stream_patch, &win_n, + &ndrop, req, &n_shadow, bar0, epoch); + + win3d = stream_patch; + win3d_n = win_n; + /* + * SESSION 153-CONTINUED-69: drop the CALL_MME_MACRO/DATA runs + * of stubbed slots (see gsp_phys_gr_mme_drop_calls). With the + * stub as a pure NOP terminator the parameter port no longer + * drains; the r1142 stall parked the PBDMA on the 16-arg + * CALL_MME_MACRO(57) run and the block never completed. The + * drop turns those calls into nothing instead of a call whose + * parameters have nowhere to go. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_call_drop) && + gsp_sema_gr_sa_stub_value()) { + u32 sa_ent[64]; + u64 sa_mask = gsp_sema_mme_sa_get(sa_ent, 64); + u64 dmask = 0; + u32 ndrop_c = gsp_phys_gr_mme_drop_calls( + stream_patch, win3d_n, sa_mask, sa_ent, + &dmask); + + if (ndrop_c) { + win3d_n -= ndrop_c; + pr_info("vgsp-gr: MME-CALL-DROP %u word(s) of stubbed-slot CALL runs (window=%u) dropped_mask=0x%llx h=0x%x push=0x%llx\n", + ndrop_c, win3d_n, + (unsigned long long)dmask, + req->h_object, req->push_va); + } + } + pr_info("vgsp-gr: 3D window relocated %u operand pair(s) to shadow VAs (window=%u shadow=%u dropped=%u subdrop=%u sdmdrop=%u progkeep=%u)\n", + nrel, win3d_n, n_shadow, ndrop, + gsp_phys_rmcall_gr_3d_last_subdrop, + gsp_phys_rmcall_gr_3d_last_sdmdrop, + gsp_phys_rmcall_gr_3d_last_progkeep); + /* + * SESSION 153-CONTINUED-87: emit the client's missing + * pipeline-stage program binds (see the sema_gl_sph_bind + * comment). The window carries only the j=1 (VTG) pair; the + * real client state also binds j=2 (TESS_CONTROL at + * family+0x200) and j=5 (PIXEL at family+0x0) - both present + * verbatim in the init-push image scraped from guest RAM + * (`20020825 [0,04400200]`, `20020855 [0,04400000]`). Build + * the two runs and push them through the same window + * relocation pass (miss -> the pair stays raw and the whole + * bind is skipped, never a raw client VA to the engine), + * then prepend them ahead of the window. Beef client + * channels only: the family layout is that client's. + */ + bool sph_bind_all = + READ_ONCE(gsp_phys_rmcall_gr_sph_bind); + u32 sph_bind_mask = + READ_ONCE(gsp_phys_rmcall_gr_sph_bind_mask); + + if ((sph_bind_all || sph_bind_mask) && + (req->h_object & 0xfffff000u) == 0xbeef0000u && + req->prog_va && + win3d_n + 96u <= GSP_PHYS_RMCALL_GR_METHOD_WORDS) { + static u32 sph_bind_log; + u64 fam = req->prog_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + u32 in[96], out[96]; + u32 nfin = 0, nrun = 0, on = 0, odrop = 0, orel; + u32 bi; + bool bind_binding = + READ_ONCE(gsp_phys_rmcall_gr_sph_bind_binding); + bool pair_safe = + READ_ONCE(gsp_phys_rmcall_gr_sph_bind_pair_safe); + /* + * SESSION 155 CONT 10: per-stage disable mask + * (bit j = synth slot j). Only meaningful when the + * all-slots knob is off, matching the r1561 bool's + * !sph_bind_all gate. + */ + u32 stage_disable_mask = !sph_bind_all ? + (READ_ONCE(gsp_phys_rmcall_gr_sph_stage_disable) & + 0x3fu) : 0u; + bool stage_disable = stage_disable_mask != 0u; + bool emit_j2 = !(stage_disable_mask & 0x04u) && + (sph_bind_all || (sph_bind_mask & 0x04u)); + bool emit_j5 = !(stage_disable_mask & 0x20u) && + (sph_bind_all || (sph_bind_mask & 0x20u)); + /* + * SESSION 155 CONT 8: the DS flags the UNBOUND VTG-class + * stages (VSA/VSB/TCS/GS) in SPH bits 1/2/6/9/10/11/18. + * Mask bits 0x01 and 0x10 add the same synthesized + * per-stage record for VSA (j=0) and GS (j=4), binding + * the client's own VTG image (family+0x900) with the + * GS/VS-CULL types. Default off: bits 0/4 were unused. + * CONT 9: stage_disable suppresses j0/j2/j4 (a disabled + * stage has no program to bind). + */ + bool emit_j0 = !(stage_disable_mask & 0x01u) && + (sph_bind_all || (sph_bind_mask & 0x01u)); + bool emit_j4 = !(stage_disable_mask & 0x10u) && + (sph_bind_all || (sph_bind_mask & 0x10u)); + /* + * SESSION 155 CONT 11: synthesized GS image bind + * (see the sema_gl_sph_gs_img comment). Independent + * of the j4 slot mask: it replaces the disable-only + * GS state with a disabled stage that still has a + * valid SPH operand. + */ + bool emit_gsimg = + READ_ONCE(gsp_phys_rmcall_gr_sph_gs_img); + u32 gs_mode = + READ_ONCE(gsp_phys_rmcall_gr_sph_gs_mode); + bool emit_raster = + READ_ONCE(gsp_phys_rmcall_gr_raster_pre); + bool emit_raster_extra = + READ_ONCE(gsp_phys_rmcall_gr_raster_extra); + u32 gs_addr = READ_ONCE(gsp_phys_rmcall_gr_sph_gs_addr); + u32 gs_ctx = READ_ONCE(gsp_phys_rmcall_gr_sph_gs_ctx); + + /* + * SESSION 155 CONT 9: the prelude is prepended ahead of + * the window, so if the tracker still holds a pair open + * from the previous emitted window every synthesized + * method would run inside BEGIN/END (r1558 EC 0x0d at + * 0x2018). Emit the class's own END as the FIRST word + * of the prelude, matching the event the open BEGIN's + * split mode requires. The pair-chain walk later sees + * this END close the open pair and starts a fresh one at + * the window's own BEGIN. Skipped when + * sema_gl_pair_close_entry is on: that helper inserts + * the same END ahead of the whole window (preludes + * included), so emitting both would double-close. + */ + if (pair_safe && + !READ_ONCE(gsp_phys_rmcall_gr_pair_close_entry) && + READ_ONCE(gsp_phys_gr_mme_pair_open)) { + u32 osplit = + READ_ONCE(gsp_phys_gr_mme_pair_open_split); + u32 ev = (osplit == 1u || osplit == 2u) ? 1u : 0u; + + in[nfin++] = 0x20000000u | (1u << 16) | + (0x1614u >> 2); + in[nfin++] = ev; + pr_info_ratelimited("vgsp-gr: SPH-BIND-PAIR-SAFE close split=%u ev=%u h=0x%x push=0x%llx\n", + osplit, ev, req->h_object, req->push_va); + } + + /* + * SESSION 155 CONT 9: the window is a non-tessellated, + * non-geometry draw. The client's own state disables + * VSA/TCS/TES/GS; the replay channel inherits the + * setup-push records instead, so the DS validates VSB + * in tessellation mode (SPH5/7) and a stale GS program + * (SPH18). Emit the client's disable records FIRST (so + * no later synth bind re-enables them), with the exact + * values the driver's disabled-stage paths write. + * CONT 10: per-bit mask - only the stages whose bit is + * set get a record (bit1 VSB and bit5 PS have no + * unchanged-program record here; bit5 only suppresses + * the synth PS bind). + */ + if (stage_disable) { + if (stage_disable_mask & 0x01u) { + in[nfin++] = 0x20000000u | (1u << 16) | + (0x2000u >> 2); + in[nfin++] = 0x00u; + } + if (stage_disable_mask & 0x04u) { + in[nfin++] = 0x20000000u | (1u << 16) | + (0x2080u >> 2); + in[nfin++] = 0x20u; + } + if (stage_disable_mask & 0x08u) { + in[nfin++] = 0x20000000u | (1u << 16) | + (0x20c0u >> 2); + in[nfin++] = 0x30u; + } + if (stage_disable_mask & 0x10u) { + in[nfin++] = 0x20000000u | (1u << 16) | + (0x2100u >> 2); + in[nfin++] = 0x40u; + } + pr_info_ratelimited("vgsp-gr: SPH-STAGE-DISABLE mask=0x%x h=0x%x push=0x%llx\n", + stage_disable_mask, req->h_object, + req->push_va); + } + + /* + * SESSION 153-CONTINUED-89 (2nd half): the client's + * full channel-init pipeline block, harvested from the + * guest-RAM init-push image (guest PA 0x81efc5b4, this + * boot) and identical in the X channel's own captured + * window (methods 0x21c, 0x155c, 0x1574 in that exact + * order). The per-frame window carries none of it - + * on real hardware the client wrote it ONCE at + * channel init, and the replay channel's fresh context + * never receives it, so the DS validates the fetched + * SPHs against unset context state (9/11). Emitted + * verbatim in the client's own order, ahead of the + * stage binds: + * 0x21c <- 0x1001 INVALIDATE_SHADER_CACHES + * (INSTRUCTION|CONSTANT) + * 0x155c <- 1, 0x214fa200, 0 + * TEX_SAMPLER_POOL A/B/C + * 0x1574 <- 1, 0x214fa220, 0x10 + * TEX_HEADER_POOL A/B/C + * 0x310 <- 0x705 SET_SPA_VERSION + * 0x22c <- 0x800 SET_GO_IDLE_TIMEOUT + * 0x1128 <- ffffffff, ffffffff + * DA_OUTPUT_ATTRIBUTE_SKIP_MASK + * ZCULL_BOUNDS(0x196c) is already carried by the + * window itself. All constant runs (no relocation). + */ + in[nfin++] = 0x20000000u | (1u << 16) | + (0x21cu >> 2); + in[nfin++] = 0x1001u; + { + /* + * The pool runs are address-bearing (the patch + * pass's own rules: addr 0x155c/0x1574, cnt >= + * 3) and count toward the relocated-pair + * guard, so emit them only when their page is + * in this launch's page set (r1290: emitted + * unconditionally -> unaccounted pairs -> + * `SPH-BIND skipped (reloc=6/4)` and the whole + * prelude was lost). Both VAs share one page. + */ + bool pool_pg = false; + u32 pk; + + for (pk = 0; pk < req->n_pages; pk++) + if ((req->pages[pk].guest_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) == + (0x214fa200u & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1))) + pool_pg = true; + if (pool_pg) { + in[nfin++] = 0x20000000u | (3u << 16) | + (0x155cu >> 2); + in[nfin++] = 0x1u; + in[nfin++] = 0x214fa200u; + in[nfin++] = 0x0u; + in[nfin++] = 0x20000000u | (3u << 16) | + (0x1574u >> 2); + in[nfin++] = 0x1u; + in[nfin++] = 0x214fa220u; + in[nfin++] = 0x10u; + nrun += 2; + } + } + in[nfin++] = 0x20000000u | (1u << 16) | + (0x310u >> 2); + in[nfin++] = 0x705u; + in[nfin++] = 0x20000000u | (1u << 16) | + (0x22cu >> 2); + in[nfin++] = 0x800u; + in[nfin++] = 0x20000000u | (2u << 16) | + (0x1128u >> 2); + in[nfin++] = 0xffffffffu; + in[nfin++] = 0xffffffffu; + /* + * SESSION 155 CONT 14: the render-enable state the + * client's channel carried from its init push + * (see the sema_gl_raster_prelude comment). + */ + if (emit_raster) { + u32 ri5; + + in[nfin++] = 0x20000000u | (8u << 16) | + (0x1a00u >> 2); + for (ri5 = 0; ri5 < 8u; ri5++) + in[nfin++] = 0x1111u; + in[nfin++] = 0x20000000u | (1u << 16) | + (0x12ccu >> 2); + in[nfin++] = 0u; + in[nfin++] = 0x20000000u | (1u << 16) | + (0x12e8u >> 2); + in[nfin++] = 0u; + in[nfin++] = 0x20000000u | (1u << 16) | + (0x1380u >> 2); + in[nfin++] = 0u; + in[nfin++] = 0x20000000u | (1u << 16) | + (0x0e00u >> 2); + in[nfin++] = 0u; + } + /* + * SESSION 155 CONT 22: neutral blend/alpha/cull/ + * point/ZCULL state (see sema_gl_raster_extra). + */ + if (emit_raster_extra) { + static const u32 rxm[] = { + 0x12e4u, 0x1e00u, 0x12ecu, 0x1308u, + 0x1520u, 0x02e4u, 0x02e8u, 0x02fcu, + }; + u32 ri6; + + for (ri6 = 0; ri6 < 8u; ri6++) { + in[nfin++] = 0x20000000u | + (1u << 16) | + (rxm[ri6] >> 2); + in[nfin++] = 0u; + } + } + + /* + * SESSION 153-CONTINUED-89: the client's init-push + * pipeline state for the window's own stage 1. The + * guest-RAM init-push image (live at guest PA + * 0x81efc5b4, this boot) carries, beside the SPH + * version pair: + * 0x2040 <- 0x11 SET_PIPELINE_SHADER(1) + * ENABLE|TYPE=VERTEX + * 0x2054 <- 1, 0x21201b00 program address pair + * 0x204c <- 0x1a, 0x0 REGISTER_COUNT(1)=26, + * BINDING(1)=0 + * The per-frame window carries only the address pair, + * so the replay channel's TYPE/COUNT/BINDING stage + * registers stay unset - the state the DS validates + * the fetched SPH against (9/11). Emit the TYPE run + * before the binds and the COUNT/BINDING pair after + * them, in the client's own order. Both are constant + * runs (no relocation). + */ + in[nfin++] = 0x20000000u | (1u << 16) | + (0x2040u >> 2); + in[nfin++] = 0x11u; + + /* + * SESSION 153-CONTINUED-87 (second half): the window's + * OWN binds first. The captured per-frame window + * carries draws (BEGIN at word 93) BEFORE its program + * binds (j=1 pair at word 184): on real hardware the + * bind happened in an earlier push, but the replay + * channel's fresh context has nothing bound when the + * early draws validate their stages - the + * Shader-Program-Header poison (r1277+ 9/11). Re-emit + * every bind the window itself carries ahead of it. + */ + for (bi = 0; sph_bind_all && + bi < req->n_prog_binds && bi < 8u; bi++) { + u64 va = req->prog_bind_va[bi]; + u32 sl = req->prog_bind_slot[bi] & 0x1fu; + + if (!va || nfin + 3u > 96u) + continue; + in[nfin++] = 0x20000000u | (2u << 16) | + ((0x2014u + sl * 0x40u) >> 2); + in[nfin++] = (u32)(va >> 32) & 0xffu; + in[nfin++] = (u32)va & 0xfffffffcu; + nrun++; + } + /* + * Then the client's missing stage binds (see the + * sema_gl_sph_bind comment): TESS_CONTROL (j=2 at + * family+0x200) and PIXEL (j=5 at family+0x0), both + * present verbatim in the init-push image scraped + * from guest RAM (`20020825 [0,04400200]`, + * `20020855 [0,04400000]`). + */ + if (nfin + 3u + + (emit_j0 ? (bind_binding ? 8u : 7u) : 0u) + + (emit_j2 ? (bind_binding ? 8u : 7u) : 0u) + + (emit_j4 ? (bind_binding ? 8u : 7u) : 0u) + + (emit_j5 ? (bind_binding ? 8u : 7u) : 0u) + + (emit_gsimg ? (9u + ((gs_mode & 1u) ? 11u : 0u) + + ((gs_ctx & 1u) ? 2u : 0u) + + ((gs_ctx & 2u) ? 2u : 0u)) : 0u) + + (emit_raster ? 17u : 0u) + + (emit_raster_extra ? 16u : 0u) + <= 96u) { + /* + * SESSION 155 CONT 9: pair_safe emits each + * synthesized stage's SET_PIPELINE_SHADER + * TYPE and REGISTER_COUNT *before* its SPH + * address pair, the order the client's own + * stage-1 init push uses. The legacy path + * below stays address-first (bytes + * unchanged when the knob is off). + */ + if (pair_safe) { + if (emit_j0) { + in[nfin++] = 0x20000000u | (1u << 16) | + (0x2000u >> 2); + in[nfin++] = 1u | (0u << 4); + in[nfin++] = 0x20000000u | + ((bind_binding ? 2u : 1u) << 16) | + (0x200cu >> 2); + in[nfin++] = 0x1au; + if (bind_binding) + in[nfin++] = 0x0u; + } + if (emit_j4) { + in[nfin++] = 0x20000000u | (1u << 16) | + (0x2100u >> 2); + in[nfin++] = 1u | (4u << 4); + in[nfin++] = 0x20000000u | + ((bind_binding ? 2u : 1u) << 16) | + (0x210cu >> 2); + in[nfin++] = 0x1au; + if (bind_binding) + in[nfin++] = 0x0u; + } + if (emit_j2) { + in[nfin++] = 0x20000000u | (1u << 16) | + (0x2080u >> 2); + in[nfin++] = 1u | (2u << 4); + in[nfin++] = 0x20000000u | + ((bind_binding ? 2u : 1u) << 16) | + (0x208cu >> 2); + in[nfin++] = 0x1au; + if (bind_binding) + in[nfin++] = 0x0u; + } + if (emit_j5) { + in[nfin++] = 0x20000000u | (1u << 16) | + (0x2140u >> 2); + in[nfin++] = 1u | (5u << 4); + in[nfin++] = 0x20000000u | + ((bind_binding ? 2u : 1u) << 16) | + (0x214cu >> 2); + in[nfin++] = 0x1au; + if (bind_binding) + in[nfin++] = 0x0u; + } + } + if (emit_j0) { + in[nfin++] = 0x20000000u | (2u << 16) | + (0x2014u >> 2); + in[nfin++] = (u32)((fam + 0x900u) >> 32) & 0xffu; + in[nfin++] = (u32)(fam + 0x900u) & 0xfffffffcu; + } + if (emit_j4) { + in[nfin++] = 0x20000000u | (2u << 16) | + (0x2114u >> 2); + in[nfin++] = (u32)((fam + 0x900u) >> 32) & 0xffu; + in[nfin++] = (u32)(fam + 0x900u) & 0xfffffffcu; + } + if (emit_j2) { + in[nfin++] = 0x20000000u | (2u << 16) | + (0x2094u >> 2); + in[nfin++] = (u32)((fam + 0x200u) >> 32) & 0xffu; + in[nfin++] = (u32)(fam + 0x200u) & 0xfffffffcu; + } + if (emit_j5) { + in[nfin++] = 0x20000000u | (2u << 16) | + (0x2154u >> 2); + in[nfin++] = (u32)(fam >> 32) & 0xffu; + in[nfin++] = (u32)fam & 0xfffffffcu; + } + if (!pair_safe) { + /* + * SESSION 153-CONTINUED-88: the per-slot + * SET_PIPELINE_SHADER(j) TYPE state. The + * client's init push writes it once per + * channel (init-push image: `20010810 + * 00000011` = SET_PIPELINE_SHADER(1) <- + * ENABLE|TYPE=VTG); the captured per-frame + * windows carry only the program-address + * pairs, so the replay context's stage + * registers for the synthesized binds would + * stay unset. Emit them for the two + * synthesized slots with the known types + * (j=2 TESS_CONTROL -> 2, j=5 PIXEL -> 5). + */ + if (emit_j0) { + in[nfin++] = 0x20000000u | (1u << 16) | + (0x2000u >> 2); + in[nfin++] = 1u | (0u << 4); + } + if (emit_j4) { + in[nfin++] = 0x20000000u | (1u << 16) | + (0x2100u >> 2); + in[nfin++] = 1u | (4u << 4); + } + if (emit_j2) { + in[nfin++] = 0x20000000u | (1u << 16) | + (0x2080u >> 2); + in[nfin++] = 1u | (2u << 4); + } + if (emit_j5) { + in[nfin++] = 0x20000000u | (1u << 16) | + (0x2140u >> 2); + in[nfin++] = 1u | (5u << 4); + } + /* + * SESSION 153-CONTINUED-90: the per-stage + * REGISTER_COUNT/BINDING records for the + * synthesized j=2/j=5 slots - the DS SPH + * validation's last missing per-stage input; + * same shape as the captured stage-1 pair + * (reg count 26, group 0). + * CONT-119 (r1450): the client never emits a + * stage-2/5 BINDING, and the FE walking the + * window rejects our synthesized BINDING=0 + * word (0x2090) with Class Error EC4. Drop + * the BINDING data writes by default and keep + * the 1-word REG_COUNT runs only; the knob + * restores the legacy CONT-90 2-word runs. + */ + if (bind_binding) { + if (emit_j0) { + in[nfin++] = 0x20000000u | (2u << 16) | + (0x200cu >> 2); + in[nfin++] = 0x1au; + in[nfin++] = 0x0u; + } + if (emit_j4) { + in[nfin++] = 0x20000000u | (2u << 16) | + (0x210cu >> 2); + in[nfin++] = 0x1au; + in[nfin++] = 0x0u; + } + if (emit_j2) { + in[nfin++] = 0x20000000u | (2u << 16) | + (0x208cu >> 2); + in[nfin++] = 0x1au; + in[nfin++] = 0x0u; + } + if (emit_j5) { + in[nfin++] = 0x20000000u | (2u << 16) | + (0x214cu >> 2); + in[nfin++] = 0x1au; + in[nfin++] = 0x0u; + } + } else { + if (emit_j0) { + in[nfin++] = 0x20000000u | (1u << 16) | + (0x200cu >> 2); + in[nfin++] = 0x1au; + } + if (emit_j4) { + in[nfin++] = 0x20000000u | (1u << 16) | + (0x210cu >> 2); + in[nfin++] = 0x1au; + } + if (emit_j2) { + in[nfin++] = 0x20000000u | (1u << 16) | + (0x208cu >> 2); + in[nfin++] = 0x1au; + } + if (emit_j5) { + in[nfin++] = 0x20000000u | (1u << 16) | + (0x214cu >> 2); + in[nfin++] = 0x1au; + } + } + } + /* + * SESSION 153-CONTINUED-89: the client's + * init-push stage-1 REGISTER_COUNT/BINDING + * pair (`20020813 0000001a 00000000` in the + * guest-RAM init image): REGISTER_COUNT(1) = + * 26 (the window SPH's reg count), BINDING(1) + * = group 0. Emitted after the address pair, + * exactly as the client's own push orders it. + */ + in[nfin++] = 0x20000000u | (2u << 16) | + (0x204cu >> 2); + in[nfin++] = 0x1au; + in[nfin++] = 0x0u; + /* + * SESSION 155 CONT 11: the synthesized + * header-only GEOMETRY SPH (128 B at + * family+0x780, authored by the sema + * service). Bind it to stage 4 and keep the + * stage disabled: the DS then validates a + * real GS SPH (its SPH18 check2 attribute + * map) while the engine does not execute the + * header-only image. Emitted after the + * stage-1 pair, before the address-pair + * census. The pair is the only relocatable + * run here (nrun++ below). + */ + if (emit_gsimg) { + u64 gva = fam + (gs_addr ? 0xe80u : + 0x780u); + + in[nfin++] = 0x20000000u | (1u << 16) | + (0x2100u >> 2); + in[nfin++] = 0x40u; + in[nfin++] = 0x20000000u | (1u << 16) | + (0x210cu >> 2); + in[nfin++] = 0x1au; + in[nfin++] = 0x20000000u | (2u << 16) | + (0x2114u >> 2); + in[nfin++] = (u32)(gva >> 32) & 0xffu; + in[nfin++] = (u32)gva & 0xfffffffcu; + /* + * CONT 13: the mesh-dummy GS + * contract also writes the GS mode + * and the post-VTG attribute skip + * mask (both zero default state). + */ + if (gs_mode & 1u) { + in[nfin++] = 0x20000000u | + (1u << 16) | + (0x0208u >> 2); + in[nfin++] = 0x1u; + in[nfin++] = 0x20000000u | + (8u << 16) | + (0x1240u >> 2); + in[nfin++] = 0u; + in[nfin++] = 0u; + in[nfin++] = 0u; + in[nfin++] = 0u; + in[nfin++] = 0u; + in[nfin++] = 0u; + in[nfin++] = 0u; + in[nfin++] = 0u; + } + if (gs_ctx & 1u) { + in[nfin++] = 0x20000000u | + (1u << 16) | + (0x114cu >> 2); + in[nfin++] = 0u; + } + if (gs_ctx & 2u) { + in[nfin++] = 0x20000000u | + (1u << 16) | + (0x0208u >> 2); + in[nfin++] = 0u; + } + in[nfin++] = 0x20000000u | (1u << 16) | + (0x2100u >> 2); + in[nfin++] = (gs_mode & 2u) ? 0x41u : + 0x40u; + } + /* + * nrun counts only the ADDRESS-bearing + * binds: the guard below matches it + * against the relocation pass's + * relocated-pair count, and the constant + * type runs can never be relocated. + */ + if (emit_j0) + nrun++; + if (emit_j2) + nrun++; + if (emit_j4) + nrun++; + if (emit_j5) + nrun++; + if (emit_gsimg) + nrun++; + } + orel = gsp_phys_rmcall_gr_3d_patch_window( + in, nfin, out, &on, &odrop, req, &n_shadow, + bar0, epoch); + if (orel == nrun && on == nfin) { + memmove(&stream_patch[nfin], stream_patch, + (size_t)win3d_n * 4u); + memcpy(stream_patch, out, (size_t)nfin * 4u); + win3d_n += nfin; + if (sph_bind_log < 64u) { + sph_bind_log++; + pr_info("vgsp-gr: SPH-BIND emitted fam=0x%llx win-binds=%u j0=0x%llx j2=0x%llx j4=0x%llx j5=0x%llx j1-type=0x11 rc=0x1a initblk=17 win=%u (pre=%u) h=0x%x push=0x%llx\n", + fam, req->n_prog_binds, + fam + 0x900u, fam + 0x200u, + fam + 0x900u, fam, + win3d_n, nfin, + req->h_object, + req->push_va); + } + } else if (sph_bind_log < 64u) { + sph_bind_log++; + pr_info("vgsp-gr: SPH-BIND skipped (reloc=%u/%u kept=%u/%u fam=0x%llx h=0x%x push=0x%llx)\n", + orel, nrun, on, nfin, fam, + req->h_object, req->push_va); + } + pr_info("vgsp-gr: SPH-BIND-MASK mask=0x%x all=%u j0=%u j2=%u j4=%u j5=%u h=0x%x push=0x%llx\n", + sph_bind_mask, sph_bind_all ? 1u : 0u, + emit_j0 ? 1u : 0u, emit_j2 ? 1u : 0u, + emit_j4 ? 1u : 0u, emit_j5 ? 1u : 0u, + req->h_object, req->push_va); + } + /* + * SESSION 153-CONTINUED-62: surface (render-target) prelude - + * see the gsp_phys_gr_surf_pre comment. Injected only when the + * window carries no usable colour-target bind of its own; the + * alias scan + relocation run exactly as they would for a + * window-carried bind (the captured VA is the surface the + * machine's draws target, X screen observed). The miss census + * feeds the MISS-ABORT below, so an unresolvable surface pair + * holds the launch (fail-closed) instead of faulting. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_surf_prelude) && + !gsp_phys_gr_window_has_surface(req->stream, + req->stream_n)) { + const u32 *sp; + u32 spn = 0, spg = 0; + + /* + * r1510: if the guest's code-heap upload overlapped + * the captured X screen, restore the screen from the + * capture-time baseline before this launch's replay. + * Content-gated (see the sema service's + * gsp_sema_surf_clean_restore). + */ + { + u64 cva = 0; + u32 cnp = 0; + + if (gsp_sema_surf_va_get(&cva, &cnp)) + (void)gsp_sema_surf_clean_restore(cva, + cnp); + } + sp = gsp_sema_surf_prelude_get(&spn, &spg); + if (sp && spn && spg) { + bool repatch = + spg != READ_ONCE(gsp_phys_gr_surf_pre_gen); + u64 want = 0; + + /* + * r1483/r1490: the rec-over-alias preference + * below rewrites the pair to THIS launch's + * data-rec bounce VA, and the fill alternates + * its relocation half per capture push. A + * buffer patched once per capture generation + * would keep the first launch's target + * forever - including the X server's 900-page + * rec lane when the next launch is the beef + * client whose partial 10-page allocation sits + * at the same VA. Re-patch whenever this + * launch's EFFECTIVE surface target (complete + * rec run or the contiguous alias, see + * gsp_phys_gr_surf_launch_target) differs + * from the cached words, so a partial launch + * always falls back to the real screen alias + * and never to its own code/operand buffer. + */ + if (!repatch && gsp_phys_gr_surf_pre_n >= 3) { + want = gsp_phys_gr_surf_launch_target( + req); + if (want) { + u64 have = + ((u64)(gsp_phys_gr_surf_pre[1] & + 0xffu) << 32) | + (u64)(gsp_phys_gr_surf_pre[2] & + 0xfffffffcu); + + if (have != want) { + static u32 retarget_log; + + repatch = true; + if (retarget_log < 16u) { + retarget_log++; + pr_info("vgsp-gr: SURF-PRELUDE retarget have=0x%llx want=0x%llx h=0x%x push=0x%llx\n", + have, want, + req->h_object, + req->push_va); + } + } + } + } + if (repatch) { + u32 sdrop = 0, sn = spn; + u32 srel; + + if (sn > GSP_PHYS_GR_SURF_PRE_WORDS) + sn = GSP_PHYS_GR_SURF_PRE_WORDS; + memcpy(gsp_phys_gr_surf_pre_in, sp, + (size_t)sn * 4u); + gsp_phys_gr_surface_alias_scan_locked( + gsp_phys_gr_surf_pre_in, sn, + req); + srel = gsp_phys_rmcall_gr_3d_patch_window( + gsp_phys_gr_surf_pre_in, sn, + gsp_phys_gr_surf_pre, + &gsp_phys_gr_surf_pre_n, + &sdrop, req, &n_shadow, + bar0, epoch); + WRITE_ONCE(gsp_phys_gr_surf_pre_gen, + spg); + { + static u32 surf_patch_log; + + if (surf_patch_log < 16u) { + surf_patch_log++; + pr_info("vgsp-gr: SURF-PRELUDE patched gen=%u raw=%u kept=%u dropped=%u reloc=%u target=0x%llx h=0x%x push=0x%llx\n", + spg, sn, + gsp_phys_gr_surf_pre_n, + sdrop, srel, + ((u64)(gsp_phys_gr_surf_pre[1] & 0xffu) << 32) | + (u64)(gsp_phys_gr_surf_pre[2] & 0xfffffffcu), + req->h_object, + req->push_va); + } + } + } + if (gsp_phys_gr_surf_pre_n && + win3d_n + gsp_phys_gr_surf_pre_n + 2u + + 8u <= + GSP_PHYS_RMCALL_GR_METHOD_WORDS) { + static u32 surf_emit_log; + + memmove(&stream_patch[gsp_phys_gr_surf_pre_n], + stream_patch, + (size_t)win3d_n * 4u); + memcpy(stream_patch, + gsp_phys_gr_surf_pre, + (size_t)gsp_phys_gr_surf_pre_n * + 4u); + win3d_n += gsp_phys_gr_surf_pre_n; + surf_pre_emitted = true; + if (surf_emit_log < 16) { + surf_emit_log++; + pr_info("vgsp-gr: SURF-PRELUDE emitted words=%u window=%u h=0x%x push=0x%llx\n", + gsp_phys_gr_surf_pre_n, + win3d_n, + req->h_object, + req->push_va); + } + } + } + } + /* + * SESSION 154c: synthesized viewport preamble (see the + * sema_gl_vp_prelude comment). Insert the chosen literal + * sequence at sema_gl_vp_prelude_pos: the window head + * (legacy), immediately ahead of the first CALL run + * (0x3800..0x39fc) so every state bind - including the + * 0x2080 SET_PIPELINE(2)=0x21 run - has landed before the + * macro calls, or ahead of the first 0x3910 run. The scan + * uses the MME-WINRUN run grammar; a window without the + * requested run is left unchanged (non-draw window); only + * pos 0 inserts at the head without a CALL run. The words + * carry no address operand, so no relocation pass is + * applied - they are emitted verbatim. Same block-size + * guard as SPH-BIND / SURF-PRELUDE. + */ + { + int vp_mode = READ_ONCE(gsp_phys_rmcall_gr_vp_prelude); + int vp_pos = READ_ONCE(gsp_phys_rmcall_gr_vp_prelude_pos); + u32 vpn = gsp_phys_gr_vp_pre_len(vp_mode); + const u32 *vp_src = gsp_phys_gr_vp_pre_src(vp_mode); + + if (vpn && vp_src) { + u32 vp_at = 0; + bool vp_skip = false; + + if (vp_pos == 1 || vp_pos == 2) { + vp_at = gsp_phys_gr_vp_insert_off( + stream_patch, win3d_n, vp_pos); + if (vp_at >= win3d_n) { + static bool vp_nd_log; + + vp_skip = true; + if (!vp_nd_log) { + vp_nd_log = true; + pr_info("vgsp-gr: VP-PRELUDE skipped (no CALL run; non-draw window) window=%u pos=%d mode=%d\n", + win3d_n, vp_pos, + vp_mode); + } + } + } + if (!vp_skip && + win3d_n + vpn + 2u + 8u <= + GSP_PHYS_RMCALL_GR_METHOD_WORDS) { + static u32 vp_emit_log; + u32 vp_old = win3d_n; + + memmove(&stream_patch[vp_at + vpn], + &stream_patch[vp_at], + (size_t)(win3d_n - vp_at) * + 4u); + memcpy(&stream_patch[vp_at], + vp_src, + (size_t)vpn * 4u); + win3d_n += vpn; + if (vp_emit_log < 16u) { + vp_emit_log++; + pr_info("vgsp-gr: VP-PRELUDE mode=%d pos=%d words=%u window=%u->%u at=%u\n", + vp_mode, vp_pos, vpn, + vp_old, win3d_n, + vp_at); + } + } else if (!vp_skip) { + static u32 vp_skip_log; + + if (vp_skip_log < 8u) { + vp_skip_log++; + pr_info("vgsp-gr: VP-PRELUDE skipped (no room) mode=%d pos=%d words=%u window=%u cap=%u\n", + vp_mode, vp_pos, vpn, + win3d_n, + GSP_PHYS_RMCALL_GR_METHOD_WORDS); + } + } + } + } + /* + * Missing RT0 bind (see the sema_gl_rt0_bind comment). + * Insert the RT0 colour-target run and the count-2 CT + * select ahead of the window's first 0x840 (RT1) run. + * Skip when the window binds 0x800 itself; skip when it + * has no 0x840 anchor. Same run grammar and block-size + * guard as the viewport prelude; constant words, no + * relocation pass. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_rt0_bind)) { + static u32 rt0_emit_log; + static u32 rt0_skip_log; + static bool rt0_nd_log; + u32 rt0_at = gsp_phys_gr_method_run_off( + stream_patch, win3d_n, 0x840u); + u32 rt0_have = gsp_phys_gr_method_run_off( + stream_patch, win3d_n, 0x800u); + u32 rt0_call = gsp_phys_gr_vp_insert_off( + stream_patch, win3d_n, 1); + + if (rt0_call >= win3d_n) { + if (!rt0_nd_log) { + rt0_nd_log = true; + pr_info("vgsp-gr: RT0-BIND skipped (non-draw window) window=%u\n", + win3d_n); + } + } else if (rt0_have < win3d_n) { + if (rt0_skip_log < 8u) { + rt0_skip_log++; + pr_info("vgsp-gr: RT0-BIND skipped (window already binds 0x800) window=%u at=%u\n", + win3d_n, rt0_at); + } + } else if (rt0_at >= win3d_n) { + if (rt0_skip_log < 8u) { + rt0_skip_log++; + pr_info("vgsp-gr: RT0-BIND skipped (no 0x840 run) window=%u\n", + win3d_n); + } + } else if (win3d_n + + (READ_ONCE(gsp_phys_rmcall_gr_rt0_third_dim) ? + GSP_PHYS_GR_RT0_BIND_EXT_WORDS : + GSP_PHYS_GR_RT0_BIND_WORDS) + + 2u + 8u <= + GSP_PHYS_RMCALL_GR_METHOD_WORDS) { + u32 rt0_old = win3d_n; + bool rt0_td = + READ_ONCE(gsp_phys_rmcall_gr_rt0_third_dim); + u32 rt0_n = rt0_td ? + GSP_PHYS_GR_RT0_BIND_EXT_WORDS : + GSP_PHYS_GR_RT0_BIND_WORDS; + u32 rt0_words[GSP_PHYS_GR_RT0_BIND_EXT_WORDS]; + + if (rt0_td) + memcpy(rt0_words, gsp_phys_gr_rt0_bind_ext, + sizeof(gsp_phys_gr_rt0_bind_ext)); + else + memcpy(rt0_words, gsp_phys_gr_rt0_bind, + sizeof(gsp_phys_gr_rt0_bind)); + /* + * r1483: RT0 and RT1 must name the SAME + * surface. The SURF prelude just emitted + * ahead of this run already carries the + * launch's real target after relocation + * (the data-rec bounce base when the + * surface rec set is complete, the + * contiguous alias otherwise); copy its + * geometry words so the PS's RT0 stores + * land in the tracked target on BOTH + * relocation halves. Without the SURF + * prelude (the window binds its own + * target) keep the historical constant. + */ + if (surf_pre_emitted && + gsp_phys_gr_surf_pre_n >= 7u) + memcpy(&rt0_words[1], + &gsp_phys_gr_surf_pre[1], + 6u * sizeof(u32)); + memmove(&stream_patch[rt0_at + rt0_n], + &stream_patch[rt0_at], + (size_t)(win3d_n - rt0_at) * 4u); + memcpy(&stream_patch[rt0_at], rt0_words, + rt0_n * 4u); + win3d_n += rt0_n; + if (rt0_emit_log < 16u) { + rt0_emit_log++; + pr_info("vgsp-gr: RT0-BIND emitted window=%u->%u at=%u target=0x%llx surf=%u td=%u\n", + rt0_old, win3d_n, rt0_at, + ((u64)(rt0_words[1] & 0xffu) << 32) | + (u64)(rt0_words[2] & 0xfffffffcu), + surf_pre_emitted ? 1u : 0u, + rt0_td ? 1u : 0u); + } + } else { + if (rt0_skip_log < 8u) { + rt0_skip_log++; + pr_info("vgsp-gr: RT0-BIND skipped (no room) window=%u cap=%u\n", + win3d_n, + GSP_PHYS_RMCALL_GR_METHOD_WORDS); + } + } + } + /* + * SESSION 153-CONTINUED-57: patch the attr/pipeline prelude for + * this launch (relocation exactly like the window: the + * SET_PIPELINE_PROGRAM_ADDRESS pairs and any other recognised + * operand land in engine-visible shadow VAs; sub!=0 and + * win_drop runs are removed by the same pass). Only done when + * the capture generation moved - the shadow VAs are stable + * per HPA for the boot, so an older patch stays valid while it + * drains across launches. See gsp_phys_gr_attr_pre*. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_attr_prelude)) { + const u32 *ap; + u32 apn = 0, apg = 0; + + ap = gsp_sema_attr_prelude_get(&apn, &apg); + if (ap && apn && apg && + apg != READ_ONCE(gsp_phys_gr_attr_pre_gen)) { + u32 adrop = 0, an = apn; + + if (an > GSP_PHYS_GR_ATTR_PRE_WORDS) + an = GSP_PHYS_GR_ATTR_PRE_WORDS; + gsp_phys_rmcall_gr_3d_patch_window(ap, an, + gsp_phys_gr_attr_pre, + &gsp_phys_gr_attr_pre_n, + &adrop, req, &n_shadow, + bar0, epoch); + if (gsp_phys_gr_attr_pre_n) { + WRITE_ONCE(gsp_phys_gr_attr_pre_gen, + apg); + pr_info("vgsp-gr: ATTR-PRELUDE patched gen=%u raw=%u kept=%u dropped=%u h=0x%x push=0x%llx\n", + apg, an, gsp_phys_gr_attr_pre_n, + adrop, req->h_object, + req->push_va); + } + } + } + if (ndrop) { + static u32 gr3d_drop_log; + + if (gr3d_drop_log < 32) { + gr3d_drop_log++; + pr_info("vgsp-gr: 3D win-drop h=0x%x push=0x%llx raw=%u kept=%u dropped=%u list=[%s]\n", + req->h_object, req->push_va, + req->stream_n, win3d_n, ndrop, + gsp_phys_rmcall_gr_win_drop ? + gsp_phys_rmcall_gr_win_drop : ""); + } + } + /* + * SESSION 153-CONTINUED-59: pre-doorbell MISS abort (see the + * gsp_phys_rmcall_gr_miss_abort comment). A window/attr + * operand whose relocation stayed a raw client VA after the + * level-2 resolver would make the physical engine read an + * unmapped page (r1104 MMU Fault HUBCLIENT_PD @0x4350000 -> + * guest channel reset; r1105 index-buffer misses). Ride the + * same failure path as RELOC-GAP-STRICT: the launch is not + * submitted, the caller falls back and authors the software + * release, the page is already in the pin demand set, and + * the next forward retries. Bounded: after 3000 aborts the + * check fails open so a page the resolver can never see + * cannot livelock the lane forever. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_miss_abort) && + gsp_phys_rmcall_gr_3d_miss_n && + gsp_phys_gr_miss_abort_n < 3000u) { + static u32 miss_abort_log; + + gsp_phys_gr_miss_abort_n++; + if (miss_abort_log < 16u || + !(gsp_phys_gr_miss_abort_n % 256u)) { + miss_abort_log++; + pr_info("vgsp-gr: MISS-ABORT h=0x%x push=0x%llx miss=%u n=%llu (operand page unresolvable; demanded via pin path, launch not submitted - retry next forward)\n", + req->h_object, req->push_va, + gsp_phys_rmcall_gr_3d_miss_n, + gsp_phys_gr_miss_abort_n); + } + mutex_unlock(&gsp_phys_rm_lock); + gsp_phys_gr_refused++; + /* + * SESSION 153-CONTINUED-61: the dedicated verdict lets + * the sema service keep the captured candidate armed + * and re-forward it (re-arm) instead of consuming it + * with a software completion: the demanded pages + * resolve through the pin path and the retry then runs + * the real launch. + */ + ret = GSP_PHYS_GR_RC_MISSABORT; + goto out; + } + /* + * SESSION 153-CONTINUED-56: bounded RAW (pre-drop, pre-reloc) + * window dump so the emitted window can be diffed against the + * client's authentic bytes. r1091 faulted at method 0x1168 + * (SET_VERTEX_ATTRIBUTE_A(2)) with Data 00000000 in the emitted + * stream; this copy tells whether that zero is the client's own + * value or an emission artifact (drop/reloc/align). + */ + { + static u32 gr3d_raw_log; + u32 wi; + + if (gr3d_raw_log < 8) { + gr3d_raw_log++; + pr_info("vgsp-gr: 3D raw dump h=0x%x push=0x%llx words=%u\n", + req->h_object, req->push_va, + req->stream_n); + for (wi = 0; + wi + 16 <= req->stream_n && wi < 448; + wi += 16) + pr_info("vgsp-gr: 3D raw[%3u]: %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n", + wi, + req->stream[wi + 0], + req->stream[wi + 1], + req->stream[wi + 2], + req->stream[wi + 3], + req->stream[wi + 4], + req->stream[wi + 5], + req->stream[wi + 6], + req->stream[wi + 7], + req->stream[wi + 8], + req->stream[wi + 9], + req->stream[wi + 10], + req->stream[wi + 11], + req->stream[wi + 12], + req->stream[wi + 13], + req->stream[wi + 14], + req->stream[wi + 15]); + } + } + /* + * SESSION 153-CONTINUED-11: bounded raw window dump - the + * exact C597 method stream the engine walks (program binds + * 0x2014/0x2018+j*0x40, SET_PIPELINE_SHADER 0x2000+j*0x40, + * draws 0x1618/0xd74). Two windows per boot. + */ + { + static u32 gr3d_win_log; + u32 wi; + + if (gr3d_win_log < 48) { + gr3d_win_log++; + pr_info("vgsp-gr: 3D window dump h=0x%x push=0x%llx words=%u\n", + req->h_object, req->push_va, win3d_n); + for (wi = 0; wi < win3d_n && wi < 384; + wi += 16) + pr_info("vgsp-gr: 3D win[%3u]: %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n", + wi, + win3d[wi + 0], win3d[wi + 1], + win3d[wi + 2], win3d[wi + 3], + win3d[wi + 4], win3d[wi + 5], + win3d[wi + 6], win3d[wi + 7], + win3d[wi + 8], win3d[wi + 9], + win3d[wi + 10], win3d[wi + 11], + win3d[wi + 12], win3d[wi + 13], + win3d[wi + 14], win3d[wi + 15]); + } + } + } + + /* + * r1453 PAIR-CHAIN: drain the final window ONCE per launch, after + * every prepend (SPH-BIND, SURF-PRELUDE) and the relocation pass, + * so the same bytes feed both the first build and the outcome + * retry. The pass only inserts WAIT_FOR_IDLE(0x110) runs ahead of + * back-to-back normal/normal BEGIN pairs; the guest's BEGIN data + * words are carried verbatim. Knob 0 = no pass (legacy bytes). + */ + if (READ_ONCE(gsp_phys_rmcall_gr_pair_chain) && win3d && win3d_n) { + u32 pn = win3d_n; + + (void)gsp_phys_gr_pair_chain_fix((u32 *)win3d, &pn, + GSP_PHYS_RMCALL_GR_METHOD_WORDS); + win3d_n = pn; + } words = gsp_phys_rmcall_build_gr_launch(methods, req, qmd, qmd_dst, - ch_sem_va, want_ch); - memcpy(gsp_phys_chunk_page_va(pb, 0), methods, words * 4); + ch_sem_va, want_ch, + win3d, win3d_n, + gsp_phys_gr_attr_pre, + gsp_phys_gr_attr_pre_n); + gsp_phys_gr_pb_copy(methods, words); + /* SESSION 153-CONTINUED-12: identify this block to a later PBDMA dump. */ + WRITE_ONCE(gsp_phys_gr_last_words, words); + WRITE_ONCE(gsp_phys_gr_last_h, req->h_object); + WRITE_ONCE(gsp_phys_gr_last_push_va, req->push_va); + WRITE_ONCE(gsp_phys_gr_last_pb_dma, + (u64)gsp_phys_chunk_page_dma(pb, 0)); + /* + * SESSION 153-CONTINUED-52: stream validator for the block the engine + * will walk. The r1072 fault class ("Channel ID 4 intr0 00040000" = + * PBDMA PBENTRY, "invalid method encoding used in method", + * fifo_tu102.c) is raised by a method header the PBDMA refuses, or by + * a run whose data is not fully present so the walk desyncs into + * following words. Walk the exact published dwords with the FIFO DMA + * grammar (dev_ram.ref NV_FIFO_DMA_*: SEC_OP 31:29; 1/3/5 carry 1+cnt + * data words, 4/7 and TERT mask ops are single-word, 6 is RESERVED) + * and flag the first occurrences of a reserved encoding or a run + * overrunning the block. Read-only; bounded per boot. + */ + { + static u32 blkval_log; + const u32 *bv = methods; + u32 bo = 0, nbad = 0; + + if (blkval_log < 48) { + while (bo < words && nbad < 4) { + u32 h = bv[bo]; + u32 bop = (h >> 29) & 7u; + u32 bcnt = (h >> 16) & 0xfffu; + u32 bspan = 1u; + + if (bop == 1u || bop == 3u || bop == 5u) + bspan = 1u + bcnt; + if (bop == 6u) { + nbad++; + pr_warn("vgsp-gr: BLOCKVAL reserved-encoding word=%u/%u val=%08x (h=%x push=0x%llx prev=%08x %08x)\n", + bo, words, h, req->h_object, + req->push_va, + bo ? bv[bo - 1] : 0, + bo + 1 < words ? + bv[bo + 1] : 0); + } + if (bo + bspan > words) { + nbad++; + pr_warn("vgsp-gr: BLOCKVAL run-overrun word=%u/%u hdr=%08x span=%u (needs %u more) (h=%x push=0x%llx)\n", + bo, words, h, bspan, + bo + bspan - words, + req->h_object, req->push_va); + } + bo += bspan; + } + blkval_log++; + if (nbad) + pr_warn("vgsp-gr: BLOCKVAL done words=%u bad=%u (h=%x push=0x%llx)\n", + words, nbad, req->h_object, + req->push_va); + } + } pr_info("vgsp-gr: qmd w0..3=%08x %08x %08x %08x w4..7=%08x %08x %08x %08x w23..25=%08x %08x %08x w32/33=%08x %08x w48/49=%08x %08x\n", qmd[0], qmd[1], qmd[2], qmd[3], qmd[4], qmd[5], qmd[6], qmd[7], qmd[23], qmd[24], qmd[25], qmd[32], qmd[33], qmd[48], qmd[49]); @@ -20160,6 +31173,22 @@ gr_page_unmapped: (u64)gcopied << 12); } + /* + * CONT-73: fingerprint the direct-write witness set (relocated + * operand destinations that map guest pages) immediately before the + * doorbell - after every pre-launch writer (copy-in, mirrors) has + * run, so the retire re-scan measures the ENGINE's stores alone. + */ + gsp_phys_gr_wit_capture(req); + + if (gsp_phys_gr_vidmem_bracket) { + int hret = gsp_sema_service_vidmem_hash_snapshot(); + + if (hret) + pr_warn_ratelimited("vgsp-gr: VIDMEM-BRACKET snapshot failed rc=%d\n", + hret); + } + ret = gsp_phys_rmcall_publish_locked(bar0, boot_stage.rmcall_va_pb, words, &sub); mutex_unlock(&gsp_phys_rm_lock); @@ -20293,19 +31322,53 @@ gr_page_unmapped: break; } } - pr_info("vgsp-gr: GR LAUNCH submitted h=0x%x push=0x%llx len=%u pages=%u shadow=%u qmd_va=0x%llx slot=%u words=%u prelude=%d qmdvirt=%d token=0x%x; polling qmd_rel=%u ch_rel=0x%08x (<=%u ms)%s\n", + pr_info("vgsp-gr: GR LAUNCH submitted h=0x%x push=0x%llx len=%u pages=%u shadow=%u qmd_va=0x%llx slot=%u gpget=%u gpput=%u words=%u e0=0x%08x e1=0x%08x prelude=%d qmdvirt=%d token=0x%x; polling qmd_rel=%u ch_rel=0x%08x (<=%u ms)%s\n", req->h_object, req->push_va, req->push_len, req->n_pages, - n_shadow, req->qmd_va, sub.slot, words, + n_shadow, req->qmd_va, sub.slot, sub.gpget, sub.gpput, words, + sub.e0, sub.e1, gsp_phys_rmcall_gr_prelude_value(), gsp_phys_rmcall_gr_qmdvirt_value(), boot_stage.rmcall_token, qmd_rel, want_ch, poll_ms, qmdwin_note); + { + /* + * r596 diag: the raw-prelude geometry the verbatim gate tests. + * The verbatim replay needs + * raw && raw_words >= hdr + 67 && + * raw_qmd_end == hdr + 67 && + * INIT_WORDS + hdr + 67 + TAIL_WORDS <= METHOD_WORDS + * Bounded to the first eight launches so the geometry is on + * record without flooding a battery run. + */ + static u32 prelude_geom_log; + + if (gsp_phys_rmcall_gr_prelude_value() && prelude_geom_log < 8) { + prelude_geom_log++; + pr_info("vgsp-gr: prelude geom raw=%u raw_words=%u hdr=%u end=%u want_end=%u v=%d\n", + req->raw ? 1u : 0u, req->raw_words, + req->raw_qmd_hdr, req->raw_qmd_end, + req->raw_qmd_hdr + + GSP_PHYS_RMCALL_GR_QMD_WINDOW_WORDS, + gsp_phys_rmcall_gr_prelude_value()); + } + } if (qmd_win_diag) pr_info("vgsp-gr: qmd dest va=0x%llx pdeIndex=%u installed=%u win_mask=0x%llx ensure_ret=%d\n", qmd_dst, qmd_win_pde, qmd_win_installed, (unsigned long long)boot_stage.rmcall_guest_pt_win_mask, qmd_win_ret); + /* + * SESSION 153-CONTINUED-69: pre-reset MME fault sample. The engine + * faults within the first tens of ms of the launch; sample the MME + * ESR/INFO/INFO2 + DS ESR + GRFIFO/trap regs at MMIO rate from the + * moment the doorbell rang (in publish) so the faulting values are + * caught before the GSP-RM clears them. + */ + if (stream3d && bar0) + fast_poison = gsp_phys_mme_fast_sample(bar0, "launch", + req->h_object, 0); + poll_start_us = (u32)ktime_to_us(ktime_get()); deadline = jiffies + msecs_to_jiffies(poll_ms); for (;;) { @@ -20334,10 +31397,80 @@ gr_page_unmapped: sked_sample[8] = ioread32(bar0 + 0x00409b00u); sked_sample[9] = ioread32(bar0 + 0x00409b04u); sked_sample[10] = ioread32(bar0 + 0x00409400u); + /* + * SESSION 153-CONTINUED-14: FECS/FE ctxsw handshake + * lanes. fe0 = FECS_CTXSW_STATUS_FE_0 (0x409c00), + * fehost = FECS_HOST_INT_STATUS (0x409c18), fecsirq + * = FECS_IRQSTAT (0x409008). If the FECS dies in a + * switch, it should die visibly here first. + */ + sked_sample[11] = ioread32(bar0 + 0x00409c00u); + sked_sample[12] = ioread32(bar0 + 0x00409c18u); + sked_sample[13] = ioread32(bar0 + 0x00409008u); + /* + * SESSION 153-CONTINUED-52: in-flight fault registers. + * The DS shader-program-header error latch (0x405840), + * the MME exception latch (0x404490) with its PC/macro + * address (0x404494) and instruction register + * (0x40449c, fixed in SESSION 153-CONTINUED-106) are + * cleared by the real RM within ~ms of + * the fault, so the post-timeout reads are all clean. + * Sampling them every poll tick records the faulting + * values at the first change. + */ + sked_sample[14] = ioread32(bar0 + 0x00405840u); + sked_sample[15] = ioread32(bar0 + 0x00404490u); + sked_sample[16] = ioread32(bar0 + 0x00404494u); + /* SESSION 153-CONTINUED-106: INFO2 0x40449c (was CONFIG). */ + sked_sample[17] = ioread32(bar0 + 0x0040449cu); + /* + * SESSION 153-CONTINUED-52: the PBDMA host-software + * METHOD latch (NV_PPBDMA_METHOD0 0x4000c0: ADDR + * 13:2, SUBCH 18:16, INCR 0) - the method whose + * delivery raises NV_PPBDMA_INTR_0_METHOD (bit 21). + * The real driver clears the entry when it processes + * the interrupt, so only an in-flight sample can name + * it. + */ + sked_sample[18] = ioread32(bar0 + 0x004000c0u); + sked_sample[19] = ioread32(bar0 + 0x004000c4u); + /* + * SESSION 153-CONTINUED-57: the class-error trapped + * register set (gr_error_gk104.c grGetExceptionChannel): + * NV_PGRAPH_TRAPPED_ADDR 0x40704 (MTHD offset + SUBCH + + * INVALID status bit), TRAPPED_DATA_LOW 0x40708 (the + * class error's "Data" field), TRAPPED_DATA_HIGH + * 0x4070c, and TRAPPED_DATA_MME 0x40710 (PC 11:0 + + * MACRO_ADDRESS 19:12) - the last one names whether the + * offending write came from MME macro execution. The + * real RM clears these within ~ms of the fault; a + * per-tick sample catches the first change. The + * persistent 0x1168/Data=0 class error (r1091/r1098) + * needs exactly this decode. + */ + sked_sample[20] = ioread32(bar0 + 0x00400704u); + sked_sample[21] = ioread32(bar0 + 0x00400708u); + sked_sample[22] = ioread32(bar0 + 0x0040070cu); + sked_sample[23] = ioread32(bar0 + 0x00400710u); + /* + * SESSION 153-CONTINUED-58: the front-end DECODE + * STATE EX registers (NV_PGRAPH_PRI_FE_DECODE_STATE_EX + * 0x40a630..0x40a63c, 4 slots): MME_PC 11:0, + * MME_MACRO_ADDR 21:12, MME_GENERATED bit 31 - the + * front end's own record of the last MME-generated + * method write, which names the macro (address) and + * its pc for the class error even when the trapped + * registers are already back to the 0xbadf priv + * poison. Sampled per tick like the trap set. + */ + sked_sample[24] = ioread32(bar0 + 0x0040a630u); + sked_sample[25] = ioread32(bar0 + 0x0040a634u); + sked_sample[26] = ioread32(bar0 + 0x0040a638u); + sked_sample[27] = ioread32(bar0 + 0x0040a63cu); if (sked_first || memcmp(sked_sample, sked_prev, sizeof(sked_sample))) { - pr_info("vgsp-gr: sked rec +%ums cwd0=%08x cwd1=%08x cwd2=%08x info1=%08x info2=%08x act=%08x esr=%08x grfifo=%08x fecs_cur=%08x fecs_new=%08x ctxsw1=%08x\n", + pr_info("vgsp-gr: sked rec +%ums cwd0=%08x cwd1=%08x cwd2=%08x info1=%08x info2=%08x act=%08x esr=%08x grfifo=%08x fecs_cur=%08x fecs_new=%08x ctxsw1=%08x fe0=%08x fehost=%08x fecsirq=%08x ds_esr=%08x mme_esr=%08x mme_pc=%u mme_mac=0x%x mme_ir=%08x mth0=%08x mth1=%08x(pbaddr=0x%x sub=%u incr=%u) trap=%08x(data_lo=%08x hi=%08x sub=%u mmegen=%u dhv=%u) mmepc=%u mmemac=0x%x\n", sked_tick * GSP_PHYS_RMCALL_POLL_MS, sked_sample[0], sked_sample[1], @@ -20345,12 +31478,150 @@ gr_page_unmapped: sked_sample[4], sked_sample[5], sked_sample[6], sked_sample[7], sked_sample[8], sked_sample[9], - sked_sample[10]); + sked_sample[10], sked_sample[11], + sked_sample[12], sked_sample[13], + sked_sample[14], sked_sample[15], + sked_sample[16] & 0xfffu, + (sked_sample[16] >> 12) & 0xffu, + sked_sample[17], + sked_sample[18], sked_sample[19], + (sked_sample[18] >> 2) << 2, + (sked_sample[18] >> 16) & 7u, + sked_sample[18] & 1u, + sked_sample[20], sked_sample[21], + sked_sample[22], + (sked_sample[20] >> 16) & 7u, + (sked_sample[20] >> 20) & 1u, + (sked_sample[20] >> 24) & 1u, + sked_sample[23] & 0xfffu, + (sked_sample[23] >> 12) & 0xffu); + if (sked_sample[24] || sked_sample[25] || + sked_sample[26] || sked_sample[27]) + pr_info("vgsp-gr: sked fed0=%08x fed1=%08x fed2=%08x fed3=%08x (pc0=%u mac0=0x%x gen0=%u)\n", + sked_sample[24], + sked_sample[25], + sked_sample[26], + sked_sample[27], + sked_sample[24] & 0xfffu, + (sked_sample[24] >> 12) & 0x3ffu, + (sked_sample[24] >> 31) & 1u); memcpy(sked_prev, sked_sample, sizeof(sked_sample)); sked_first = false; } } + /* + * SESSION 153-CONTINUED-48: priv-error (0xbadf-tagged) wedge + * recovery - see sema_gr_wedge_recover. The poison latch is + * checked before the completion break and the ladder so a + * launch whose window faulted the GR hands the engine to the + * next launch recovered instead of wedged. One attempt per + * launch (local flag); the hard re-arm is the r445 mechanism, + * bounded by its own RPC and sem-probe timeouts. + */ + if (READ_ONCE(gsp_phys_gr_wedge_recover) && bar0 && + !wedge_recover_done) { + u32 g0 = ioread32(bar0 + 0x00400504u); + + /* + * SESSION 153-CONTINUED-72: either the poison is still + * asserted here, or the MME-FAST spin caught it before + * the FECS cleared it (fast_poison latch). Both mean + * the window priv-errored and the engine context is + * gone; the earlier live-register-only check missed + * the self-cleared case (r1149) and wedged the boot. + */ + if ((g0 & 0xffff0000u) == 0xbadf0000u || fast_poison) { + u32 g1; + + wedge_recover_done = true; + wedge_recoveries++; + pr_warn("vgsp-gr: WEDGE-RECOVER priv-error poison seen (grfifo=0x%08x%s) h=0x%x push=0x%llx - TSG preempt + resched\n", + g0, + fast_poison && + (g0 & 0xffff0000u) != 0xbadf0000u ? + " via MME-FAST latch (self-cleared)" : + "", + req->h_object, req->push_va); + gsp_phys_mme_regs_log(bar0, "wedge-poison"); + mutex_lock(&gsp_phys_rm_lock); + gsp_phys_gr_wedge_hard_rearm_locked(bar0); + mutex_unlock(&gsp_phys_rm_lock); + g1 = ioread32(bar0 + 0x00400504u); + /* + * SESSION 153-CONTINUED-61: the TSG preempt + + * resched above resets the engine context, so + * the MME program the emitted prelude loaded is + * gone. Clear the emission latch: the next + * launch that can carry the prelude re-preloads + * it (without this every later replay ran its + * macros against an unloaded MME and stalled + * pre-retire - the r1110 mme_pc=513 class). + */ + WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, 0); + /* + * SESSION 153-CONTINUED-79: also rewind the + * split-emission drain cursor, else a cleared + * latch with an exhausted cursor would skip + * the owed re-preload. + */ + WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0); + /* + * SESSION 153-CONTINUED-109: rewind the PACK + * cursor too (the r1377 deadlock: recovery reset + * pre_emit_off but left pack_off at its end, so + * the split emitter hit `poff >= maxa` -> can=0 + * -> packed=1 with zero words on every launch: + * `MME-PRELUDE part gen=3 off=0 words=0/3083` + * forever, latch never re-set, gates never + * release, MME never starts). The gate-fail + * path (gsp_phys_gr_mme_iram_gate_fail) already + * rewinds both cursor pairs. + */ + WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_sa, false); + /* + * SESSION 153-CONTINUED-109: the hard re-arm + * wiped the engine context, so the generation + * whose carrier executed is no longer resident. + * Invalidate it; the re-preload's own GP_GET + * advance (readback-free witness) re-promotes + * (fetch_miss/poll-break paths). Without this + * the stale live_gen would mark a wiped context + * live and the next window+calls launch would + * run against an unloaded MME. + */ + WRITE_ONCE(gsp_phys_gr_mme_live_gen, 0); + WRITE_ONCE(gsp_phys_gr_mme_release_gen, 0); + WRITE_ONCE(gsp_phys_gr_mme_release_h, 0); + /* r1453 PAIR-CHAIN: context wipe voids the + * pair-install proof; re-earn it. */ + WRITE_ONCE(gsp_phys_gr_mme_pairs_gen, 0); + /* r1454 PAIR-CHAIN-ENTRY-ELIDE: the wiped + * context has no open FE pair; clear the + * cross-launch tracker too. */ + WRITE_ONCE(gsp_phys_gr_mme_pair_open, false); + WRITE_ONCE(gsp_phys_gr_mme_pair_open_split, + 0); + WRITE_ONCE(gsp_phys_gr_mme_pair_open_entry, + false); + WRITE_ONCE( + gsp_phys_gr_mme_pair_open_entry_split, + 0); + /* + * r1447: the wiped context voids the arming + * window-hold and the strict grace budget too. + */ + WRITE_ONCE(gsp_phys_gr_mme_window_hold_gen, 0); + WRITE_ONCE(gsp_phys_gr_mme_window_hold_h, 0); + WRITE_ONCE(gsp_phys_gr_mme_strict_holds, 0); + pr_warn("vgsp-gr: WEDGE-RECOVER after re-arm grfifo=0x%08x (%s); recovery #%u this boot; MME prelude latch cleared (context re-preload owed)\n", + g1, + (g1 & 0xffff0000u) == 0xbadf0000u ? + "still poisoned" : "cleared", + wedge_recoveries); + } + } if (qmd_rel) { if (qmd_rel_hpa) w0 = READ_ONCE(*(u32 *)((u8 *)__va(qmd_rel_hpa) + @@ -20358,15 +31629,162 @@ gr_page_unmapped: else w0 = READ_ONCE(*(u32 *)sem_va); } + /* + * SESSION 153-CONTINUED-59: PBENTRY first-sighting watch. + * The r1102 fault was only captured via the GSP's 0x1006 + * relay (seconds later, intr0 already cleared by the RM): + * here every 5 ms poll of a 3D-stream launch scans the 12 + * PBDMA instances for NV_PPBDMA_INTR_0 bit 18 (PBENTRY) and + * on the FIRST sighting dumps the live PB_HEADER / PB_COUNT + * / PB_DATA0 / HDR_SHADOW set plus the raw 0x100..0x120 + * block for the faulting instance, before any re-arm or RM + * clear can overwrite them. Bounded (8 lines per boot); + * read-only, no behavior change to the launch. + */ + if (stream3d && bar0) { + static u32 pbentry_log; + u32 pi2; + + for (pi2 = 0; pi2 < 12 && pbentry_log < 8; pi2++) { + u32 b2 = 0x40000u + pi2 * 0x2000u; + u32 i0 = ioread32(bar0 + b2 + 0x108u); + + if (!(i0 & 0x00040000u)) + continue; + pbentry_log++; + pr_warn("vgsp-gr: PBENTRY-FIRST-SIGHTING P%u intr0=%08x gpf=%u pbfetch=0x%llx hdr=%08x cnt=%08x data0=%08x pbhd=%08x h=%x push=0x%llx words=%u tick=%u\n", + pi2, i0, + ioread32(bar0 + b2 + 0x050u), + ((u64)(ioread32(bar0 + b2 + 0x058u) & + 0xffu) << 32) | + (ioread32(bar0 + b2 + 0x054u) & + ~3u), + ioread32(bar0 + b2 + 0x118u), + ioread32(bar0 + b2 + 0x088u), + ioread32(bar0 + b2 + 0x08cu), + ioread32(bar0 + b2 + 0x084u), + req->h_object, req->push_va, words, + sked_tick); + pr_info("vgsp-gr: PBENTRY-FIRST-SIGHTING P%u raw[0x100]=%08x [104]=%08x [10c]=%08x [110]=%08x [114]=%08x [11c]=%08x [120]=%08x\n", + pi2, + ioread32(bar0 + b2 + 0x100u), + ioread32(bar0 + b2 + 0x104u), + ioread32(bar0 + b2 + 0x10cu), + ioread32(bar0 + b2 + 0x110u), + ioread32(bar0 + b2 + 0x114u), + ioread32(bar0 + b2 + 0x11cu), + ioread32(bar0 + b2 + 0x120u)); + gsp_phys_pbdma_fault_dump(0xffffffffu); + break; + } + } + /* + * SESSION 153-CONTINUED-13: early PBDMA fetch probe for a 3D + * stream launch - where is the front end while a DRAW window + * does not retire? One shot per launch at ~150 ms and ~1.2 s + * (5 ms poll), bounded per boot. pbfetch inside the module + * pushbuffer page = the walk/exec is stuck in our block; + * pbfetch past it = the whole stream was decoded and the + * stall is in execution/completion. + */ + if (stream3d && bar0 && + (sked_tick == 30 || sked_tick == 240)) { + static u32 winprobe_log; + u32 pi; + + if (winprobe_log < 64) { + winprobe_log++; + pr_info("vgsp-gr: WINPROBE +%ums push=0x%llx words=%u block_dma=0x%llx grfifo=%08x trapped=%08x trapped_lo=%08x\n", + sked_tick * GSP_PHYS_RMCALL_POLL_MS, + req->push_va, words, sub.pb_dma, + ioread32(bar0 + 0x00400504u), + ioread32(bar0 + 0x00400704u), + ioread32(bar0 + 0x00400708u)); + for (pi = 0; pi < 12; pi++) { + u32 b = 0x40000u + pi * 0x2000u; + u32 intr0 = ioread32(bar0 + b + 0x108u); + u32 gpf = ioread32(bar0 + b + 0x050u); + u32 pbf = ioread32(bar0 + b + 0x054u); + u32 pbfhi = ioread32(bar0 + b + 0x058u); + u32 hdrsh = ioread32(bar0 + b + 0x118u); + u32 wbh = ioread32(bar0 + b + 0x084u); + u32 wbcnt = ioread32(bar0 + b + 0x088u); + u32 wbd0 = ioread32(bar0 + b + 0x08cu); + u64 fetch64 = + ((u64)(pbfhi & 0xffu) << 32) | + (u64)(pbf & ~3u); + s64 woff = -1; + + if (!intr0 && !gpf && !pbf && !hdrsh) + continue; + if (gsp_phys_gr_last_pb_dma && + fetch64 >= gsp_phys_gr_last_pb_dma && + fetch64 < + gsp_phys_gr_last_pb_dma + + (u64)gsp_phys_gr_last_words * + 4u) + woff = (s64)(fetch64 - + gsp_phys_gr_last_pb_dma) / + 4; + pr_info("vgsp-gr: WINPROBE P%u intr0=%08x gp_put=%u pbfetch=0x%llx off=%lld hdr=%08x(%s m=0x%x cnt=%u sub=%u) pbhd=%08x(type=%s) pbcnt=%08x pbd0=%08x\n", + pi, intr0, gpf, fetch64, woff, hdrsh, + gsp_phys_pbdma_type_name(hdrsh >> 29), + (hdrsh & 0x1fffu) << 2, + (hdrsh >> 16) & 0xfffu, + (hdrsh >> 13) & 7u, wbh, + gsp_phys_pbdma_type_name(wbh >> 29), + wbcnt, wbd0); + } + } + } if (ch_rel_hpa) w1 = READ_ONCE(*(u32 *)((u8 *)__va(ch_rel_hpa) + ch_rel_off)); else w1 = READ_ONCE(*(u32 *)(sem_va + 4)); if ((!qmd_rel || w0 == want_qmd) && w1 == want_ch) { + /* + * r1453 PAIR-CHAIN: this launch retired. If its + * emitted window carried BEGIN/END runs, the pair + * runs are installed on the channel for this + * prelude generation - the proof the held-call + * elision extension waits for. A context wipe + * (WEDGE-RECOVER / park recovery) clears this so a + * later generation re-earns it from a drain-carrying + * window. + */ + if (gsp_phys_gr_mme_pairs_emitted && + gsp_phys_gr_mme_launch_gen && + READ_ONCE(gsp_phys_rmcall_gr_pair_chain)) + WRITE_ONCE(gsp_phys_gr_mme_pairs_gen, + gsp_phys_gr_mme_launch_gen); + /* + * SESSION 153-CONTINUED-107: if THIS launch carried + * the final packed prelude part, gate its residency on + * the IRAM readback BEFORE the break log promotes the + * generation to live (see gsp_phys_gr_mme_iram_gate). + * A failed readback clears the pending gen, the + * prelude latch and both emission cursors: the calls + * stay held and the next launch re-emits from word 0. + */ + if (bar0 && + READ_ONCE(gsp_phys_gr_mme_pending_gen) && + READ_ONCE(gsp_phys_rmcall_gr_mme_prelude_pack) && + READ_ONCE(gsp_phys_rmcall_gr_mme_iram_gate) && + READ_ONCE(gsp_phys_rmcall_gr_mme_iram_gate_break) && + !gsp_phys_gr_mme_iram_gate(bar0, req->h_object, + READ_ONCE(gsp_phys_gr_mme_pending_gen), + "break")) + gsp_phys_gr_mme_iram_gate_fail(); /* F4-a: first poll break + USERD witnesses. */ gsp_phys_gr_poll_break_log(w0, want_qmd, w1, - poll_start_us, &sub); + poll_start_us, &sub, + req->h_object); + /* + * SESSION 153-CONTINUED-72: GR-STATE readback at the + * retire (fed MME-generated-write records + latches). + */ + gsp_phys_gr_state_rb_log(req->h_object); break; } /* @@ -20384,6 +31802,9 @@ gr_page_unmapped: pr_err("vgsp-gr: GR LAUNCH FAILED: wait budget crossed (qmd_rel=0x%08x want=0x%08x ch_rel=0x%08x want=0x%08x) h=0x%x push=0x%llx; ladder stopped\n", w0, want_qmd, w1, want_ch, req->h_object, req->push_va); + gsp_phys_mme_regs_log(bar0, "wait-max"); + gsp_phys_gr_fetch_miss_log(req->h_object, &sub, + "wait-max"); ret = -ETIMEDOUT; goto out; } @@ -20395,6 +31816,23 @@ gr_page_unmapped: pr_err("vgsp-gr: GR LAUNCH FAILED: timeout after %u ms (qmd_rel=0x%08x want=0x%08x ch_rel=0x%08x want=0x%08x) h=0x%x push=0x%llx\n", poll_ms, w0, want_qmd, w1, want_ch, req->h_object, req->push_va); + gsp_phys_mme_regs_log(bar0, "timeout"); + gsp_phys_gr_fetch_miss_log(req->h_object, &sub, + "timeout"); + /* + * SESSION 153-CONTINUED-106: the r1363 trap sat in the + * ~2.6 s window AFTER this timeout (launch failed + * 403088.34, ILLEGAL_OPCODE relayed 403090.96). Re-spin + * once here, bounded to the first few timeouts of the + * boot, so the latch is caught at first sight with its + * INFO/IR word instead of being missed between spins. + */ + if (gsp_phys_mme_post_ms && post_spin_n < 8u) { + post_spin_n++; + (void)gsp_phys_mme_fast_sample(bar0, "post", + req->h_object, + gsp_phys_mme_post_ms); + } /* * F4-b: one-shot deposit readback at the FIRST timeout * (the old post-mortem only ran after attempt 3, which @@ -20499,6 +31937,80 @@ gr_page_unmapped: mutex_lock(&gsp_phys_rm_lock); sp_ret = gsp_phys_rmcall_gr_sem_push_locked(bar0); mutex_unlock(&gsp_phys_rm_lock); + /* + * SESSION 153-CONTINUED-93: parked-engine + * recovery (see the sema_gr_park_recover + * knob). A wedged probe on a poison-free, + * trap-free front-end is the r1311-r1313 + * park class: the context is gone and only a + * context reload brings it back. One bounded + * hard re-arm per launch; ALIVE afterwards + * resumes the fresh-entry retries. + */ + if (sp_ret && bar0 && + READ_ONCE(gsp_phys_gr_park_recover)) { + static u32 park_rec_n; + + mutex_lock(&gsp_phys_rm_lock); + sp_ret = + gsp_phys_gr_wedge_hard_rearm_locked(bar0); + mutex_unlock(&gsp_phys_rm_lock); + park_rec_n++; + if (!sp_ret) { + WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, 0); + WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_sa, false); + WRITE_ONCE(gsp_phys_gr_mme_live_gen, 0); + WRITE_ONCE( + gsp_phys_gr_mme_release_gen, + 0); + WRITE_ONCE( + gsp_phys_gr_mme_release_h, + 0); + /* r1453 PAIR-CHAIN: same + * context wipe - the pair + * install proof is void. */ + WRITE_ONCE( + gsp_phys_gr_mme_pairs_gen, + 0); + /* r1454 PAIR-CHAIN-ENTRY- + * ELIDE: same wipe - no + * open FE pair remains. */ + WRITE_ONCE( + gsp_phys_gr_mme_pair_open, + false); + WRITE_ONCE( + gsp_phys_gr_mme_pair_open_split, + 0); + WRITE_ONCE( + gsp_phys_gr_mme_pair_open_entry, + false); + WRITE_ONCE( + gsp_phys_gr_mme_pair_open_entry_split, + 0); + /* + * r1447: same context wipe - + * void the arming window-hold + * and the strict grace budget. + */ + WRITE_ONCE( + gsp_phys_gr_mme_window_hold_gen, + 0); + WRITE_ONCE( + gsp_phys_gr_mme_window_hold_h, + 0); + WRITE_ONCE( + gsp_phys_gr_mme_strict_holds, + 0); + pr_warn("vgsp-gr: PARK-RECOVER #%u warm sem ALIVE after TSG preempt + resched; fresh-entry retries resume push=0x%llx\n", + park_rec_n, req->push_va); + } else { + pr_warn("vgsp-gr: PARK-RECOVER #%u still wedged after hard re-arm (rc=%d) push=0x%llx\n", + park_rec_n, sp_ret, + req->push_va); + } + } if (sp_ret) { gsp_phys_gr_sem_wedged++; pr_info("vgsp-gr: timeout ladder: warm sem WEDGED (rc=%d) - failing fast instead of fresh-entry retries\n", @@ -20793,6 +32305,16 @@ gr_page_unmapped: ioread32(bar0 + 0x00405b00u), ioread32(bar0 + 0x00400100u), ioread32(bar0 + 0x0040013cu)); + if (bar0) + pr_info("vgsp-gr: FECS readback: CUR=0x%08x NEW=0x%08x ST1=0x%08x FE0=0x%08x HOSTINT=0x%08x IRQSTAT=0x%08x CPUCTL=0x%08x OS=0x%08x\n", + ioread32(bar0 + 0x00409b00u), + ioread32(bar0 + 0x00409b04u), + ioread32(bar0 + 0x00409400u), + ioread32(bar0 + 0x00409c00u), + ioread32(bar0 + 0x00409c18u), + ioread32(bar0 + 0x00409008u), + ioread32(bar0 + 0x00409100u), + ioread32(bar0 + 0x00409080u)); if (bar0) { u32 g; @@ -20830,7 +32352,7 @@ gr_page_unmapped: ret = -ETIMEDOUT; goto out; } - msleep(GSP_PHYS_RMCALL_POLL_MS); + gsp_phys_rmcall_wait_quantum(); } gsp_phys_gr_ok++; @@ -20878,6 +32400,14 @@ gr_page_unmapped: u32 wrote = gsp_phys_gr_output_retire_wait(req, bounce_csum_in, &n_tracked); + if (gsp_phys_gr_vidmem_bracket) + gsp_sema_service_vidmem_hash_diff("launch", 64); + + /* CONT-73: where did the draw write? bounce count above; + * direct guest stores (relocated alias/shadow operands and + * MME shadow-scratch destinations) here. */ + gsp_phys_gr_wit_report("retire"); + /* * r434: post-launch hardware-only QMD dump (w50/w51 INNER * get/put = the SKED's CTA dispatch counters) so the @@ -20898,9 +32428,16 @@ gr_page_unmapped: if (wrote == 0 && n_tracked) { bool ff = gsp_phys_gr_qmd_fetched(qmd_dst_mem); - pr_info("vgsp-gr: first-attempt witness: engine fetch %s (qmd sentinel %s) tracked=%u\n", - ff ? "OK" : "NONE", - ff ? "overwritten" : "intact", n_tracked); + /* CONT-103: no QMD deposit on the stream3d lane, so the + * sentinel is vacuous there; the GP_GET FETCH-WIT is the + * fetch witness for that lane. */ + if (qmd_dst_mem) + pr_info("vgsp-gr: first-attempt witness: engine fetch %s (qmd sentinel %s) tracked=%u\n", + ff ? "OK" : "NONE", + ff ? "overwritten" : "intact", n_tracked); + else + pr_info("vgsp-gr: first-attempt witness: engine fetch n/a (no QMD deposit on this lane - sentinel vacuous; see FETCH-WIT) tracked=%u\n", + n_tracked); } /* * r427 idempotent fast-accept: a zero-write launch whose identity @@ -21166,9 +32703,12 @@ gr_page_unmapped: words = gsp_phys_rmcall_build_gr_launch( methods, req, qmd, qmd_dst, ch_sem_va, - want_ch); - memcpy(gsp_phys_chunk_page_va(pb, 0), - methods, words * 4); + want_ch, win3d, + win3d_n, + gsp_phys_gr_attr_pre, + gsp_phys_gr_attr_pre_n); + gsp_phys_gr_pb_copy(methods, + words); } if (qmdwin_swap) pr_info("vgsp-gr: outcome retry %u: release words zeroed + fresh qmdwin bank %u dest 0x%llx + win0 bank %u%s + PB rebuilt\n", @@ -21188,6 +32728,9 @@ gr_page_unmapped: bounce_staged_hpa); gsp_phys_gr_fill_gap_bounce(bounce_csum_in, bounce_staged_hpa); + /* CONT-73: refingerprint the direct-write + * witness set for this retry publish. */ + gsp_phys_gr_wit_capture(req); mutex_lock(&gsp_phys_rm_lock); ret = gsp_phys_rmcall_publish_locked( bar0, boot_stage.rmcall_va_pb, @@ -21235,7 +32778,7 @@ gr_page_unmapped: } if (time_after_eq(jiffies, dl)) break; - msleep(GSP_PHYS_RMCALL_POLL_MS); + gsp_phys_rmcall_wait_quantum(); } pr_info("vgsp-gr: outcome retry %u: fresh ack %s (qmd_rel=0x%08x want=0x%08x ch_rel=0x%08x after %u ms)\n", nw_attempts, @@ -21275,6 +32818,8 @@ gr_page_unmapped: } wrote = gsp_phys_gr_output_retire_wait( req, bounce_csum_in, &n_tracked); + /* CONT-73: direct-write witness per retry. */ + gsp_phys_gr_wit_report("retry-retire"); gsp_phys_gr_qmd64_post(qmd_dst_mem, "retry-retire"); pr_info("vgsp-gr: outcome retry %u done: engine wrote %u page(s)\n", @@ -21329,7 +32874,8 @@ gr_page_unmapped: fetched ? "AMBIGUOUS" : "CONFIRMED", req->h_object, req->push_va, req->n_data, n_tracked, nw_attempts, want_qmd, - fetched ? "OK" : "NONE", + qmd_dst_mem ? (fetched ? "OK" : "NONE") : + "n/a(noQMD-stream3d)", READ_ONCE(gsp_phys_gr_nw_streak) + 1u, gsp_phys_gr_wedge_quiet, gsp_phys_gr_wedge_heals); @@ -21414,14 +32960,12 @@ gr_page_unmapped: const u8 *pp = gsp_phys_chunk_page_va( gsp_phys_gr_data_chunk(), gsp_phys_gr_prev_out_bi); - u32 pc = 0, pi; + u64 pc; if (pp) { - for (pi = 0; pi < GSP_PHYS_PAGE_SIZE; - pi += 4) - pc += get_unaligned_le32(pp + pi); + pc = gsp_phys_gr_fnv1a64(pp, GSP_PHYS_PAGE_SIZE); if (pc != gsp_phys_gr_prev_out_csum) - pr_warn("vgsp-gr: PREV-OUT-CHANGED bi=%u csum %08x -> %08x (late store after its launch shipped)\n", + pr_warn("vgsp-gr: PREV-OUT-CHANGED bi=%u fp %016llx -> %016llx (late store after its launch shipped)\n", gsp_phys_gr_prev_out_bi, gsp_phys_gr_prev_out_csum, pc); @@ -21447,6 +32991,18 @@ gr_page_unmapped: late, late_tracked, late ? "LATE-STORES - copy-out will ship them" : "NO-STORES - dispatch lost or read-only kernel"); + /* + * SESSION 153-CONTINUED-83: the no-writes moment + * is the store wall's fingerprint - snapshot the + * DS/SPH/MME/FE-decode evidence here so the + * stage that never received a program header (or + * the MME PC that never emitted the draw) is on + * the same line as the failing launch. + */ + if (!late) + gsp_phys_store_side_log(gsp_phys_gpu_bar0(), + "no-writes", + req->h_object); } if (qmd_rel_hpa) { @@ -21514,11 +33070,12 @@ gr_page_unmapped: u32 aliased = 0, unchanged = 0, skipped = 0; u32 chg[8] = { }; u32 outwit = 0; /* COPYWIT: bounded per-launch witness lines */ + u32 outwit_chg = 0; /* CONT-112: own budget, changed pages */ for (wbi = 0; wbi < req->n_data; wbi++) { u32 bi = gsp_phys_gr_data_bi(req->data[wbi].reloc_va); const u8 *src; - u32 ci, csum = 0; + u64 fp; if (bi == GSP_PHYS_GR_DATA_BI_NONE || !req->data[wbi].hpa) @@ -21539,9 +33096,8 @@ gr_page_unmapped: aliased++; continue; } - for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) - csum += get_unaligned_le32(src + ci); - if (csum == bounce_csum_in[bi]) { + fp = gsp_phys_gr_fnv1a64(src, GSP_PHYS_PAGE_SIZE); + if (fp == bounce_csum_in[bi]) { /* * r378: the engine did not touch this page, * so the guest page still holds exactly this @@ -21550,6 +33106,37 @@ gr_page_unmapped: * stale-content corruption at worst - skip. */ unchanged++; + /* + * r452: witness the UNCHANGED page's chunk + * content (first 4 words) and the guest + * page's, bounded per launch. This is the + * only place the two can be compared for a + * page the csum rule classified as "engine + * did not touch" - it separates "the store + * never landed in the chunk" from "the + * store landed but the csum rule missed + * it" (or "the reader resolves a different + * page than the writer wrote"). + */ + if (gsp_phys_gr_copywit_enabled() && + outwit < GSP_PHYS_GR_COPYWIT_MAX) { + const u8 *gp = req->data[wbi].hpa ? + (const u8 *)__va(req->data[wbi].hpa) : + NULL; + + outwit++; + pr_info("vgsp-gr: OUT-UNCH bi=%u va=0x%llx fp=%016llx pre=%016llx chunk=%08x %08x %08x %08x guest=%08x %08x %08x %08x\n", + bi, req->data[wbi].va, fp, + bounce_csum_in[bi], + get_unaligned_le32(src), + get_unaligned_le32(src + 4), + get_unaligned_le32(src + 8), + get_unaligned_le32(src + 12), + gp ? get_unaligned_le32(gp) : 0, + gp ? get_unaligned_le32(gp + 4) : 0, + gp ? get_unaligned_le32(gp + 8) : 0, + gp ? get_unaligned_le32(gp + 12) : 0); + } continue; } { @@ -21584,7 +33171,7 @@ gr_page_unmapped: * sema-bigcopy resolution lines. */ if (gsp_phys_gr_copywit_enabled() && - outwit < GSP_PHYS_GR_COPYWIT_MAX) { + outwit_chg < GSP_PHYS_GR_COPYWIT_CHG_MAX) { /* * r391 diagnostic (sum=0 class): the * landing-table resolution of this VA @@ -21599,14 +33186,14 @@ gr_page_unmapped: req->data[wbi].va, &rpa, &rap, &rval); - outwit++; - pr_info("vgsp-gr: OUT-PAGE-WIT va=0x%llx hpa=0x%llx bi=%u first8=%016llx last8=%016llx pre=%08x post=%08x res=%s pa=0x%llx ap=%u val8=%016llx vbase=%px\n", + outwit_chg++; + pr_info("vgsp-gr: OUT-PAGE-WIT va=0x%llx hpa=0x%llx bi=%u first8=%016llx last8=%016llx pre=%016llx post=%016llx res=%s pa=0x%llx ap=%u val8=%016llx vbase=%px\n", req->data[wbi].va, req->data[wbi].hpa, bi, f8, (u64)get_unaligned_le64( src + GSP_PHYS_PAGE_SIZE - 8), - bounce_csum_in[bi], csum, + bounce_csum_in[bi], fp, rhave ? "ledger" : "none", rpa, rap, rval, (void *)gsp_sema_gr_shadow_base()); @@ -21621,7 +33208,8 @@ gr_page_unmapped: * evidence). */ if (gsp_phys_gr_copywit_enabled() && - !gsp_phys_gr_page_all_zero(csum, src)) { + !gsp_phys_gr_page_all_zero( + gsp_phys_gr_page_sum32(src), src)) { u32 wi; for (wi = 0; wi < gsp_phys_gr_outwit_n; @@ -21657,7 +33245,7 @@ gr_page_unmapped: */ if (changed == 0) { gsp_phys_gr_prev_out_bi = bi; - gsp_phys_gr_prev_out_csum = csum; + gsp_phys_gr_prev_out_csum = fp; gsp_phys_gr_prev_out_valid = true; } changed++; @@ -21673,7 +33261,49 @@ gr_page_unmapped: if (!gsp_sema_gr_write_guard(req->data[wbi].va, req->data[wbi].hpa, "gr-bounce-copy-out")) { - skipped++; + /* + * CONT 18: a migrated screen page - follow + * the VA's current mapping instead of + * dropping the frame (see the knob). + */ + bool followed = false; + + if (READ_ONCE(gsp_phys_gr_stale_write)) { + u64 sva = 0; + u32 snp = 0; + + if (gsp_sema_surf_va_get(&sva, &snp) && + sva && + req->data[wbi].va >= sva && + req->data[wbi].va < + sva + ((u64)snp << 12)) { + u64 rpa = 0, rval = 0; + u32 rap = 0; + + if (gsp_sema_gr_va_snap( + req->data[wbi].va, + &rpa, &rap, &rval) && + rpa) { + memcpy((u8 *)__va(rpa), + src, + GSP_PHYS_PAGE_SIZE); + followed = true; + if (gsp_phys_gr_stale_log < + 64u) { + gsp_phys_gr_stale_log++; + pr_info("vgsp-gr: WGUARD-STALE-WRITE va=0x%llx used=0x%llx now=0x%llx h=0x%x push=0x%llx\n", + req->data[wbi].va, + req->data[wbi].hpa, + rpa, + req->h_object, + req->push_va); + } + wrote++; + } + } + } + if (!followed) + skipped++; continue; } memcpy((u8 *)__va(req->data[wbi].hpa), src, @@ -21688,13 +33318,13 @@ gr_page_unmapped: * write target and the reader's page are different * (aperture/staleness disagreement), not a lost write. */ - if (outwit < GSP_PHYS_GR_COPYWIT_MAX) { + if (outwit_chg < GSP_PHYS_GR_COPYWIT_CHG_MAX) { u64 rpa2 = 0, rval2 = 0; u32 rap2 = 0; if (gsp_sema_gr_va_snap(req->data[wbi].va, &rpa2, &rap2, &rval2)) { - outwit++; + outwit_chg++; pr_info("vgsp-gr: OUT-PAGE-POST va=0x%llx hpa=0x%llx res_pa=0x%llx ap=%u val8=%016llx vbase=%px\n", req->data[wbi].va, req->data[wbi].hpa, rpa2, @@ -21738,8 +33368,7 @@ gr_page_unmapped: if (bi == GSP_PHYS_GR_DATA_BI_NONE) continue; wp = gsp_phys_chunk_page_va(dchunk, bi); - for (k = 0; k < GSP_PHYS_PAGE_SIZE; k += 4) - wc += get_unaligned_le32(wp + k); + (void)k; (void)wc; pr_info("vgsp-gr: window[%u] reloc=0x%llx first=%08x %08x %08x %08x changed=%u\n", bi, req->data[wi].reloc_va, @@ -21747,7 +33376,9 @@ gr_page_unmapped: get_unaligned_le32(wp + 4), get_unaligned_le32(wp + 8), get_unaligned_le32(wp + 12), - wc != bounce_csum_in[bi]); + gsp_phys_gr_fnv1a64( + wp, GSP_PHYS_PAGE_SIZE) != + bounce_csum_in[bi]); } } pr_info("vgsp-gr: bounce copy-out: %u pages written back, %u changed by engine, %u unchanged-skipped, %u alias-skipped, %u stale-skipped (first bi: %u %u %u %u %u %u %u %u)\n", @@ -21770,7 +33401,7 @@ gr_page_unmapped: for (gi = 0; gi < gsp_phys_gr_gap_n; gi++) { u32 bi = gsp_phys_gr_gap[gi].bi; const u8 *src = gsp_phys_chunk_page_va(dchunk, bi); - u32 ci, csum = 0; + u64 fp; if (!src) continue; @@ -21783,9 +33414,8 @@ gr_page_unmapped: gskipped++; continue; } - for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) - csum += get_unaligned_le32(src + ci); - if (csum == bounce_csum_in[bi]) { + fp = gsp_phys_gr_fnv1a64(src, GSP_PHYS_PAGE_SIZE); + if (fp == bounce_csum_in[bi]) { /* Engine did not touch it: nothing to ship. */ gunchanged++; continue; @@ -22009,6 +33639,181 @@ static void gsp_phys_rmcall_ctx_diag_decode(const char *tag) ramfc_valid); } +/* + * SESSION 153-CONTINUED-96: secondary COPY0 lane enable. On a GR0 ladder + * the ladder channel speaks GR and CE class methods are never legal on it, + * so the ladder's own FBMEM reads (gsp_phys_ctx_readback_try) need a second + * engine. Default 1 builds the lane (best-effort: a failed build only + * disables the readback, the GR ladder is unchanged); 0 skips it entirely. + */ +static bool gsp_phys_rmcall_ce_lane = 1; +module_param_named(gsp_phys_rmcall_ce_lane, gsp_phys_rmcall_ce_lane, bool, + 0644); +MODULE_PARM_DESC(gsp_phys_rmcall_ce_lane, + "Session 153-CONTINUED-96: build and schedule the secondary COPY0 lane (COPY0 TSG + channel on the ladder VAS, engineType 9) on a GR0 ladder so the ladder's own CE copy blocks (engine FBMEM readback) have a legal engine (default 1; 0 disables the lane, the GR ladder is unchanged)"); + +static bool gsp_phys_rmcall_ce_lane_value(void) +{ + return gsp_phys_rmcall_ce_lane; +} + +/* + * SESSION 153-CONTINUED-96: secondary COPY0 lane bringup (GR0 ladders only). + * + * The ladder's own FBMEM readback (and any later diagnostic copy of engine + * state the host cannot aperture) needs a CE engine, but CE class methods + * are never legal on the GR0 ladder channel. This builds the smallest + * second lane on the SAME VAS: its own COPY0 TSG (parent device, same + * ladder VAS) and its own V5-shape channel (engineType 9, TSG parent, + * lane memdescs, GPFIFO at VA_CE_GPFIFO), then runs the historical + * COPY0 stage-4 sequence - BIND(COPY0) -> GPFIFO_SCHEDULE -> + * TSG_GPFIFO_SCHEDULE -> notif index (advisory) -> GET_WORK_SUBMIT_TOKEN - + * and stores the lane's own work-submit token. Best-effort by design: a + * failure is logged and the GR ladder continues unchanged (no stage-4 + * abort, no retry); rmcall_ce_lane_valid stays false and the readback is + * skipped. The PDB slots for the lane's GPFIFO/PB pages were planted in + * pdb_build (stage 3, before any engine fetch). + */ +static int gsp_phys_rmcall_ce_lane_bringup(void __iomem *bar0) +{ + struct gsp_phys_rmcall_tsg_params tsg; + struct gsp_phys_rmcall_channel_params ch; + struct gsp_phys_rmcall_channel_variant v; + u32 token = 0, rpc_result = 0, status = 0; + u8 sched_params[2] = { 1, 0 }; + u32 bind_params = GSP_PHYS_RMCALL_ENGINE_COPY0; + u32 notif_index = GSP_PHYS_RMCALL_NOTIF_INDEX; + int ret; + + if (!gsp_phys_rmcall_engine_is_gr()) + return 0; /* COPY0 ladder: the lane already is it */ + if (!gsp_phys_rmcall_ce_lane_value()) + return 0; + if (boot_stage.rmcall_ce_lane_valid) + return 0; + + /* (1) TSG on the ladder VAS bound to the COPY0 engine. */ + memset(&tsg, 0, sizeof(tsg)); + tsg.h_va_space = cpu_to_le32(boot_stage.rmcall_vas); + tsg.engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_COPY0); + ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client, + boot_stage.rmcall_device, + GSP_PHYS_RMCALL_HANDLE_CE_TSG, + GSP_PHYS_RMCALL_CLASS_TSG, &tsg, sizeof(tsg), + false, NULL, "KEPLER_CHANNEL_GROUP_A (CE lane)", + true, NULL, NULL); + if (ret) { + pr_warn("vgsp-boot: rm-call: CE lane: COPY0 TSG alloc failed (%d); the lane is not built and the GR ladder continues unchanged\n", + ret); + return ret; + } + boot_stage.rmcall_ce_tsg = GSP_PHYS_RMCALL_HANDLE_CE_TSG; + + /* (2) The lane channel: V5 shape, COPY0 engine, lane memdescs. */ + memset(&v, 0, sizeof(v)); + v.name = "ce-lane-tsg-vas-pdb-skip-scrubber"; + v.id = 5; + v.tsg_parent = true; + v.flags = GSP_PHYS_RMCALL_CHANNEL_FLAG_SKIP_SCRUBBER; + v.internal_flags = GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL; + v.server_mem = false; + v.handle = GSP_PHYS_RMCALL_HANDLE_CE_CHANNEL; + gsp_phys_rmcall_channel_params_init(&ch, &v); + ch.engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_COPY0); + ch.h_context_share = 0; /* a CE channel never joins a GR share */ + ch.gp_fifo_offset = cpu_to_le64(GSP_PHYS_RMCALL_VA_CE_GPFIFO); + ch.instance_mem = gsp_phys_rmcall_mem_desc( + GSP_PHYS_RMCALL_BUF_CE_INSTANCE); + ch.userd_mem = gsp_phys_rmcall_mem_desc( + GSP_PHYS_RMCALL_BUF_CE_USERD); + ch.ramfc_mem = gsp_phys_rmcall_mem_desc( + GSP_PHYS_RMCALL_BUF_CE_RAMFC); + ch.mthdbuf_mem = gsp_phys_rmcall_mem_desc( + GSP_PHYS_RMCALL_BUF_CE_MTHDBUF); + ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client, + boot_stage.rmcall_ce_tsg, + GSP_PHYS_RMCALL_HANDLE_CE_CHANNEL, + GSP_PHYS_RMCALL_CLASS_CHANNEL, &ch, sizeof(ch), + false, NULL, "CE lane channel", true, NULL, + &status); + if (ret) { + pr_warn("vgsp-boot: rm-call: CE lane: channel alloc failed (%d, status=0x%x); the lane is not built and the GR ladder continues unchanged\n", + ret, status); + return ret; + } + boot_stage.rmcall_ce_channel = GSP_PHYS_RMCALL_HANDLE_CE_CHANNEL; + + /* (3) BIND to the COPY0 engine (the historical stage-4 sequence). */ + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_ce_channel, + GSP_PHYS_RMCALL_CTRL_BIND_CHANNEL, + &bind_params, sizeof(bind_params), 16, NULL, 0, + &rpc_result, &status, + "CE lane BIND_CHANNEL COPY0"); + if (ret) { + pr_warn("vgsp-boot: rm-call: CE lane: BIND_CHANNEL (engineType=0x%x) failed (rpc_result=0x%x status=0x%x); lane unusable, GR ladder continues\n", + bind_params, rpc_result, status); + return ret; + } + + /* (4) GPFIFO_SCHEDULE + TSG_GPFIFO_SCHEDULE. */ + rpc_result = 0; + status = 0; + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_ce_channel, + GSP_PHYS_RMCALL_CTRL_GPFIFO_SCHEDULE, + sched_params, sizeof(sched_params), 16, NULL, 0, + &rpc_result, &status, "CE lane GPFIFO_SCHEDULE"); + if (ret) { + pr_warn("vgsp-boot: rm-call: CE lane: GPFIFO_SCHEDULE failed (rpc_result=0x%x status=0x%x); lane unusable, GR ladder continues\n", + rpc_result, status); + return ret; + } + rpc_result = 0; + status = 0; + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_ce_tsg, + GSP_PHYS_RMCALL_CTRL_TSG_GPFIFO_SCHEDULE, + sched_params, sizeof(sched_params), 16, NULL, 0, + &rpc_result, &status, "CE lane TSG_GPFIFO_SCHEDULE"); + if (ret) + pr_warn("vgsp-boot: rm-call: CE lane: TSG_GPFIFO_SCHEDULE advisory failure (rpc_result=0x%x status=0x%x); continuing\n", + rpc_result, status); + + /* (5) notif index (advisory) + the lane's own work-submit token. */ + rpc_result = 0; + status = 0; + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_ce_channel, + GSP_PHYS_RMCALL_CTRL_SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX, + ¬if_index, sizeof(notif_index), 16, NULL, 0, + &rpc_result, &status, + "CE lane SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX"); + if (ret) + pr_info("vgsp-boot: rm-call: CE lane: SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX advisory failure (rpc_result=0x%x status=0x%x); continuing to the token\n", + rpc_result, status); + token = 0; + rpc_result = 0; + status = 0; + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_ce_channel, + GSP_PHYS_RMCALL_CTRL_GET_WORK_SUBMIT_TOKEN, + &token, sizeof(token), 28, &token, sizeof(token), + &rpc_result, &status, "CE lane GET_WORK_SUBMIT_TOKEN"); + if (ret && token != 0) { + pr_warn("vgsp-boot: rm-call: CE lane: GET_WORK_SUBMIT_TOKEN failed (rpc_result=0x%x status=0x%x) but the echoed params carry token=0x%x; proceeding\n", + rpc_result, status, token); + ret = 0; + } + if (ret || !token) { + pr_warn("vgsp-boot: rm-call: CE lane: no work-submit token (ret=%d rpc_result=0x%x status=0x%x token=0x%x); lane unusable, GR ladder continues\n", + ret, rpc_result, status, token); + return ret ? ret : -EIO; + } + boot_stage.rmcall_ce_token = token; + boot_stage.rmcall_ce_lane_valid = true; + pr_info("vgsp-boot: rm-call: CE lane READY (secondary COPY0 on the ladder VAS): hTsg=0x%x hChannel=0x%x token=0x%x (runlistId=%u chId=0x%x) gpFifoVa=0x%llx pbVa=0x%llx; CE copies on it read FBMEM the host cannot aperture\n", + boot_stage.rmcall_ce_tsg, boot_stage.rmcall_ce_channel, token, + (token >> 16) & 0x7fu, token & 0xfffu, + GSP_PHYS_RMCALL_VA_CE_GPFIFO, GSP_PHYS_RMCALL_VA_CE_PB); + return 0; +} + static int gsp_phys_rmcall_stage4(void __iomem *bar0) { struct gsp_phys_boot_chunk *userd_c = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD]; @@ -22190,6 +33995,22 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) boot_stage.rmcall_token = token; /* + * SESSION 153-CONTINUED-96: on a GR0 ladder, build the secondary + * COPY0 lane (own TSG/channel on the same VAS, own token) so the + * ladder's own CE copy blocks - the engine-context FBMEM readback - + * have a legal engine. Best-effort: a failure disables the readback + * only; the GR ladder below is byte-identical. On a COPY0 ladder the + * bringup is a no-op (the ladder channel already is the CE lane). + */ + { + int ce_ret = gsp_phys_rmcall_ce_lane_bringup(bar0); + + if (ce_ret) + pr_warn("vgsp-boot: rm-call: CE lane bringup failed (%d); engine FBMEM readback unavailable this boot (GR ladder unchanged)\n", + ce_ret); + } + + /* * H1: a GR0 ladder releases a semaphore on the compute engine instead * of running a CE copy. The dedicated helper performs the same guarded * submission (gr=true), doorbell and module-sem poll, logs REAL GR SEM @@ -22259,7 +34080,7 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) gsp_phys_status_drain(NULL); return -ETIMEDOUT; } - msleep(GSP_PHYS_RMCALL_POLL_MS); + gsp_phys_rmcall_wait_quantum(); } if (memcmp(gsp_phys_chunk_page_va(src, 0), @@ -22736,6 +34557,8 @@ static void gsp_phys_rmcall_fini(struct pci_dev *pdev) gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_pt[i]); gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_shadow); gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_qmdwin); + gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_pb2); + gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_probe); /* * B-2: the old 208-page bounce window (previously leaked here) and the * grown data-bounce region chunks. The region's PT chunk and data chunk @@ -22880,6 +34703,17 @@ static int gsp_phys_state_show(struct seq_file *m, void *v) boot_stage.rmcall_gr_promote_step, boot_stage.rmcall_gr_promote_size, boot_stage.rmcall_gr_hvirt); + /* SESSION 153-CONTINUED-96: secondary COPY0 lane (GR0 ladders only). */ + if (boot_stage.rmcall_ce_tsg || boot_stage.rmcall_ce_lane_valid) + seq_printf(m, + "rm_call_ce_lane: valid=%u hTsg=0x%x hChannel=0x%x token=0x%x gpFifoVa=0x%llx pbVa=0x%llx ceLaneParam=%u\n", + boot_stage.rmcall_ce_lane_valid, + boot_stage.rmcall_ce_tsg, + boot_stage.rmcall_ce_channel, + boot_stage.rmcall_ce_token, + GSP_PHYS_RMCALL_VA_CE_GPFIFO, + GSP_PHYS_RMCALL_VA_CE_PB, + gsp_phys_rmcall_ce_lane_value() ? 1u : 0u); seq_printf(m, "rm_call_guest: gpa=0x%llx hpa=0x%llx g2h_ok=%u w2g_ok=%u selftest_ok=%u gpa_valid=%u phys_domain_identity=%d\n", boot_stage.rmcall_guest_gpa, boot_stage.rmcall_guest_hpa, @@ -23769,6 +35603,91 @@ static const struct file_operations gsp_phys_guest_gpa_fops = { }; /* + * gsp-phys-shared-bit0 (write-only, mode 0200): development-harness assist + * for the NVIDIA X driver's per-client shared-data slot. + * + * Every direct GLX client receives a shared memory page from the X driver + * (nvidia_drv.so creates a memfd and passes it over a unix socket) whose + * first words are {u32 zero, u32 cookie, u32 state, u32 zero, u32 hClient, + * u32 hObject}; glcore's device init reads the state word and skips its own + * class-c46f channel creation while bit 0 of it is set. In this emulated + * environment the X driver publishes state=3 (bits 0+1) and direct clients + * therefore never allocate a channel and never render. The value is written + * by the X driver before the fd reaches the client, so it cannot be + * corrected from the client side. + * + * Input: "0x<hex GPA>" (4 KB aligned) naming the guest page that carries the + * slot. The GPA must fall inside a range registered through + * gsp-phys-guest-ram; it is translated with gsp_phys_gpa_to_hpa_vgpu() + * (first open mdev) and the state word is cleared of bit 0 through the + * kernel direct map (the translator validates pfn_valid, so __va() is a + * valid kernel mapping for the shmem-backed guest RAM). As a sanity guard + * the write is only performed when the page's +0x10 word looks like an RM + * client handle (0xc1d0xxxx / 0xc1e0xxxx) or the token carries a "force" + * suffix. Old and new values are always logged. + * + * This is a diagnostic aid for the development VM, matching the spirit of + * the existing guest-GPA pin; it is not part of the product behaviour. + */ +static ssize_t gsp_phys_shared_bit0_write(struct file *file, + const char __user *ubuf, size_t len, + loff_t *ppos) +{ + char buf[48]; + u64 gpa, hpa = 0; + u32 *word; + u32 h1032; + bool force = false; + int ret; + + if (len == 0 || len >= sizeof(buf)) + return -EINVAL; + if (copy_from_user(buf, ubuf, len)) + return -EFAULT; + buf[len] = '\0'; + + { + char *p = strim(buf); + char *tok = strsep(&p, " \t"); + + if (p && strstr(p, "force")) + force = true; + if (kstrtoull(tok, 0, &gpa) || !PAGE_ALIGNED(gpa)) { + pr_info("vgsp-boot: gsp-phys-shared-bit0: write ignored: expected \"0x<hex GPA>\" (4 KB aligned, optional 'force')\n"); + return -EINVAL; + } + } + + ret = gsp_phys_gpa_to_hpa_vgpu(NULL, gpa, &hpa); + if (ret) { + pr_err("vgsp-boot: gsp-phys-shared-bit0: GPA 0x%llx translation failed (%d: no open mdev / GPA not inside a gsp-phys-guest-ram range / not RAM)\n", + gpa, ret); + return -ENOENT; + } + + word = (u32 *)__va(hpa + 8); + h1032 = *(u32 *)__va(hpa + 0x10); + if (!force && (h1032 >> 16) != 0xc1d0 && (h1032 >> 16) != 0xc1e0) { + pr_info("vgsp-boot: gsp-phys-shared-bit0: gpa=0x%llx word=0x%x but +0x10=0x%x does not look like an RM client handle; refusing (append 'force' to override)\n", + gpa, *word, h1032); + return -EINVAL; + } + + pr_info("vgsp-boot: gsp-phys-shared-bit0: gpa=0x%llx hpa=0x%llx hClient=0x%x word=0x%x -> 0x%x\n", + gpa, hpa, h1032, *word, *word & ~1u); + *word &= ~1u; + return len; +} + +static const struct file_operations gsp_phys_shared_bit0_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = gsp_phys_shared_bit0_write, + .llseek = no_llseek, +}; + + +/* * gsp-phys-guest-ram (write-only, mode 0200): configure a guest RAM backing * file that gsp_phys_gpa_to_hpa_vgpu() translates through (QEMU * memory-backend-file,...,share=yes). One write is @@ -23971,12 +35890,54 @@ static int gsp_phys_reg_probe_show(struct seq_file *m, void *unused) 0x0040070cu); fe_hww_esr = gsp_phys_reg_probe_line(m, bar0, "FE_HWW_ESR", 0x00404000u); + /* + * SESSION 153-CONTINUED-52: MME exception latches (see + * gsp_phys_mme_regs_log). + */ + (void)gsp_phys_reg_probe_line(m, bar0, "PGRAPH_PRI_MME_HWW_ESR", + 0x00404490u); + (void)gsp_phys_reg_probe_line(m, bar0, + "PGRAPH_PRI_MME_HWW_ESR_INFO", + 0x00404494u); + (void)gsp_phys_reg_probe_line(m, bar0, + "PGRAPH_PRI_MME_HWW_ESR_INFO2", + 0x0040449cu); fe_chip_def = gsp_phys_reg_probe_line(m, bar0, "FE_CHIP_DEF_INFO", 0x00404030u); fe_go_idle = gsp_phys_reg_probe_line(m, bar0, "FE_GO_IDLE_INFO", 0x00404194u); sked_hww_esr = gsp_phys_reg_probe_line(m, bar0, "SKED_HWW_ESR", 0x00407020u); + /* + * SESSION 153-CONTINUED-11: the shader-program-header fault window. + * DS_HWW_ESR bit n = "the SPH with index n failed validation" (hwref: + * NV_PGRAPH_PRI_DS_HWW_ESR_SPH0_ERR..SPH23_ERR, ESR_2 for 24..30); + * the SPH_INFO registers snapshot the START address of each stage's + * faulting SPH, so a post-fault read names the exact VA the engine + * fetched. + */ + (void)gsp_phys_reg_probe_line(m, bar0, "DS_HWW_ESR", + 0x00405840u); + (void)gsp_phys_reg_probe_line(m, bar0, "DS_HWW_ESR_2", + 0x00405848u); + (void)gsp_phys_reg_probe_line(m, bar0, + "DS_HWW_ESR_SPH_INFO_PROGRAM_REGION_A", + 0x00405c1cu); + (void)gsp_phys_reg_probe_line(m, bar0, + "DS_HWW_ESR_SPH_INFO_PROGRAM_REGION_B", + 0x00405c20u); + { + u32 si; + + for (si = 0; si < 6u; si++) { + gsp_phys_reg_probe_line(m, bar0, + "DS_HWW_ESR_SPH_INFO_STAGE_START", + 0x00405c00u + si * 4u); + gsp_phys_reg_probe_line(m, bar0, + "DS_HWW_ESR_SPH_INFO_STAGE_START_UPPER", + 0x00405c24u + si * 4u); + } + } sked_activity = gsp_phys_reg_probe_line(m, bar0, "SKED_ACTIVITY", 0x00407054u); sked_cwd0 = gsp_phys_reg_probe_line(m, bar0, @@ -24131,6 +36092,306 @@ static const struct file_operations gsp_phys_reg_probe_fops = { .release = single_release, }; +/* + * SESSION 153-CONTINUED-91: DS SPH fault spy. + * + * The DS_HWW_ESR SPH<n>_ERR bits (bit n = "the SPH with index n failed + * validation", hwref NV_PGRAPH_PRI_DS_HWW_ESR_SPH0_ERR..SPH23_ERR) and the + * per-stage SPH_INFO snapshot (STAGE0..STAGE5 START/UPPER at 0x405c00+si*4 / + * 0x405c24+si*4, PROGRAM_REGION_A/B at 0x405c1c/c20) are latched by the DS + * at fault time and consumed+cleared by the real GSP-RM exception handler + * before our relayed fault packet arrives (the fault-time register read in + * gsp_phys_fault_regs_log sees ESR=0x80000000 and all-zero stage starts + * one millisecond later). The RM's own diagnostic prints those STAGE + * registers (grErrorPrintDsSphInfo_GM200), so they name the exact SPH VAs + * the DS rejected - the missing datum behind the r1277+ Shader Program + * Header wall (CONT-90/91 could only move the fault SET by content, never + * identify the SPH entries). + * + * Reading this file runs a bounded tight poll of DS_HWW_ESR and snapshots + * the ESR pair, the six stage starts and the region pair on every sample + * that has any SPH bit set. Development-only, read-only, no state touched; + * run `cat /sys/kernel/debug/vgsp/gsp-phys-ds-spy` while the client draws. + * Duration via sema_ds_spy_ms (default 20 s). + */ +static u32 gsp_phys_ds_spy_ms = 20000; +module_param_named(sema_ds_spy_ms, gsp_phys_ds_spy_ms, uint, 0644); +MODULE_PARM_DESC(sema_ds_spy_ms, + "gsp-phys-ds-spy sampling duration in milliseconds (default 20000)"); + +#define GSP_PHYS_DS_ESR 0x00405840u +#define GSP_PHYS_DS_ESR_2 0x00405848u +#define GSP_PHYS_DS_REPORT_MASK 0x00405844u +#define GSP_PHYS_DS_REPORT_MASK_2 0x0040584cu +#define GSP_PHYS_DS_SPH_STAGE_LO 0x00405c00u +#define GSP_PHYS_DS_SPH_STAGE_UP 0x00405c24u +#define GSP_PHYS_DS_SPH_REGION_A 0x00405c1cu +#define GSP_PHYS_DS_SPH_REGION_B 0x00405c20u + +static int gsp_phys_ds_spy_show(struct seq_file *m, void *unused) +{ + void __iomem *bar0 = gsp_phys_gpu_bar0(); + u64 t0; + u32 hits = 0, samples = 0, maskholds = 0, mask0_last = 0, mask2_last = 0; + u32 ms = READ_ONCE(gsp_phys_ds_spy_ms); + + if (!bar0) + return -ENODEV; + if (!ms) + ms = 20000; + seq_printf(m, "# DS-SPY begin duration_ms=%u (poll DS_HWW_ESR 0x%08x)\n", + ms, GSP_PHYS_DS_ESR); + seq_printf(m, "# DS-SPY masks esr=%08x esr2=%08x mask=%08x mask2=%08x\n", + readl(bar0 + GSP_PHYS_DS_ESR), + readl(bar0 + GSP_PHYS_DS_ESR_2), + readl(bar0 + GSP_PHYS_DS_REPORT_MASK), + readl(bar0 + GSP_PHYS_DS_REPORT_MASK_2)); + t0 = ktime_get_ns(); + while (ktime_get_ns() - t0 < (u64)ms * 1000000ull) { + u32 esr = readl(bar0 + GSP_PHYS_DS_ESR); + u32 m0 = readl(bar0 + GSP_PHYS_DS_REPORT_MASK); + u32 m2 = readl(bar0 + GSP_PHYS_DS_REPORT_MASK_2); + + /* + * SESSION 153-CONTINUED-97: hold the HWW report masks + * cleared while sampling. A pending report POISONS the + * whole PRI block (every read returns 0xbadf1201), which + * hid the SPH stage snapshot at the validation instant + * (r1325 spy2: all 16+ hits badf1201, masks back at + * 00ffffff). The mask is (re)armed by the context load / + * the RM setup, not only by our replay path, so re-clear + * it every sample; the re-clear count is logged so the + * capture's provenance is visible. NOTE (r1326): poison + * still appeared on one pass despite 29 re-clears, so a + * clean latch is not guaranteed and the store-side read + * remains the authoritative fault-time datum. + * Development-only diagnostic tap. + */ + if (m0 || m2) { + iowrite32(0u, bar0 + GSP_PHYS_DS_REPORT_MASK); + iowrite32(0u, bar0 + GSP_PHYS_DS_REPORT_MASK_2); + maskholds++; + mask0_last = m0; + mask2_last = m2; + } + samples++; + if (esr & 0x7fffffffu) { + u32 esr2 = readl(bar0 + GSP_PHYS_DS_ESR_2); + u32 esr_a, s[6], u[6], ra, rb; + u32 si; + + for (si = 0; si < 6u; si++) { + s[si] = readl(bar0 + + GSP_PHYS_DS_SPH_STAGE_LO + + si * 4u); + u[si] = readl(bar0 + + GSP_PHYS_DS_SPH_STAGE_UP + + si * 4u); + } + ra = readl(bar0 + GSP_PHYS_DS_SPH_REGION_A); + rb = readl(bar0 + GSP_PHYS_DS_SPH_REGION_B); + esr_a = readl(bar0 + GSP_PHYS_DS_ESR); + hits++; + seq_printf(m, + "DS-SPY hit %u t=%lldus esr=%08x esr2=%08x esr_after=%08x region=%08x/%08x stage=%08x,%08x,%08x,%08x,%08x,%08x upper=%02x,%02x,%02x,%02x,%02x,%02x\n", + hits, div_u64(ktime_get_ns() - t0, 1000), + esr, esr2, esr_a, ra, rb, + s[0], s[1], s[2], s[3], s[4], s[5], + u[0] & 0xffu, u[1] & 0xffu, u[2] & 0xffu, + u[3] & 0xffu, u[4] & 0xffu, u[5] & 0xffu); + udelay(20); + } + } + seq_printf(m, "# DS-SPY end samples=%u hits=%u maskholds=%u last_mask=%08x/%08x\n", + samples, hits, maskholds, mask0_last, mask2_last); + return 0; +} + +static int gsp_phys_ds_spy_open(struct inode *inode, struct file *file) +{ + return single_open(file, gsp_phys_ds_spy_show, NULL); +} + +static const struct file_operations gsp_phys_ds_spy_fops = { + .owner = THIS_MODULE, + .open = gsp_phys_ds_spy_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +/* + * SESSION 153-CONTINUED-93: live PGRAPH register lab (debugfs + * gsp-phys-pgraph-rw). Read: a fixed snapshot of the DS/PD/FIFO registers + * the SPH investigation reads. Write: "0xADDR [0xVALUE]" (hex) inside the + * PGRAPH PRI window [0x400000,0x410000) - reads the old value, writes the + * new one (when given) and reads back, logging both. Diagnostic only; + * development bring-up tool. Writes outside the window are refused. + */ +static int gsp_phys_pgraph_show(struct seq_file *m, void *unused) +{ + void __iomem *bar0 = gsp_phys_gpu_bar0(); + + if (!bar0) + return -ENODEV; + seq_printf(m, "# PGRAPH register snapshot\n"); + seq_printf(m, "PD_CYA 0x00406020 %08x\n", readl(bar0 + 0x00406020u)); + seq_printf(m, "PD_HWW_ESR 0x00406018 %08x\n", readl(bar0 + 0x00406018u)); + seq_printf(m, "DS_HWW_ESR 0x00405840 %08x\n", readl(bar0 + 0x00405840u)); + seq_printf(m, "DS_HWW_ESR_2 0x00405848 %08x\n", readl(bar0 + 0x00405848u)); + seq_printf(m, "DS_REPORT_MASK 0x00405844 %08x\n", readl(bar0 + 0x00405844u)); + seq_printf(m, "DS_REPORT_MASK2 0x0040584c %08x\n", readl(bar0 + 0x0040584cu)); + seq_printf(m, "DS_SPH_REGION_A 0x00405c1c %08x\n", readl(bar0 + 0x00405c1cu)); + seq_printf(m, "DS_SPH_REGION_B 0x00405c20 %08x\n", readl(bar0 + 0x00405c20u)); + seq_printf(m, "DS_SPH_STAGE1 0x00405c04 %08x\n", readl(bar0 + 0x00405c04u)); + seq_printf(m, "DS_SPH_STAGE2 0x00405c08 %08x\n", readl(bar0 + 0x00405c08u)); + seq_printf(m, "DS_SPH_STAGE5 0x00405c14 %08x\n", readl(bar0 + 0x00405c14u)); + seq_printf(m, "PGRAPH_STATUS 0x00400700 %08x\n", readl(bar0 + 0x00400700u)); + seq_printf(m, "GR_FIFO_STATUS 0x00400504 %08x\n", readl(bar0 + 0x00400504u)); + return 0; +} + +static int gsp_phys_pgraph_open(struct inode *inode, struct file *file) +{ + return single_open(file, gsp_phys_pgraph_show, NULL); +} + +static ssize_t gsp_phys_pgraph_write(struct file *file, + const char __user *ubuf, size_t len, + loff_t *off) +{ + char buf[64]; + unsigned long long addr = 0, val = 0; + void __iomem *bar0 = gsp_phys_gpu_bar0(); + int n, has_val; + u32 old, now; + + if (!bar0) + return -ENODEV; + if (len == 0 || len >= sizeof(buf)) + return -EINVAL; + if (copy_from_user(buf, ubuf, len)) + return -EFAULT; + buf[len] = '\0'; + n = sscanf(buf, "%llx %llx", &addr, &val); + if (n < 1) + return -EINVAL; + has_val = (n >= 2); + if (addr < 0x00400000ull || addr >= 0x00410000ull) + return -EINVAL; + old = readl(bar0 + addr); + if (has_val) { + writel((u32)val, bar0 + addr); + now = readl(bar0 + addr); + pr_warn("vgsp-boot: pgraph-rw 0x%08llx old=%08x new=%08x (wrote %08llx)\n", + addr, old, now, val); + } else { + pr_info("vgsp-boot: pgraph-rw 0x%08llx = %08x (read)\n", + addr, old); + } + return (ssize_t)len; +} + +static const struct file_operations gsp_phys_pgraph_fops = { + .owner = THIS_MODULE, + .open = gsp_phys_pgraph_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, + .write = gsp_phys_pgraph_write, +}; + +/* + * SESSION 153-CONTINUED-94: gsp-phys-init-push - the first large beef push + * captured at decode time (see svc_init_push_scan). Prints a header with + * the push address/length, the local-memory-run count (SET_SHADER_LOCAL_ + * MEMORY_A..D seen in ANY beef push so far; 0 = the client never emitted + * the download-state local-memory block) and the bytes as 16-byte rows. + */ +static int gsp_phys_init_push_show(struct seq_file *m, void *unused) +{ + const u8 *buf; + u32 len, h, lmem, off; + u64 va; + + buf = gsp_sema_init_cap_get(&len, &h, &va, &lmem); + if (!len) { + seq_puts(m, "vgsp-init-push: empty (no >=4 KiB beef push with non-zero head captured yet)\n"); + return 0; + } + seq_printf(m, "vgsp-init-push: h=0x%x va=0x%llx len=%u lmem_runs=%u\n", + h, va, len, lmem); + for (off = 0; off + 4u <= len; off += 16u) { + u32 k, n = min_t(u32, 4u, (len - off) / 4u); + + seq_printf(m, "%06x:", off); + for (k = 0; k < n; k++) { + u32 w; + + memcpy(&w, buf + off + k * 4u, 4); + seq_printf(m, " %08x", w); + } + seq_putc(m, '\n'); + } + return 0; +} + +static int gsp_phys_init_push_open(struct inode *inode, struct file *file) +{ + return single_open(file, gsp_phys_init_push_show, NULL); +} + +static const struct file_operations gsp_phys_init_push_fops = { + .owner = THIS_MODULE, + .open = gsp_phys_init_push_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +/* + * SESSION 153-CONTINUED-95: gsp-phys-ctx-dump - raw binary readback of the + * engine context buffer (see gsp_phys_ctx_readback_try). Empty until the + * first launch of the boot has run the CE readback. + */ +static ssize_t gsp_phys_ctx_dump_read(struct file *filp, char __user *ubuf, + size_t count, loff_t *ppos) +{ + const u8 *buf; + u32 len = 0; + + buf = gsp_phys_ctx_readback_get(&len); + if (!buf || !len) + return 0; + return simple_read_from_buffer(ubuf, count, ppos, buf, len); +} + +static const struct file_operations gsp_phys_ctx_dump_fops = { + .owner = THIS_MODULE, + .read = gsp_phys_ctx_dump_read, + .llseek = default_llseek, +}; + +/* + * SESSION 153-CONTINUED-96: gsp-phys-ctx-read - write-only re-arm for the + * one-shot engine-context readback. `echo 1 > .../gsp-phys-ctx-read` clears + * the tried flag; the next forwarded launch re-copies MAIN+PATCH with the + * current server state. The copy itself still runs only in the launch path. + */ +static ssize_t gsp_phys_ctx_read_write(struct file *filp, + const char __user *ubuf, + size_t count, loff_t *ppos) +{ + gsp_phys_ctx_readback_rearm(); + pr_info("vgsp-gr: CTX-READBACK re-armed; the next forwarded launch re-copies the engine ctx (MAIN+PATCH)\n"); + return count; +} + +static const struct file_operations gsp_phys_ctx_read_fops = { + .owner = THIS_MODULE, + .write = gsp_phys_ctx_read_write, +}; + int gsp_phys_boot_debugfs_init(void) { struct dentry *file; @@ -24223,6 +36484,20 @@ int gsp_phys_boot_debugfs_init(void) err); return (int)err; } + file = debugfs_create_file("gsp-phys-shared-bit0", 0200, + gsp_phys_debugfs_root, NULL, + &gsp_phys_shared_bit0_fops); + if (IS_ERR_OR_NULL(file)) { + long err = IS_ERR(file) ? PTR_ERR(file) : -ENODEV; + + /* + * The shared-slot bit0 assist is optional; the other + * triggers and the state snapshot stay registered. + */ + pr_warn("vgsp-boot: debugfs file gsp-phys-shared-bit0 creation failed: %ld (other triggers and state snapshot stay available)\n", + err); + return (int)err; + } file = debugfs_create_file("gsp-phys-guest-ram", 0200, gsp_phys_debugfs_root, NULL, &gsp_phys_guest_ram_fops); @@ -24279,7 +36554,57 @@ int gsp_phys_boot_debugfs_init(void) err); return (int)err; } - pr_info("vgsp-boot: debugfs files /sys/kernel/debug/vgsp/{gsp-phys-state read-only, gsp-phys-log-dump read-only, gsp-phys-status-drain read-only, gsp-phys-reg-probe read-only, gsp-phys-unload-rpc write-only, gsp-phys-rm-probe write-only, gsp-phys-rm-call write-only, gsp-phys-guest-gpa write-only, gsp-phys-guest-ram write-only} ready\n"); + /* + * SESSION 153-CONTINUED-91: the DS SPH fault spy (see the show fn). + * Diagnostic only; failure to create it must not fail the init. + */ + file = debugfs_create_file("gsp-phys-ds-spy", 0444, + gsp_phys_debugfs_root, NULL, + &gsp_phys_ds_spy_fops); + if (IS_ERR_OR_NULL(file)) + pr_warn("vgsp-boot: debugfs file gsp-phys-ds-spy creation failed (DS SPH fault spy unavailable)\n"); + /* + * SESSION 153-CONTINUED-93: live PGRAPH register lab (read snapshot + + * bounded writes inside the PGRAPH PRI window). Diagnostic only. + */ + file = debugfs_create_file("gsp-phys-pgraph-rw", 0644, + gsp_phys_debugfs_root, NULL, + &gsp_phys_pgraph_fops); + if (IS_ERR_OR_NULL(file)) + pr_warn("vgsp-boot: debugfs file gsp-phys-pgraph-rw creation failed (PGRAPH register lab unavailable)\n"); + /* + * SESSION 153-CONTINUED-94: the captured first large beef push (the + * GL client's channel-init stream) verbatim, so the local-memory + * census and any other method in it can be verified offline after + * the guest's transient pushbuffer is gone. Diagnostic only. + */ + file = debugfs_create_file("gsp-phys-init-push", 0444, + gsp_phys_debugfs_root, NULL, + &gsp_phys_init_push_fops); + if (IS_ERR_OR_NULL(file)) + pr_warn("vgsp-boot: debugfs file gsp-phys-init-push creation failed (init-push capture unavailable)\n"); + /* + * SESSION 153-CONTINUED-95: raw binary dump of the engine context + * buffer readback (64 KB from GSP_PHYS_RMCALL_VA_GR_CTX, taken once + * per boot at the first launch). 0444; empty until the first launch + * happens. Read with `dd if=...gsp-phys-ctx-dump of=/tmp/ctx.bin`. + */ + file = debugfs_create_file("gsp-phys-ctx-dump", 0444, + gsp_phys_debugfs_root, NULL, + &gsp_phys_ctx_dump_fops); + if (IS_ERR_OR_NULL(file)) + pr_warn("vgsp-boot: debugfs file gsp-phys-ctx-dump creation failed (engine ctx readback unavailable)\n"); + /* + * SESSION 153-CONTINUED-96: write-only re-arm for the readback (see + * gsp_phys_ctx_read_write). `echo 1 >` it to re-copy MAIN+PATCH at + * the next forwarded launch without a module reload. + */ + file = debugfs_create_file("gsp-phys-ctx-read", 0200, + gsp_phys_debugfs_root, NULL, + &gsp_phys_ctx_read_fops); + if (IS_ERR_OR_NULL(file)) + pr_warn("vgsp-boot: debugfs file gsp-phys-ctx-read creation failed (readback re-arm unavailable)\n"); + pr_info("vgsp-boot: debugfs files /sys/kernel/debug/vgsp/{gsp-phys-state read-only, gsp-phys-log-dump read-only, gsp-phys-status-drain read-only, gsp-phys-reg-probe read-only, gsp-phys-ds-spy read-only, gsp-phys-ctx-dump read-only, gsp-phys-ctx-read write-only, gsp-phys-unload-rpc write-only, gsp-phys-rm-probe write-only, gsp-phys-rm-call write-only, gsp-phys-guest-gpa write-only, gsp-phys-guest-ram write-only} ready\n"); return 0; } @@ -24633,8 +36958,8 @@ static_assert(offsetof(struct gsp_phys_rmcall_channel_params, process_id) == 296 "NV_CHANNEL_ALLOC_PARAMS.ProcessID must be at offset 296"); static_assert(offsetof(struct gsp_phys_rmcall_channel_params, tpc_config_id) == 360, "NV_CHANNEL_ALLOC_PARAMS.tpcConfigID must be at offset 360"); -static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 81, - "the rm-call ladder registers exactly eighty-one DMA buffers (14 ladder + 16 single-page guest scratch PTs + 48 single-page extension scratch PTs + 3 golden9g VAS levels)"); +static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 87, + "the rm-call ladder registers exactly eighty-seven DMA buffers (14 ladder + 16 single-page guest scratch PTs + 48 single-page extension scratch PTs + 3 golden9g VAS levels + 6 secondary CE lane buffers)"); static_assert((GSP_PHYS_RMCALL_HANDLE_MEM_BASE + (GSP_PHYS_RMCALL_BUF_COUNT - 1) * GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE) < 0xcaf80000u, @@ -24648,11 +36973,26 @@ static_assert(((GSP_PHYS_RMCALL_VA_BASE >> 47) & 0x3) == 0 && ((GSP_PHYS_RMCALL_VA_BASE >> 21) & 0xff) == 0 && ((GSP_PHYS_RMCALL_VA_SEM >> 21) & 0xff) == 0, "the channel PDB VA window must sit in root/PD2/PD1/PD0 index 0 (GP10X 48:47/46:38/37:29/28:21 splits)"); -static_assert(((GSP_PHYS_RMCALL_VA_GPFIFO >> 12) & 0x1ff) != - ((GSP_PHYS_RMCALL_VA_PB >> 12) & 0x1ff) && - (((GSP_PHYS_RMCALL_VA_GPFIFO >> 12) & 0x1ff) + 4) == - ((GSP_PHYS_RMCALL_VA_SEM >> 12) & 0x1ff), - "the GPFIFO, pushbuffer and CE src/dst/semaphore VAs must take the five consecutive small-PT slots 0x100..0x104"); +static_assert(((GSP_PHYS_RMCALL_VA_GPFIFO >> 12) & 0x1ff) == 0x100 && + ((GSP_PHYS_RMCALL_VA_PB >> 12) & 0x1ff) == 0x101 && + ((GSP_PHYS_RMCALL_VA_PB2 >> 12) & 0x1ff) == 0x102 && + ((GSP_PHYS_RMCALL_VA_PB3 >> 12) & 0x1ff) == 0x103 && + ((GSP_PHYS_RMCALL_VA_PB4 >> 12) & 0x1ff) == 0x104, + "the GPFIFO + four pushbuffer pages must occupy the consecutive small-PT slots 0x100..0x104 for the engine's linear fetch walk (SESSION 153-CONTINUED-77: CE src/dst/sem moved off the walk to 0x109/0x10b/0x10c)"); +/* + * SESSION 153-CONTINUED-66: the two-page pushbuffer and the moved CE SRC / + * probe pages must stay inside the same 2 MB PD0 slot and the same 128 B PTE + * fetch line as the ladder, and must not collide with each other or with the + * QMD-window pair (0x107/0x108). + */ +static_assert(((GSP_PHYS_RMCALL_VA_PB2 >> 12) & 0x1ff) == + ((GSP_PHYS_RMCALL_VA_PB >> 12) & 0x1ff) + 1, + "the pushbuffer's second page must sit directly above the first (one contiguous GPU VA fetch)"); +static_assert(((GSP_PHYS_RMCALL_VA_SRC >> 21) & 0xff) == 0 && + ((GSP_PHYS_RMCALL_VA_GR_PROBE >> 21) & 0xff) == 0 && + ((GSP_PHYS_RMCALL_VA_SRC >> 12) & 0x1ff) == 0x109 && + ((GSP_PHYS_RMCALL_VA_GR_PROBE >> 12) & 0x1ff) == 0x10a, + "the moved CE SRC page and the warm-semaphore probe page must take the free slots 0x109/0x10a in the ladder's PTE line"); static_assert(((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 47) & 0x3) == 0 && ((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 38) & 0x1ff) == 0 && ((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 29) & 0x1ff) == 0 && @@ -24689,18 +37029,20 @@ static_assert(GSP_PHYS_RMCALL_VA_GR_CTX + static_assert(sizeof(struct gsp_phys_gr_page) == 24, "the GR forwarding page descriptor must stay {u64,u64,u32,u32} = 24 bytes"); static_assert(offsetof(struct gsp_phys_gr_inline, data) == 12 && - sizeof(struct gsp_phys_gr_inline) == 80, - "the GR forwarding inline record must stay {u64,u32,u8[64]} padded to 80 bytes"); + sizeof(struct gsp_phys_gr_inline) >= + 12 + GSP_PHYS_GR_INLINE_BYTES, + "the GR forwarding inline record must stay {u64,u32,u8[N]}"); static_assert((2u + 1u + GSP_PHYS_RMCALL_GR_QMD_INLINE_WORDS + 8u) <= GSP_PHYS_RMCALL_GR_METHOD_WORDS, "the GR launch method block (compute init + SET_OBJECT + inline-QMD run + PCAS + REPORT_SEMAPHORE) must fit the bound"); -static_assert(GSP_PHYS_RMCALL_GR_INIT_WORDS + GSP_PHYS_GR_RAW_WORDS + +static_assert(GSP_PHYS_RMCALL_GR_INIT_WORDS + + GSP_PHYS_RMCALL_GR_QMD_WINDOW_WORDS + GSP_PHYS_RMCALL_GR_TAIL_WORDS <= GSP_PHYS_RMCALL_GR_METHOD_WORDS, - "the verbatim GR launch block (compute init + raw tail + PCAS + REPORT_SEMAPHORE) must fit the bound"); + "the verbatim GR launch block minimum (compute init + window + PCAS + REPORT_SEMAPHORE) must fit the bound; the per-launch runtime check bounds raw_qmd_hdr for larger preludes"); static_assert(GSP_PHYS_RMCALL_GR_METHOD_WORDS * 4u <= - GSP_PHYS_RMCALL_COPY_SIZE, - "the GR launch method block must fit one module pushbuffer page"); + 4u * GSP_PHYS_RMCALL_COPY_SIZE, + "the GR launch method block may span the four module pushbuffer pages (SESSION 153-CONTINUED-77: prelude cap 3584 + largest chunked window)"); /* * GspSystemInfo wire ABI, pinned against the reference header compile |